From 84e1b01ceb2954a74c4ea67f8695fe41b736d85a Mon Sep 17 00:00:00 2001 From: Constance Date: Mon, 28 Jun 2021 08:39:27 -0700 Subject: [PATCH 01/55] Result settings: Fix restore defaults copy (#103413) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../components/result_settings/result_settings_logic.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/result_settings/result_settings_logic.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/result_settings/result_settings_logic.ts index 13530c2c29ef0..216c43e1d3072 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/result_settings/result_settings_logic.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/result_settings/result_settings_logic.ts @@ -92,7 +92,7 @@ const RESET_CONFIRMATION_MESSAGE = i18n.translate( 'xpack.enterpriseSearch.appSearch.engine.resultSettings.confirmResetMessage', { defaultMessage: - 'This will revert your settings back to the default: all fields set to raw. The default will take over immediately and impact your search results.', + 'Are you sure you want to restore result settings defaults? This will set all fields back to raw with no limits.', } ); From 96fe9c23f878354df5b9122d48093e696f014c57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20S=C3=A1nchez?= Date: Mon, 28 Jun 2021 17:48:54 +0200 Subject: [PATCH 02/55] [Security solution][Endpoint] Get os name from host.os.name when agent type endpoint (#103450) * When type endpoint gets os type from os name instead of os family * Allow users add event filters only for endpoint events * Fixes error with wrong map function Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../pages/event_filters/store/utils.ts | 16 +++++++++------- .../pages/event_filters/test_utils/index.ts | 1 + .../components/timeline/body/actions/index.tsx | 8 ++++---- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/security_solution/public/management/pages/event_filters/store/utils.ts b/x-pack/plugins/security_solution/public/management/pages/event_filters/store/utils.ts index 6adc490b40e78..e0f9a6bcc965c 100644 --- a/x-pack/plugins/security_solution/public/management/pages/event_filters/store/utils.ts +++ b/x-pack/plugins/security_solution/public/management/pages/event_filters/store/utils.ts @@ -10,6 +10,14 @@ import type { CreateExceptionListItemSchema } from '@kbn/securitysolution-io-ts- import { Ecs } from '../../../../../common/ecs'; import { ENDPOINT_EVENT_FILTERS_LIST_ID } from '../constants'; +const osTypeBasedOnAgentType = (data?: Ecs) => { + if (data?.agent?.type?.includes('endpoint')) { + return (data?.host?.os?.name || ['windows']).map((name) => name.toLowerCase()); + } else { + return data?.host?.os?.family ?? ['windows']; + } +}; + export const getInitialExceptionFromEvent = (data?: Ecs): CreateExceptionListItemSchema => ({ comments: [], description: '', @@ -46,11 +54,5 @@ export const getInitialExceptionFromEvent = (data?: Ecs): CreateExceptionListIte namespace_type: 'agnostic', tags: ['policy:all'], type: 'simple', - // TODO: Try to fix this type casting - os_types: [ - (data && data.host ? data.host.os?.family ?? ['windows'] : ['windows'])[0] as - | 'windows' - | 'linux' - | 'macos', - ], + os_types: osTypeBasedOnAgentType(data) as Array<'windows' | 'linux' | 'macos'>, }); diff --git a/x-pack/plugins/security_solution/public/management/pages/event_filters/test_utils/index.ts b/x-pack/plugins/security_solution/public/management/pages/event_filters/test_utils/index.ts index dc235cf511157..c45d0f88927be 100644 --- a/x-pack/plugins/security_solution/public/management/pages/event_filters/test_utils/index.ts +++ b/x-pack/plugins/security_solution/public/management/pages/event_filters/test_utils/index.ts @@ -50,6 +50,7 @@ export const ecsEventMock = (): Ecs => ({ name: ['Host-tvs68wo3qc'], os: { family: ['windows'], + name: ['Windows'], }, id: ['a563b365-2bee-40df-adcd-ae84d889f523'], ip: ['10.242.233.187'], diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/actions/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/actions/index.tsx index 0a3a1cd88accc..29e00d169b4e4 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/actions/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/actions/index.tsx @@ -87,9 +87,9 @@ const ActionsComponent: React.FC = ({ ); const eventType = getEventType(ecsData); - const isEventContextMenuEnabled = useMemo( - () => !!ecsData.event?.kind && ecsData.event?.kind[0] === 'event', - [ecsData.event?.kind] + const isEventContextMenuEnabledForEndpoint = useMemo( + () => ecsData.event?.kind?.includes('event') && ecsData.agent?.type?.includes('endpoint'), + [ecsData.event?.kind, ecsData.agent?.type] ); return ( @@ -174,7 +174,7 @@ const ActionsComponent: React.FC = ({ key="alert-context-menu" ecsRowData={ecsData} timelineId={timelineId} - disabled={eventType !== 'signal' && !isEventContextMenuEnabled} + disabled={eventType !== 'signal' && !isEventContextMenuEnabledForEndpoint} refetch={refetch ?? noop} onRuleChange={onRuleChange} /> From 15b0dbff7bc27cee3bfbd938f698ac5053dc854e Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Mon, 28 Jun 2021 19:18:30 +0300 Subject: [PATCH 03/55] [TSVB] Fix references to the index pattern are not embedded when exporting a saved object (#103255) * [TSVB] Importing a dashboard with only TSVB viz on another space, breaks the dashboard Closes: #103059 * move index-pattern to constant Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- ...na-plugin-plugins-data-public.esfilters.md | 8 ++-- ...bana-plugin-plugins-data-public.eskuery.md | 2 +- ...bana-plugin-plugins-data-public.esquery.md | 2 +- ...lugin-plugins-data-public.iindexpattern.md | 2 +- ...-public.index_pattern_saved_object_type.md | 13 +++++++ .../kibana-plugin-plugins-data-public.md | 1 + ...na-plugin-plugins-data-server.esfilters.md | 8 ++-- ...bana-plugin-plugins-data-server.eskuery.md | 2 +- ...bana-plugin-plugins-data-server.esquery.md | 2 +- ...-server.index_pattern_saved_object_type.md | 13 +++++++ .../kibana-plugin-plugins-data-server.md | 1 + .../saved_objects/dashboard_migrations.ts | 29 ++++++++------ .../replace_index_pattern_reference.test.ts | 39 +++++++++++++++++++ .../replace_index_pattern_reference.ts | 22 +++++++++++ src/plugins/data/common/constants.ts | 3 ++ .../index_patterns/index_patterns.ts | 23 ++++++----- .../common/index_patterns/lib/get_title.ts | 6 ++- .../data/common/index_patterns/utils.ts | 4 +- .../search_source/extract_references.ts | 6 ++- src/plugins/data/public/index.ts | 1 + src/plugins/data/public/public.api.md | 31 ++++++++------- src/plugins/data/server/index.ts | 7 +++- .../data/server/index_patterns/utils.ts | 9 ++++- .../server/saved_objects/index_patterns.ts | 5 ++- src/plugins/data/server/server.api.md | 31 ++++++++------- .../controls_references.ts | 3 +- .../timeseries_references.ts | 6 +-- ...ualization_saved_object_migrations.test.ts | 30 ++++++++++++++ .../visualization_saved_object_migrations.ts | 29 +++++++++++--- 29 files changed, 257 insertions(+), 81 deletions(-) create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.index_pattern_saved_object_type.md create mode 100644 docs/development/plugins/data/server/kibana-plugin-plugins-data-server.index_pattern_saved_object_type.md create mode 100644 src/plugins/dashboard/server/saved_objects/replace_index_pattern_reference.test.ts create mode 100644 src/plugins/dashboard/server/saved_objects/replace_index_pattern_reference.ts diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.esfilters.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.esfilters.md index 2ca4847d6dc39..80c321ce6b320 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.esfilters.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.esfilters.md @@ -13,11 +13,11 @@ esFilters: { FILTERS: typeof FILTERS; FilterStateStore: typeof FilterStateStore; buildEmptyFilter: (isPinned: boolean, index?: string | undefined) => import("../common").Filter; - buildPhrasesFilter: (field: import("../common").IFieldType, params: any[], indexPattern: import("../common").MinimalIndexPattern) => import("../common").PhrasesFilter; - buildExistsFilter: (field: import("../common").IFieldType, indexPattern: import("../common").MinimalIndexPattern) => import("../common").ExistsFilter; - buildPhraseFilter: (field: import("../common").IFieldType, value: any, indexPattern: import("../common").MinimalIndexPattern) => import("../common").PhraseFilter; + buildPhrasesFilter: (field: import("../common").IFieldType, params: any[], indexPattern: import("../common").IndexPatternBase) => import("../common").PhrasesFilter; + buildExistsFilter: (field: import("../common").IFieldType, indexPattern: import("../common").IndexPatternBase) => import("../common").ExistsFilter; + buildPhraseFilter: (field: import("../common").IFieldType, value: any, indexPattern: import("../common").IndexPatternBase) => import("../common").PhraseFilter; buildQueryFilter: (query: any, index: string, alias: string) => import("../common").QueryStringFilter; - buildRangeFilter: (field: import("../common").IFieldType, params: import("../common").RangeFilterParams, indexPattern: import("../common").MinimalIndexPattern, formattedValue?: string | undefined) => import("../common").RangeFilter; + buildRangeFilter: (field: import("../common").IFieldType, params: import("../common").RangeFilterParams, indexPattern: import("../common").IndexPatternBase, formattedValue?: string | undefined) => import("../common").RangeFilter; isPhraseFilter: (filter: any) => filter is import("../common").PhraseFilter; isExistsFilter: (filter: any) => filter is import("../common").ExistsFilter; isPhrasesFilter: (filter: any) => filter is import("../common").PhrasesFilter; diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.eskuery.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.eskuery.md index 881a1fa803ca6..332114e637586 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.eskuery.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.eskuery.md @@ -10,6 +10,6 @@ esKuery: { nodeTypes: import("../common/es_query/kuery/node_types").NodeTypes; fromKueryExpression: (expression: any, parseOptions?: Partial) => import("../common").KueryNode; - toElasticsearchQuery: (node: import("../common").KueryNode, indexPattern?: import("../common").MinimalIndexPattern | undefined, config?: Record | undefined, context?: Record | undefined) => import("@kbn/common-utils").JsonObject; + toElasticsearchQuery: (node: import("../common").KueryNode, indexPattern?: import("../common").IndexPatternBase | undefined, config?: Record | undefined, context?: Record | undefined) => import("@kbn/common-utils").JsonObject; } ``` diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.esquery.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.esquery.md index 70805aaaaee8c..0bc9c0c12fc3a 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.esquery.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.esquery.md @@ -10,7 +10,7 @@ esQuery: { buildEsQuery: typeof buildEsQuery; getEsQueryConfig: typeof getEsQueryConfig; - buildQueryFromFilters: (filters: import("../common").Filter[] | undefined, indexPattern: import("../common").MinimalIndexPattern | undefined, ignoreFilterIfFieldNotInIndex?: boolean) => { + buildQueryFromFilters: (filters: import("../common").Filter[] | undefined, indexPattern: import("../common").IndexPatternBase | undefined, ignoreFilterIfFieldNotInIndex?: boolean) => { must: never[]; filter: import("../common").Filter[]; should: never[]; diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.iindexpattern.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.iindexpattern.md index 88d8520a373c6..ec29ef81a6e69 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.iindexpattern.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.iindexpattern.md @@ -12,7 +12,7 @@ Signature: ```typescript -export interface IIndexPattern extends MinimalIndexPattern +export interface IIndexPattern extends IndexPatternBase ``` ## Properties diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.index_pattern_saved_object_type.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.index_pattern_saved_object_type.md new file mode 100644 index 0000000000000..552d131984517 --- /dev/null +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.index_pattern_saved_object_type.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [INDEX\_PATTERN\_SAVED\_OBJECT\_TYPE](./kibana-plugin-plugins-data-public.index_pattern_saved_object_type.md) + +## INDEX\_PATTERN\_SAVED\_OBJECT\_TYPE variable + +\* + +Signature: + +```typescript +INDEX_PATTERN_SAVED_OBJECT_TYPE = "index-pattern" +``` diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.md index 7c023e756ebd5..65c4601d5faec 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.md @@ -118,6 +118,7 @@ | [fieldFormats](./kibana-plugin-plugins-data-public.fieldformats.md) | | | [fieldList](./kibana-plugin-plugins-data-public.fieldlist.md) | | | [getKbnTypeNames](./kibana-plugin-plugins-data-public.getkbntypenames.md) | Get the esTypes known by all kbnFieldTypes {Array} | +| [INDEX\_PATTERN\_SAVED\_OBJECT\_TYPE](./kibana-plugin-plugins-data-public.index_pattern_saved_object_type.md) | \* | | [indexPatterns](./kibana-plugin-plugins-data-public.indexpatterns.md) | | | [injectSearchSourceReferences](./kibana-plugin-plugins-data-public.injectsearchsourcereferences.md) | | | [isCompleteResponse](./kibana-plugin-plugins-data-public.iscompleteresponse.md) | | diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.esfilters.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.esfilters.md index d951cb2426943..d009cad9ec601 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.esfilters.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.esfilters.md @@ -11,11 +11,11 @@ esFilters: { buildQueryFilter: (query: any, index: string, alias: string) => import("../common").QueryStringFilter; buildCustomFilter: typeof buildCustomFilter; buildEmptyFilter: (isPinned: boolean, index?: string | undefined) => import("../common").Filter; - buildExistsFilter: (field: import("../common").IFieldType, indexPattern: import("../common").MinimalIndexPattern) => import("../common").ExistsFilter; + buildExistsFilter: (field: import("../common").IFieldType, indexPattern: import("../common").IndexPatternBase) => import("../common").ExistsFilter; buildFilter: typeof buildFilter; - buildPhraseFilter: (field: import("../common").IFieldType, value: any, indexPattern: import("../common").MinimalIndexPattern) => import("../common").PhraseFilter; - buildPhrasesFilter: (field: import("../common").IFieldType, params: any[], indexPattern: import("../common").MinimalIndexPattern) => import("../common").PhrasesFilter; - buildRangeFilter: (field: import("../common").IFieldType, params: import("../common").RangeFilterParams, indexPattern: import("../common").MinimalIndexPattern, formattedValue?: string | undefined) => import("../common").RangeFilter; + buildPhraseFilter: (field: import("../common").IFieldType, value: any, indexPattern: import("../common").IndexPatternBase) => import("../common").PhraseFilter; + buildPhrasesFilter: (field: import("../common").IFieldType, params: any[], indexPattern: import("../common").IndexPatternBase) => import("../common").PhrasesFilter; + buildRangeFilter: (field: import("../common").IFieldType, params: import("../common").RangeFilterParams, indexPattern: import("../common").IndexPatternBase, formattedValue?: string | undefined) => import("../common").RangeFilter; isFilterDisabled: (filter: import("../common").Filter) => boolean; } ``` diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.eskuery.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.eskuery.md index 6274eb5f4f4a5..fce25a899de8e 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.eskuery.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.eskuery.md @@ -10,6 +10,6 @@ esKuery: { nodeTypes: import("../common/es_query/kuery/node_types").NodeTypes; fromKueryExpression: (expression: any, parseOptions?: Partial) => import("../common").KueryNode; - toElasticsearchQuery: (node: import("../common").KueryNode, indexPattern?: import("../common").MinimalIndexPattern | undefined, config?: Record | undefined, context?: Record | undefined) => import("@kbn/common-utils").JsonObject; + toElasticsearchQuery: (node: import("../common").KueryNode, indexPattern?: import("../common").IndexPatternBase | undefined, config?: Record | undefined, context?: Record | undefined) => import("@kbn/common-utils").JsonObject; } ``` diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.esquery.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.esquery.md index 0d1baecb014f5..68507f3fb9b81 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.esquery.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.esquery.md @@ -8,7 +8,7 @@ ```typescript esQuery: { - buildQueryFromFilters: (filters: import("../common").Filter[] | undefined, indexPattern: import("../common").MinimalIndexPattern | undefined, ignoreFilterIfFieldNotInIndex?: boolean) => { + buildQueryFromFilters: (filters: import("../common").Filter[] | undefined, indexPattern: import("../common").IndexPatternBase | undefined, ignoreFilterIfFieldNotInIndex?: boolean) => { must: never[]; filter: import("../common").Filter[]; should: never[]; diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.index_pattern_saved_object_type.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.index_pattern_saved_object_type.md new file mode 100644 index 0000000000000..34f76d4ab13b1 --- /dev/null +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.index_pattern_saved_object_type.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) > [INDEX\_PATTERN\_SAVED\_OBJECT\_TYPE](./kibana-plugin-plugins-data-server.index_pattern_saved_object_type.md) + +## INDEX\_PATTERN\_SAVED\_OBJECT\_TYPE variable + +\* + +Signature: + +```typescript +INDEX_PATTERN_SAVED_OBJECT_TYPE = "index-pattern" +``` diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.md index 9816b884c4614..ab14abdd74e87 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.md @@ -83,6 +83,7 @@ | [esQuery](./kibana-plugin-plugins-data-server.esquery.md) | | | [exporters](./kibana-plugin-plugins-data-server.exporters.md) | | | [fieldFormats](./kibana-plugin-plugins-data-server.fieldformats.md) | | +| [INDEX\_PATTERN\_SAVED\_OBJECT\_TYPE](./kibana-plugin-plugins-data-server.index_pattern_saved_object_type.md) | \* | | [indexPatterns](./kibana-plugin-plugins-data-server.indexpatterns.md) | | | [mergeCapabilitiesWithFields](./kibana-plugin-plugins-data-server.mergecapabilitieswithfields.md) | | | [search](./kibana-plugin-plugins-data-server.search.md) | | diff --git a/src/plugins/dashboard/server/saved_objects/dashboard_migrations.ts b/src/plugins/dashboard/server/saved_objects/dashboard_migrations.ts index 4ebca5ba8965e..0bd100b3d5803 100644 --- a/src/plugins/dashboard/server/saved_objects/dashboard_migrations.ts +++ b/src/plugins/dashboard/server/saved_objects/dashboard_migrations.ts @@ -7,7 +7,7 @@ */ import semver from 'semver'; -import { get, flow } from 'lodash'; +import { get, flow, identity } from 'lodash'; import { SavedObjectAttributes, SavedObjectMigrationFn, @@ -25,7 +25,9 @@ import { convertSavedDashboardPanelToPanelState, } from '../../common/embeddable/embeddable_saved_object_converters'; import { SavedObjectEmbeddableInput } from '../../../embeddable/common'; +import { INDEX_PATTERN_SAVED_OBJECT_TYPE } from '../../../data/common'; import { SerializableValue } from '../../../kibana_utils/common'; +import { replaceIndexPatternReference } from './replace_index_pattern_reference'; function migrateIndexPattern(doc: DashboardDoc700To720) { const searchSourceJSON = get(doc, 'attributes.kibanaSavedObjectMeta.searchSourceJSON'); @@ -43,7 +45,7 @@ function migrateIndexPattern(doc: DashboardDoc700To720) { searchSource.indexRefName = 'kibanaSavedObjectMeta.searchSourceJSON.index'; doc.references.push({ name: searchSource.indexRefName, - type: 'index-pattern', + type: INDEX_PATTERN_SAVED_OBJECT_TYPE, id: searchSource.index, }); delete searchSource.index; @@ -56,7 +58,7 @@ function migrateIndexPattern(doc: DashboardDoc700To720) { filterRow.meta.indexRefName = `kibanaSavedObjectMeta.searchSourceJSON.filter[${i}].meta.index`; doc.references.push({ name: filterRow.meta.indexRefName, - type: 'index-pattern', + type: INDEX_PATTERN_SAVED_OBJECT_TYPE, id: filterRow.meta.index, }); delete filterRow.meta.index; @@ -214,12 +216,14 @@ export interface DashboardSavedObjectTypeMigrationsDeps { export const createDashboardSavedObjectTypeMigrations = ( deps: DashboardSavedObjectTypeMigrationsDeps ): SavedObjectMigrationMap => { - const embeddableMigrations = deps.embeddable - .getMigrationVersions() - .filter((version) => semver.gt(version, '7.12.0')) - .map((version): [string, SavedObjectMigrationFn] => { - return [version, migrateByValuePanels(deps, version)]; - }); + const embeddableMigrations = Object.fromEntries( + deps.embeddable + .getMigrationVersions() + .filter((version) => semver.gt(version, '7.12.0')) + .map((version): [string, SavedObjectMigrationFn] => { + return [version, migrateByValuePanels(deps, version)]; + }) + ); return { /** @@ -237,12 +241,15 @@ export const createDashboardSavedObjectTypeMigrations = ( '7.3.0': flow(migrations730), '7.9.3': flow(migrateMatchAllQuery), '7.11.0': flow(createExtractPanelReferencesMigration(deps)), - ...Object.fromEntries(embeddableMigrations), + + ...embeddableMigrations, /** * Any dashboard saved object migrations that come after this point will have to be wary of * potentially overwriting embeddable migrations. An example of how to mitigate this follows: */ - // '7.x': flow(yourNewMigrationFunction, embeddableMigrations['7.x']) + // '7.x': flow(yourNewMigrationFunction, embeddableMigrations['7.x'] ?? identity), + + '7.14.0': flow(replaceIndexPatternReference, embeddableMigrations['7.14.0'] ?? identity), }; }; diff --git a/src/plugins/dashboard/server/saved_objects/replace_index_pattern_reference.test.ts b/src/plugins/dashboard/server/saved_objects/replace_index_pattern_reference.test.ts new file mode 100644 index 0000000000000..01207fb4e3404 --- /dev/null +++ b/src/plugins/dashboard/server/saved_objects/replace_index_pattern_reference.test.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import type { SavedObjectMigrationContext, SavedObjectMigrationFn } from 'kibana/server'; + +import { replaceIndexPatternReference } from './replace_index_pattern_reference'; + +describe('replaceIndexPatternReference', () => { + const savedObjectMigrationContext = (null as unknown) as SavedObjectMigrationContext; + + test('should replace index_pattern to index-pattern', () => { + const migratedDoc = replaceIndexPatternReference( + { + references: [ + { + name: 'name', + type: 'index_pattern', + }, + ], + } as Parameters[0], + savedObjectMigrationContext + ); + + expect(migratedDoc).toMatchInlineSnapshot(` + Object { + "references": Array [ + Object { + "name": "name", + "type": "index-pattern", + }, + ], + } + `); + }); +}); diff --git a/src/plugins/dashboard/server/saved_objects/replace_index_pattern_reference.ts b/src/plugins/dashboard/server/saved_objects/replace_index_pattern_reference.ts new file mode 100644 index 0000000000000..ddd1c45841b9c --- /dev/null +++ b/src/plugins/dashboard/server/saved_objects/replace_index_pattern_reference.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { SavedObjectMigrationFn } from 'kibana/server'; +import { INDEX_PATTERN_SAVED_OBJECT_TYPE } from '../../../data/common'; + +export const replaceIndexPatternReference: SavedObjectMigrationFn = (doc) => ({ + ...doc, + references: Array.isArray(doc.references) + ? doc.references.map((reference) => { + if (reference.type === 'index_pattern') { + reference.type = INDEX_PATTERN_SAVED_OBJECT_TYPE; + } + return reference; + }) + : doc.references, +}); diff --git a/src/plugins/data/common/constants.ts b/src/plugins/data/common/constants.ts index 79a9e0ac5451b..c6bfbfc75c290 100644 --- a/src/plugins/data/common/constants.ts +++ b/src/plugins/data/common/constants.ts @@ -9,6 +9,9 @@ export const DEFAULT_QUERY_LANGUAGE = 'kuery'; export const KIBANA_USER_QUERY_LANGUAGE_KEY = 'kibana.userQueryLanguage'; +/** @public **/ +export const INDEX_PATTERN_SAVED_OBJECT_TYPE = 'index-pattern'; + export const UI_SETTINGS = { META_FIELDS: 'metaFields', DOC_HIGHLIGHT: 'doc_table:highlight', diff --git a/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts b/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts index e67e72f295b8e..cecf3b8c07d1a 100644 --- a/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts +++ b/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts @@ -8,7 +8,7 @@ import { i18n } from '@kbn/i18n'; import { PublicMethodsOf } from '@kbn/utility-types'; -import { SavedObjectsClientCommon } from '../..'; +import { INDEX_PATTERN_SAVED_OBJECT_TYPE, SavedObjectsClientCommon } from '../..'; import { createIndexPatternCache } from '.'; import type { RuntimeField } from '../types'; @@ -38,7 +38,6 @@ import { DuplicateIndexPatternError } from '../errors'; import { castEsToKbnFieldTypeName } from '../../kbn_field_types'; const MAX_ATTEMPTS_TO_RESOLVE_CONFLICTS = 3; -const savedObjectType = 'index-pattern'; export interface IndexPatternSavedObjectAttrs { title: string; @@ -94,7 +93,7 @@ export class IndexPatternsService { */ private async refreshSavedObjectsCache() { const so = await this.savedObjectsClient.find({ - type: 'index-pattern', + type: INDEX_PATTERN_SAVED_OBJECT_TYPE, fields: ['title'], perPage: 10000, }); @@ -137,7 +136,7 @@ export class IndexPatternsService { */ find = async (search: string, size: number = 10): Promise => { const savedObjects = await this.savedObjectsClient.find({ - type: 'index-pattern', + type: INDEX_PATTERN_SAVED_OBJECT_TYPE, fields: ['title'], search, searchFields: ['title'], @@ -395,12 +394,16 @@ export class IndexPatternsService { private getSavedObjectAndInit = async (id: string): Promise => { const savedObject = await this.savedObjectsClient.get( - savedObjectType, + INDEX_PATTERN_SAVED_OBJECT_TYPE, id ); if (!savedObject.version) { - throw new SavedObjectNotFound(savedObjectType, id, 'management/kibana/indexPatterns'); + throw new SavedObjectNotFound( + INDEX_PATTERN_SAVED_OBJECT_TYPE, + id, + 'management/kibana/indexPatterns' + ); } return this.initFromSavedObject(savedObject); @@ -546,7 +549,7 @@ export class IndexPatternsService { const body = indexPattern.getAsSavedObjectBody(); const response: SavedObject = (await this.savedObjectsClient.create( - savedObjectType, + INDEX_PATTERN_SAVED_OBJECT_TYPE, body, { id: indexPattern.id, @@ -587,7 +590,9 @@ export class IndexPatternsService { }); return this.savedObjectsClient - .update(savedObjectType, indexPattern.id, body, { version: indexPattern.version }) + .update(INDEX_PATTERN_SAVED_OBJECT_TYPE, indexPattern.id, body, { + version: indexPattern.version, + }) .then((resp) => { indexPattern.id = resp.id; indexPattern.version = resp.version; @@ -655,7 +660,7 @@ export class IndexPatternsService { */ async delete(indexPatternId: string) { this.indexPatternCache.clear(indexPatternId); - return this.savedObjectsClient.delete('index-pattern', indexPatternId); + return this.savedObjectsClient.delete(INDEX_PATTERN_SAVED_OBJECT_TYPE, indexPatternId); } } diff --git a/src/plugins/data/common/index_patterns/lib/get_title.ts b/src/plugins/data/common/index_patterns/lib/get_title.ts index 2dd122092f688..69afad486a745 100644 --- a/src/plugins/data/common/index_patterns/lib/get_title.ts +++ b/src/plugins/data/common/index_patterns/lib/get_title.ts @@ -7,12 +7,16 @@ */ import { SavedObjectsClientContract, SimpleSavedObject } from '../../../../../core/public'; +import { INDEX_PATTERN_SAVED_OBJECT_TYPE } from '../../constants'; export async function getTitle( client: SavedObjectsClientContract, indexPatternId: string ): Promise> { - const savedObject = (await client.get('index-pattern', indexPatternId)) as SimpleSavedObject; + const savedObject = (await client.get( + INDEX_PATTERN_SAVED_OBJECT_TYPE, + indexPatternId + )) as SimpleSavedObject; if (savedObject.error) { throw new Error(`Unable to get index-pattern title: ${savedObject.error.message}`); diff --git a/src/plugins/data/common/index_patterns/utils.ts b/src/plugins/data/common/index_patterns/utils.ts index 941ad3c47066b..925f646b83bb7 100644 --- a/src/plugins/data/common/index_patterns/utils.ts +++ b/src/plugins/data/common/index_patterns/utils.ts @@ -9,6 +9,8 @@ import type { IndexPatternSavedObjectAttrs } from './index_patterns'; import type { SavedObjectsClientCommon } from '../types'; +import { INDEX_PATTERN_SAVED_OBJECT_TYPE } from '../constants'; + /** * Returns an object matching a given title * @@ -19,7 +21,7 @@ import type { SavedObjectsClientCommon } from '../types'; export async function findByTitle(client: SavedObjectsClientCommon, title: string) { if (title) { const savedObjects = await client.find({ - type: 'index-pattern', + type: INDEX_PATTERN_SAVED_OBJECT_TYPE, perPage: 10, search: `"${title}"`, searchFields: ['title'], diff --git a/src/plugins/data/common/search/search_source/extract_references.ts b/src/plugins/data/common/search/search_source/extract_references.ts index 1b4d1732a5e37..b63b8ed1cfee2 100644 --- a/src/plugins/data/common/search/search_source/extract_references.ts +++ b/src/plugins/data/common/search/search_source/extract_references.ts @@ -10,6 +10,8 @@ import { SavedObjectReference } from 'src/core/types'; import { Filter } from '../../es_query/filters'; import { SearchSourceFields } from './types'; +import { INDEX_PATTERN_SAVED_OBJECT_TYPE } from '../../constants'; + export const extractReferences = ( state: SearchSourceFields ): [SearchSourceFields & { indexRefName?: string }, SavedObjectReference[]] => { @@ -20,7 +22,7 @@ export const extractReferences = ( const refName = 'kibanaSavedObjectMeta.searchSourceJSON.index'; references.push({ name: refName, - type: 'index-pattern', + type: INDEX_PATTERN_SAVED_OBJECT_TYPE, id: indexId, }); searchSourceFields = { @@ -40,7 +42,7 @@ export const extractReferences = ( const refName = `kibanaSavedObjectMeta.searchSourceJSON.filter[${i}].meta.index`; references.push({ name: refName, - type: 'index-pattern', + type: INDEX_PATTERN_SAVED_OBJECT_TYPE, id: filterRow.meta.index, }); return { diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts index d7667f20d517e..e9e50ebfaf138 100644 --- a/src/plugins/data/public/index.ts +++ b/src/plugins/data/public/index.ts @@ -268,6 +268,7 @@ export { IndexPatternSpec, IndexPatternLoadExpressionFunctionDefinition, fieldList, + INDEX_PATTERN_SAVED_OBJECT_TYPE, } from '../common'; export { DuplicateIndexPatternError } from '../common/index_patterns/errors'; diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index 2849b93b14483..6a49fab0e33ff 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -1361,6 +1361,9 @@ export interface IKibanaSearchResponse { // @public (undocumented) export type IMetricAggType = MetricAggType; +// @public (undocumented) +export const INDEX_PATTERN_SAVED_OBJECT_TYPE = "index-pattern"; + // Warning: (ae-missing-release-tag) "IndexPattern" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -2772,20 +2775,20 @@ export interface WaitUntilNextSessionCompletesOptions { // src/plugins/data/public/index.ts:238:27 - (ae-forgotten-export) The symbol "validateIndexPattern" needs to be exported by the entry point index.d.ts // src/plugins/data/public/index.ts:238:27 - (ae-forgotten-export) The symbol "flattenHitWrapper" needs to be exported by the entry point index.d.ts // src/plugins/data/public/index.ts:238:27 - (ae-forgotten-export) The symbol "formatHitProvider" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:408:20 - (ae-forgotten-export) The symbol "getResponseInspectorStats" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:408:20 - (ae-forgotten-export) The symbol "tabifyAggResponse" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:408:20 - (ae-forgotten-export) The symbol "tabifyGetColumns" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:410:1 - (ae-forgotten-export) The symbol "CidrMask" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:411:1 - (ae-forgotten-export) The symbol "dateHistogramInterval" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:420:1 - (ae-forgotten-export) The symbol "InvalidEsCalendarIntervalError" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:421:1 - (ae-forgotten-export) The symbol "InvalidEsIntervalFormatError" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:422:1 - (ae-forgotten-export) The symbol "IpAddress" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:423:1 - (ae-forgotten-export) The symbol "isDateHistogramBucketAggConfig" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:427:1 - (ae-forgotten-export) The symbol "isValidEsInterval" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:428:1 - (ae-forgotten-export) The symbol "isValidInterval" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:431:1 - (ae-forgotten-export) The symbol "parseInterval" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:432:1 - (ae-forgotten-export) The symbol "propFilter" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:435:1 - (ae-forgotten-export) The symbol "toAbsoluteDates" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:409:20 - (ae-forgotten-export) The symbol "getResponseInspectorStats" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:409:20 - (ae-forgotten-export) The symbol "tabifyAggResponse" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:409:20 - (ae-forgotten-export) The symbol "tabifyGetColumns" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:411:1 - (ae-forgotten-export) The symbol "CidrMask" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:412:1 - (ae-forgotten-export) The symbol "dateHistogramInterval" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:421:1 - (ae-forgotten-export) The symbol "InvalidEsCalendarIntervalError" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:422:1 - (ae-forgotten-export) The symbol "InvalidEsIntervalFormatError" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:423:1 - (ae-forgotten-export) The symbol "IpAddress" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:424:1 - (ae-forgotten-export) The symbol "isDateHistogramBucketAggConfig" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:428:1 - (ae-forgotten-export) The symbol "isValidEsInterval" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:429:1 - (ae-forgotten-export) The symbol "isValidInterval" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:432:1 - (ae-forgotten-export) The symbol "parseInterval" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:433:1 - (ae-forgotten-export) The symbol "propFilter" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:436:1 - (ae-forgotten-export) The symbol "toAbsoluteDates" needs to be exported by the entry point index.d.ts // src/plugins/data/public/query/state_sync/connect_to_query_state.ts:34:5 - (ae-forgotten-export) The symbol "FilterStateStore" needs to be exported by the entry point index.d.ts // src/plugins/data/public/search/session/session_service.ts:56:5 - (ae-forgotten-export) The symbol "UrlGeneratorStateMapping" needs to be exported by the entry point index.d.ts diff --git a/src/plugins/data/server/index.ts b/src/plugins/data/server/index.ts index dd60951e6d228..143400a2c09d3 100644 --- a/src/plugins/data/server/index.ts +++ b/src/plugins/data/server/index.ts @@ -117,7 +117,12 @@ export const fieldFormats = { HistogramFormat, }; -export { IFieldFormatsRegistry, FieldFormatsGetConfigFn, FieldFormatConfig } from '../common'; +export { + IFieldFormatsRegistry, + FieldFormatsGetConfigFn, + FieldFormatConfig, + INDEX_PATTERN_SAVED_OBJECT_TYPE, +} from '../common'; /* * Index patterns: diff --git a/src/plugins/data/server/index_patterns/utils.ts b/src/plugins/data/server/index_patterns/utils.ts index bb16be23edc7d..7f1a953c482d0 100644 --- a/src/plugins/data/server/index_patterns/utils.ts +++ b/src/plugins/data/server/index_patterns/utils.ts @@ -7,7 +7,12 @@ */ import { SavedObjectsClientContract } from 'kibana/server'; -import { IFieldType, IndexPatternAttributes, SavedObject } from '../../common'; +import { + IFieldType, + INDEX_PATTERN_SAVED_OBJECT_TYPE, + IndexPatternAttributes, + SavedObject, +} from '../../common'; export const getFieldByName = ( fieldName: string, @@ -24,7 +29,7 @@ export const findIndexPatternById = async ( index: string ): Promise | undefined> => { const savedObjectsResponse = await savedObjectsClient.find({ - type: 'index-pattern', + type: INDEX_PATTERN_SAVED_OBJECT_TYPE, fields: ['fields'], search: `"${index}"`, searchFields: ['title'], diff --git a/src/plugins/data/server/saved_objects/index_patterns.ts b/src/plugins/data/server/saved_objects/index_patterns.ts index f570e239c3c64..a809f2ce73e1b 100644 --- a/src/plugins/data/server/saved_objects/index_patterns.ts +++ b/src/plugins/data/server/saved_objects/index_patterns.ts @@ -6,11 +6,12 @@ * Side Public License, v 1. */ -import { SavedObjectsType } from 'kibana/server'; +import type { SavedObjectsType } from 'kibana/server'; import { indexPatternSavedObjectTypeMigrations } from './index_pattern_migrations'; +import { INDEX_PATTERN_SAVED_OBJECT_TYPE } from '../../common'; export const indexPatternSavedObjectType: SavedObjectsType = { - name: 'index-pattern', + name: INDEX_PATTERN_SAVED_OBJECT_TYPE, hidden: false, namespaceType: 'single', management: { diff --git a/src/plugins/data/server/server.api.md b/src/plugins/data/server/server.api.md index 5ca19f9e1e509..86aaf64dea852 100644 --- a/src/plugins/data/server/server.api.md +++ b/src/plugins/data/server/server.api.md @@ -745,6 +745,9 @@ export interface IFieldType { // @public (undocumented) export type IMetricAggType = MetricAggType; +// @public (undocumented) +export const INDEX_PATTERN_SAVED_OBJECT_TYPE = "index-pattern"; + // Warning: (ae-missing-release-tag) "IndexPattern" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -1543,20 +1546,20 @@ export function usageProvider(core: CoreSetup_2): SearchUsage; // src/plugins/data/server/index.ts:101:26 - (ae-forgotten-export) The symbol "StringFormat" needs to be exported by the entry point index.d.ts // src/plugins/data/server/index.ts:101:26 - (ae-forgotten-export) The symbol "TruncateFormat" needs to be exported by the entry point index.d.ts // src/plugins/data/server/index.ts:101:26 - (ae-forgotten-export) The symbol "HistogramFormat" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:128:27 - (ae-forgotten-export) The symbol "isFilterable" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:128:27 - (ae-forgotten-export) The symbol "isNestedField" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:245:20 - (ae-forgotten-export) The symbol "tabifyAggResponse" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:245:20 - (ae-forgotten-export) The symbol "tabifyGetColumns" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:247:1 - (ae-forgotten-export) The symbol "CidrMask" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:248:1 - (ae-forgotten-export) The symbol "dateHistogramInterval" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:257:1 - (ae-forgotten-export) The symbol "InvalidEsCalendarIntervalError" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:258:1 - (ae-forgotten-export) The symbol "InvalidEsIntervalFormatError" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:259:1 - (ae-forgotten-export) The symbol "IpAddress" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:263:1 - (ae-forgotten-export) The symbol "isValidEsInterval" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:264:1 - (ae-forgotten-export) The symbol "isValidInterval" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:268:1 - (ae-forgotten-export) The symbol "propFilter" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:271:1 - (ae-forgotten-export) The symbol "toAbsoluteDates" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:272:1 - (ae-forgotten-export) The symbol "calcAutoIntervalLessThan" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:133:27 - (ae-forgotten-export) The symbol "isFilterable" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:133:27 - (ae-forgotten-export) The symbol "isNestedField" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:250:20 - (ae-forgotten-export) The symbol "tabifyAggResponse" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:250:20 - (ae-forgotten-export) The symbol "tabifyGetColumns" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:252:1 - (ae-forgotten-export) The symbol "CidrMask" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:253:1 - (ae-forgotten-export) The symbol "dateHistogramInterval" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:262:1 - (ae-forgotten-export) The symbol "InvalidEsCalendarIntervalError" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:263:1 - (ae-forgotten-export) The symbol "InvalidEsIntervalFormatError" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:264:1 - (ae-forgotten-export) The symbol "IpAddress" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:268:1 - (ae-forgotten-export) The symbol "isValidEsInterval" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:269:1 - (ae-forgotten-export) The symbol "isValidInterval" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:273:1 - (ae-forgotten-export) The symbol "propFilter" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:276:1 - (ae-forgotten-export) The symbol "toAbsoluteDates" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:277:1 - (ae-forgotten-export) The symbol "calcAutoIntervalLessThan" needs to be exported by the entry point index.d.ts // src/plugins/data/server/plugin.ts:81:74 - (ae-forgotten-export) The symbol "DataEnhancements" needs to be exported by the entry point index.d.ts // src/plugins/data/server/search/types.ts:115:5 - (ae-forgotten-export) The symbol "ISearchStartSearchSource" needs to be exported by the entry point index.d.ts diff --git a/src/plugins/visualizations/public/saved_visualizations/saved_visualization_references/controls_references.ts b/src/plugins/visualizations/public/saved_visualizations/saved_visualization_references/controls_references.ts index d116fd2e2e9a7..7a0bb4584e83a 100644 --- a/src/plugins/visualizations/public/saved_visualizations/saved_visualization_references/controls_references.ts +++ b/src/plugins/visualizations/public/saved_visualizations/saved_visualization_references/controls_references.ts @@ -8,6 +8,7 @@ import { SavedObjectReference } from '../../../../../core/types'; import { VisParams } from '../../../common'; +import { INDEX_PATTERN_SAVED_OBJECT_TYPE } from '../../../../data/public'; const isControlsVis = (visType: string) => visType === 'input_control_vis'; @@ -25,7 +26,7 @@ export const extractControlsReferences = ( control.indexPatternRefName = `${prefix}_${i}_index_pattern`; references.push({ name: control.indexPatternRefName, - type: 'index-pattern', + type: INDEX_PATTERN_SAVED_OBJECT_TYPE, id: control.indexPattern, }); delete control.indexPattern; diff --git a/src/plugins/visualizations/public/saved_visualizations/saved_visualization_references/timeseries_references.ts b/src/plugins/visualizations/public/saved_visualizations/saved_visualization_references/timeseries_references.ts index 57706ee824e8d..98970a0127c71 100644 --- a/src/plugins/visualizations/public/saved_visualizations/saved_visualization_references/timeseries_references.ts +++ b/src/plugins/visualizations/public/saved_visualizations/saved_visualization_references/timeseries_references.ts @@ -8,13 +8,11 @@ import { SavedObjectReference } from '../../../../../core/types'; import { VisParams } from '../../../common'; +import { INDEX_PATTERN_SAVED_OBJECT_TYPE } from '../../../../data/public'; /** @internal **/ const REF_NAME_POSTFIX = '_ref_name'; -/** @internal **/ -const INDEX_PATTERN_REF_TYPE = 'index_pattern'; - /** @internal **/ type Action = (object: Record, key: string) => void; @@ -51,7 +49,7 @@ export const extractTimeSeriesReferences = ( object[key + REF_NAME_POSTFIX] = name; references.push({ name, - type: INDEX_PATTERN_REF_TYPE, + type: INDEX_PATTERN_SAVED_OBJECT_TYPE, id: object[key].id, }); delete object[key]; diff --git a/src/plugins/visualizations/server/migrations/visualization_saved_object_migrations.test.ts b/src/plugins/visualizations/server/migrations/visualization_saved_object_migrations.test.ts index 7debc9412925e..869a9add89066 100644 --- a/src/plugins/visualizations/server/migrations/visualization_saved_object_migrations.test.ts +++ b/src/plugins/visualizations/server/migrations/visualization_saved_object_migrations.test.ts @@ -2163,6 +2163,36 @@ describe('migration visualization', () => { }); }); + describe('7.14.0 replaceIndexPatternReference', () => { + const migrate = (doc: any) => + visualizationSavedObjectTypeMigrations['7.14.0']( + doc as Parameters[0], + savedObjectMigrationContext + ); + + test('should replace index_pattern to index-pattern', () => { + expect( + migrate({ + references: [ + { + name: 'name', + type: 'index_pattern', + }, + ], + } as Parameters[0]) + ).toMatchInlineSnapshot(` + Object { + "references": Array [ + Object { + "name": "name", + "type": "index-pattern", + }, + ], + } + `); + }); + }); + describe('7.14.0 update tagcloud defaults', () => { const migrate = (doc: any) => visualizationSavedObjectTypeMigrations['7.14.0']( diff --git a/src/plugins/visualizations/server/migrations/visualization_saved_object_migrations.ts b/src/plugins/visualizations/server/migrations/visualization_saved_object_migrations.ts index 7fb54b0425935..1f50e26ea9ec1 100644 --- a/src/plugins/visualizations/server/migrations/visualization_saved_object_migrations.ts +++ b/src/plugins/visualizations/server/migrations/visualization_saved_object_migrations.ts @@ -8,9 +8,9 @@ import { cloneDeep, get, omit, has, flow, forOwn } from 'lodash'; -import { SavedObjectMigrationFn } from 'kibana/server'; +import type { SavedObjectMigrationFn } from 'kibana/server'; -import { DEFAULT_QUERY_LANGUAGE } from '../../../data/common'; +import { DEFAULT_QUERY_LANGUAGE, INDEX_PATTERN_SAVED_OBJECT_TYPE } from '../../../data/common'; import { commonAddSupportOfDualIndexSelectionModeInTSVB, commonHideTSVBLastValueIndicator, @@ -37,7 +37,7 @@ const migrateIndexPattern: SavedObjectMigrationFn = (doc) => { searchSource.indexRefName = 'kibanaSavedObjectMeta.searchSourceJSON.index'; doc.references.push({ name: searchSource.indexRefName, - type: 'index-pattern', + type: INDEX_PATTERN_SAVED_OBJECT_TYPE, id: searchSource.index, }); delete searchSource.index; @@ -50,7 +50,7 @@ const migrateIndexPattern: SavedObjectMigrationFn = (doc) => { filterRow.meta.indexRefName = `kibanaSavedObjectMeta.searchSourceJSON.filter[${i}].meta.index`; doc.references.push({ name: filterRow.meta.indexRefName, - type: 'index-pattern', + type: INDEX_PATTERN_SAVED_OBJECT_TYPE, id: filterRow.meta.index, }); delete filterRow.meta.index; @@ -648,7 +648,7 @@ const migrateControls: SavedObjectMigrationFn = (doc) => { control.indexPatternRefName = `control_${i}_index_pattern`; doc.references.push({ name: control.indexPatternRefName, - type: 'index-pattern', + type: INDEX_PATTERN_SAVED_OBJECT_TYPE, id: control.indexPattern, }); delete control.indexPattern; @@ -1038,6 +1038,18 @@ const migrateTagCloud: SavedObjectMigrationFn = (doc) => { return doc; }; +export const replaceIndexPatternReference: SavedObjectMigrationFn = (doc) => ({ + ...doc, + references: Array.isArray(doc.references) + ? doc.references.map((reference) => { + if (reference.type === 'index_pattern') { + reference.type = INDEX_PATTERN_SAVED_OBJECT_TYPE; + } + return reference; + }) + : doc.references, +}); + export const visualizationSavedObjectTypeMigrations = { /** * We need to have this migration twice, once with a version prior to 7.0.0 once with a version @@ -1084,5 +1096,10 @@ export const visualizationSavedObjectTypeMigrations = { hideTSVBLastValueIndicator, removeDefaultIndexPatternAndTimeFieldFromTSVBModel ), - '7.14.0': flow(addEmptyValueColorRule, migrateVislibPie, migrateTagCloud), + '7.14.0': flow( + addEmptyValueColorRule, + migrateVislibPie, + migrateTagCloud, + replaceIndexPatternReference + ), }; From dfeecb902fd26d6e41bab81fc4a31c45663f8894 Mon Sep 17 00:00:00 2001 From: Melissa Alvarez Date: Mon, 28 Jun 2021 12:31:10 -0400 Subject: [PATCH 04/55] [ML] Data Frame Analytics creation wizard: ensure included fields table updates correctly (#103191) * fix includes table rerender loop * remove unnecessary comment --- .../analysis_fields_table.tsx | 2 - .../configuration_step_form.tsx | 41 ++++++++----------- .../configuration_step/job_type.tsx | 1 - 3 files changed, 16 insertions(+), 28 deletions(-) diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/analysis_fields_table.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/analysis_fields_table.tsx index 0b6843d49e95c..9dd4c5c42cca7 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/analysis_fields_table.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/analysis_fields_table.tsx @@ -84,7 +84,6 @@ const checkboxDisabledCheck = (item: FieldSelectionItem) => export const AnalysisFieldsTable: FC<{ dependentVariable?: string; includes: string[]; - loadingItems: boolean; setFormState: React.Dispatch>; minimumFieldsRequiredMessage?: string; setMinimumFieldsRequiredMessage: React.Dispatch>; @@ -94,7 +93,6 @@ export const AnalysisFieldsTable: FC<{ }> = ({ dependentVariable, includes, - loadingItems, setFormState, minimumFieldsRequiredMessage, setMinimumFieldsRequiredMessage, diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx index 930c32ce7e4da..9b68b03853990 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx @@ -105,7 +105,6 @@ export const ConfigurationStepForm: FC = ({ const { currentSavedSearch, currentIndexPattern } = mlContext; const { savedSearchQuery, savedSearchQueryStr } = useSavedSearch(); - const [loadingFieldOptions, setLoadingFieldOptions] = useState(false); const [fieldOptionsFetchFail, setFieldOptionsFetchFail] = useState(false); const [loadingDepVarOptions, setLoadingDepVarOptions] = useState(false); const [dependentVariableFetchFail, setDependentVariableFetchFail] = useState(false); @@ -247,21 +246,17 @@ export const ConfigurationStepForm: FC = ({ if (firstUpdate.current) { firstUpdate.current = false; } - // Reset if jobType changes (jobType requires dependent_variable to be set - - // which won't be the case if switching from outlier detection) - if (jobTypeChanged) { - setLoadingFieldOptions(true); - } + + const depVarNotIncluded = + isJobTypeWithDepVar && includes.length > 0 && includes.includes(dependentVariable) === false; // Ensure runtime field is in 'includes' table if it is set as dependent variable const depVarIsRuntimeField = - isJobTypeWithDepVar && + depVarNotIncluded && runtimeMappings && - Object.keys(runtimeMappings).includes(dependentVariable) && - includes.length > 0 && - includes.includes(dependentVariable) === false; + Object.keys(runtimeMappings).includes(dependentVariable); let formToUse = form; - if (depVarIsRuntimeField) { + if (depVarIsRuntimeField || depVarNotIncluded) { formToUse = cloneDeep(form); formToUse.includes = [...includes, dependentVariable]; } @@ -279,24 +274,22 @@ export const ConfigurationStepForm: FC = ({ (field) => field.is_included === true && field.is_required === false ); + const formStateUpdated = { + ...(shouldUpdateModelMemoryLimit ? { modelMemoryLimit: expectedMemory } : {}), + ...(depVarIsRuntimeField || jobTypeChanged || depVarNotIncluded + ? { includes: formToUse.includes } + : {}), + requiredFieldsError: !hasRequiredFields ? requiredFieldsErrorText : undefined, + }; + if (jobTypeChanged) { - setLoadingFieldOptions(false); setFieldOptionsFetchFail(false); setMaxDistinctValuesError(undefined); setUnsupportedFieldsError(undefined); - setFormState({ - ...(shouldUpdateModelMemoryLimit ? { modelMemoryLimit: expectedMemory } : {}), - requiredFieldsError: !hasRequiredFields ? requiredFieldsErrorText : undefined, - includes: formToUse.includes, - }); setIncludesTableItems(fieldSelection ? fieldSelection : []); - } else { - setFormState({ - ...(shouldUpdateModelMemoryLimit ? { modelMemoryLimit: expectedMemory } : {}), - requiredFieldsError: !hasRequiredFields ? requiredFieldsErrorText : undefined, - includes: formToUse.includes, - }); } + + setFormState(formStateUpdated); setFetchingExplainData(false); } else { const { @@ -319,7 +312,6 @@ export const ConfigurationStepForm: FC = ({ : DEFAULT_MODEL_MEMORY_LIMIT.outlier_detection; setEstimatedModelMemoryLimit(fallbackModelMemoryLimit); - setLoadingFieldOptions(false); setFieldOptionsFetchFail(true); setMaxDistinctValuesError(maxDistinctValuesErrorMessage); setUnsupportedFieldsError(unsupportedFieldsErrorMessage); @@ -650,7 +642,6 @@ export const ConfigurationStepForm: FC = ({ tableItems={includesTableItems} unsupportedFieldsError={unsupportedFieldsError} setUnsupportedFieldsError={setUnsupportedFieldsError} - loadingItems={loadingFieldOptions} setFormState={setFormState} /> {showScatterplotMatrix && ( diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/job_type.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/job_type.tsx index 443e2cfacbb5e..5f54ba3c2bb7c 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/job_type.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/job_type.tsx @@ -82,7 +82,6 @@ export const JobType: FC = ({ type, setFormState }) => { setFormState({ previousJobType: type, jobType, - includes: [], requiredFieldsError: undefined, }); setSelectedCard({ [jobType]: !selectedCard[jobType] }); From bd32299c13036ac28e1a7dd1120dbe4f241a6c15 Mon Sep 17 00:00:00 2001 From: Greg Thompson Date: Mon, 28 Jun 2021 11:31:23 -0500 Subject: [PATCH 05/55] Upgrade EUI to v34.5.1 (#103297) * eui to 34.5.0 * snapshot updates * Fix some page layouts * eui to 34.5.1 Co-authored-by: cchaos --- package.json | 2 +- .../page_template/solution_nav/solution_nav.tsx | 2 +- .../public/components/home/home.component.tsx | 1 - .../settings/__snapshots__/settings.test.tsx.snap | 14 +++++++------- .../home_integration/tutorial_directory_notice.tsx | 2 +- .../entity_by_expression.test.tsx.snap | 2 +- yarn.lock | 11 ++++++----- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/package.json b/package.json index b589153d2af90..b071b587a3620 100644 --- a/package.json +++ b/package.json @@ -103,7 +103,7 @@ "@elastic/datemath": "link:bazel-bin/packages/elastic-datemath", "@elastic/elasticsearch": "npm:@elastic/elasticsearch-canary@^8.0.0-canary.13", "@elastic/ems-client": "7.14.0", - "@elastic/eui": "34.3.0", + "@elastic/eui": "34.5.1", "@elastic/filesaver": "1.1.2", "@elastic/good": "^9.0.1-kibana3", "@elastic/maki": "6.3.0", diff --git a/src/plugins/kibana_react/public/page_template/solution_nav/solution_nav.tsx b/src/plugins/kibana_react/public/page_template/solution_nav/solution_nav.tsx index bd9ee8eb4d0e8..4aa456f716dbd 100644 --- a/src/plugins/kibana_react/public/page_template/solution_nav/solution_nav.tsx +++ b/src/plugins/kibana_react/public/page_template/solution_nav/solution_nav.tsx @@ -17,7 +17,7 @@ import { KibanaPageTemplateSolutionNavAvatarProps, } from './solution_nav_avatar'; -export type KibanaPageTemplateSolutionNavProps = Partial> & { +export type KibanaPageTemplateSolutionNavProps = EuiSideNavProps<{}> & { /** * Name of the solution, i.e. "Observability" */ diff --git a/x-pack/plugins/canvas/public/components/home/home.component.tsx b/x-pack/plugins/canvas/public/components/home/home.component.tsx index 96a773186da2b..6e98439a0c530 100644 --- a/x-pack/plugins/canvas/public/components/home/home.component.tsx +++ b/x-pack/plugins/canvas/public/components/home/home.component.tsx @@ -31,7 +31,6 @@ export const Home = ({ activeTab = 'workpads' }: Props) => { pageHeader={{ pageTitle: 'Canvas', rightSideItems: [], - bottomBorder: true, tabs: [ { label: strings.getMyWorkpadsTabLabel(), diff --git a/x-pack/plugins/canvas/shareable_runtime/components/footer/settings/__snapshots__/settings.test.tsx.snap b/x-pack/plugins/canvas/shareable_runtime/components/footer/settings/__snapshots__/settings.test.tsx.snap index 27f0d3610fb9f..075c0cd386759 100644 --- a/x-pack/plugins/canvas/shareable_runtime/components/footer/settings/__snapshots__/settings.test.tsx.snap +++ b/x-pack/plugins/canvas/shareable_runtime/components/footer/settings/__snapshots__/settings.test.tsx.snap @@ -31,7 +31,7 @@ exports[` can navigate Autoplay Settings 1`] = ` >
can navigate Autoplay Settings 2`] = ` >
can navigate Autoplay Settings 2`] = `
`; -exports[` can navigate Toolbar Settings, closes when activated 3`] = `"

You are in a dialog. To close this dialog, hit escape.

Settings
Hide Toolbar
Hide the toolbar when the mouse is not within the Canvas?
"`; +exports[` can navigate Toolbar Settings, closes when activated 3`] = `"

You are in a dialog. To close this dialog, hit escape.

Settings
Hide Toolbar
Hide the toolbar when the mouse is not within the Canvas?
"`; diff --git a/x-pack/plugins/fleet/public/components/home_integration/tutorial_directory_notice.tsx b/x-pack/plugins/fleet/public/components/home_integration/tutorial_directory_notice.tsx index 5e4357d95235b..23754571c5bc1 100644 --- a/x-pack/plugins/fleet/public/components/home_integration/tutorial_directory_notice.tsx +++ b/x-pack/plugins/fleet/public/components/home_integration/tutorial_directory_notice.tsx @@ -66,7 +66,6 @@ const TutorialDirectoryNotice: TutorialDirectoryNoticeComponent = memo(() => { return hasIngestManager && !hasSeenNotice ? ( <> - { + ) : null; }); diff --git a/x-pack/plugins/stack_alerts/public/alert_types/geo_containment/query_builder/expressions/__snapshots__/entity_by_expression.test.tsx.snap b/x-pack/plugins/stack_alerts/public/alert_types/geo_containment/query_builder/expressions/__snapshots__/entity_by_expression.test.tsx.snap index d9dd6ec4a0be5..8f59fdd7df000 100644 --- a/x-pack/plugins/stack_alerts/public/alert_types/geo_containment/query_builder/expressions/__snapshots__/entity_by_expression.test.tsx.snap +++ b/x-pack/plugins/stack_alerts/public/alert_types/geo_containment/query_builder/expressions/__snapshots__/entity_by_expression.test.tsx.snap @@ -141,7 +141,7 @@ exports[`should render entity by expression with aggregatable field options for value="FlightNum" >
{detailFields.length > 0 ? ( - detailFields.map(({ fieldName, label }, index) => ( -
- -

{label}

- - -
{result[fieldName]}
- -
- )) + detailFields.map(({ fieldName, label }, index) => { + const value = result[fieldName] as string; + const dateValue = getAsLocalDateTimeString(value); + + return ( +
+ +

{label}

+ + +
{dateValue || value}
+ +
+ ); + }) ) : ( )} diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/overview.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/overview.tsx index cc890e0f104ac..e9b8574032916 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/overview.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/overview.tsx @@ -198,12 +198,16 @@ export const Overview: React.FC = () => { {!custom && ( - {status} {activityDetails && } + + {status} {activityDetails && } + )} - {time} + + {time} + ))} @@ -453,7 +457,7 @@ export const Overview: React.FC = () => { - + @@ -465,7 +469,7 @@ export const Overview: React.FC = () => { )} - + {groups.length > 0 && groupsSummary} {details.length > 0 && {detailsSummary}} From bc097856e61be2dc5285bd11eb22704a8c056647 Mon Sep 17 00:00:00 2001 From: Bhavya RM Date: Mon, 28 Jun 2021 19:33:35 -0400 Subject: [PATCH 41/55] Unskip the reporting screenshots.ts by fixing unable to update UI settings error. (#103184) --- .../functional/apps/dashboard/reporting/screenshots.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/x-pack/test/functional/apps/dashboard/reporting/screenshots.ts b/x-pack/test/functional/apps/dashboard/reporting/screenshots.ts index 7eb2ef74000e0..881b847f1180b 100644 --- a/x-pack/test/functional/apps/dashboard/reporting/screenshots.ts +++ b/x-pack/test/functional/apps/dashboard/reporting/screenshots.ts @@ -29,9 +29,12 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { const kibanaServer = getService('kibanaServer'); const ecommerceSOPath = 'x-pack/test/functional/fixtures/kbn_archiver/reporting/ecommerce.json'; - // https://github.com/elastic/kibana/issues/102911 - describe.skip('Dashboard Reporting Screenshots', () => { + describe('Dashboard Reporting Screenshots', () => { before('initialize tests', async () => { + await kibanaServer.uiSettings.replace({ + defaultIndex: '5193f870-d861-11e9-a311-0fa548c5f953', + }); + await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/reporting/ecommerce'); await kibanaServer.importExport.load(ecommerceSOPath); await browser.setWindowSize(1600, 850); @@ -215,7 +218,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { }); it('downloads a PDF file with saved search given EuiDataGrid enabled', async function () { - await kibanaServer.uiSettings.replace({ 'doc_table:legacy': false }); + await kibanaServer.uiSettings.update({ 'doc_table:legacy': false }); this.timeout(300000); await PageObjects.common.navigateToApp('dashboard'); await PageObjects.dashboard.loadSavedDashboard('Ecom Dashboard'); From 1b5cc2a7bc046f8f9870a6f9f74481df7a7b588b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ece=20=C3=96zalp?= Date: Mon, 28 Jun 2021 19:43:38 -0400 Subject: [PATCH 42/55] [Security Solution] Disables loadPrebuiltRulesAndTemplatesButton if loading is in progress (#103568) --- .../load_empty_prompt.test.tsx | 22 +++++++++++++++++++ .../pre_packaged_rules/load_empty_prompt.tsx | 4 ++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/pre_packaged_rules/load_empty_prompt.test.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/pre_packaged_rules/load_empty_prompt.test.tsx index 9e482b228018e..cbdfe5b246aff 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/pre_packaged_rules/load_empty_prompt.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/pre_packaged_rules/load_empty_prompt.test.tsx @@ -127,4 +127,26 @@ describe('LoadPrebuiltRulesAndTemplatesButton', () => { ); }); }); + + it('renders disabled button if loading is true', async () => { + (getPrePackagedRulesStatus as jest.Mock).mockResolvedValue({ + rules_not_installed: 0, + rules_installed: 0, + rules_not_updated: 0, + timelines_not_installed: 3, + timelines_installed: 0, + timelines_not_updated: 0, + }); + + const wrapper: ReactWrapper = mount( + + ); + await waitFor(() => { + wrapper.update(); + + expect( + wrapper.find('[data-test-subj="load-prebuilt-rules"] button').props().disabled + ).toEqual(true); + }); + }); }); diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/pre_packaged_rules/load_empty_prompt.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/pre_packaged_rules/load_empty_prompt.tsx index 9a011da9aff05..56875bcc4f88c 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/pre_packaged_rules/load_empty_prompt.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/pre_packaged_rules/load_empty_prompt.tsx @@ -64,12 +64,12 @@ const PrePackagedRulesPromptComponent: React.FC = ( const loadPrebuiltRulesAndTemplatesButton = useMemo( () => getLoadPrebuiltRulesAndTemplatesButton({ - isDisabled: !userHasPermissions, + isDisabled: !userHasPermissions || loading, onClick: handlePreBuiltCreation, fill: true, 'data-test-subj': 'load-prebuilt-rules', }), - [getLoadPrebuiltRulesAndTemplatesButton, handlePreBuiltCreation, userHasPermissions] + [getLoadPrebuiltRulesAndTemplatesButton, handlePreBuiltCreation, userHasPermissions, loading] ); return ( From 7442a99168b56a02514519e0ed3e143a60822811 Mon Sep 17 00:00:00 2001 From: Spencer Date: Mon, 28 Jun 2021 16:44:29 -0700 Subject: [PATCH 43/55] [dev_docs] add tutorial for setting up a development env (#103566) Co-authored-by: Jonathan Budzenski Co-authored-by: spalger Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../setting_up_a_development_env.mdx | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 dev_docs/tutorials/setting_up_a_development_env.mdx diff --git a/dev_docs/tutorials/setting_up_a_development_env.mdx b/dev_docs/tutorials/setting_up_a_development_env.mdx new file mode 100644 index 0000000000000..449e8b886a44d --- /dev/null +++ b/dev_docs/tutorials/setting_up_a_development_env.mdx @@ -0,0 +1,89 @@ +--- +id: kibDevTutorialSetupDevEnv +slug: /kibana-dev-docs/tutorial/setup-dev-env +title: Setting up a Development Environment +summary: Learn how to setup a development environemnt for contributing to the Kibana repository +date: 2021-04-26 +tags: ['kibana', 'onboarding', 'dev', 'architecture', 'setup'] +--- + +Setting up a development environment is pretty easy. + + + In order to support Windows development we currently require you to use one of the following: + + - [Git Bash](https://git-scm.com/download/win) + - [Windows Subsystem for Linux](https://docs.microsoft.com/en-us/windows/wsl/about) + + + Before running the steps below, please make sure you have installed [Visual C++ Redistributable for Visual Studio 2015](https://www.microsoft.com/en-us/download/details.aspx?id=48145) and that you are running all commands in either Git Bash or WSL. + + +## Get the code + +Start by forking [the Kibana repository](https://github.com/elastic/kibana) on Github so that you have a place to stage pull requests and create branches for development. + +Then clone the repository to your machine: + +```sh +git clone https://github.com/[YOUR_USERNAME]/kibana.git kibana +cd kibana +``` + +## Install dependencies + +Install the version of Node.js listed in the `.node-version` file. This can be automated with tools such as [nvm](https://github.com/creationix/nvm) or [nvm-windows](https://github.com/coreybutler/nvm-windows). As we also include a `.nvmrc` file you can switch to the correct version when using nvm by running: + +```sh +nvm use +``` + +Then, install the latest version of yarn using: + +```sh +npm install -g yarn +``` + +Finally, boostrap Kibana and install all of the remaining dependencies: + +```sh +yarn kbn bootstrap +``` + +Node.js native modules could be in use and node-gyp is the tool used to build them. There are tools you need to install per platform and python versions you need to be using. Please follow the [node-gyp installation steps](https://github.com/nodejs/node-gyp#installation) for your platform. + +## Run Elasticsearch + +In order to start Kibana you need to run a local version of Elasticsearch. You can startup and initialize the latest Elasticsearch snapshot of the correct version for Kibana by running the following in a new terminal tab/window: + +```sh +yarn es snapshot +``` + +You can pass `--license trial` to start Elasticsearch with a trial license, or use the Kibana UI to switch the local version to a trial version which includes all features. + +Read about more options for [Running Elasticsearch during development](https://www.elastic.co/guide/en/kibana/current/running-elasticsearch.html), like connecting to a remote host, running from source, preserving data inbetween runs, running remote cluster, etc. + +## Run Kibana + +In another terminal tab/window you can start Kibana. + +```sh +yarn start +``` + +If you include the `--run-examples` flag then all of the [developer examples](https://github.com/elastic/kibana/tree/{branch}/examples). Read more about the advanced options for [Running Kibana](https://www.elastic.co/guide/en/kibana/current/running-kibana-advanced.html). + +## Code away! + +You are now ready to start developing. Changes to the source files should be picked up automatically and either cause the server to restart, or be served to the browser on the next page refresh. + +## Install pre-commit hook (optional) + +In case you want to run a couple of checks like linting or check the file casing of the files to commit, we provide a way to install a pre-commit hook. To configure it you just need to run the following: + +```sh +node scripts/register_git_hook +``` + +After the script completes the pre-commit hook will be created within the file `.git/hooks/pre-commit`. If you choose to not install it, don’t worry, we still run a quick CI check to provide feedback earliest as we can about the same checks. From d7d4a14c8d0d51a6085afd8a30b6e192650d2887 Mon Sep 17 00:00:00 2001 From: Frank Hassanabad Date: Mon, 28 Jun 2021 18:11:10 -0600 Subject: [PATCH 44/55] [Security Solutions][Detection Engine] Implements best effort merging of constant_keyword, runtime fields, aliases, and copy_to fields (#102280) ## Summary This adds utilities and two strategies for merging using the [fields API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-fields.html) and the `_source` document during signal generation. This gives us the ability to support `constant_keyword`, field alias value support, some runtime fields support, and `copy_to` support. Previously we did not copy any of these values and only generated signals based on the `_source` record values. This changes the behavior to allow us to copy some of the mentioned values above. The folder of `source_fields_merging` contains a `strategy` folder and a `utils` folder which contains both the strategies and the utilities for this implementation. The two strategies are `merge_all_fields_with_source` and `merge_missing_fields_with_source`. The defaulted choice for this PR is we use `merge_missing_fields_with_source` and not the `merge_all_fields_with_source`. The reasoning is that this is much lower risk and lower behavior changes to the signals detection engine. The main driving force behind this PR is that ECS has introduced `constant_keyword` and that field has the possibility of only showing up in the fields section of a document and not `_source` when index authors do not push the `constant_keyword` into the `_source` section. The secondary driving forces behind this behavioral change is that some users have been expecting their runtime fields, `copy_to` fields, and field alias values of their indexes to be copied into the signals index. Both strategies of `merge_missing_fields_with_source` and `merge_all_fields_with_source` are considered Best Effort meaning that both strategies will not always merge as expected when they encounter ambiguous use cases as outlined in the `README.md` text at the top of `source_fields_merging` in detail. The default used strategy of `merge_missing_fields_with_source` which has the simplest behavior will work in most common use cases. This is simply if the `_source` document is missing a value that is present in the `fields`, and the `fields` value is a primitive concrete value such as a `string` or `number` or `boolean` and the `_source` document does not contain an existing object or ambiguous array, then the value will be merged into `_source` and a new reference is returned. If you call the strategy twice it should be idempotent meaning that the second call will detect a value is now present in `_source` and not re-merge a second time. * 301 unit tests were added * Extensive README.md docs are added * e2e tests are updated to test scenarios and ambiguity and conflicts from previously to support this effort. * Other e2e tests were updated * One bug with EQL and fields was found with a workaround implemented. See https://github.com/elastic/elasticsearch/issues/74582 * SearchTypes adjusted to use recursive TypeScript types * Changed deprecated for `@deprecated` in a few spots * Removed some `ts-expect-error` in favor of `??` in a few areas * Added a new handling of epoch strings and tests to `detection_engine/signals/utils.ts` since fields returns `epoch_millis` as a string instead of as a number. * Uses lodash safer set to reduce changes of prototype pollution ### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios ### Risk Matrix | Risk | Probability | Severity | Mitigation/Notes | |---------------------------|-------------|----------|-------------------------| | Prototype pollution | Low | High | Used lodash safer set | | Users which have existing rules that work, upgrade and now we do not generate signals due to bad merging of fields and _source | Mid | High | We use the safer strategy method, `merge_missing_fields_with_source `, that is lighter weight to start with. We might add a follow up PR which enables a key in Kibana to turn off merging of fields with source. We added extensive unit tests and e2e tests. However, unexpected unknowns and behaviors from runtime fields and fields API such as geo-points looking like nested fields or `epoch_milliseconds` being a string value or runtime fields allowing invalid values were uncovered and tests and utilities around that have been added which makes this PR risky | | Found a bug with using fields and EQL which caused EQL rules to not run. | Low | High | Implemented workaround for tests to pass and created an Elastic ticket and communicated the bug to EQL developers. | --- .../detection_engine/get_query_filter.test.ts | 30 + .../detection_engine/get_query_filter.ts | 14 + .../common/detection_engine/types.ts | 18 +- .../__mocks__/empty_signal_source_hit.ts | 17 + .../signals/build_bulk_body.test.ts | 24 + .../signals/build_bulk_body.ts | 34 +- .../signals/source_fields_merging/README.md | 389 +++++ .../signals/source_fields_merging/index.ts | 9 + .../source_fields_merging/strategies/index.ts | 8 + .../merge_all_fields_with_source.test.ts | 1478 +++++++++++++++++ .../merge_all_fields_with_source.ts | 113 ++ .../merge_missing_fields_with_source.test.ts | 1379 +++++++++++++++ .../merge_missing_fields_with_source.ts | 88 + .../signals/source_fields_merging/types.ts | 11 + .../utils/array_in_path_exists.test.ts | 42 + .../utils/array_in_path_exists.ts | 23 + .../utils/filter_field_entries.test.ts | 83 + .../utils/filter_field_entries.ts | 32 + .../source_fields_merging/utils/index.ts | 16 + .../utils/is_array_of_primitives.test.ts | 51 + .../utils/is_array_of_primitives.ts | 21 + .../utils/is_invalid_key.test.ts | 51 + .../utils/is_invalid_key.ts | 16 + .../utils/is_multifield.test.ts | 40 + .../utils/is_multifield.ts | 34 + .../utils/is_nested_object.test.ts | 46 + .../utils/is_nested_object.ts | 22 + ...objectlike_or_array_of_objectlikes.test.ts | 71 + .../is_objectlike_or_array_of_objectlikes.ts | 25 + .../utils/is_primitive.test.ts | 42 + .../utils/is_primitive.ts | 16 + .../utils/is_type_object.test.ts | 34 + .../utils/is_type_object.ts | 25 + .../utils/recursive_unboxing_fields.test.ts | 292 ++++ .../utils/recursive_unboxing_fields.ts | 60 + .../lib/detection_engine/signals/types.ts | 27 +- .../detection_engine/signals/utils.test.ts | 39 + .../lib/detection_engine/signals/utils.ts | 9 + .../security_and_spaces/tests/aliases.ts | 7 +- .../security_and_spaces/tests/create_ml.ts | 7 + .../tests/keyword_family/const_keyword.ts | 6 +- .../keyword_mixed_with_const.ts | 6 +- .../security_and_spaces/tests/runtime.ts | 67 +- .../security_solution/alias/data.json | 8 +- .../runtime_conflicting_fields/mappings.json | 5 +- 45 files changed, 4773 insertions(+), 62 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/empty_signal_source_hit.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/README.md create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/index.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/index.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/types.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/array_in_path_exists.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/array_in_path_exists.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/filter_field_entries.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/filter_field_entries.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/index.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_array_of_primitives.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_array_of_primitives.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_invalid_key.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_invalid_key.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_multifield.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_multifield.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_nested_object.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_nested_object.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_objectlike_or_array_of_objectlikes.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_objectlike_or_array_of_objectlikes.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_primitive.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_primitive.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_type_object.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_type_object.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/recursive_unboxing_fields.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/recursive_unboxing_fields.ts diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts index 63a38ad7d71c1..7de082e778a07 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts @@ -1143,6 +1143,16 @@ describe('get_filter', () => { ], }, }, + fields: [ + { + field: '*', + include_unmapped: true, + }, + { + field: '@timestamp', + format: 'epoch_millis', + }, + ], }, }); }); @@ -1180,6 +1190,16 @@ describe('get_filter', () => { ], }, }, + fields: [ + { + field: '*', + include_unmapped: true, + }, + { + field: '@timestamp', + format: 'epoch_millis', + }, + ], }, }); }); @@ -1262,6 +1282,16 @@ describe('get_filter', () => { ], }, }, + fields: [ + { + field: '*', + include_unmapped: true, + }, + { + field: '@timestamp', + format: 'epoch_millis', + }, + ], }, }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts index 6a61f892747b4..86e66577abd45 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts @@ -121,6 +121,20 @@ export const buildEqlSearchRequest = ( }, }, event_category_field: eventCategoryOverride, + fields: [ + { + field: '*', + include_unmapped: true, + }, + { + field: '@timestamp', + // BUG: We have to format @timestamp until this bug is fixed with epoch_millis + // https://github.com/elastic/elasticsearch/issues/74582 + // TODO: Remove epoch and use the same techniques from x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts + // where we format both the timestamp and any overrides as ISO8601 + format: 'epoch_millis', + }, + ], }, }; }; diff --git a/x-pack/plugins/security_solution/common/detection_engine/types.ts b/x-pack/plugins/security_solution/common/detection_engine/types.ts index c0e502312b2ff..e7b8cca8d5a97 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/types.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/types.ts @@ -11,16 +11,14 @@ export type RuleAlertAction = Omit & { action_type_id: string; }; -export type SearchTypes = - | string - | string[] - | number - | number[] - | boolean - | boolean[] - | object - | object[] - | undefined; +/** + * Defines the search types you can have from Elasticsearch within a + * doc._source. It uses recursive types of "| SearchTypes[]" to designate + * anything can also be of a type array, and it uses the recursive type of + * "| { [property: string]: SearchTypes }" to designate you can can sub-objects + * or sub-sub-objects, etc... + */ +export type SearchTypes = string | number | boolean | object | SearchTypes[] | undefined; export interface Explanation { value: number; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/empty_signal_source_hit.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/empty_signal_source_hit.ts new file mode 100644 index 0000000000000..805a401f782fa --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/empty_signal_source_hit.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SignalSourceHit } from '../types'; + +/** + * Simple empty Elasticsearch result for testing + * @returns Empty Elasticsearch result for testing + */ +export const emptyEsResult = (): SignalSourceHit => ({ + _index: 'index', + _id: '123', +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts index 4d3ca26f5a71e..4053d64539c49 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts @@ -77,6 +77,9 @@ describe('buildBulkBody', () => { rule: expectedRule(), depth: 1, }, + source: { + ip: '127.0.0.1', + }, }; expect(fakeSignalSourceHit).toEqual(expected); }); @@ -160,6 +163,9 @@ describe('buildBulkBody', () => { }, depth: 1, }, + source: { + ip: '127.0.0.1', + }, }; expect(fakeSignalSourceHit).toEqual(expected); }); @@ -222,6 +228,9 @@ describe('buildBulkBody', () => { rule: expectedRule(), depth: 1, }, + source: { + ip: '127.0.0.1', + }, }; expect(fakeSignalSourceHit).toEqual(expected); }); @@ -282,6 +291,9 @@ describe('buildBulkBody', () => { rule: expectedRule(), depth: 1, }, + source: { + ip: '127.0.0.1', + }, }; expect(fakeSignalSourceHit).toEqual(expected); }); @@ -335,6 +347,9 @@ describe('buildBulkBody', () => { rule: expectedRule(), depth: 1, }, + source: { + ip: '127.0.0.1', + }, }; expect(fakeSignalSourceHit).toEqual(expected); }); @@ -388,6 +403,9 @@ describe('buildBulkBody', () => { rule: expectedRule(), depth: 1, }, + source: { + ip: '127.0.0.1', + }, }; expect(fakeSignalSourceHit).toEqual(expected); }); @@ -441,6 +459,9 @@ describe('buildBulkBody', () => { rule: expectedRule(), depth: 1, }, + source: { + ip: '127.0.0.1', + }, }; expect(fakeSignalSourceHit).toEqual(expected); }); @@ -673,6 +694,9 @@ describe('buildSignalFromEvent', () => { rule: expectedRule(), depth: 2, }, + source: { + ip: '127.0.0.1', + }, }; expect(signal).toEqual(expected); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts index 10cc168700447..819e1f3eb6df1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts @@ -6,6 +6,7 @@ */ import { SavedObject } from 'src/core/types'; +import { mergeMissingFieldsWithSource } from './source_fields_merging/strategies/merge_missing_fields_with_source'; import { AlertAttributes, SignalSourceHit, @@ -21,18 +22,27 @@ import { buildEventTypeSignal } from './build_event_type_signal'; import { EqlSequence } from '../../../../common/detection_engine/types'; import { generateSignalId, wrapBuildingBlocks, wrapSignal } from './utils'; -// format search_after result for signals index. +/** + * Formats the search_after result for insertion into the signals index. We first create a + * "best effort" merged "fields" with the "_source" object, then build the signal object, + * then the event object, and finally we strip away any additional temporary data that was added + * such as the "threshold_result". + * @param ruleSO The rule saved object to build overrides + * @param doc The SignalSourceHit with "_source", "fields", and additional data such as "threshold_result" + * @returns The body that can be added to a bulk call for inserting the signal. + */ export const buildBulkBody = ( ruleSO: SavedObject, doc: SignalSourceHit ): SignalHit => { - const rule = buildRuleWithOverrides(ruleSO, doc._source!); + const mergedDoc = mergeMissingFieldsWithSource({ doc }); + const rule = buildRuleWithOverrides(ruleSO, mergedDoc._source ?? {}); const signal: Signal = { - ...buildSignal([doc], rule), - ...additionalSignalFields(doc), + ...buildSignal([mergedDoc], rule), + ...additionalSignalFields(mergedDoc), }; - const event = buildEventTypeSignal(doc); - const { threshold_result: thresholdResult, ...filteredSource } = doc._source || { + const event = buildEventTypeSignal(mergedDoc); + const { threshold_result: thresholdResult, ...filteredSource } = mergedDoc._source || { threshold_result: null, }; const signalHit: SignalHit = { @@ -122,18 +132,18 @@ export const buildSignalFromEvent = ( ruleSO: SavedObject, applyOverrides: boolean ): SignalHit => { + const mergedEvent = mergeMissingFieldsWithSource({ doc: event }); const rule = applyOverrides - ? // @ts-expect-error @elastic/elasticsearch _source is optional - buildRuleWithOverrides(ruleSO, event._source) + ? buildRuleWithOverrides(ruleSO, mergedEvent._source ?? {}) : buildRuleWithoutOverrides(ruleSO); const signal: Signal = { - ...buildSignal([event], rule), - ...additionalSignalFields(event), + ...buildSignal([mergedEvent], rule), + ...additionalSignalFields(mergedEvent), }; - const eventFields = buildEventTypeSignal(event); + const eventFields = buildEventTypeSignal(mergedEvent); // TODO: better naming for SignalHit - it's really a new signal to be inserted const signalHit: SignalHit = { - ...event._source, + ...mergedEvent._source, '@timestamp': new Date().toISOString(), event: eventFields, signal, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/README.md b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/README.md new file mode 100644 index 0000000000000..eb72fc2b32687 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/README.md @@ -0,0 +1,389 @@ +Set of utilities for merging between `_source` and `fields` are within this folder as well as strategies for merging between these two. + +See `strategies` for the strategies for merging between `_source` and `fields`. See the `utils` folder for the different utilities +which the strategies utilize for help in building their merged documents. + +If we run into problems such as ambiguities, uncertainties, or data type contradictions then we will prefer the value within +"doc.fields" when we can. If "doc.fields" contradicts its self or is too ambiguous, then we assume that +there a problem within "doc.fields" due to a malformed runtime field definition and omit the last seen +contradiction. In some cases we might have to omit the merging of the field altogether and instead utilize +the value from "doc._source" + +Hence, these are labeled as "best effort" since we could run into conditions where we should have taken the value +from "doc.fields" but instead did not and took the value from "doc._source". + +If "doc.fields" does not exist we return "doc._source" untouched as-is. If "doc._source" does not exist but +"doc.fields" does exist then we will do a "best effort" to merge "doc.fields" into a fully functional object as +if it was a "doc._source". But again, if we run into contradictions or ambiguities from the +"doc.fields" we will remove that field or omit one of the contradictions. + +If a "doc.field" is found that does not exist in "doc._source" then we merge that "doc.field" into our +return object. + +If we find that a "field" contradicts the "doc._source" object in which we cannot create a regular +JSON such as a keyword trying to override an object or an object trying to override a keyword: + +``` +"fields": { 'foo': 'value_1', foo.bar': 'value_2' } <-- Foo cannot be both an object and a value +``` +Then you will get an object such as + +``` +{ "foo": "value_1" } +``` + +We cannot merge both together as this is a contradiction and no longer capable of being a JSON object. +This happens when we have multiFields since multiFields are represented in fields as well as when runtime +fields tries to add multiple overrides or invalid multiFields. + +Invalid field names such as ".", "..", ".foo", "foo.", ".foo." will be skipped as those cause errors if +we tried to insert them into Elasticsearch as a new field. + +If we encounter an array within "doc._source" that contains an object with more than 1 value and a "field" +tries to add a new element we will not merge that in as we do not know which array to merge that value into. + +If we encounter a flattened array in the fields object which is not a nested fields such as: +``` +"fields": { "object_name.first" : [ "foo", "bar" ], "object_name.second" : [ "mars", "bar" ] } +``` + +and no "doc._source" with the name "object_name", the assumption is that we these are not related and we construct the object as this: + +``` +{ "object.name": { "first": ["foo", "bar" }, "second": ["mars", "bar"] } +``` + +If we detect a "doc._source with a single flattened array sub objects we will prefer the "fields" flattened +array elements and copy them over as-is, which means we could be subtracting elements, adding elements, or +completely changing the items from the array. + +If we detect an object within the "doc._source" inside of the array, we will not take anything from the +"fields" flattened array elements even if they exist as it is ambiguous where we would put those elements +within the ""doc._source" as an override. + +It is best to feed this both the "doc._source" and "doc.fields" values to get the best chances of merging the document correctly. + +Using these strategies will get you these value types merged that you would otherwise not get directly on your +``` +"doc._source": + - constant_keyword field + - runtime fields + - field aliases + - copy_to +``` + +References: +--- + * https://www.elastic.co/guide/en/elasticsearch/reference/7.13/keyword.html#constant-keyword-field-type + * https://www.elastic.co/guide/en/elasticsearch/reference/7.13/runtime.html + * https://www.elastic.co/guide/en/elasticsearch/reference/7.13/search-fields.html + +Ambiguities and issues +--- +* geo data points/types and nested fields look the same. +* multi-fields such as `host.name` and `host.name.keyword` can lead to misinterpreting valid values vs multi-fields +* All data is an array with at least 1 value we call "boxed", meaning that it is difficult to determine if the user wanted the fields as an array or not. + +Existing bugs and ambiguities +--- +* We currently filter out the geo data points by looking at "type" on the object and filter it out. We could transform it to be valid input at some point. + +Tests +--- +Some tests in this folder use a special table and nomenclature in the comments to show the enumerations and tests for each type. + +Key for the nomenclature is: +``` +undefined means non-existent +p_[] means primitive key and empty array +p_p1 or p_p2 means primitive key and primitive value +p_[p1] or p_[p2] means primitive key and primitive array with a single array value +p[p1, ...1] or p[p2, ...2] means primitive array with 2 or more values +p_{}1 or p_{}2 means a primitive key with a single object +p_[{}1] or p_[{}2] means a primitive key with an array of exactly 1 object +p_[{}1, ...1] or p_[{}2, ...2] means a primitive key with 2 or more array elements +f_[] means a flattened object key and empty array +f_p1 or f_p2 means a flattened object key and a primitive value +f_[p1] or f_[p2] means a flattened object key and a single primitive value in an array +f_[p1, ...1] or f_[p2, ...2] means a flattened object key and 2 or more primitive values in an array +f_{}1 or f_{}2 means a flattened object key with 1 object +f_[{}1] or f_[{}2] means a flattened object key with a single object in a single array +f_[{}1, ...1] or f_[{}2, ...2] means a flattened object key with 2 or more objects in an array +``` + +`_source` documents can contain the following values: +``` +undefined +p_[] +p_p1 +p_[p1] +p_[p1, ...1] +p_{}1 +p_[{}1] +p_[{}1, ...1] +f_[] +f_p1 +f_[p1] +f_[p1, ...1] +f_{}1 +f_[{}1] +f_[{}1, ...1] +``` + +fields arrays can contain the following values: +``` +undefined +f_[] +f_[p2] +f_[p2, ...2] +f_[{}2] +f_[{}2, ...2] +``` + +When fields is undefined or empty array f_[] you never overwrite +the source and source is always the same as before the merge for all the strategies +``` +source | fields | value after merge +----- | --------- | ----- +undefined | undefined | undefined +undefined | f_[] | undefined +p_[] | undefined | p_[] +p_[] | f_[] | p_[] +p_p1 | undefined | p_p1 +p_p1 | f_[] | p_p1 +p_[p1] | undefined | p_[p1] +p_[p1] | f_[] | p_[p1] +p_[p1, ...1] | undefined | p_[p1, ...1] +p_[p1, ...1] | f_[] | p_[p1, ...1] +p_{}1 | undefined | p_{}1 +p_{}1 | f_[] | p_{}1 +p_[{}1] | undefined | p_{}1 +p_[{}1] | f_[] | p_{}1 +p_[{}1, ...1] | undefined | p_[{}1, ...1] +p_[{}1, ...1] | f_[] | p_[{}1, ...1] +f_[] | undefined | f_[] +f_[] | f_[] | f_[] +f_p1 | undefined | f_p1 +f_p1 | f_[] | f_p1 +f_[p1] | undefined | f_[p1] +f_[p1] | f_[] | f_[p1] +f_[p1, ...1] | undefined | f_[p1, ...1] +f_[p1, ...1] | f_[] | f_[p1, ...1] +f_{}1 | undefined | f_{}1 +f_{}1 | f_[] | f_{}1 +f_[{}1] | undefined | f_{}1 +f_[{}1] | f_[] | f_{}1 +f_[{}1, ...1] | undefined | f_[{}1, ...1] +f_[{}1, ...1] | f_[] | f_[{}1, ...1] +``` + +When source key and source value does not exist but field keys and values do exist, then you +you will always get field keys and values replacing the source key and value. Caveat is that +fields will create a single item rather than an array item if field keys and value only has a single +array element. Also, we prefer to create an object structure in source (e.x. p_p2 instead of a flattened object f_p2) +for the `merge_all_fields_with_source` strategy +``` +source | fields | value after merge +----- | --------- | ----- +undefined | f_[p2] | p_p2 <-- Unboxed from array +undefined | f_[p2, ...2] | p_[p2, ...2] +undefined | f_[{}2] | p_{}2 <-- Unboxed from array +undefined | f_[{}2, ...2] | p_[{}2, ...2] +``` + +For the `merge_missing_fields_with_source` it will be that we completely skip the fields that contain nested +fields or type fields such as geo points. + +``` +source | fields | value after merge +----- | --------- | ----- +undefined | f_[p2] | p_p2 <-- Unboxed from array +undefined | f_[p2, ...2] | p_[p2, ...2] +undefined | f_[{}2] | {} <-- We have an empty object since we only merge primitives +undefined | f_[{}2, ...2] | {} <-- We have an empty object since we only merge primitives +``` + +When source key is either a primitive key or a flattened object key with a primitive value (p_p1 or f_p1), +then we overwrite source value with fields value as an unboxed value array if fields value is a +single array element (f_[p2] or f[{}2]), otherwise we overwrite source as an array. + +``` +source | fields | value after merge +----- | --------- | ----- +p_p1 | f_[p2] | p_p2 <-- Unboxed from array +p_p1 | f_[p2, ...2] | p_[p2, ...2] +p_p1 | f_[{}2] | p_{}2 <-- Unboxed from array +p_p1 | f_[{}2, ...2] | p_[{}2, ...2] + +f_p1 | f_[p2] | f_p2 <-- Unboxed from array +f_p1 | f_[p2, ...2] | f_[p2, ...2] +f_p1 | f_[{}2] | f_{}2 <-- Unboxed from array +f_p1 | f_[{}2, ...2] | f_[{}2, ...2] +``` + +For the `merge_missing_fields_with_source` none of these will be merged since the source has values such as + +``` +source | fields | value after merge +----- | --------- | ----- +p_p1 | f_[p2] | p_p1 +p_p1 | f_[p2, ...2] | p_p1 +p_p1 | f_[{}2] | p_p1 +p_p1 | f_[{}2, ...2] | p_p1 + +f_p1 | f_[p2] | f_p1 +f_p1 | f_[p2, ...2] | f_p1 +f_p1 | f_[{}2] | f_p1 +f_p1 | f_[{}2, ...2] | f_p1 +``` + +When source key is a primitive key or a flattened object key and the source value is any +type of array (p_[], p_p[p1], or p_p[p1, ...1]) of primitives then we always copy the +fields value as is and keep the source key as it was originally (primitive or flattened) + +``` +source | fields | value after merge +----- | --------- | ----- +p_[] | f_[p2] | p_[p2] +p_[] | f_[p2, ...2] | p_[p2, ...2] +p_[] | f_[{}2] | p_[{}2] +p_[] | f_[{}2, ...2] | p_[{}2, ...2] + +f_[] | f_[p2] | f_[p2] +f_[] | f_[p2, ...2] | f_[p2, ...2] +f_[] | f_[{}2] | f_[{}2] +f_[] | f_[{}2, ...2] | f_[{}2, ...2] + +p_[p1] | f_[p2] | p_[p2] +p_[p1] | f_[p2, ...2] | p_[p2, ...2] +p_[p1] | f_[{}2] | p_[{}2] +p_[p1] | f_[{}2, ...2] | p_[{}2, ...2] + +f_[p1] | f_[p2] | f_[p2] +f_[p1] | f_[p2, ...2] | f_[p2, ...2] +f_[p1] | f_[{}2] | f_{}2 +f_[p1] | f_[{}2, ...2] | f_[{}2, ...2] + +p_[p1, ...1] | f_[p2] | p_[p2] +p_[p1, ...1] | f_[p2, ...2] | p_[p2, ...2] +p_[p1, ...1] | f_[{}2] | p_[{}2] +p_[p1, ...1] | f_[{}2, ...2] | p_[{}2, ...2] + +f_[p1, ...1] | f_[p2] | f_[p2] +f_[p1, ...1] | f_[p2, ...2] | f_[p2, ...2] +f_[p1, ...1] | f_[{}2] | f_[{}2] +f_[p1, ...1] | f_[{}2, ...2] | f_[{}2, ...2] +``` + +For the `merge_missing_fields_with_source` none of these will be merged since the source has values such as + +``` +source | fields | value after merge +----- | --------- | ----- +p_[] | f_[p2] | p_[] +p_[] | f_[p2, ...2] | p_[] +p_[] | f_[{}2] | p_[] +p_[] | f_[{}2, ...2] | p_[] + +f_[] | f_[p2] | f_[] +f_[] | f_[p2, ...2] | f_[] +f_[] | f_[{}2] | f_[] +f_[] | f_[{}2, ...2] | f_[] + +p_[p1] | f_[p2] | p_[p1] +p_[p1] | f_[p2, ...2] | p_[p1] +p_[p1] | f_[{}2] | p_[p1] +p_[p1] | f_[{}2, ...2] | p_[p1] + +f_[p1] | f_[p2] | f_[p1] +f_[p1] | f_[p2, ...2] | f_[p1] +f_[p1] | f_[{}2] | f_[p1] +f_[p1] | f_[{}2, ...2] | f_[p1] + +p_[p1, ...1] | f_[p2] | p_[p1, ...1] +p_[p1, ...1] | f_[p2, ...2] | p_[p1, ...1] +p_[p1, ...1] | f_[{}2] | p_[p1, ...1] +p_[p1, ...1] | f_[{}2, ...2] | p_[p1, ...1] + +f_[p1, ...1] | f_[p2] | f_[p1, ...1] +f_[p1, ...1] | f_[p2, ...2] | f_[p1, ...1] +f_[p1, ...1] | f_[{}2] | f_[p1, ...1] +f_[p1, ...1] | f_[{}2, ...2] | f_[p1, ...1] +``` + +When source key is a primitive key or flattened key and the source value is an object (p_{}1, f_{}1) or +an array containing objects ([p_{1}], f_{}1, p_[{}1, ...1], f_[{}1, ...1]), we only copy the +field value if we detect that field value is also an object meaning that it is a nested field, +(f_[{}]2 or f[{}2, ...2]). We never allow a field to convert an object back into a value. +We never try to merge field values into the array either since they're flattened in the fields and we +will have too many ambiguities and issues between the flattened array values and the source objects. + +``` +source | fields | value after merge +----- | --------- | ----- +p_{}1 | f_[p2] | p_{}1 +p_{}1 | f_[p2, ...2] | p_{}1 +p_{}1 | f_[{}2] | p_{}2 <-- Copied and unboxed array since we detected a nested field +p_{}1 | f_[{}2, ...2] | p_[{}2, ...2] <-- Copied since we detected a nested field + +f_{}1 | f_[p2] | f_{}1 +f_{}1 | f_[p2, ...2] | f_{}1 +f_{}1 | f_[{}2] | f_{}2 <-- Copied and unboxed array since we detected a nested field +f_{}1 | f_[{}2, ...2] | f_[{}2, ...2] <-- Copied since we detected a nested field + +p_[{}1] | f_[p2] | p_[{}1] +p_[{}1] | f_[p2, ...2] | p_[{}1] +p_[{}1] | f_[{}2] | p_[{}2] <-- Copied since we detected a nested field +p_[{}1] | f_[{}2, ...2] | p_[{}2, ...2] <-- Copied since we detected a nested field + +f_[{}1] | f_[p2] | f_[{}1] +f_[{}1] | f_[p2, ...2] | f_[{}1] +f_[{}1] | f_[{}2] | f_[{}2] <-- Copied since we detected a nested field +f_[{}1] | f_[{}2, ...2] | f_[{}2, ...2] <-- Copied since we detected a nested field + +p_[{}1, ...1] | f_[p2] | p_[{}1, ...1] +p_[{}1, ...1] | f_[p2, ...2] | p_[{}1, ...1] +p_[{}1, ...1] | f_[{}2] | p_[{}2] <-- Copied since we detected a nested field +p_[{}1, ...1] | f_[{}2, ...2] | p_[{}2, ...2] <-- Copied since we detected a nested field + +f_[{}1, ...1] | f_[p2] | f_[{}1, ...1] +f_[{}1, ...1] | f_[p2, ...2] | f_[{}1, ...1] +f_[{}1, ...1] | f_[{}2] | f_[{}2] <-- Copied since we detected a nested field +f_[{}1, ...1] | f_[{}2, ...2] | f_[{}2, ...2] <-- Copied since we detected a nested field +``` + +For the `merge_missing_fields_with_source` none of these will be merged since the source has values such as + +``` +source | fields | value after merge +----- | --------- | ----- +p_{}1 | f_[p2] | p_{}1 +p_{}1 | f_[p2, ...2] | p_{}1 +p_{}1 | f_[{}2] | p_{}1 +p_{}1 | f_[{}2, ...2] | p_{}1 + +f_{}1 | f_[p2] | f_{}1 +f_{}1 | f_[p2, ...2] | f_{}1 +f_{}1 | f_[{}2] | f_{}1 +f_{}1 | f_[{}2, ...2] | f_{}1 + +p_[{}1] | f_[p2] | p_[{}1] +p_[{}1] | f_[p2, ...2] | p_[{}1] +p_[{}1] | f_[{}2] | p_[{}1] +p_[{}1] | f_[{}2, ...2] | p_[{}1] + +f_[{}1] | f_[p2] | f_[{}1] +f_[{}1] | f_[p2, ...2] | f_[{}1] +f_[{}1] | f_[{}2] | f_[{}1] +f_[{}1] | f_[{}2, ...2] | f_[{}1] + +p_[{}1, ...1] | f_[p2] | p_[{}1, ...1] +p_[{}1, ...1] | f_[p2, ...2] | p_[{}1, ...1] +p_[{}1, ...1] | f_[{}2] | p_[{}1, ...1] +p_[{}1, ...1] | f_[{}2, ...2] | p_[{}1, ...1] + +f_[{}1, ...1] | f_[p2] | f_[{}1, ...1] +f_[{}1, ...1] | f_[p2, ...2] | f_[{}1, ...1] +f_[{}1, ...1] | f_[{}2] | f_[{}1, ...1] +f_[{}1, ...1] | f_[{}2, ...2] | f_[{}1, ...1] +``` diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/index.ts new file mode 100644 index 0000000000000..ff07c898a3a24 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export * from './types'; +export * from './strategies'; +export * from './utils'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/index.ts new file mode 100644 index 0000000000000..212eba9c6c3be --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export * from './merge_all_fields_with_source'; +export * from './merge_missing_fields_with_source'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.test.ts new file mode 100644 index 0000000000000..b900ea268fd6e --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.test.ts @@ -0,0 +1,1478 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { mergeAllFieldsWithSource } from './merge_all_fields_with_source'; +import { SignalSourceHit } from '../../types'; +import { emptyEsResult } from '../../__mocks__/empty_signal_source_hit'; + +/** + * See ../README.md for the nomenclature of any notes within tests below + */ +describe('merge_all_fields_with_source', () => { + beforeAll(() => { + jest.resetAllMocks(); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + /** Get the return type of the mergeAllFieldsWithSource for TypeScript checks against expected */ + type ReturnTypeMergeFieldsWithSource = ReturnType['_source']; + + describe('fields is "undefined"', () => { + /** + * source | fields | value after merge + * ----- | --------- | ----- + * undefined | undefined | undefined + * p_[] | undefined | p_[] + * p_p1 | undefined | p_p1 + * p_[p1] | undefined | p_[p1] + * p_[p1, ...1] | undefined | p_[p1, ...1] + * p_{}1 | undefined | p_{}1 + * p_[{}1] | undefined | p_{}1 + * p_[{}1, ...1] | undefined | p_[{}1, ...1] + */ + describe('primitive keys in the _source document', () => { + /** fields is "undefined" for all tests below */ + const fields: SignalSourceHit['fields'] = {}; + + test('when source is "undefined", merged doc is "undefined"', () => { + const _source: SignalSourceHit['_source'] = {}; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an empty array (p_[]), merged doc is empty array (p_[])"', () => { + const _source: SignalSourceHit['_source'] = { + foo: [], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is a primitive (p_p1), merged doc is primitive (p_p1)', () => { + const _source: SignalSourceHit['_source'] = { + foo: 'value', + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array with a single primitive (p_[p1]), merged doc is primitive (p_[p1])', () => { + const _source: SignalSourceHit['_source'] = { + foo: ['value'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array with 2 or more primitives (p_[p1, ..1]), merged doc is primitive (p_[p1, ...1])', () => { + const _source: SignalSourceHit['_source'] = { + foo: ['value_1', 'value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is a single object (p_{}), merged doc is single object (p_{})', () => { + const _source: SignalSourceHit['_source'] = { + foo: { bar: 'some value' }, + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array with single object (p_[{}1]), merged doc is single object (p_[{}1])', () => { + const _source: SignalSourceHit['_source'] = { + foo: [{ bar: 'some value' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array of 1 or more objects (p_[{}, ...1]), merged doc is the same (p_[{}, ...1])', () => { + const _source: SignalSourceHit['_source'] = { + foo: [{ bar: 'some value' }, { foo: 'some other value' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + }); + + /** + * source | fields | value after merge + * ----- | --------- | ----- + * undefined | undefined | undefined + * f_[] | undefined | f_[] + * f_p1 | undefined | f_p1 + * f_[p1] | undefined | f_[p1] + * f_[p1, ...1] | undefined | f_[p1, ...1] + * f_{}1 | undefined | f_{}1 + * f_[{}1] | undefined | f_{}1 + * f_[{}1, ...1] | undefined | f_[{}1, ...1] + */ + describe('flattened object keys in the _source document', () => { + /** fields is "undefined" for all tests below */ + const fields: SignalSourceHit['fields'] = {}; + + test('when source is an empty array (f_[]), merged doc is empty array (f_[])"', () => { + const _source: SignalSourceHit['_source'] = { + 'foo.bar': [], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is a primitive (f_p1), merged doc is primitive (f_p1)', () => { + const _source: SignalSourceHit['_source'] = { + 'foo.bar': 'value', + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array with a single primitive (f_[p1]), merged doc is primitive (f_[p1])', () => { + const _source: SignalSourceHit['_source'] = { + 'foo.bar': ['value'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array with 2 or more primitives (f_[p1, ...1]), merged doc is primitive (f_[p1, ...1])', () => { + const _source: SignalSourceHit['_source'] = { + 'foo.bar': ['value_1', 'value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is a single object (f_{}1), merged doc is single object (f_{}1)', () => { + const _source: SignalSourceHit['_source'] = { + foo: { bar: 'some value' }, + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array with single object ([f_{}1]), merged doc is single object ([f_{}1])', () => { + const _source: SignalSourceHit['_source'] = { + foo: [{ bar: 'some value' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array of 1 or more objects (f_[{}1, ...1]), merged doc is the same (f_[{}1, ...1])', () => { + const _source: SignalSourceHit['_source'] = { + foo: [{ bar: 'some value' }, { foo: 'some other value' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + }); + }); + + describe('fields is "[]"', () => { + /** + * source | fields | value after merge + * ----- | --------- | ----- + * undefined | f_[] | undefined + * p_[] | f_[] | p_[] + * p_p1 | f_[] | p_p1 + * p_[p1] | f_[] | p_[p1] + * p_[p1, ...1] | f_[] | p_[p1, ...1] + * p_{}1 | f_[] | p_{}1 + * p_[{}1] | f_[] | p_{}1 + * p_[{}1, ...1] | f_[] | p_[{}1, ...1] + */ + describe('primitive keys in the _source document', () => { + /** fields is a flattened object key and an empty array value (p_[]) */ + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [], + }; + + test('when source is an empty array (p_[]), merged doc is empty array (p_[])"', () => { + const _source: SignalSourceHit['_source'] = { + foo: { bar: [] }, + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is a primitive (p_p1), merged doc is primitive (p_p1)', () => { + const _source: SignalSourceHit['_source'] = { + foo: { bar: 'value' }, + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array with a single primitive (p_[p1]), merged doc is primitive (p_[p1])', () => { + const _source: SignalSourceHit['_source'] = { + foo: { bar: ['value'] }, + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array with 2 or more primitives (p_[p1, ..1]), merged doc is primitive (p_[p1, ...1])', () => { + const _source: SignalSourceHit['_source'] = { + foo: { bar: ['value_1', 'value_2'] }, + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is a single object (p_{}), merged doc is single object (p_{})', () => { + const _source: SignalSourceHit['_source'] = { + foo: { bar: { mars: 'some value' } }, + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array with single object (p_[{}1]), merged doc is single object (p_[{}1])', () => { + const _source: SignalSourceHit['_source'] = { + foo: { bar: [{ mars: 'some value' }] }, + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array of 1 or more objects (p_[{}, ...1]), merged doc is the same (p_[{}, ...1])', () => { + const _source: SignalSourceHit['_source'] = { + foo: { bar: [{ mars: 'some value' }, { mars: 'some other value' }] }, + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + }); + + /** + * source | fields | value after merge + * ----- | --------- | ----- + * undefined | f_[] | undefined + * f_[] | f_[] | f_[] + * f_p1 | f_[] | f_p1 + * f_[p1] | f_[] | f_[p1] + * f_[p1, ...1] | f_[] | f_[p1, ...1] + * f_{}1 | f_[] | f_{}1 + * f_[{}1] | f_[] | f_{}1 + * f_[{}1, ...1] | f_[] | f_[{}1, ...1] + */ + describe('flattened object keys in the _source document', () => { + /** fields is flattened object key with an empty array for a value (f_[]) */ + const fields: SignalSourceHit['fields'] = { + 'bar.foo': [], + }; + + test('when source is an empty array (f_[]), merged doc is empty array (f_[])"', () => { + const _source: SignalSourceHit['_source'] = { + 'bar.foo': [], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is a primitive (f_p1), merged doc is primitive (f_p1)', () => { + const _source: SignalSourceHit['_source'] = { + 'bar.foo': 'value', + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array with a single primitive (f_[p1]), merged doc is primitive (f_[p1])', () => { + const _source: SignalSourceHit['_source'] = { + 'bar.foo': ['value'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array with 2 or more primitives (f_[p1, ...1]), merged doc is primitive (f_[p1, ...1])', () => { + const _source: SignalSourceHit['_source'] = { + 'bar.foo': ['value_1', 'value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is a single object (f_{}1), merged doc is single object (f_{}1)', () => { + const _source: SignalSourceHit['_source'] = { + foo: { bar: 'some value' }, + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array with single object ([f_{}1]), merged doc is single object ([f_{}1])', () => { + const _source: SignalSourceHit['_source'] = { + foo: [{ bar: 'some value' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array of 1 or more objects (f_[{}1, ...1]), merged doc is the same (f_[{}1, ...1])', () => { + const _source: SignalSourceHit['_source'] = { + foo: [{ bar: 'some value' }, { foo: 'some other value' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + }); + }); + + /** + * source | fields | value after merge + * ----- | --------- | ----- + * undefined | f_[p2] | p_p2 <-- Unboxed from array + * undefined | f_[p2, ...2] | p_[p2, ...2] + * undefined | f_[{}2] | p_{}2 <-- Unboxed from array + * undefined | f_[{}2, ...2] | p_[{}2, ...2] + */ + describe('source is "undefined"', () => { + /** _source is "undefined" for all tests below */ + const _source: SignalSourceHit['_source'] = {}; + + test('fields is a single primitive value (f_[p2]), merged doc is an unboxed array element p_p2"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: { + bar: 'other_value_1', + }, + }); + }); + + test('fields is a multiple primitive values (f_[p2, ...2]), merged doc is the array (f_[p2, ...2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1', 'other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: { + bar: ['other_value_1', 'other_value_2'], + }, + }); + }); + + test('fields is a single nested field value (f_[{}2]), merged doc is the unboxed array element (p_{}2)"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: { bar: { zed: 'other_value_1' } }, + }); + }); + + test('fields is multiple nested field values (f_[{}2, ...2]), merged doc is the array (f_[{}2, ...2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }] }, + }); + }); + }); + + describe('source is either primitive or flattened keys, with primitive values', () => { + /** + * source | fields | value after merge + * ----- | --------- | ----- + * p_p1 | f_[p2] | p_p2 <-- Unboxed from array + * p_p1 | f_[p2, ...2] | p_[p2, ...2] + * p_p1 | f_[{}2] | p_{}2 <-- Unboxed from array + * p_p1 | f_[{}2, ...2] | p_[{}2, ...2] + */ + describe('primitive keys in the _source document with the value of "value" (p_p1)', () => { + /** _source is a single primitive key with a primitive value for all tests below (p_p1) */ + const _source: SignalSourceHit['_source'] = { + foo: { bar: 'value' }, + }; + + test('fields is single array primitive value (f_[p2]), merged doc is unboxed primitive key and value (p_p2)"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: { + bar: 'other_value_1', + }, + }); + }); + + test('fields has single primitive values (f_[p2, ...2]), merged doc is the array (p_[p2, ...2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1', 'other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: { bar: ['other_value_1', 'other_value_2'] }, + }); + }); + + test('fields has a single nested object (f_[{}2]), merged doc is the unboxed array (p_{}2)"', () => { + const fields: SignalSourceHit['fields'] = { + foo: [{ bar: 'other_value_1' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: { + bar: 'other_value_1', + }, + }); + }); + + test('fields has multiple nested objects (f_[{}2, ...2]), merged doc is the array (f_[{}2, ...2])"', () => { + const fields: SignalSourceHit['fields'] = { + foo: [{ bar: 'other_value_1' }, { bar: 'other_value_2' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: [{ bar: 'other_value_1' }, { bar: 'other_value_2' }], + }); + }); + }); + + /** + * source | fields | value after merge + * ----- | --------- | ----- + * f_p1 | f_[p2] | f_p2 <-- Unboxed from array + * f_p1 | f_[p2, ...2] | f_[p2, ...2] + * f_p1 | f_[{}2] | f_{}2 <-- Unboxed from array + * f_p1 | f_[{}2, ...2] | f_[{}2, ...2] + */ + describe('flattened object keys in the _source document (f_p1)', () => { + /** _source is a flattened object key with a primitive value for all tests below (f_p1) */ + const _source: SignalSourceHit['_source'] = { + 'foo.bar': 'value', + }; + + test('fields is flattened object key with single array value (f_[p2]), merged doc is unboxed primitive key and value (f_p2)"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + 'foo.bar': 'other_value_1', + }); + }); + + test('fields has single primitive value (f_[p2, ...2]), merged doc is the array (f_[p2, ...2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1', 'other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(fields); + }); + + test('fields has a single nested object (f_[{}2]), merged doc is the unboxed array (f_{}2)"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + 'foo.bar': { zed: 'other_value_1' }, + }); + }); + + test('fields has multiple nested objects (f_[{}2, ...2]), merged doc is the array (f_[{}2, ...2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(fields); + }); + }); + }); + + describe('source is either primitive or flattened keys, with primitive array values', () => { + /** + * source | fields | value after merge + * ----- | --------- | ----- + * p_[] | f_[p2] | p_[p2] + * p_[] | f_[p2, ...2] | p_[p2, ...2] + * p_[] | f_[{}2] | p_[{}2] + * p_[] | f_[{}2, ...2] | p_[{}2, ...2] + */ + describe('primitive keys in the _source document with empty array (p_[])', () => { + /** _source is a primitive key with an empty array for all tests below (p_[]) */ + const _source: SignalSourceHit['_source'] = { + foo: { bar: [] }, + }; + + test('fields is flattened object key with single array value (f_[p2]), merged doc is array value (p_[p2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: { bar: ['other_value_1'] }, + }); + }); + + test('fields has single primitive values (f_[p2, ...2]), merged doc is the array (p_[p2, ...2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1', 'other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: { bar: ['other_value_1', 'other_value_2'] }, + }); + }); + + test('fields has a single nested object (f_[{}2]), merged doc is the array value (p_[{}2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: { bar: [{ zed: 'other_value_1' }] }, + }); + }); + + test('fields has multiple nested objects (f_[{}2, ...2]), merged doc is the array (p_[{}2, ...2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }] }, + }); + }); + }); + + /** + * source | fields | value after merge + * ----- | --------- | ----- + * f_[] | f_[p2] | f_[p2] + * f_[] | f_[p2, ...2] | f_[p2, ...2] + * f_[] | f_[{}2] | f_[{}2] + * f_[] | f_[{}2, ...2] | f_[{}2, ...2] + */ + describe('flattened object keys in the _source document with empty array (f_[])', () => { + /** _source is a flattened object key with an empty array for all tests below (f_[]) */ + const _source: SignalSourceHit['_source'] = { + 'foo.bar': [], + }; + + test('fields is flattened object key with single array value (f_[p2]), merged doc is array (f_[p2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(fields); + }); + + test('fields has multiple primitive values (f_[p2, ...2]), merged doc is the array (f_[p2, ...2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1', 'other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(fields); + }); + + test('fields has a single nested object (f_[{}2]), merged doc is the array (f_[{}2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(fields); + }); + + test('fields has multiple nested objects (f_[{}2, ...2]), merged doc is the array (f_[{}2, ...2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(fields); + }); + }); + + /** + * source | fields | value after merge + * ----- | --------- | ----- + * p_[p1] | f_[p2] | p_[p2] + * p_[p1] | f_[p2, ...2] | p_[p2, ...2] + * p_[p1] | f_[{}2] | p_[{}2] + * p_[p1] | f_[{}2, ...2] | p_[{}2, ...2] + */ + describe('primitive keys in the _source document with single primitive value in an array (p_[p1])', () => { + /** _source is a primitive key with a single primitive array value for all tests below (p_[p1]) */ + const _source: SignalSourceHit['_source'] = { + foo: { bar: ['value'] }, + }; + + test('fields is flattened object key with single array value (f_[p2]), merged doc is the array value (p_[p2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: { bar: ['other_value_1'] }, + }); + }); + + test('fields has single primitive value (f_[p2, ...2]), merged doc is the array (p_[p2, ...2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1', 'other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: { bar: ['other_value_1', 'other_value_2'] }, + }); + }); + + test('fields has a single nested object (f_[{}2]), merged doc is the array (p_[{}2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: { bar: [{ zed: 'other_value_1' }] }, + }); + }); + + test('fields has multiple nested objects (f_[{}2, ...2]), merged doc is the array (p_[{}2, ...2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }] }, + }); + }); + }); + + /** + * source | fields | value after merge + * ----- | --------- | ----- + * f_[p1] | f_[p2] | f_[p2] + * f_[p1] | f_[p2, ...2] | f_[p2, ...2] + * f_[p1] | f_[{}2] | f_[{}2] + * f_[p1] | f_[{}2, ...2] | f_[{}2, ...2] + */ + describe('flattened keys in the _source document with single flattened value in an array (f_[p1])', () => { + /** _source is a flattened object key with a single primitive value for all tests below (f_p[p1]) */ + const _source: SignalSourceHit['_source'] = { + 'foo.bar': ['value'], + }; + + test('fields is flattened object key with single array value (f_[p2]), merged doc is array value (f_[p2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(fields); + }); + + test('fields has single primitive value (f_[p2, ...2]), merged doc is the array (f_[p2, ...2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1', 'other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(fields); + }); + + test('fields has a single nested object (f_[{}2]), merged doc is the array (f_[{}2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(fields); + }); + + test('fields has multiple nested objects (f_[{}2, ...2]), merged doc is the array (f_[{}2, ...2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(fields); + }); + }); + + /** + * source | fields | value after merge + * ----- | --------- | ----- + * p_[p1, ...1] | f_[p2] | p_[p2] + * p_[p1, ...1] | f_[p2, ...2] | p_[p2, ...2] + * p_[p1, ...1] | f_[{}2] | p_[{}2] + * p_[p1, ...1] | f_[{}2, ...2] | p_[{}2, ...2] + */ + describe('primitive keys in the _source document with multiple array values in an array (p_[p1, ...1])', () => { + /** _source is a primitive key with an array of 2 or more elements for all tests below (p_[p1, ...1]) */ + const _source: SignalSourceHit['_source'] = { + foo: { + bar: ['value_1', 'value_2'], + }, + }; + + test('fields is single array value (f_[p2]), merged doc is array (p_[p2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: { + bar: ['other_value_1'], + }, + }); + }); + + test('fields is multiple primitive values (f_[p2, ...2]), merged doc is the array (p_[p2, ...2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1', 'other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: { + bar: ['other_value_1', 'other_value_2'], + }, + }); + }); + + test('fields has a single nested object (f_[{}2]), merged doc is the unboxed value (p_[{}2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: { + bar: [{ zed: 'other_value_1' }], + }, + }); + }); + + test('fields has multiple nested objects (f_[{}2, ...2]), merged doc is the array (p_[{}2, ...2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: { + bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], + }, + }); + }); + }); + + /** + * source | fields | value after merge + * ----- | --------- | ----- + * f_[p1, ...1] | f_[p2] | f_[p2] + * f_[p1, ...1] | f_[p2, ...2] | f_[p2, ...2] + * f_[p1, ...1] | f_[{}2] | f_[{}2] + * f_[p1, ...1] | f_[{}2, ...2] | f_[{}2, ...2] + */ + describe('flattened keys in the _source document with multiple array values in an array (f_[p1, ...1])', () => { + /** _source is a flattened object key with an array of 2 or more elements for all tests below (f_[p1, ...1]) */ + const _source: SignalSourceHit['_source'] = { + 'foo.bar': ['value_1', 'value_2'], + }; + + test('fields is flattened object key with single array value (f_[p2]), merged doc is unboxed primitive key and value (f_p2)"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(fields); + }); + + test('fields has multiple primitive values (f_[p2, ...2]), merged doc is the array (f_[p2, ...2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1', 'other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(fields); + }); + + test('fields has a single nested object (f_[{}2]), merged doc is the array (f_{}2)"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(fields); + }); + + test('fields has multiple nested objects (f_[{}2, ...2]), merged doc is the array (f_[{}2, ...2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(fields); + }); + }); + }); + + describe('source is either primitive or flattened keys, with object values', () => { + /** + * source | fields | value after merge + * ----- | --------- | ----- + * p_{}1 | f_[p2] | p_{}1 + * p_{}1 | f_[p2, ...2] | p_{}1 + * p_{}1 | f_[{}2] | p_{}2 <-- Copied and unboxed array since we detected a nested field + * p_{}1 | f_[{}2, ...2] | p_[{}2, ...2] <-- Copied since we detected a nested field + */ + describe('primitive keys in the _source document with the value of "value" (p_{}1)', () => { + /** _source is a primitive key with an object value for all tests below (p_{}1) */ + const _source: SignalSourceHit['_source'] = { + foo: { bar: { mars: 'value_1' } }, + }; + + test('fields is flattened object key with single array value (f_[p2]), merged doc is the same source (p_{}1)"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has single primitive values (f_[p2, ...2]), merged doc is the same _source (p_{}1)"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1', 'other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has a single nested object (f_[{}2]), merged doc is the unboxed array value (p_{}2)"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: { bar: { zed: 'other_value_1' } }, + }); + }); + + test('fields has multiple nested objects (f_[{}2, ...2]), merged doc is the array (p_[{}2, ...2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }] }, + }); + }); + }); + + /** + * source | fields | value after merge + * ----- | --------- | ----- + * f_{}1 | f_[p2] | f_{}1 + * f_{}1 | f_[p2, ...2] | f_{}1 + * f_{}1 | f_[{}2] | f_{}2 <-- Copied and unboxed array since we detected a nested field + * f_{}1 | f_[{}2, ...2] | f_[{}2, ...2] <-- Copied since we detected a nested field + */ + describe('flattened object keys in the _source document with the value of "value" (f_{}1)', () => { + /** _source is a flattened object key with an object value for all tests below (f_{}1) */ + const _source: SignalSourceHit['_source'] = { + 'foo.bar': { mars: 'value_1' }, + }; + + test('fields is flattened object key with single array value (f_[p2]), merged doc is the same source (f_{}1)"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has primitive values (f_[p2, ...2]), merged doc is the same _source (f_{}1)"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1', 'other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has a single nested object (f_[{}2]), merged doc is unboxed array value (f_{}2)"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ mars: 'other_value_1' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + 'foo.bar': { mars: 'other_value_1' }, + }); + }); + + test('fields has multiple nested objects (f_[{}2, ...2]), merged doc is the array (f_[{}2, ...2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], + }); + }); + }); + }); + + describe('source is either primitive or flattened keys, with object array values', () => { + /** + * source | fields | value after merge + * ----- | --------- | ----- + * p_[{}1] | f_[p2] | p_[{}1] + * p_[{}1] | f_[p2, ...2] | p_[{}1] + * p_[{}1] | f_[{}2] | p_[{}2] <-- Copied since we detected a nested field + * p_[{}1] | f_[{}2, ...2] | p_[{}2, ...2] <-- Copied since we detected a nested field + */ + describe('primitive keys in the _source document with a single array value with an object (p_[{}1])', () => { + /** _source is a primitive key with a single array value with an object for all tests below (p_[{}1]) */ + const _source: SignalSourceHit['_source'] = { + foo: { bar: [{ mars: ['value_1'] }] }, + }; + + test('fields has a single primitive value (f_[p2]), merged doc is the same _source (p_[{}1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has 2 or more primitive values (f_[p2, ...2]), merged doc is the same _source (p_[{}1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1', 'other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has a single nested object (f_[{}2]), merged doc is the array value (p_[{}2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: { bar: [{ zed: 'other_value_1' }] }, + }); + }); + + test('fields has multiple nested objects (f_[{}2, ...2]), merged doc is the array (p_[{}2, ...2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }] }, + }); + }); + }); + + /** + * source | fields | value after merge + * ----- | --------- | ----- + * p_[{}1, ...1] | f_[p2] | p_[{}1, ...1] + * p_[{}1, ...1] | f_[p2, ...2] | p_[{}1, ...1] + * p_[{}1, ...1] | f_[{}2] | p_[{}2] <-- Copied since we detected a nested field + * p_[{}1, ...1] | f_[{}2, ...2] | p_[{}2, ...2] <-- Copied since we detected a nested field + */ + describe('primitive keys in the _source document with multiple array objects (p_[{}1, ...1])', () => { + /** _source is a primitive key with a 2 or more array values with an object for all tests below (p_[{}1, ...1]) */ + const _source: SignalSourceHit['_source'] = { + foo: { bar: [{ mars: ['value_1'] }, { mars: ['value_1'] }] }, + }; + + test('fields has a single primitive value (f_[p2]), merged doc is the same _source (p_[{}1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has 2 or more primitive values (f_[p2, ...2]), merged doc is the same _source (p_[{}1, ...1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1', 'other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has a single nested object (f_[{}2]), merged doc is the array value (p_[{}2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: { bar: [{ zed: 'other_value_1' }] }, + }); + }); + + test('fields has multiple nested objects (f_[{}2, ...2]), merged doc is the array (p_[{}2, ...2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }] }, + }); + }); + }); + + /** + * source | fields | value after merge + * ----- | --------- | ----- + * f_[{}1] | f_[p2] | f_[{}1] + * f_[{}1] | f_[p2, ...2] | f_[{}1] + * f_[{}1] | f_[{}2] | f_[{}2] <-- Copied since we detected a nested field + * f_[{}1] | f_[{}2, ...2] | f_[{}2, ...2] <-- Copied since we detected a nested field + */ + describe('flattened object keys in the _source document with the single value of "value" (f_[{}1])', () => { + /** _source is a flattened object key with a single array object for all tests below (f_[{}1]) */ + const _source: SignalSourceHit['_source'] = { + 'foo.bar': [{ mars: 'value_1' }], + }; + + test('fields is flattened object key with single array value (f_[p2]), merged doc is the same _source (f_[{}1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has primitive values (f_[p2, ...2]), merged doc is the same _source (f_[{}1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1', 'other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has a single nested object (f_[{}2]), merged doc is array value (f_[{}2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ mars: 'other_value_1' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(fields); + }); + + test('fields has multiple nested objects (f_[{}2, ...2]), merged doc is the array (f_[{}2, ...2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(fields); + }); + }); + + /** + * source | fields | value after merge + * ----- | --------- | ----- + * f_[{}1, ...1] | f_[p2] | f_[{}1, ...1] + * f_[{}1, ...1] | f_[p2, ...2] | f_[{}1, ...1] + * f_[{}1, ...1] | f_[{}2] | f_[{}2] <-- Copied since we detected a nested field + * f_[{}1, ...1] | f_[{}2, ...2] | f_[{}2, ...2] <-- Copied since we detected a nested field + */ + describe('flattened object keys in the _source document with multiple values of "value" (f_[{}1, ...1])', () => { + /** _source is a flattened object key with 2 or more array objects for all tests below (f_[{}1]) */ + const _source: SignalSourceHit['_source'] = { + 'foo.bar': [{ mars: 'value_1' }, { mars: 'value_2' }], + }; + + test('fields is flattened object key with single array value (f_[p2]), merged doc is the same _source (f_[{}1, ...1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has primitive values (f_[p2, ...2]), merged doc is the same _source (f_[{}1, ...1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1', 'other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has a single nested object (f_[{}2]), merged doc is array value (f_[{}2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ mars: 'other_value_1' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(fields); + }); + + test('fields has multiple nested objects (f_[{}2, ...2]), merged doc is the array (f_[{}2, ...2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(fields); + }); + }); + }); + + /** + * It is possible to have a mixture of flattened keys and primitive keys within a _source document. + * These tests cover those cases and these test cases should be considered hopefully rare occurrences. + * If these become more common place, update the top table with all the permutations and combinations + * of tests for these. For now, expect the flattened object keys to get the values added to them vs. + * the other value. These tests show the behaviors of this but also the existing bugs of what happens + * when we merge. + */ + describe('miscellaneous tests of mixed flattened and source objects within _source', () => { + /** _source has a primitive key mixed with an object with the same path information which causes ambiguity */ + const _source: SignalSourceHit['_source'] = { + foo: { bar: 'value_1' }, + 'foo.bar': 'value_2', + }; + + test('fields has a single primitive value f_[p2] which is to override one of the values above"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: { bar: 'value_1' }, + 'foo.bar': 'other_value_1', + }); + }); + + /** + * This is an ambiguous situation in which we produce incorrect results. + */ + test('fields has the same list of values as that of the original document and we actually do not understand if this is a new value or not"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['value_1', 'value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: { bar: 'value_1' }, // <--- We have duplicated value_1 twice which is a bug + 'foo.bar': ['value_1', 'value_2'], // <-- We have merged the array value because we do not understand if we should or not + }); + }); + }); + + /** + * These tests show the behaviors around overriding fields with other fields such as objects overriding + * values and values overriding objects. This occurs with multi fields where you can have "foo" and "foo.keyword" + * in the fields + */ + describe('Fields overriding fields', () => { + describe('primitive keys for the _source', () => { + test('removes multi-field values such "foo.keyword" mixed with "foo" and prefers just "foo" for 1st level', () => { + const _source: SignalSourceHit['_source'] = { + foo: 'foo_value_1', + bar: 'bar_value_1', + }; + const fields: SignalSourceHit['fields'] = { + foo: ['foo_other_value_1'], + 'foo.keyword': ['foo_other_value_keyword_1'], + bar: ['bar_other_value_1'], + 'bar.keyword': ['bar_other_value_keyword_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: 'foo_other_value_1', + bar: 'bar_other_value_1', + }); + }); + + test('removes multi-field values such "host.name.keyword" mixed with "host.name" and prefers just "host.name" for 2nd level', () => { + const _source: SignalSourceHit['_source'] = { + host: { + name: 'host_value_1', + hostname: 'host_name_value_1', + }, + }; + const fields: SignalSourceHit['fields'] = { + 'host.name': ['host_name_other_value_1'], + 'host.name.keyword': ['host_name_other_value_keyword_1'], + 'host.hostname': ['hostname_other_value_1'], + 'host.hostname.keyword': ['hostname_other_value_keyword_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + host: { + hostname: 'hostname_other_value_1', + name: 'host_name_other_value_1', + }, + }); + }); + + test('removes multi-field values such "foo.host.name.keyword" mixed with "foo.host.name" and prefers just "foo.host.name" for 3rd level', () => { + const _source: SignalSourceHit['_source'] = { + foo: { + host: { + name: 'host_value_1', + hostname: 'host_name_value_1', + }, + }, + }; + const fields: SignalSourceHit['fields'] = { + 'foo.host.name': ['host_name_other_value_1'], + 'foo.host.name.keyword': ['host_name_other_value_keyword_1'], + 'foo.host.hostname': ['hostname_other_value_1'], + 'foo.host.hostname.keyword': ['hostname_other_value_keyword_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: { + host: { + hostname: 'hostname_other_value_1', + name: 'host_name_other_value_1', + }, + }, + }); + }); + + test('multi-field values mixed with regular values will not be merged accidentally"', () => { + const _source: SignalSourceHit['_source'] = {}; + const fields: SignalSourceHit['fields'] = { + foo: ['other_value_1'], + 'foo.bar': ['other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: 'other_value_1', + }); + }); + }); + + describe('flattened keys for the _source', () => { + test('removes multi-field values such "host.name.keyword" mixed with "host.name" and prefers just "host.name" for 2nd level', () => { + const _source: SignalSourceHit['_source'] = { + 'host.name': 'host_value_1', + 'host.hostname': 'host_name_value_1', + }; + const fields: SignalSourceHit['fields'] = { + 'host.name': ['host_name_other_value_1'], + 'host.name.keyword': ['host_name_other_value_keyword_1'], + 'host.hostname': ['hostname_other_value_1'], + 'host.hostname.keyword': ['hostname_other_value_keyword_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + 'host.name': 'host_name_other_value_1', + 'host.hostname': 'hostname_other_value_1', + }); + }); + + test('removes multi-field values such "foo.host.name.keyword" mixed with "foo.host.name" and prefers just "foo.host.name" for 3rd level', () => { + const _source: SignalSourceHit['_source'] = { + 'foo.host.name': 'host_value_1', + 'foo.host.hostname': 'host_name_value_1', + }; + const fields: SignalSourceHit['fields'] = { + 'foo.host.name': ['host_name_other_value_1'], + 'foo.host.name.keyword': ['host_name_other_value_keyword_1'], + 'foo.host.hostname': ['hostname_other_value_1'], + 'foo.host.hostname.keyword': ['hostname_other_value_keyword_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + 'foo.host.name': 'host_name_other_value_1', + 'foo.host.hostname': 'hostname_other_value_1', + }); + }); + + test('invalid fields of several levels mixed with regular values will not be merged accidentally due to runtime fields being liberal"', () => { + const _source: SignalSourceHit['_source'] = {}; + const fields: SignalSourceHit['fields'] = { + foo: ['other_value_1'], + 'foo.bar': ['other_value_2'], + 'foo.bar.zed': ['zed_other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: 'other_value_1', + }); + }); + }); + }); + + /** + * These tests are around parent objects that are not nested but are array types. We do not try to merge + * into these as this causes ambiguities between array types and object types. + */ + describe('parent array objects', () => { + test('parent array objects will not be overridden since that is an ambiguous use case for a top level value', () => { + const _source: SignalSourceHit['_source'] = { + foo: [ + { + bar: 'value_1', + mars: ['value_1'], + }, + ], + }; + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + 'foo.mars': ['other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('parent array objects will not be overridden since that is an ambiguous use case for a deeply nested value', () => { + const _source: SignalSourceHit['_source'] = { + foo: { + zed: [ + { + bar: 'value_1', + mars: ['value_1'], + }, + ], + }, + }; + const fields: SignalSourceHit['fields'] = { + 'foo.zed.bar': ['other_value_1'], + 'foo.zed.mars': ['other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + }); + + /** + * Specific tests around nested field types such as ensuring we are unboxing when we can + */ + describe('nested fields', () => { + test('unboxes deeply nested fields from a single array items when source is non-existent', () => { + const _source: SignalSourceHit['_source'] = {}; + const fields: SignalSourceHit['fields'] = { + foo: [{ bar: ['single_value'], zed: ['single_value'] }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: { bar: 'single_value', zed: 'single_value' }, + }); + }); + + test('does not unbox when source is exists and has arrays for the same values with primitive keys', () => { + const _source: SignalSourceHit['_source'] = { + foo: [ + { + bar: [], + zed: [], + }, + ], + }; + const fields: SignalSourceHit['fields'] = { + foo: [{ bar: ['single_value'], zed: ['single_value'] }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: [{ bar: ['single_value'], zed: ['single_value'] }], + }); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.ts new file mode 100644 index 0000000000000..de8d3ba820e23 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.ts @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { get } from 'lodash/fp'; +import { set } from '@elastic/safer-lodash-set/fp'; +import { SignalSource, SignalSourceHit } from '../../types'; +import { filterFieldEntries } from '../utils/filter_field_entries'; +import type { FieldsType } from '../types'; +import { isObjectLikeOrArrayOfObjectLikes } from '../utils/is_objectlike_or_array_of_objectlikes'; +import { isNestedObject } from '../utils/is_nested_object'; +import { recursiveUnboxingFields } from '../utils/recursive_unboxing_fields'; +import { isPrimitive } from '../utils/is_primitive'; +import { isArrayOfPrimitives } from '../utils/is_array_of_primitives'; +import { arrayInPathExists } from '../utils/array_in_path_exists'; +import { isTypeObject } from '../utils/is_type_object'; + +/** + * Merges all of "doc._source" with its "doc.fields" on a "best effort" basis. See ../README.md for more information + * on this function and the general strategies. + * + * @param doc The document with "_source" and "fields" + * @param throwOnFailSafe Defaults to false, but if set to true it will cause a throw if the fail safe is triggered to indicate we need to add a new explicit test condition + * @returns The two merged together in one object where we can + */ +export const mergeAllFieldsWithSource = ({ doc }: { doc: SignalSourceHit }): SignalSourceHit => { + const source = doc._source ?? {}; + const fields = doc.fields ?? {}; + const fieldEntries = Object.entries(fields); + const filteredEntries = filterFieldEntries(fieldEntries); + + const transformedSource = filteredEntries.reduce( + (merged, [fieldsKey, fieldsValue]: [string, FieldsType]) => { + if ( + hasEarlyReturnConditions({ + fieldsValue, + fieldsKey, + merged, + }) + ) { + return merged; + } + + const valueInMergedDocument = get(fieldsKey, merged); + if (valueInMergedDocument === undefined) { + const valueToMerge = recursiveUnboxingFields(fieldsValue, valueInMergedDocument); + return set(fieldsKey, valueToMerge, merged); + } else if (isPrimitive(valueInMergedDocument)) { + const valueToMerge = recursiveUnboxingFields(fieldsValue, valueInMergedDocument); + return set(fieldsKey, valueToMerge, merged); + } else if (isArrayOfPrimitives(valueInMergedDocument)) { + const valueToMerge = recursiveUnboxingFields(fieldsValue, valueInMergedDocument); + return set(fieldsKey, valueToMerge, merged); + } else if ( + isObjectLikeOrArrayOfObjectLikes(valueInMergedDocument) && + isNestedObject(fieldsValue) && + !Array.isArray(valueInMergedDocument) + ) { + const valueToMerge = recursiveUnboxingFields(fieldsValue, valueInMergedDocument); + return set(fieldsKey, valueToMerge, merged); + } else if ( + isObjectLikeOrArrayOfObjectLikes(valueInMergedDocument) && + isNestedObject(fieldsValue) && + Array.isArray(valueInMergedDocument) + ) { + const valueToMerge = recursiveUnboxingFields(fieldsValue, valueInMergedDocument); + return set(fieldsKey, valueToMerge, merged); + } else { + // fail safe catch all condition for production, but we shouldn't try to reach here and + // instead write tests if we encounter this situation. + return merged; + } + }, + { ...source } + ); + + return { + ...doc, + _source: transformedSource, + }; +}; + +/** + * Returns true if any early return conditions are met which are + * - If the fieldsValue is an empty array return + * - If we have an array within the path return and the value in our merged documented is non-existent + * - If the value is an object or is an array of object types and we don't have a nested field + * @param fieldsValue The field value to check + * @param fieldsKey The key of the field we are checking + * @param merged The merge document which is what we are testing conditions against + * @returns true if we should return early, otherwise false + */ +const hasEarlyReturnConditions = ({ + fieldsValue, + fieldsKey, + merged, +}: { + fieldsValue: FieldsType; + fieldsKey: string; + merged: SignalSource; +}) => { + const valueInMergedDocument = get(fieldsKey, merged); + return ( + fieldsValue.length === 0 || + (valueInMergedDocument === undefined && arrayInPathExists(fieldsKey, merged)) || + (isObjectLikeOrArrayOfObjectLikes(valueInMergedDocument) && + !isNestedObject(fieldsValue) && + !isTypeObject(fieldsValue)) + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.test.ts new file mode 100644 index 0000000000000..70d1e79580e84 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.test.ts @@ -0,0 +1,1379 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { mergeMissingFieldsWithSource } from './merge_missing_fields_with_source'; +import { SignalSourceHit } from '../../types'; +import { emptyEsResult } from '../../__mocks__/empty_signal_source_hit'; + +/** + * See ../README.md for the nomenclature of any notes within tests below + */ +describe('merge_missing_fields_with_source', () => { + beforeAll(() => { + jest.resetAllMocks(); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + /** Get the return type of the mergeMissingFieldsWithSource for TypeScript checks against expected */ + type ReturnTypeMergeFieldsWithSource = ReturnType['_source']; + + describe('fields is "undefined"', () => { + /** + * source | fields | value after merge + * ----- | --------- | ----- + * undefined | undefined | undefined + * p_[] | undefined | p_[] + * p_p1 | undefined | p_p1 + * p_[p1] | undefined | p_[p1] + * p_[p1, ...1] | undefined | p_[p1, ...1] + * p_{}1 | undefined | p_{}1 + * p_[{}1] | undefined | p_{}1 + * p_[{}1, ...1] | undefined | p_[{}1, ...1] + */ + describe('primitive keys in the _source document', () => { + /** fields is "undefined" for all tests below */ + const fields: SignalSourceHit['fields'] = {}; + + test('when source is "undefined", merged doc is "undefined"', () => { + const _source: SignalSourceHit['_source'] = {}; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an empty array (p_[]), merged doc is empty array (p_[])"', () => { + const _source: SignalSourceHit['_source'] = { + foo: [], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is a primitive (p_p1), merged doc is primitive (p_p1)', () => { + const _source: SignalSourceHit['_source'] = { + foo: 'value', + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array with a single primitive (p_[p1]), merged doc is primitive (p_[p1])', () => { + const _source: SignalSourceHit['_source'] = { + foo: ['value'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array with 2 or more primitives (p_[p1, ..1]), merged doc is primitive (p_[p1, ...1])', () => { + const _source: SignalSourceHit['_source'] = { + foo: ['value_1', 'value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is a single object (p_{}), merged doc is single object (p_{})', () => { + const _source: SignalSourceHit['_source'] = { + foo: { bar: 'some value' }, + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array with single object (p_[{}1]), merged doc is single object (p_[{}1])', () => { + const _source: SignalSourceHit['_source'] = { + foo: [{ bar: 'some value' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array of 1 or more objects (p_[{}, ...1]), merged doc is the same (p_[{}, ...1])', () => { + const _source: SignalSourceHit['_source'] = { + foo: [{ bar: 'some value' }, { foo: 'some other value' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + }); + + /** + * source | fields | value after merge + * ----- | --------- | ----- + * undefined | undefined | undefined + * f_[] | undefined | f_[] + * f_p1 | undefined | f_p1 + * f_[p1] | undefined | f_[p1] + * f_[p1, ...1] | undefined | f_[p1, ...1] + * f_{}1 | undefined | f_{}1 + * f_[{}1] | undefined | f_{}1 + * f_[{}1, ...1] | undefined | f_[{}1, ...1] + */ + describe('flattened object keys in the _source document', () => { + /** fields is "undefined" for all tests below */ + const fields: SignalSourceHit['fields'] = {}; + + test('when source is an empty array (f_[]), merged doc is empty array (f_[])"', () => { + const _source: SignalSourceHit['_source'] = { + 'foo.bar': [], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is a primitive (f_p1), merged doc is primitive (f_p1)', () => { + const _source: SignalSourceHit['_source'] = { + 'foo.bar': 'value', + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array with a single primitive (f_[p1]), merged doc is primitive (f_[p1])', () => { + const _source: SignalSourceHit['_source'] = { + 'foo.bar': ['value'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array with 2 or more primitives (f_[p1, ...1]), merged doc is primitive (f_[p1, ...1])', () => { + const _source: SignalSourceHit['_source'] = { + 'foo.bar': ['value_1', 'value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is a single object (f_{}1), merged doc is single object (f_{}1)', () => { + const _source: SignalSourceHit['_source'] = { + foo: { bar: 'some value' }, + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array with single object ([f_{}1]), merged doc is single object ([f_{}1])', () => { + const _source: SignalSourceHit['_source'] = { + foo: [{ bar: 'some value' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array of 1 or more objects (f_[{}1, ...1]), merged doc is the same (f_[{}1, ...1])', () => { + const _source: SignalSourceHit['_source'] = { + foo: [{ bar: 'some value' }, { foo: 'some other value' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + }); + }); + + describe('fields is "[]"', () => { + /** + * source | fields | value after merge + * ----- | --------- | ----- + * undefined | f_[] | undefined + * p_[] | f_[] | p_[] + * p_p1 | f_[] | p_p1 + * p_[p1] | f_[] | p_[p1] + * p_[p1, ...1] | f_[] | p_[p1, ...1] + * p_{}1 | f_[] | p_{}1 + * p_[{}1] | f_[] | p_{}1 + * p_[{}1, ...1] | f_[] | p_[{}1, ...1] + */ + describe('primitive keys in the _source document', () => { + /** fields is a flattened object key and an empty array value (p_[]) */ + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [], + }; + + test('when source is an empty array (p_[]), merged doc is empty array (p_[])"', () => { + const _source: SignalSourceHit['_source'] = { + foo: { bar: [] }, + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is a primitive (p_p1), merged doc is primitive (p_p1)', () => { + const _source: SignalSourceHit['_source'] = { + foo: { bar: 'value' }, + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array with a single primitive (p_[p1]), merged doc is primitive (p_[p1])', () => { + const _source: SignalSourceHit['_source'] = { + foo: { bar: ['value'] }, + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array with 2 or more primitives (p_[p1, ..1]), merged doc is primitive (p_[p1, ...1])', () => { + const _source: SignalSourceHit['_source'] = { + foo: { bar: ['value_1', 'value_2'] }, + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is a single object (p_{}), merged doc is single object (p_{})', () => { + const _source: SignalSourceHit['_source'] = { + foo: { bar: { mars: 'some value' } }, + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array with single object (p_[{}1]), merged doc is single object (p_[{}1])', () => { + const _source: SignalSourceHit['_source'] = { + foo: { bar: [{ mars: 'some value' }] }, + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array of 1 or more objects (p_[{}, ...1]), merged doc is the same (p_[{}, ...1])', () => { + const _source: SignalSourceHit['_source'] = { + foo: { bar: [{ mars: 'some value' }, { mars: 'some other value' }] }, + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + }); + + /** + * source | fields | value after merge + * ----- | --------- | ----- + * undefined | f_[] | undefined + * f_[] | f_[] | f_[] + * f_p1 | f_[] | f_p1 + * f_[p1] | f_[] | f_[p1] + * f_[p1, ...1] | f_[] | f_[p1, ...1] + * f_{}1 | f_[] | f_{}1 + * f_[{}1] | f_[] | f_{}1 + * f_[{}1, ...1] | f_[] | f_[{}1, ...1] + */ + describe('flattened object keys in the _source document', () => { + /** fields is flattened object key with an empty array for a value (f_[]) */ + const fields: SignalSourceHit['fields'] = { + 'bar.foo': [], + }; + + test('when source is an empty array (f_[]), merged doc is empty array (f_[])"', () => { + const _source: SignalSourceHit['_source'] = { + 'bar.foo': [], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is a primitive (f_p1), merged doc is primitive (f_p1)', () => { + const _source: SignalSourceHit['_source'] = { + 'bar.foo': 'value', + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array with a single primitive (f_[p1]), merged doc is primitive (f_[p1])', () => { + const _source: SignalSourceHit['_source'] = { + 'bar.foo': ['value'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array with 2 or more primitives (f_[p1, ...1]), merged doc is primitive (f_[p1, ...1])', () => { + const _source: SignalSourceHit['_source'] = { + 'bar.foo': ['value_1', 'value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is a single object (f_{}1), merged doc is single object (f_{}1)', () => { + const _source: SignalSourceHit['_source'] = { + foo: { bar: 'some value' }, + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array with single object ([f_{}1]), merged doc is single object ([f_{}1])', () => { + const _source: SignalSourceHit['_source'] = { + foo: [{ bar: 'some value' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('when source is an array of 1 or more objects (f_[{}1, ...1]), merged doc is the same (f_[{}1, ...1])', () => { + const _source: SignalSourceHit['_source'] = { + foo: [{ bar: 'some value' }, { foo: 'some other value' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + }); + }); + + /** + * source | fields | value after merge + * ----- | --------- | ----- + * undefined | f_[p2] | p_p2 <-- Unboxed from array + * undefined | f_[p2, ...2] | p_[p2, ...2] + * undefined | f_[{}2] | {} <-- We have an empty object since we only merge primitives + * undefined | f_[{}2, ...2] | {} <-- We have an empty object since we only merge primitives + */ + describe('source is "undefined"', () => { + /** _source is "undefined" for all tests below */ + const _source: SignalSourceHit['_source'] = {}; + + test('fields is a single primitive value (f_[p2]), merged doc is an unboxed array element p_p2"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: { + bar: 'other_value_1', + }, + }); + }); + + test('fields is a multiple primitive values (f_[p2, ...2]), merged doc is the array (p_[p2, ...2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1', 'other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: { + bar: ['other_value_1', 'other_value_2'], + }, + }); + }); + + test('fields is a single nested field value (f_[{}2]), merged doc is empty object ({})"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual({}); + }); + + test('fields is multiple nested field values (f_[{}2, ...2]), merged doc is the empty object ({})"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual({}); + }); + }); + + describe('source is either primitive or flattened keys, with primitive values', () => { + /** + * source | fields | value after merge + * ----- | --------- | ----- + * p_p1 | f_[p2] | p_p1 + * p_p1 | f_[p2, ...2] | p_p1 + * p_p1 | f_[{}2] | p_p1 + * p_p1 | f_[{}2, ...2] | p_p1 + */ + describe('primitive keys in the _source document with the value of "value" (p_p1)', () => { + /** _source is a single primitive key with a primitive value for all tests below (p_p1) */ + const _source: SignalSourceHit['_source'] = { + foo: { bar: 'value' }, + }; + + test('fields is single array primitive value (f_[p2]), merged doc is unboxed primitive key and value (p_p1)"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has single primitive values (f_[p2, ...2]), merged doc is the array (p_p1)"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1', 'other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has a single nested object (f_[{}2]), merged doc is the unboxed array (p_p1)"', () => { + const fields: SignalSourceHit['fields'] = { + foo: [{ bar: 'other_value_1' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has multiple nested objects (f_[{}2, ...2]), merged doc is the array (p_p1)"', () => { + const fields: SignalSourceHit['fields'] = { + foo: [{ bar: 'other_value_1' }, { bar: 'other_value_2' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + }); + + /** + * source | fields | value after merge + * ----- | --------- | ----- + * f_p1 | f_[p2] | f_p1 + * f_p1 | f_[p2, ...2] | f_p1 + * f_p1 | f_[{}2] | f_p1 + * f_p1 | f_[{}2, ...2] | f_p1 + */ + describe('flattened object keys in the _source document (f_p1)', () => { + /** _source is a flattened object key with a primitive value for all tests below (f_p1) */ + const _source: SignalSourceHit['_source'] = { + 'foo.bar': 'value', + }; + + test('fields is flattened object key with single array value (f_[p2]), merged doc is unboxed primitive key and value (f_p1)"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has single primitive value (f_[p2, ...2]), merged doc is the array (f_p1)"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1', 'other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has a single nested object (f_[{}2]), merged doc is the unboxed array (f_p1)"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has multiple nested objects (f_[{}2, ...2]), merged doc is the array (f_p1)"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + }); + }); + + describe('source is either primitive or flattened keys, with primitive array values', () => { + /** + * source | fields | value after merge + * ----- | --------- | ----- + * p_[] | f_[p2] | p_[] + * p_[] | f_[p2, ...2] | p_[] + * p_[] | f_[{}2] | p_[] + * p_[] | f_[{}2, ...2] | p_[] + */ + describe('primitive keys in the _source document with empty array (p_[])', () => { + /** _source is a primitive key with an empty array for all tests below (p_[]) */ + const _source: SignalSourceHit['_source'] = { + foo: { bar: [] }, + }; + + test('fields is flattened object key with single array value (f_[p2]), merged doc is array value (p_[]])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has single primitive values (f_[p2, ...2]), merged doc is the array (p_[])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1', 'other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has a single nested object (f_[{}2]), merged doc is the array value (p_[])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has multiple nested objects (f_[{}2, ...2]), merged doc is the array (p_[])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + }); + + /** + * source | fields | value after merge + * ----- | --------- | ----- + * f_[] | f_[p2] | f_[] + * f_[] | f_[p2, ...2] | f_[] + * f_[] | f_[{}2] | f_[] + * f_[] | f_[{}2, ...2] | f_[] + */ + describe('flattened object keys in the _source document with empty array (f_[])', () => { + /** _source is a flattened object key with an empty array for all tests below (f_[]) */ + const _source: SignalSourceHit['_source'] = { + 'foo.bar': [], + }; + + test('fields is flattened object key with single array value (f_[p2]), merged doc is array (f_[p2])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has multiple primitive values (f_[p2, ...2]), merged doc is the array (f_[])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1', 'other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has a single nested object (f_[{}2]), merged doc is the array (f_[])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has multiple nested objects (f_[{}2, ...2]), merged doc is the array (f_[])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + }); + + /** + * source | fields | value after merge + * ----- | --------- | ----- + * p_[p1] | f_[p2] | p_[p1] + * p_[p1] | f_[p2, ...2] | p_[p1] + * p_[p1] | f_[{}2] | p_[p1] + * p_[p1] | f_[{}2, ...2] | p_[p1] + */ + describe('primitive keys in the _source document with single primitive value in an array (p_[p1])', () => { + /** _source is a primitive key with a single primitive array value for all tests below (p_[p1]) */ + const _source: SignalSourceHit['_source'] = { + foo: { bar: ['value'] }, + }; + + test('fields is flattened object key with single array value (f_[p2]), merged doc is the array value (p_[p1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has single primitive value (f_[p2, ...2]), merged doc is the array (p_[p1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1', 'other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has a single nested object (f_[{}2]), merged doc is the array (p_[p1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has multiple nested objects (f_[{}2, ...2]), merged doc is the array (p_[p1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + }); + + /** + * source | fields | value after merge + * ----- | --------- | ----- + * f_[p1] | f_[p2] | f_[p1] + * f_[p1] | f_[p2, ...2] | f_[p1] + * f_[p1] | f_[{}2] | f_[p1] + * f_[p1] | f_[{}2, ...2] | f_[p1] + */ + describe('flattened keys in the _source document with single flattened value in an array (f_[p1])', () => { + /** _source is a flattened object key with a single primitive value for all tests below (f_p[p1]) */ + const _source: SignalSourceHit['_source'] = { + 'foo.bar': ['value'], + }; + + test('fields is flattened object key with single array value (f_[p2]), merged doc is array value (f_[p1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has single primitive value (f_[p2, ...2]), merged doc is the array (f_[p1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1', 'other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has a single nested object (f_[{}2]), merged doc is the array (f_[p1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has multiple nested objects (f_[{}2, ...2]), merged doc is the array (f_[p1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + }); + + /** + * source | fields | value after merge + * ----- | --------- | ----- + * p_[p1, ...1] | f_[p2] | p_[p1, ...1] + * p_[p1, ...1] | f_[p2, ...2] | p_[p1, ...1] + * p_[p1, ...1] | f_[{}2] | p_[p1, ...1] + * p_[p1, ...1] | f_[{}2, ...2] | p_[p1, ...1] + */ + describe('primitive keys in the _source document with multiple array values in an array (p_[p1, ...1])', () => { + /** _source is a primitive key with an array of 2 or more elements for all tests below (p_[p1, ...1]) */ + const _source: SignalSourceHit['_source'] = { + foo: { + bar: ['value_1', 'value_2'], + }, + }; + + test('fields is single array value (f_[p2]), merged doc is array (p_[p1, ...1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields is multiple primitive values (f_[p2, ...2]), merged doc is the array (p_[p1, ...1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1', 'other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has a single nested object (f_[{}2]), merged doc is the unboxed value (p_[p1, ...1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has multiple nested objects (f_[{}2, ...2]), merged doc is the array (p_[p1, ...1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + }); + + /** + * source | fields | value after merge + * ----- | --------- | ----- + * f_[p1, ...1] | f_[p2] | f_[p1, ...1] + * f_[p1, ...1] | f_[p2, ...2] | f_[p1, ...1] + * f_[p1, ...1] | f_[{}2] | f_[p1, ...1] + * f_[p1, ...1] | f_[{}2, ...2] | f_[p1, ...1] + */ + describe('flattened keys in the _source document with multiple array values in an array (f_[p1, ...1])', () => { + /** _source is a flattened object key with an array of 2 or more elements for all tests below (f_[p1, ...1]) */ + const _source: SignalSourceHit['_source'] = { + 'foo.bar': ['value_1', 'value_2'], + }; + + test('fields is flattened object key with single array value (f_[p2]), merged doc is unboxed primitive key and value (f_[p1, ...1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has multiple primitive values (f_[p2, ...2]), merged doc is the array (f_[p1, ...1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1', 'other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has a single nested object (f_[{}2]), merged doc is the array (f_[p1, ...1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has multiple nested objects (f_[{}2, ...2]), merged doc is the array (f_[p1, ...1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + }); + }); + + describe('source is either primitive or flattened keys, with object values', () => { + /** + * source | fields | value after merge + * ----- | --------- | ----- + * p_{}1 | f_[p2] | p_{}1 + * p_{}1 | f_[p2, ...2] | p_{}1 + * p_{}1 | f_[{}2] | p_{}1 + * p_{}1 | f_[{}2, ...2] | p_{}1 + */ + describe('primitive keys in the _source document with the value of "value" (p_{}1)', () => { + /** _source is a primitive key with an object value for all tests below (p_{}1) */ + const _source: SignalSourceHit['_source'] = { + foo: { bar: { mars: 'value_1' } }, + }; + + test('fields is flattened object key with single array value (f_[p2]), merged doc is the same source (p_{}1)"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has single primitive values (f_[p2, ...2]), merged doc is the same _source (p_{}1)"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1', 'other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has a single nested object (f_[{}2]), merged doc is the unboxed array value (p_{}1)"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has multiple nested objects (f_[{}2, ...2]), merged doc is the array (p_{}1)"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + }); + + /** + * source | fields | value after merge + * ----- | --------- | ----- + * f_{}1 | f_[p2] | f_{}1 + * f_{}1 | f_[p2, ...2] | f_{}1 + * f_{}1 | f_[{}2] | f_{}1 + * f_{}1 | f_[{}2, ...2] | f_{}1 + */ + describe('flattened object keys in the _source document with the value of "value" (f_{}1)', () => { + /** _source is a flattened object key with an object value for all tests below (f_{}1) */ + const _source: SignalSourceHit['_source'] = { + 'foo.bar': { mars: 'value_1' }, + }; + + test('fields is flattened object key with single array value (f_[p2]), merged doc is the same source (f_{}1)"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has primitive values (f_[p2, ...2]), merged doc is the same _source (f_{}1)"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1', 'other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has a single nested object (f_[{}2]), merged doc is unboxed array value (f_{}1)"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ mars: 'other_value_1' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has multiple nested objects (f_[{}2, ...2]), merged doc is the array (f_{}1)"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + }); + }); + + describe('source is either primitive or flattened keys, with object array values', () => { + /** + * source | fields | value after merge + * ----- | --------- | ----- + * p_[{}1] | f_[p2] | p_[{}1] + * p_[{}1] | f_[p2, ...2] | p_[{}1] + * p_[{}1] | f_[{}2] | p_[{}1] + * p_[{}1] | f_[{}2, ...2] | p_[{}1] + */ + describe('primitive keys in the _source document with a single array value with an object (p_[{}1])', () => { + /** _source is a primitive key with a single array value with an object for all tests below (p_[{}1]) */ + const _source: SignalSourceHit['_source'] = { + foo: { bar: [{ mars: ['value_1'] }] }, + }; + + test('fields has a single primitive value (f_[p2]), merged doc is the same _source (p_[{}1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has 2 or more primitive values (f_[p2, ...2]), merged doc is the same _source (p_[{}1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1', 'other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has a single nested object (f_[{}2]), merged doc is the array value (p_[{}1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has multiple nested objects (f_[{}2, ...2]), merged doc is the array (p_[{}1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + }); + + /** + * source | fields | value after merge + * ----- | --------- | ----- + * p_[{}1, ...1] | f_[p2] | p_[{}1, ...1] + * p_[{}1, ...1] | f_[p2, ...2] | p_[{}1, ...1] + * p_[{}1, ...1] | f_[{}2] | p_[{}1, ...1] + * p_[{}1, ...1] | f_[{}2, ...2] | p_[{}1, ...1] + */ + describe('primitive keys in the _source document with multiple array objects (p_[{}1, ...1])', () => { + /** _source is a primitive key with a 2 or more array values with an object for all tests below (p_[{}1, ...1]) */ + const _source: SignalSourceHit['_source'] = { + foo: { bar: [{ mars: ['value_1'] }, { mars: ['value_1'] }] }, + }; + + test('fields has a single primitive value (f_[p2]), merged doc is the same _source (p_[{}1, ...1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has 2 or more primitive values (f_[p2, ...2]), merged doc is the same _source (p_[{}1, ...1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1', 'other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has a single nested object (f_[{}2]), merged doc is the array value (p_[{}1, ...1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has multiple nested objects (f_[{}2, ...2]), merged doc is the array (p_[{}1, ...1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + }); + + /** + * source | fields | value after merge + * ----- | --------- | ----- + * f_[{}1] | f_[p2] | f_[{}1] + * f_[{}1] | f_[p2, ...2] | f_[{}1] + * f_[{}1] | f_[{}2] | f_[{}1] + * f_[{}1] | f_[{}2, ...2] | f_[{}1] + */ + describe('flattened object keys in the _source document with the single value of "value" (f_[{}1])', () => { + /** _source is a flattened object key with a single array object for all tests below (f_[{}1]) */ + const _source: SignalSourceHit['_source'] = { + 'foo.bar': [{ mars: 'value_1' }], + }; + + test('fields is flattened object key with single array value (f_[p2]), merged doc is the same _source (f_[{}1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has primitive values (f_[p2, ...2]), merged doc is the same _source (f_[{}1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1', 'other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has a single nested object (f_[{}2]), merged doc is array value (f_[{}1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ mars: 'other_value_1' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has multiple nested objects (f_[{}2, ...2]), merged doc is the array (f_[{}1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + }); + + /** + * source | fields | value after merge + * ----- | --------- | ----- + * f_[{}1, ...1] | f_[p2] | f_[{}1, ...1] + * f_[{}1, ...1] | f_[p2, ...2] | f_[{}1, ...1] + * f_[{}1, ...1] | f_[{}2] | f_[{}1, ...1] + * f_[{}1, ...1] | f_[{}2, ...2] | f_[{}1, ...1] + */ + describe('flattened object keys in the _source document with multiple values of "value" (f_[{}1, ...1])', () => { + /** _source is a flattened object key with 2 or more array objects for all tests below (f_[{}1]) */ + const _source: SignalSourceHit['_source'] = { + 'foo.bar': [{ mars: 'value_1' }, { mars: 'value_2' }], + }; + + test('fields is flattened object key with single array value (f_[p2]), merged doc is the same _source (f_[{}1, ...1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has primitive values (f_[p2, ...2]), merged doc is the same _source (f_[{}1, ...1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1', 'other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has a single nested object (f_[{}2]), merged doc is array value (f_[{}1, ...1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ mars: 'other_value_1' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('fields has multiple nested objects (f_[{}2, ...2]), merged doc is the array (f_[{}1, ...1])"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + }); + }); + + /** + * It is possible to have a mixture of flattened keys and primitive keys within a _source document. + * These tests cover those cases and these test cases should be considered hopefully rare occurrences. + * If these become more common place, update the top table with all the permutations and combinations + * of tests for these. For now, expect the flattened object keys to get the values added to them vs. + * the other value. These tests show the behaviors of this but also the existing bugs of what happens + * when we merge. + */ + describe('miscellaneous tests of mixed flattened and source objects within _source', () => { + /** _source has a primitive key mixed with an object with the same path information which causes ambiguity */ + const _source: SignalSourceHit['_source'] = { + foo: { bar: 'value_1' }, + 'foo.bar': 'value_2', + }; + + test('fields has a single primitive value f_[p2] which is not overridden"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + /** + * This is an ambiguous situation in which we produce correct results since _source is defined. + */ + test('fields has the same list of values as that of the original document"', () => { + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['value_1', 'value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + }); + + /** + * These tests show the behaviors around overriding fields with other fields such as objects overriding + * values and values overriding objects. This occurs with multi fields where you can have "foo" and "foo.keyword" + * in the fields + */ + describe('Fields overriding fields', () => { + describe('primitive keys for the _source', () => { + test('DOES NOT remove multi-field values such "foo.keyword" mixed with "foo" and prefers just "foo" for 1st level', () => { + const _source: SignalSourceHit['_source'] = { + foo: 'foo_value_1', + bar: 'bar_value_1', + }; + const fields: SignalSourceHit['fields'] = { + foo: ['foo_other_value_1'], + 'foo.keyword': ['foo_other_value_keyword_1'], + bar: ['bar_other_value_1'], + 'bar.keyword': ['bar_other_value_keyword_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('DOES NOT remove multi-field values such "host.name.keyword" mixed with "host.name" and prefers just "host.name" for 2nd level', () => { + const _source: SignalSourceHit['_source'] = { + host: { + name: 'host_value_1', + hostname: 'host_name_value_1', + }, + }; + const fields: SignalSourceHit['fields'] = { + 'host.name': ['host_name_other_value_1'], + 'host.name.keyword': ['host_name_other_value_keyword_1'], + 'host.hostname': ['hostname_other_value_1'], + 'host.hostname.keyword': ['hostname_other_value_keyword_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('DOES NOT remove multi-field values such "foo.host.name.keyword" mixed with "foo.host.name" and prefers just "foo.host.name" for 3rd level', () => { + const _source: SignalSourceHit['_source'] = { + foo: { + host: { + name: 'host_value_1', + hostname: 'host_name_value_1', + }, + }, + }; + const fields: SignalSourceHit['fields'] = { + 'foo.host.name': ['host_name_other_value_1'], + 'foo.host.name.keyword': ['host_name_other_value_keyword_1'], + 'foo.host.hostname': ['hostname_other_value_1'], + 'foo.host.hostname.keyword': ['hostname_other_value_keyword_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('multi-field values mixed with regular values will not be merged accidentally"', () => { + const _source: SignalSourceHit['_source'] = {}; + const fields: SignalSourceHit['fields'] = { + foo: ['other_value_1'], + 'foo.bar': ['other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: 'other_value_1', + }); + }); + }); + + describe('flattened keys for the _source', () => { + test('DOES NOT remove multi-field values such "host.name.keyword" mixed with "host.name" and prefers just "host.name" for 2nd level', () => { + const _source: SignalSourceHit['_source'] = { + 'host.name': 'host_value_1', + 'host.hostname': 'host_name_value_1', + }; + const fields: SignalSourceHit['fields'] = { + 'host.name': ['host_name_other_value_1'], + 'host.name.keyword': ['host_name_other_value_keyword_1'], + 'host.hostname': ['hostname_other_value_1'], + 'host.hostname.keyword': ['hostname_other_value_keyword_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('DOES NOT remove multi-field values such "foo.host.name.keyword" mixed with "foo.host.name" and prefers just "foo.host.name" for 3rd level', () => { + const _source: SignalSourceHit['_source'] = { + 'foo.host.name': 'host_value_1', + 'foo.host.hostname': 'host_name_value_1', + }; + const fields: SignalSourceHit['fields'] = { + 'foo.host.name': ['host_name_other_value_1'], + 'foo.host.name.keyword': ['host_name_other_value_keyword_1'], + 'foo.host.hostname': ['hostname_other_value_1'], + 'foo.host.hostname.keyword': ['hostname_other_value_keyword_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('invalid fields of several levels mixed with regular values will not be merged accidentally due to runtime fields being liberal"', () => { + const _source: SignalSourceHit['_source'] = {}; + const fields: SignalSourceHit['fields'] = { + foo: ['other_value_1'], + 'foo.bar': ['other_value_2'], + 'foo.bar.zed': ['zed_other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual({ + foo: 'other_value_1', + }); + }); + }); + }); + + /** + * These tests are around parent objects that are not nested but are array types. We do not try to merge + * into these as this causes ambiguities between array types and object types. + */ + describe('parent array objects', () => { + test('parent array objects will not be overridden since that is an ambiguous use case for a top level value', () => { + const _source: SignalSourceHit['_source'] = { + foo: [ + { + bar: 'value_1', + mars: ['value_1'], + }, + ], + }; + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + 'foo.mars': ['other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + + test('parent array objects will not be overridden since that is an ambiguous use case for a deeply nested value', () => { + const _source: SignalSourceHit['_source'] = { + foo: { + zed: [ + { + bar: 'value_1', + mars: ['value_1'], + }, + ], + }, + }; + const fields: SignalSourceHit['fields'] = { + 'foo.zed.bar': ['other_value_1'], + 'foo.zed.mars': ['other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + }); + + /** + * Specific tests around nested field types such as ensuring we are unboxing when we can + */ + describe('nested fields', () => { + test('returns empty object since we only consider merging in primitive values and not nested fields', () => { + const _source: SignalSourceHit['_source'] = {}; + const fields: SignalSourceHit['fields'] = { + foo: [{ bar: ['single_value'], zed: ['single_value'] }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual({}); + }); + + test('does not touch the source object when it is empty arrays', () => { + const _source: SignalSourceHit['_source'] = { + foo: [ + { + bar: [], + zed: [], + }, + ], + }; + const fields: SignalSourceHit['fields'] = { + foo: [{ bar: ['single_value'], zed: ['single_value'] }], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc })._source; + expect(merged).toEqual(_source); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.ts new file mode 100644 index 0000000000000..bf541acbe7e33 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.ts @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { get } from 'lodash/fp'; +import { set } from '@elastic/safer-lodash-set/fp'; +import { SignalSource, SignalSourceHit } from '../../types'; +import { filterFieldEntries } from '../utils/filter_field_entries'; +import type { FieldsType } from '../types'; +import { recursiveUnboxingFields } from '../utils/recursive_unboxing_fields'; +import { isTypeObject } from '../utils/is_type_object'; +import { arrayInPathExists } from '../utils/array_in_path_exists'; +import { isNestedObject } from '../utils/is_nested_object'; + +/** + * Merges only missing sections of "doc._source" with its "doc.fields" on a "best effort" basis. See ../README.md for more information + * on this function and the general strategies. + * @param doc The document with "_source" and "fields" + * @param throwOnFailSafe Defaults to false, but if set to true it will cause a throw if the fail safe is triggered to indicate we need to add a new explicit test condition + * @returns The two merged together in one object where we can + */ +export const mergeMissingFieldsWithSource = ({ + doc, +}: { + doc: SignalSourceHit; +}): SignalSourceHit => { + const source = doc._source ?? {}; + const fields = doc.fields ?? {}; + const fieldEntries = Object.entries(fields); + const filteredEntries = filterFieldEntries(fieldEntries); + + const transformedSource = filteredEntries.reduce( + (merged, [fieldsKey, fieldsValue]: [string, FieldsType]) => { + if ( + hasEarlyReturnConditions({ + fieldsValue, + fieldsKey, + merged, + }) + ) { + return merged; + } + + const valueInMergedDocument = get(fieldsKey, merged); + const valueToMerge = recursiveUnboxingFields(fieldsValue, valueInMergedDocument); + return set(fieldsKey, valueToMerge, merged); + }, + { ...source } + ); + + return { + ...doc, + _source: transformedSource, + }; +}; + +/** + * Returns true if any early return conditions are met which are + * - If the fieldsValue is an empty array return + * - If the value to merge in is not undefined, return early + * - If an array within the path exists, do an early return + * - If the value matches a type object, do an early return + * @param fieldsValue The field value to check + * @param fieldsKey The key of the field we are checking + * @param merged The merge document which is what we are testing conditions against + * @returns true if we should return early, otherwise false + */ +const hasEarlyReturnConditions = ({ + fieldsValue, + fieldsKey, + merged, +}: { + fieldsValue: FieldsType; + fieldsKey: string; + merged: SignalSource; +}) => { + const valueInMergedDocument = get(fieldsKey, merged); + return ( + fieldsValue.length === 0 || + valueInMergedDocument !== undefined || + arrayInPathExists(fieldsKey, merged) || + isNestedObject(fieldsValue) || + isTypeObject(fieldsValue) + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/types.ts new file mode 100644 index 0000000000000..e8142e41715e2 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/types.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * A bit stricter typing since the default fields type is an "any" + */ +export type FieldsType = string[] | number[] | boolean[] | object[]; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/array_in_path_exists.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/array_in_path_exists.test.ts new file mode 100644 index 0000000000000..1b2accd0a16a1 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/array_in_path_exists.test.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { arrayInPathExists } from './array_in_path_exists'; + +describe('array_in_path_exists', () => { + beforeAll(() => { + jest.resetAllMocks(); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + test('returns false when empty string and empty object', () => { + expect(arrayInPathExists('', {})).toEqual(false); + }); + + test('returns false when a path and empty object', () => { + expect(arrayInPathExists('a.b.c', {})).toEqual(false); + }); + + test('returns true when a path and an array exists', () => { + expect(arrayInPathExists('a', { a: [] })).toEqual(true); + }); + + test('returns true when a path and an array exists within the parent path at level 1', () => { + expect(arrayInPathExists('a.b', { a: [] })).toEqual(true); + }); + + test('returns true when a path and an array exists within the parent path at level 3', () => { + expect(arrayInPathExists('a.b.c', { a: [] })).toEqual(true); + }); + + test('returns true when a path and an array exists within the parent path at level 2', () => { + expect(arrayInPathExists('a.b.c', { a: { b: [] } })).toEqual(true); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/array_in_path_exists.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/array_in_path_exists.ts new file mode 100644 index 0000000000000..b8e742fbaba61 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/array_in_path_exists.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { get } from 'lodash/fp'; +import { SignalSource } from '../../types'; + +/** + * Returns true if an array within the path exists anywhere. + * @param fieldsKey The fields key to check if an array exists along the path + * @param source The source document to check for an array anywhere along the path + * @returns true if we detect an array along the path, otherwise false + */ +export const arrayInPathExists = (fieldsKey: string, source: SignalSource): boolean => { + const splitPath = fieldsKey.split('.'); + return splitPath.some((_, index, array) => { + const newPath = [...array].splice(0, index + 1).join('.'); + return Array.isArray(get(newPath, source)); + }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/filter_field_entries.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/filter_field_entries.test.ts new file mode 100644 index 0000000000000..9cc2478290885 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/filter_field_entries.test.ts @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { filterFieldEntries } from './filter_field_entries'; +import { FieldsType } from '../types'; + +describe('filter_field_entries', () => { + beforeAll(() => { + jest.resetAllMocks(); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + /** Dummy test value */ + const dummyValue = ['value']; + + /** + * Get the return type of the mergeFieldsWithSource for TypeScript checks against expected + */ + type ReturnTypeFilterFieldEntries = ReturnType; + + test('returns a single valid fieldEntries as expected', () => { + const fieldEntries: Array<[string, FieldsType]> = [['foo.bar', dummyValue]]; + expect(filterFieldEntries(fieldEntries)).toEqual(fieldEntries); + }); + + test('removes invalid dotted entries', () => { + const fieldEntries: Array<[string, FieldsType]> = [ + ['.', dummyValue], + ['foo.bar', dummyValue], + ['..', dummyValue], + ['foo..bar', dummyValue], + ]; + expect(filterFieldEntries(fieldEntries)).toEqual([ + ['foo.bar', dummyValue], + ]); + }); + + test('removes multi-field values such "foo.keyword" mixed with "foo" and prefers just "foo" for 1st level', () => { + const fieldEntries: Array<[string, FieldsType]> = [ + ['foo', dummyValue], + ['foo.keyword', dummyValue], // <-- "foo.keyword" multi-field should be removed + ['bar.keyword', dummyValue], // <-- "bar.keyword" multi-field should be removed + ['bar', dummyValue], + ]; + expect(filterFieldEntries(fieldEntries)).toEqual([ + ['foo', dummyValue], + ['bar', dummyValue], + ]); + }); + + test('removes multi-field values such "host.name.keyword" mixed with "host.name" and prefers just "host.name" for 2nd level', () => { + const fieldEntries: Array<[string, FieldsType]> = [ + ['host.name', dummyValue], + ['host.name.keyword', dummyValue], // <-- multi-field should be removed + ['host.hostname', dummyValue], + ['host.hostname.keyword', dummyValue], // <-- multi-field should be removed + ]; + expect(filterFieldEntries(fieldEntries)).toEqual([ + ['host.name', dummyValue], + ['host.hostname', dummyValue], + ]); + }); + + test('removes multi-field values such "foo.host.name.keyword" mixed with "foo.host.name" and prefers just "foo.host.name" for 3rd level', () => { + const fieldEntries: Array<[string, FieldsType]> = [ + ['foo.host.name', dummyValue], + ['foo.host.name.keyword', dummyValue], // <-- multi-field should be removed + ['foo.host.hostname', dummyValue], + ['foo.host.hostname.keyword', dummyValue], // <-- multi-field should be removed + ]; + expect(filterFieldEntries(fieldEntries)).toEqual([ + ['foo.host.name', dummyValue], + ['foo.host.hostname', dummyValue], + ]); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/filter_field_entries.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/filter_field_entries.ts new file mode 100644 index 0000000000000..221cdabc62847 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/filter_field_entries.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isMultiField } from './is_multifield'; +import { isInvalidKey } from './is_invalid_key'; +import { isTypeObject } from './is_type_object'; +import { FieldsType } from '../types'; + +/** + * Filters field entries by removing invalid field entries such as any invalid characters + * in the keys or if there are sub-objects that are trying to override regular objects and + * are invalid runtime field names. Also matches type objects such as geo-points and we ignore + * those and don't try to merge those. + * + * @param fieldEntries The field entries to filter + * @returns The field entries filtered + */ +export const filterFieldEntries = ( + fieldEntries: Array<[string, FieldsType]> +): Array<[string, FieldsType]> => { + return fieldEntries.filter(([fieldsKey, fieldsValue]: [string, FieldsType]) => { + return ( + !isInvalidKey(fieldsKey) && + !isMultiField(fieldsKey, fieldEntries) && + !isTypeObject(fieldsValue) // TODO: Look at not filtering this and instead transform it so it can be inserted correctly in the strategies which does an overwrite of everything from fields + ); + }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/index.ts new file mode 100644 index 0000000000000..baf9efca511e2 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/index.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export * from './array_in_path_exists'; +export * from './filter_field_entries'; +export * from './is_array_of_primitives'; +export * from './is_invalid_key'; +export * from './is_multifield'; +export * from './is_nested_object'; +export * from './is_objectlike_or_array_of_objectlikes'; +export * from './is_primitive'; +export * from './is_type_object'; +export * from './recursive_unboxing_fields'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_array_of_primitives.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_array_of_primitives.test.ts new file mode 100644 index 0000000000000..22b9903d30de6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_array_of_primitives.test.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isArrayOfPrimitives } from './is_array_of_primitives'; + +describe('is_array_of_primitives', () => { + beforeAll(() => { + jest.resetAllMocks(); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + test('returns true when an empty array is passed in', () => { + expect(isArrayOfPrimitives([])).toEqual(true); + }); + + test('returns true when an array of primitives are passed in', () => { + expect(isArrayOfPrimitives([null, 2, 'string', 5, undefined])).toEqual(true); + }); + + /** + * Simple table test of values of primitive arrays which should all pass + */ + test.each([ + [[null]], + [[1]], + [['string']], + [['string', null, 5, false, String('hi'), Boolean(true), undefined]], + ])('returns true when a primitive array of %o is passed in', (arrayValue) => { + expect(isArrayOfPrimitives(arrayValue)).toEqual(true); + }); + + /** + * Simple table test of values of all objects which should not pass + */ + test.each([ + [[{}]], + [[{ a: 1 }]], + [[1, {}]], + [[[], 'string']], + [['string', null, 5, false, String('hi'), {}, Boolean(true), undefined]], + ])('returns false when the array of %o contains an object is passed in', (arrayValue) => { + expect(isArrayOfPrimitives(arrayValue)).toEqual(false); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_array_of_primitives.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_array_of_primitives.ts new file mode 100644 index 0000000000000..c65c88c40b9bb --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_array_of_primitives.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SearchTypes } from '../../../../../../common/detection_engine/types'; +import { isPrimitive } from './is_primitive'; + +/** + * Returns true if this is an array and all elements of the array are primitives and not objects + * @param valueInMergedDocument The search type to check if everything is primitive or not + * @returns true if is an array and everything in the array is a primitive type + */ +export const isArrayOfPrimitives = (valueInMergedDocument: SearchTypes | null): boolean => { + return ( + Array.isArray(valueInMergedDocument) && + valueInMergedDocument.every((value) => isPrimitive(value)) + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_invalid_key.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_invalid_key.test.ts new file mode 100644 index 0000000000000..7035732238775 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_invalid_key.test.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isInvalidKey } from './is_invalid_key'; + +describe('matches_invalid_key', () => { + beforeAll(() => { + jest.resetAllMocks(); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + test('it returns true if a value is a single dot', () => { + expect(isInvalidKey('.')).toEqual(true); + }); + + test('it returns true if a value starts with a dot', () => { + expect(isInvalidKey('.invalidName')).toEqual(true); + }); + + test('it returns true if a value is 2 dots', () => { + expect(isInvalidKey('..')).toEqual(true); + }); + + test('it returns true if a value is 3 dots', () => { + expect(isInvalidKey('...')).toEqual(true); + }); + + test('it returns true if a value has two dots in its name', () => { + expect(isInvalidKey('host..name')).toEqual(true); + }); + + test('it returns false if a value has a single dot', () => { + expect(isInvalidKey('host.name')).toEqual(false); + }); + + test('it returns false if a value is a regular path', () => { + expect(isInvalidKey('a.b.c.d')).toEqual(false); + }); + + /** Yes, this is a valid key in elastic */ + test('it returns false if a value ends with a dot', () => { + expect(isInvalidKey('validName.')).toEqual(false); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_invalid_key.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_invalid_key.ts new file mode 100644 index 0000000000000..358dc3e148547 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_invalid_key.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Matches any invalid keys from runtime fields such as runtime fields which can start with a + * "." or runtime fields which can have ".." two or more dots. + * @param fieldsKey The fields key to match against + * @returns true if it is invalid key, otherwise false + */ +export const isInvalidKey = (fieldsKey: string): boolean => { + return fieldsKey.startsWith('.') || fieldsKey.match(/[\.]{2,}/) != null; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_multifield.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_multifield.test.ts new file mode 100644 index 0000000000000..a8050b600b464 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_multifield.test.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isMultiField } from './is_multifield'; + +describe('is_multifield', () => { + beforeAll(() => { + jest.resetAllMocks(); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + const dummyValue = ['value']; + + test('it returns true if the string "foo.bar" is a multiField', () => { + expect(isMultiField('foo.bar', [['foo', dummyValue]])).toEqual(true); + }); + + test('it returns false if the string "foo" is not a multiField', () => { + expect(isMultiField('foo', [['foo', dummyValue]])).toEqual(false); + }); + + test('it returns false if we have a sibling string and are not a multiField', () => { + expect(isMultiField('foo.bar', [['foo.mar', dummyValue]])).toEqual(false); + }); + + test('it returns true for a 3rd level match of being a sub-object. Runtime fields can have multiple layers of multiFields', () => { + expect(isMultiField('foo.mars.bar', [['foo', dummyValue]])).toEqual(true); + }); + + test('it returns true for a 3rd level match against a 2nd level sub-object. Runtime fields can have multiple layers of multiFields', () => { + expect(isMultiField('foo.mars.bar', [['foo.mars', dummyValue]])).toEqual(true); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_multifield.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_multifield.ts new file mode 100644 index 0000000000000..feee6026c60b3 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_multifield.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FieldsType } from '../types'; + +/** + * Returns true if we are a multiField when passed in a fields entry and a fields key, + * otherwise false. Notice that runtime fields can have multiple levels of multiFields which is kind a problem + * but we compensate and test for that here as well. So technically this matches both multiFields and + * invalid multiple-multiFields. + * @param fieldsKey The key to check against the entries to see if it is a multiField + * @param fieldEntries The entries to check against. + * @returns True if we are a subObject, otherwise false. + */ +export const isMultiField = ( + fieldsKey: string, + fieldEntries: Array<[string, FieldsType]> +): boolean => { + const splitPath = fieldsKey.split('.'); + return splitPath.some((_, index, array) => { + if (index + 1 === array.length) { + return false; + } else { + const newPath = [...array].splice(0, index + 1).join('.'); + return fieldEntries.some(([fieldKeyToCheck]) => { + return fieldKeyToCheck === newPath; + }); + } + }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_nested_object.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_nested_object.test.ts new file mode 100644 index 0000000000000..d083fb8bdf845 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_nested_object.test.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isNestedObject } from './is_nested_object'; + +describe('is_nested_object', () => { + beforeAll(() => { + jest.resetAllMocks(); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + test('returns false when an empty array is passed in', () => { + expect(isNestedObject([])).toEqual(false); + }); + + /** + * Simple table test of values of primitive arrays which should all return false + */ + test.each([ + [[1]], + [['string']], + [[true]], + [[String('hi')]], + [[Number(5)]], + [[{ type: 'point' }]], + ])('returns false when a primitive array of %o is passed in', (arrayValues) => { + expect(isNestedObject(arrayValues)).toEqual(false); + }); + + /** + * Simple table test of values of primitive arrays which should all return true + */ + test.each([[[{}]], [[{ a: 'foo' }]]])( + 'returns false when a primitive array of %o is passed in', + (arrayValues) => { + expect(isNestedObject(arrayValues)).toEqual(true); + } + ); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_nested_object.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_nested_object.ts new file mode 100644 index 0000000000000..38a0f871279eb --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_nested_object.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isObjectLike } from 'lodash/fp'; +import { isTypeObject } from './is_type_object'; +import { FieldsType } from '../types'; + +/** + * Returns true if the first value is object-like but does not contain the shape of + * a "type object" such as geo point, then it makes an assumption everything is objectlike + * and not "type object" for all the array values. This should be used only for checking + * for nested object types within fields. + * @param fieldsValue The value to check if the first element is object like or not + * @returns True if this is a nested object, otherwise false. + */ +export const isNestedObject = (fieldsValue: FieldsType): boolean => { + return isObjectLike(fieldsValue[0]) && !isTypeObject(fieldsValue); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_objectlike_or_array_of_objectlikes.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_objectlike_or_array_of_objectlikes.test.ts new file mode 100644 index 0000000000000..454b70e69d0a8 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_objectlike_or_array_of_objectlikes.test.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isObjectLikeOrArrayOfObjectLikes } from './is_objectlike_or_array_of_objectlikes'; + +describe('is_objectlike_or_array_of_objectlikes', () => { + beforeAll(() => { + jest.resetAllMocks(); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + test('returns false when an empty array is passed in', () => { + expect(isObjectLikeOrArrayOfObjectLikes([])).toEqual(false); + }); + + test('returns false when an array of primitives are passed in', () => { + expect(isObjectLikeOrArrayOfObjectLikes([null, 2, 'string', 5, undefined])).toEqual(false); + }); + + /** + * Simple table test of values of primitive arrays which should all fail + */ + test.each([ + [[null]], + [[1]], + [['string']], + [['string', null, 5, false, String('hi'), Boolean(true), undefined]], + ])('returns true when a primitive array of %o is passed in', (arrayValue) => { + expect(isObjectLikeOrArrayOfObjectLikes(arrayValue)).toEqual(false); + }); + + /** + * Simple table test of values of primitives which should all fail + */ + test.each([[null], [1], ['string'], [null], [String('hi')], [Boolean(true)], [undefined]])( + 'returns true when a primitive array of %o is passed in', + (arrayValue) => { + expect(isObjectLikeOrArrayOfObjectLikes(arrayValue)).toEqual(false); + } + ); + + /** + * Simple table test of values of all array of objects which should pass + */ + test.each([ + [[{}]], + [[{ a: 1 }]], + [[1, {}]], + [[[], 'string']], + [['string', null, 5, false, String('hi'), {}, Boolean(true), undefined]], + ])('returns false when the array of %o contains an object is passed in', (arrayValue) => { + expect(isObjectLikeOrArrayOfObjectLikes(arrayValue)).toEqual(true); + }); + + /** + * Simple table test of objects which should pass + */ + test.each([[{}], [{ a: 1 }]])( + 'returns false when the array of %o contains an object is passed in', + (arrayValue) => { + expect(isObjectLikeOrArrayOfObjectLikes(arrayValue)).toEqual(true); + } + ); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_objectlike_or_array_of_objectlikes.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_objectlike_or_array_of_objectlikes.ts new file mode 100644 index 0000000000000..3f57eda31ca3a --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_objectlike_or_array_of_objectlikes.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isObjectLike } from 'lodash/fp'; +import { SearchTypes } from '../../../../../../common/detection_engine/types'; + +/** + * Returns true if at least one element is an object, otherwise false if they all are not objects + * if this is an array. If it is not an array, this will check that single type + * @param valueInMergedDocument The search type to check if it is object like or not + * @returns true if is object like and not an array, or true if it is an array and at least 1 element is object like + */ +export const isObjectLikeOrArrayOfObjectLikes = ( + valueInMergedDocument: SearchTypes | null +): boolean => { + if (Array.isArray(valueInMergedDocument)) { + return valueInMergedDocument.some((value) => isObjectLike(value)); + } else { + return isObjectLike(valueInMergedDocument); + } +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_primitive.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_primitive.test.ts new file mode 100644 index 0000000000000..6d1b273df4ad4 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_primitive.test.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isPrimitive } from './is_primitive'; + +describe('is_primitives', () => { + beforeAll(() => { + jest.resetAllMocks(); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + test('returns false when an empty array is passed in', () => { + expect(isPrimitive([])).toEqual(false); + }); + + /** + * Simple table test of values of primitive values which should all pass + */ + test.each([[null], [1], ['string'], [true], [Boolean('true')]])( + 'returns true when a primitive array of %o is passed in', + (arrayValue) => { + expect(isPrimitive(arrayValue)).toEqual(true); + } + ); + + /** + * Simple table test of values of objects which should not pass + */ + test.each([[{}], [{ a: 1 }]])( + 'returns false when the array of %o contains an object is passed in', + (arrayValue) => { + expect(isPrimitive(arrayValue)).toEqual(false); + } + ); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_primitive.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_primitive.ts new file mode 100644 index 0000000000000..c74b5f085989b --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_primitive.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isObjectLike } from 'lodash/fp'; +import { SearchTypes } from '../../../../../../common/detection_engine/types'; + +/** + * Returns true if it is a primitive type, otherwise false + */ +export const isPrimitive = (valueInMergedDocument: SearchTypes | null): boolean => { + return !isObjectLike(valueInMergedDocument); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_type_object.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_type_object.test.ts new file mode 100644 index 0000000000000..ee5ba60e91e1a --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_type_object.test.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isTypeObject } from './is_type_object'; + +describe('is_type_object', () => { + beforeAll(() => { + jest.resetAllMocks(); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + test('returns false when an empty array is passed in', () => { + expect(isTypeObject([])).toEqual(false); + }); + + test('returns true when a type object is in the array', () => { + expect(isTypeObject([{ type: 'Point' }])).toEqual(true); + }); + + test('returns false when a type object is not in the array', () => { + expect(isTypeObject([{ foo: 'a' }])).toEqual(false); + }); + + test('returns false when a primitive is passed in', () => { + expect(isTypeObject(['string'])).toEqual(false); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_type_object.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_type_object.ts new file mode 100644 index 0000000000000..68afad9ff4fe3 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_type_object.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { get } from 'lodash/fp'; +import { FieldsType } from '../types'; + +/** + * Returns true if we match a "type" object which could be a geo-point when we are parsing field + * values and we encounter a geo-point. + * @param fieldsValue The value to test the shape of the data and see if it is a geo-point or not + * @returns True if we match a geo-point or another type or not. + */ +export const isTypeObject = (fieldsValue: FieldsType): boolean => { + return (fieldsValue as Array).some((value) => { + if (typeof value === 'object' && value != null) { + return get('type', value); + } else { + return false; + } + }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/recursive_unboxing_fields.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/recursive_unboxing_fields.test.ts new file mode 100644 index 0000000000000..130990393b743 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/recursive_unboxing_fields.test.ts @@ -0,0 +1,292 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SearchTypes } from '../../../../../../common/detection_engine/types'; +import { recursiveUnboxingFields } from './recursive_unboxing_fields'; +import { FieldsType } from '../types'; + +describe('recursive_unboxing_fields', () => { + beforeAll(() => { + jest.resetAllMocks(); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + describe('valueInMergedDocument is "undefined"', () => { + const valueInMergedDocument: SearchTypes = undefined; + test('it will return an empty array as is', () => { + const nested: FieldsType = []; + expect(recursiveUnboxingFields(nested, valueInMergedDocument)).toEqual([]); + }); + + test('it will return an empty object as is', () => { + const nested: FieldsType[0] = {}; + expect(recursiveUnboxingFields(nested, valueInMergedDocument)).toEqual({}); + }); + + test('it will unbox a single array field', () => { + const nested: FieldsType = ['foo_value_1']; + expect(recursiveUnboxingFields(nested, valueInMergedDocument)).toEqual('foo_value_1'); + }); + + test('it will not unbox an array with two fields', () => { + const nested: FieldsType = ['foo_value_1', 'foo_value_2']; + expect(recursiveUnboxingFields(nested, valueInMergedDocument)).toEqual([ + 'foo_value_1', + 'foo_value_2', + ]); + }); + + test('it will unbox a nested structure of 3 single arrays', () => { + const nested: FieldsType = [ + { + foo: ['foo_value_1'], + bar: { + zed: ['zed_value_1'], + }, + }, + ]; + const recursed = recursiveUnboxingFields(nested, valueInMergedDocument); + expect(recursed).toEqual({ bar: { zed: 'zed_value_1' }, foo: 'foo_value_1' }); + }); + + test('it will not unbox a nested structure of 2 array values at the top most level', () => { + const nested: FieldsType = [ + { + foo: ['foo_value_1'], + bar: { + zed: ['zed_value_1'], + }, + }, + { + foo: ['foo_value_1'], + bar: { + zed: ['zed_value_1'], + }, + }, + ]; + const recursed = recursiveUnboxingFields(nested, valueInMergedDocument); + expect(recursed).toEqual([ + { bar: { zed: 'zed_value_1' }, foo: 'foo_value_1' }, + { bar: { zed: 'zed_value_1' }, foo: 'foo_value_1' }, + ]); + }); + + test('it will not unbox a nested structure of mixed values at different levels', () => { + const nested: FieldsType = [ + { + foo: ['foo_value_1'], + bar: { + zed: ['zed_value_1'], + fred: { + yolo: ['deep_1', 'deep_2'], + }, + }, + }, + ]; + const recursed = recursiveUnboxingFields(nested, valueInMergedDocument); + expect(recursed).toEqual({ + bar: { fred: { yolo: ['deep_1', 'deep_2'] }, zed: 'zed_value_1' }, + foo: 'foo_value_1', + }); + }); + }); + + describe('valueInMergedDocument is an empty object', () => { + const valueInMergedDocument: SearchTypes = {}; + test('it will return an empty array as is', () => { + const nested: FieldsType = []; + expect(recursiveUnboxingFields(nested, valueInMergedDocument)).toEqual([]); + }); + + test('it will return an empty object as is', () => { + const nested: FieldsType[0] = {}; + expect(recursiveUnboxingFields(nested, valueInMergedDocument)).toEqual({}); + }); + + test('it will unbox a single array field', () => { + const nested: FieldsType = ['foo_value_1']; + expect(recursiveUnboxingFields(nested, valueInMergedDocument)).toEqual('foo_value_1'); + }); + + test('it will not unbox an array with two fields', () => { + const nested: FieldsType = ['foo_value_1', 'foo_value_2']; + expect(recursiveUnboxingFields(nested, valueInMergedDocument)).toEqual([ + 'foo_value_1', + 'foo_value_2', + ]); + }); + + test('it will unbox a nested structure of 3 single arrays', () => { + const nested: FieldsType = [ + { + foo: ['foo_value_1'], + bar: { + zed: ['zed_value_1'], + }, + }, + ]; + const recursed = recursiveUnboxingFields(nested, valueInMergedDocument); + expect(recursed).toEqual({ bar: { zed: 'zed_value_1' }, foo: 'foo_value_1' }); + }); + + test('it will not unbox a nested structure of 2 array values at the top most level', () => { + const nested: FieldsType = [ + { + foo: ['foo_value_1'], + bar: { + zed: ['zed_value_1'], + }, + }, + { + foo: ['foo_value_1'], + bar: { + zed: ['zed_value_1'], + }, + }, + ]; + const recursed = recursiveUnboxingFields(nested, valueInMergedDocument); + expect(recursed).toEqual([ + { bar: { zed: 'zed_value_1' }, foo: 'foo_value_1' }, + { bar: { zed: 'zed_value_1' }, foo: 'foo_value_1' }, + ]); + }); + + test('it will not unbox a nested structure of mixed values at different levels', () => { + const nested: FieldsType = [ + { + foo: ['foo_value_1'], + bar: { + zed: ['zed_value_1'], + fred: { + yolo: ['deep_1', 'deep_2'], + }, + }, + }, + ]; + const recursed = recursiveUnboxingFields(nested, valueInMergedDocument); + expect(recursed).toEqual({ + bar: { fred: { yolo: ['deep_1', 'deep_2'] }, zed: 'zed_value_1' }, + foo: 'foo_value_1', + }); + }); + }); + + describe('valueInMergedDocument mirrors the nested field in different ways', () => { + test('it will not unbox when the valueInMergedDocument is an array value', () => { + const valueInMergedDocument: SearchTypes = ['foo_value_1']; + const nested: FieldsType = ['foo_value_1']; + expect(recursiveUnboxingFields(nested, valueInMergedDocument)).toEqual(['foo_value_1']); + }); + + test('it will not unbox when the valueInMergedDocument is an empty array value', () => { + const valueInMergedDocument: SearchTypes = []; + const nested: FieldsType = ['foo_value_1']; + expect(recursiveUnboxingFields(nested, valueInMergedDocument)).toEqual(['foo_value_1']); + }); + + test('it will not unbox an array with two fields', () => { + const valueInMergedDocument: SearchTypes = ['foo_value_1', 'foo_value_2']; + const nested: FieldsType = ['foo_value_1', 'foo_value_2']; + expect(recursiveUnboxingFields(nested, valueInMergedDocument)).toEqual([ + 'foo_value_1', + 'foo_value_2', + ]); + }); + + test('it will not unbox a nested structure of 3 single arrays when valueInMergedDocument has empty array values', () => { + const valueInMergedDocument: SearchTypes = [ + { + foo: [], + bar: { + zed: [], + }, + }, + ]; + const nested: FieldsType = [ + { + foo: ['foo_value_1'], + bar: { + zed: ['zed_value_1'], + }, + }, + ]; + const recursed = recursiveUnboxingFields(nested, valueInMergedDocument); + expect(recursed).toEqual([{ bar: { zed: ['zed_value_1'] }, foo: ['foo_value_1'] }]); + }); + + test('it will not unbox a nested structure of 3 single arrays when valueInMergedDocument has array values', () => { + const valueInMergedDocument: SearchTypes = [ + { + foo: ['foo_value_1'], + bar: { + zed: ['zed_value_1'], + }, + }, + ]; + const nested: FieldsType = [ + { + foo: ['foo_value_1'], + bar: { + zed: ['zed_value_1'], + }, + }, + ]; + const recursed = recursiveUnboxingFields(nested, valueInMergedDocument); + expect(recursed).toEqual([{ bar: { zed: ['zed_value_1'] }, foo: ['foo_value_1'] }]); + }); + + test('it will not overwrite a nested structure of 3 single arrays when valueInMergedDocument has array values that are different', () => { + const valueInMergedDocument: SearchTypes = [ + { + foo: ['other_value_1'], + bar: { + zed: ['other_value_2'], + }, + }, + ]; + const nested: FieldsType = [ + { + foo: ['foo_value_1'], + bar: { + zed: ['zed_value_1'], + }, + }, + ]; + const recursed = recursiveUnboxingFields(nested, valueInMergedDocument); + expect(recursed).toEqual([{ bar: { zed: ['zed_value_1'] }, foo: ['foo_value_1'] }]); + }); + + test('it will work with mixed array values between "nested" and "valueInMergedDocument"', () => { + const valueInMergedDocument: SearchTypes = [ + { + foo: ['foo_value_1'], + bar: { + zed: ['zed_value_1'], + }, + }, + ]; + const nested: FieldsType = [ + { + foo: ['foo_value_1', 'foo_value_2', 'foo_value_3'], + bar: { + zed: ['zed_value_1', 'zed_value_1', 'zed_value_2'], + }, + }, + ]; + const recursed = recursiveUnboxingFields(nested, valueInMergedDocument); + expect(recursed).toEqual([ + { + bar: { zed: ['zed_value_1', 'zed_value_1', 'zed_value_2'] }, + foo: ['foo_value_1', 'foo_value_2', 'foo_value_3'], + }, + ]); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/recursive_unboxing_fields.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/recursive_unboxing_fields.ts new file mode 100644 index 0000000000000..9cd0ebcb5a427 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/recursive_unboxing_fields.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { get } from 'lodash/fp'; +import { set } from '@elastic/safer-lodash-set/fp'; +import { SearchTypes } from '../../../../../../common/detection_engine/types'; +import { FieldsType } from '../types'; + +/** + * Recursively unboxes fields from an array when it is common sense to unbox them and safe to + * make an assumption to unbox them when we compare them to the "fieldsValue" and the "valueInMergedDocument" + * + * NOTE: We use "typeof fieldsValue === 'object' && fieldsValue != null" instead of lodash "objectLike" + * so that we can do type narrowing into an object to get the keys from it. + * + * @param fieldsValue The fields value that contains the nested field or not. + * @param valueInMergedDocument The document to compare against fields value to see if it is also an array or not + * @returns + */ +export const recursiveUnboxingFields = ( + fieldsValue: FieldsType | FieldsType[0], + valueInMergedDocument: SearchTypes +): FieldsType | FieldsType[0] => { + if (Array.isArray(fieldsValue)) { + const fieldsValueMapped = (fieldsValue as Array).map( + (value, index) => { + if (Array.isArray(valueInMergedDocument)) { + return recursiveUnboxingFields(value, valueInMergedDocument[index]); + } else { + return recursiveUnboxingFields(value, undefined); + } + } + ); + + if (fieldsValueMapped.length === 1) { + if (Array.isArray(valueInMergedDocument)) { + return fieldsValueMapped; + } else { + return fieldsValueMapped[0]; + } + } else { + return fieldsValueMapped; + } + } else if (typeof fieldsValue === 'object' && fieldsValue != null) { + const reducedFromKeys = Object.keys(fieldsValue).reduce((accum, key) => { + const recursed = recursiveUnboxingFields( + get(key, fieldsValue), + get(key, valueInMergedDocument) + ); + return set(key, recursed, accum); + }, {}); + return reducedFromKeys; + } else { + return fieldsValue; + } +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index c399454b9888b..3fc36d5930a0a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -83,14 +83,25 @@ export interface RuleRangeTuple { maxSignals: number; } +/** + * SignalSource is being used as both a type for documents that match detection engine queries as well as + * for queries that could be on top of signals. In cases where it is matched against detection engine queries, + * '@timestamp' might not be there since it is not required and we have timestamp override capabilities. Also + * the signal addition object, "signal?: {" will not be there unless it's a conflicting field when we are running + * queries on events. + * + * For cases where we are running queries against signals (signals on signals) "@timestamp" should always be there + * and the "signal?: {" sub-object should always be there. + */ export interface SignalSource { [key: string]: SearchTypes; - // TODO: SignalSource is being used as the type for documents matching detection engine queries, but they may not - // actually have @timestamp if a timestamp override is used - '@timestamp': string; + '@timestamp'?: string; signal?: { - // parent is deprecated: new signals should populate parents instead - // both are optional until all signals with parent are gone and we can safely remove it + /** + * "parent" is deprecated: new signals should populate "parents" instead. Both are optional + * until all signals with parent are gone and we can safely remove it. + * @deprecated Use parents instead + */ parent?: Ancestor; parents?: Ancestor[]; ancestors: Ancestor[]; @@ -101,7 +112,7 @@ export interface SignalSource { rule: { id: string; }; - // signal.depth doesn't exist on pre-7.10 signals + /** signal.depth was introduced in 7.10 and pre-7.10 signals do not have it. */ depth?: number; original_time?: string; threshold_result?: ThresholdResult; @@ -202,7 +213,9 @@ export interface Signal { version: number; }; rule: RulesSchema; - // DEPRECATED: use parents instead of parent + /** + * @deprecated Use "parents" instead of "parent" + */ parent?: Ancestor; parents: Ancestor[]; ancestors: Ancestor[]; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts index 616cf714d6a8c..4d5ac05957a4b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts @@ -1368,6 +1368,45 @@ describe('utils', () => { const date = getValidDateFromDoc({ doc, timestampOverride: 'different_timestamp' }); expect(date?.toISOString()).toEqual(override); }); + + test('It returns the timestamp if the timestamp happens to be a string of an epoch when it has it in _source and fields', () => { + const doc = sampleDocNoSortId(); + const testDateString = '2021-06-25T15:53:56.590Z'; + const testDate = `${new Date(testDateString).valueOf()}`; + doc._source['@timestamp'] = testDate; + if (doc.fields != null) { + doc.fields['@timestamp'] = [testDate]; + } + const date = getValidDateFromDoc({ doc, timestampOverride: undefined }); + expect(date?.toISOString()).toEqual(testDateString); + }); + + test('It returns the timestamp if the timestamp happens to be a string of an epoch when it has it in _source and fields is nonexistent', () => { + const doc = sampleDocNoSortId(); + const testDateString = '2021-06-25T15:53:56.590Z'; + const testDate = `${new Date(testDateString).valueOf()}`; + doc._source['@timestamp'] = testDate; + doc.fields = undefined; + const date = getValidDateFromDoc({ doc, timestampOverride: undefined }); + expect(date?.toISOString()).toEqual(testDateString); + }); + + test('It returns the timestamp if the timestamp happens to be a string of an epoch in an override field', () => { + const override = '2020-10-07T19:36:31.110Z'; + const testDate = `${new Date(override).valueOf()}`; + let doc = sampleDocNoSortId(); + if (doc == null) { + throw new TypeError('Test requires one element'); + } + doc = { + ...doc, + fields: { + different_timestamp: [testDate], + }, + }; + const date = getValidDateFromDoc({ doc, timestampOverride: 'different_timestamp' }); + expect(date?.toISOString()).toEqual(override); + }); }); describe('createSearchAfterReturnType', () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index 6d67bab6eb2f7..4dd434156288f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -625,6 +625,15 @@ export const getValidDateFromDoc = ({ const tempMoment = moment(lastTimestamp); if (tempMoment.isValid()) { return tempMoment.toDate(); + } else if (typeof timestampValue === 'string') { + // worse case we have a string from fields API or other areas of Elasticsearch that have given us a number as a string, + // so we try one last time to parse this best we can by converting from string to a number + const maybeDate = moment(+lastTimestamp); + if (maybeDate.isValid()) { + return maybeDate.toDate(); + } else { + return undefined; + } } else { return undefined; } diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/aliases.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/aliases.ts index ca1281e0d2da9..790dc2b725a72 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/aliases.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/aliases.ts @@ -40,7 +40,7 @@ export default ({ getService }: FtrProviderContext) => { }); it('should keep the original alias value such as "host_alias" from a source index when the value is indexed', async () => { - const rule = getRuleForSignalTesting(['alias']); + const rule = getRuleForSignalTesting(['host_alias']); const { id } = await createRule(supertest, rule); await waitForRuleSuccessOrStatus(supertest, id); await waitForSignalsToBePresent(supertest, 4, [id]); @@ -51,9 +51,8 @@ export default ({ getService }: FtrProviderContext) => { expect(hits).to.eql(['host name 1', 'host name 2', 'host name 3', 'host name 4']); }); - // TODO: Make aliases work to where we can have ECS fields such as host.name filled out - it.skip('should copy alias data from a source index into the signals index in the same position when the target is ECS compatible', async () => { - const rule = getRuleForSignalTesting(['alias']); + it('should copy alias data from a source index into the signals index in the same position when the target is ECS compatible', async () => { + const rule = getRuleForSignalTesting(['host_alias']); const { id } = await createRule(supertest, rule); await waitForRuleSuccessOrStatus(supertest, id); await waitForSignalsToBePresent(supertest, 4, [id]); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_ml.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_ml.ts index 2294d51537fb1..6a6822ba7eb2d 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_ml.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_ml.ts @@ -191,6 +191,13 @@ export default ({ getService }: FtrProviderContext) => { }, original_time: '2020-11-16T22:58:08.000Z', }, + all_field_values: [ + 'store', + 'linux_anomalous_network_activity_ecs', + 'root', + 'store', + 'mothra', + ], }); }); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/keyword_family/const_keyword.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/keyword_family/const_keyword.ts index fccfe4d74e241..b793fc635843e 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/keyword_family/const_keyword.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/keyword_family/const_keyword.ts @@ -60,8 +60,7 @@ export default ({ getService }: FtrProviderContext) => { expect(signalsOpen.hits.hits.length).to.eql(4); }); - // TODO: Fix this bug and make this work. We currently do not write out the dataset name when it is not in _source - it.skip('should copy the dataset_name_1 from the index into the signal', async () => { + it('should copy the dataset_name_1 from the index into the signal', async () => { const rule = { ...getRuleForSignalTesting(['const_keyword']), query: 'event.dataset: "dataset_name_1"', @@ -99,8 +98,7 @@ export default ({ getService }: FtrProviderContext) => { expect(signalsOpen.hits.hits.length).to.eql(4); }); - // TODO: Fix this bug and make this work. We currently do not write out the dataset name when it is not in _source - it.skip('should copy the "dataset_name_1" from "event.dataset"', async () => { + it('should copy the "dataset_name_1" from "event.dataset"', async () => { const rule: EqlCreateSchema = { ...getRuleForSignalTesting(['const_keyword']), rule_id: 'eql-rule', diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/keyword_family/keyword_mixed_with_const.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/keyword_family/keyword_mixed_with_const.ts index 3802d1f7a7bef..2ce88da13afab 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/keyword_family/keyword_mixed_with_const.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/keyword_family/keyword_mixed_with_const.ts @@ -62,8 +62,7 @@ export default ({ getService }: FtrProviderContext) => { expect(signalsOpen.hits.hits.length).to.eql(8); }); - // TODO: Fix this bug and make this work. We currently do not write out the dataset name when it is not in _source - it.skip('should copy the dataset_name_1 from the index into the signal', async () => { + it('should copy the dataset_name_1 from the index into the signal', async () => { const rule = { ...getRuleForSignalTesting(['keyword', 'const_keyword']), query: 'event.dataset: "dataset_name_1"', @@ -105,8 +104,7 @@ export default ({ getService }: FtrProviderContext) => { expect(signalsOpen.hits.hits.length).to.eql(8); }); - // TODO: Fix this bug and make this work. We currently do not write out the dataset name when it is not in _source - it.skip('should copy the "dataset_name_1" from "event.dataset"', async () => { + it('should copy the "dataset_name_1" from "event.dataset"', async () => { const rule: EqlCreateSchema = { ...getRuleForSignalTesting(['keyword', 'const_keyword']), rule_id: 'eql-rule', diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/runtime.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/runtime.ts index 94cc390e0e6ef..0015a41f911d4 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/runtime.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/runtime.ts @@ -23,7 +23,7 @@ import { export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const esArchiver = getService('esArchiver'); - interface HostAlias { + interface Runtime { name: string; hostname: string; } @@ -47,19 +47,18 @@ export default ({ getService }: FtrProviderContext) => { await waitForRuleSuccessOrStatus(supertest, id); await waitForSignalsToBePresent(supertest, 4, [id]); const signalsOpen = await getSignalsById(supertest, id); - const hits = signalsOpen.hits.hits.map((signal) => (signal._source.host as HostAlias).name); + const hits = signalsOpen.hits.hits.map((signal) => (signal._source.host as Runtime).name); expect(hits).to.eql(['host name 1', 'host name 2', 'host name 3', 'host name 4']); }); - // TODO: Make runtime fields able to be copied to where we can have ECS fields such as host.name filled out by them within the mapping directly - it.skip('should copy "runtime mapping" data from a source index into the signals index in the same position when the target is ECS compatible', async () => { + it('should copy "runtime mapping" data from a source index into the signals index in the same position when the target is ECS compatible', async () => { const rule = getRuleForSignalTesting(['runtime']); const { id } = await createRule(supertest, rule); await waitForRuleSuccessOrStatus(supertest, id); await waitForSignalsToBePresent(supertest, 4, [id]); const signalsOpen = await getSignalsById(supertest, id); const hits = signalsOpen.hits.hits.map( - (signal) => (signal._source.host_alias as HostAlias).hostname + (signal) => (signal._source.host as Runtime).hostname ); expect(hits).to.eql(['host name 1', 'host name 2', 'host name 3', 'host name 4']); }); @@ -81,33 +80,69 @@ export default ({ getService }: FtrProviderContext) => { ); }); - // TODO: Make the overrides of runtime fields override the host.name in this use case. - it.skip('should copy normal non-runtime data set from the source index into the signals index in the same position when the target is ECS compatible', async () => { + /** + * Note, this test shows that we do not shadow or overwrite runtime fields on-top of regular fields as we reduced + * risk with overwriting fields in the strategy we are currently using in detection engine. If you swap, change the strategies + * because we decide to overwrite "_source" values with "fields", then expect to change this test. + */ + it('should NOT copy normal non-runtime data set from the source index into the signals index in the same position when the target is ECS compatible', async () => { const rule = getRuleForSignalTesting(['runtime_conflicting_fields']); const { id } = await createRule(supertest, rule); await waitForRuleSuccessOrStatus(supertest, id); await waitForSignalsToBePresent(supertest, 4, [id]); const signalsOpen = await getSignalsById(supertest, id); - const hits = signalsOpen.hits.hits.map((signal) => (signal._source.host as HostAlias).name); + const hits = signalsOpen.hits.hits.map((signal) => signal._source.host); expect(hits).to.eql([ - 'I am the [host.name] field value which shadows the original host.name value', - 'I am the [host.name] field value which shadows the original host.name value', - 'I am the [host.name] field value which shadows the original host.name value', - 'I am the [host.name] field value which shadows the original host.name value', + [ + { + name: 'host name 1_1', + }, + { + name: 'host name 1_2', + }, + ], + [ + { + name: 'host name 2_1', + }, + { + name: 'host name 2_2', + }, + ], + [ + { + name: 'host name 3_1', + }, + { + name: 'host name 3_2', + }, + ], + [ + { + name: 'host name 4_1', + }, + { + name: 'host name 4_2', + }, + ], ]); }); - // TODO: Make runtime fields able to be copied to where we can have ECS fields such as host.name filled out by them within the mapping directly - it.skip('should copy "runtime mapping" data from a source index into the signals index in the same position when the target is ECS compatible', async () => { + /** + * Note, this test shows that we do NOT shadow or overwrite runtime fields on-top of regular fields when we detect those + * fields as arrays of objects since the objects are flattened in "fields" and we detect something already there so we skip + * this shadowed runtime data as it is ambiguous of where we would put it in the array. + */ + it('should NOT copy "runtime mapping" data from a source index into the signals index in the same position when the target is ECS compatible', async () => { const rule = getRuleForSignalTesting(['runtime_conflicting_fields']); const { id } = await createRule(supertest, rule); await waitForRuleSuccessOrStatus(supertest, id); await waitForSignalsToBePresent(supertest, 4, [id]); const signalsOpen = await getSignalsById(supertest, id); const hits = signalsOpen.hits.hits.map( - (signal) => (signal._source.host_alias as HostAlias).hostname + (signal) => (signal._source.host as Runtime).hostname ); - expect(hits).to.eql(['host name 1', 'host name 2', 'host name 3', 'host name 4']); + expect(hits).to.eql([undefined, undefined, undefined, undefined]); }); }); }); diff --git a/x-pack/test/functional/es_archives/security_solution/alias/data.json b/x-pack/test/functional/es_archives/security_solution/alias/data.json index a8bd64cb044eb..f0f74a9094df1 100644 --- a/x-pack/test/functional/es_archives/security_solution/alias/data.json +++ b/x-pack/test/functional/es_archives/security_solution/alias/data.json @@ -2,7 +2,7 @@ "type": "doc", "value": { "id": "1", - "index": "alias", + "index": "host_alias", "source": { "@timestamp": "2020-10-28T05:00:53.000Z", "host_alias": { @@ -17,7 +17,7 @@ "type": "doc", "value": { "id": "2", - "index": "alias", + "index": "host_alias", "source": { "@timestamp": "2020-10-28T05:01:53.000Z", "host_alias": { @@ -32,7 +32,7 @@ "type": "doc", "value": { "id": "3", - "index": "alias", + "index": "host_alias", "source": { "@timestamp": "2020-10-28T05:02:53.000Z", "host_alias": { @@ -47,7 +47,7 @@ "type": "doc", "value": { "id": "4", - "index": "alias", + "index": "host_alias", "source": { "@timestamp": "2020-10-28T05:03:53.000Z", "host_alias": { diff --git a/x-pack/test/functional/es_archives/security_solution/runtime_conflicting_fields/mappings.json b/x-pack/test/functional/es_archives/security_solution/runtime_conflicting_fields/mappings.json index 04a538a332953..2e34eae159a7f 100644 --- a/x-pack/test/functional/es_archives/security_solution/runtime_conflicting_fields/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/runtime_conflicting_fields/mappings.json @@ -5,7 +5,7 @@ "mappings": { "dynamic": "strict", "runtime": { - "host_alias": { + "host.hostname": { "type": "keyword", "script": { "source": "emit(doc['host.name'].value)" @@ -98,6 +98,9 @@ "properties": { "name": { "type": "keyword" + }, + "hostname": { + "type": "keyword" } } } From f28bfa71ad1ef9803d9a5fa59fcbe70c50564373 Mon Sep 17 00:00:00 2001 From: Aaron Caldwell Date: Mon, 28 Jun 2021 18:13:12 -0600 Subject: [PATCH 45/55] [Maps] Move edit tools to beta and remove experimental config flags (#103556) --- x-pack/plugins/maps/config.ts | 2 -- .../public/classes/layers/layer_wizard_registry.ts | 2 ++ .../public/classes/layers/load_layer_wizards.ts | 5 +---- .../layers/new_vector_layer_wizard/config.tsx | 4 +++- .../sources/es_search_source/es_search_source.tsx | 10 +--------- .../flyout_body/layer_wizard_select.tsx | 1 + x-pack/plugins/maps/server/index.ts | 1 - x-pack/plugins/maps/server/plugin.ts | 9 +-------- x-pack/plugins/maps/server/routes.js | 13 ++----------- x-pack/test/functional/config.js | 1 - 10 files changed, 11 insertions(+), 37 deletions(-) diff --git a/x-pack/plugins/maps/config.ts b/x-pack/plugins/maps/config.ts index 781967714be03..104ba00263545 100644 --- a/x-pack/plugins/maps/config.ts +++ b/x-pack/plugins/maps/config.ts @@ -10,7 +10,6 @@ import { schema, TypeOf } from '@kbn/config-schema'; export interface MapsConfigType { enabled: boolean; showMapVisualizationTypes: boolean; - enableDrawingFeature: boolean; showMapsInspectorAdapter: boolean; preserveDrawingBuffer: boolean; } @@ -18,7 +17,6 @@ export interface MapsConfigType { export const configSchema = schema.object({ enabled: schema.boolean({ defaultValue: true }), showMapVisualizationTypes: schema.boolean({ defaultValue: false }), - enableDrawingFeature: schema.boolean({ defaultValue: false }), // flag used in functional testing showMapsInspectorAdapter: schema.boolean({ defaultValue: false }), // flag used in functional testing diff --git a/x-pack/plugins/maps/public/classes/layers/layer_wizard_registry.ts b/x-pack/plugins/maps/public/classes/layers/layer_wizard_registry.ts index 824d9835380ec..b14ba7cf693b0 100644 --- a/x-pack/plugins/maps/public/classes/layers/layer_wizard_registry.ts +++ b/x-pack/plugins/maps/public/classes/layers/layer_wizard_registry.ts @@ -33,6 +33,7 @@ export type LayerWizard = { description: string; disabledReason?: string; getIsDisabled?: () => Promise | boolean; + isBeta?: boolean; icon: string | FunctionComponent; prerequisiteSteps?: Array<{ id: string; label: string }>; renderWizard(renderWizardArguments: RenderWizardArguments): ReactElement; @@ -54,6 +55,7 @@ export function registerLayerWizard(layerWizard: LayerWizard) { getIsDisabled: async () => { return false; }, + isBeta: false, ...layerWizard, }); } diff --git a/x-pack/plugins/maps/public/classes/layers/load_layer_wizards.ts b/x-pack/plugins/maps/public/classes/layers/load_layer_wizards.ts index 2c7f09ce9dfeb..3c86c57343a06 100644 --- a/x-pack/plugins/maps/public/classes/layers/load_layer_wizards.ts +++ b/x-pack/plugins/maps/public/classes/layers/load_layer_wizards.ts @@ -29,7 +29,6 @@ import { ObservabilityLayerWizardConfig } from './solution_layers/observability' import { SecurityLayerWizardConfig } from './solution_layers/security'; import { choroplethLayerWizardConfig } from './choropleth_layer_wizard'; import { newVectorLayerWizardConfig } from './new_vector_layer_wizard'; -import { getMapAppConfig } from '../../kibana_services'; let registered = false; export function registerLayerWizards() { @@ -39,9 +38,7 @@ export function registerLayerWizards() { // Registration order determines display order registerLayerWizard(uploadLayerWizardConfig); - if (getMapAppConfig().enableDrawingFeature) { - registerLayerWizard(newVectorLayerWizardConfig); - } + registerLayerWizard(newVectorLayerWizardConfig); registerLayerWizard(esDocumentsLayerWizardConfig); // @ts-ignore registerLayerWizard(choroplethLayerWizardConfig); diff --git a/x-pack/plugins/maps/public/classes/layers/new_vector_layer_wizard/config.tsx b/x-pack/plugins/maps/public/classes/layers/new_vector_layer_wizard/config.tsx index 2a0400c3d6bee..c4464c8787c19 100644 --- a/x-pack/plugins/maps/public/classes/layers/new_vector_layer_wizard/config.tsx +++ b/x-pack/plugins/maps/public/classes/layers/new_vector_layer_wizard/config.tsx @@ -18,7 +18,8 @@ const ADD_VECTOR_DRAWING_LAYER = 'ADD_VECTOR_DRAWING_LAYER'; export const newVectorLayerWizardConfig: LayerWizard = { categories: [LAYER_WIZARD_CATEGORY.ELASTICSEARCH], description: i18n.translate('xpack.maps.newVectorLayerWizard.description', { - defaultMessage: 'Creates a new empty layer. Use this to add shapes to the map', + defaultMessage: + 'Create an empty layer. Use this to create documents by drawing shapes on the map', }), disabledReason: i18n.translate('xpack.maps.newVectorLayerWizard.disabledDesc', { defaultMessage: @@ -31,6 +32,7 @@ export const newVectorLayerWizardConfig: LayerWizard = { }); return !hasImportPermission; }, + isBeta: true, icon: DrawLayerIcon, prerequisiteSteps: [ { diff --git a/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.tsx b/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.tsx index 019c3c1b4943b..343c366b548f6 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.tsx +++ b/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.tsx @@ -12,12 +12,7 @@ import { i18n } from '@kbn/i18n'; import { IFieldType, IndexPattern } from 'src/plugins/data/public'; import { GeoJsonProperties, Geometry, Position } from 'geojson'; import { AbstractESSource } from '../es_source'; -import { - getHttp, - getMapAppConfig, - getSearchService, - getTimeFilter, -} from '../../../kibana_services'; +import { getHttp, getSearchService, getTimeFilter } from '../../../kibana_services'; import { addFieldToDSL, getField, @@ -424,9 +419,6 @@ export class ESSearchSource extends AbstractESSource implements ITiledSingleLaye } async supportsFeatureEditing(): Promise { - if (!getMapAppConfig().enableDrawingFeature) { - return false; - } await this.getIndexPattern(); if (!(this.indexPattern && this.indexPattern.title)) { return false; diff --git a/x-pack/plugins/maps/public/connected_components/add_layer_panel/flyout_body/layer_wizard_select.tsx b/x-pack/plugins/maps/public/connected_components/add_layer_panel/flyout_body/layer_wizard_select.tsx index 5350b0e2ccf38..1e92dd16d9dac 100644 --- a/x-pack/plugins/maps/public/connected_components/add_layer_panel/flyout_body/layer_wizard_select.tsx +++ b/x-pack/plugins/maps/public/connected_components/add_layer_panel/flyout_body/layer_wizard_select.tsx @@ -161,6 +161,7 @@ export class LayerWizardSelect extends Component { = { exposeToBrowser: { enabled: true, showMapVisualizationTypes: true, - enableDrawingFeature: true, showMapsInspectorAdapter: true, preserveDrawingBuffer: true, }, diff --git a/x-pack/plugins/maps/server/plugin.ts b/x-pack/plugins/maps/server/plugin.ts index b8676559a4e2b..c5fc602864f96 100644 --- a/x-pack/plugins/maps/server/plugin.ts +++ b/x-pack/plugins/maps/server/plugin.ts @@ -170,14 +170,7 @@ export class MapsPlugin implements Plugin { lastLicenseId = license.uid; }); - initRoutes( - core, - () => lastLicenseId, - emsSettings, - this.kibanaVersion, - this._logger, - currentConfig.enableDrawingFeature - ); + initRoutes(core, () => lastLicenseId, emsSettings, this.kibanaVersion, this._logger); this._initHomeData(home, core.http.basePath.prepend, emsSettings); diff --git a/x-pack/plugins/maps/server/routes.js b/x-pack/plugins/maps/server/routes.js index 39ce9979870c5..1b669a5bcdbee 100644 --- a/x-pack/plugins/maps/server/routes.js +++ b/x-pack/plugins/maps/server/routes.js @@ -54,14 +54,7 @@ const EMPTY_EMS_CLIENT = { addQueryParams() {}, }; -export async function initRoutes( - core, - getLicenseId, - emsSettings, - kbnVersion, - logger, - drawingFeatureEnabled -) { +export async function initRoutes(core, getLicenseId, emsSettings, kbnVersion, logger) { let emsClient; let lastLicenseId; const router = core.http.createRouter(); @@ -624,7 +617,5 @@ export async function initRoutes( } initMVTRoutes({ router, logger }); - if (drawingFeatureEnabled) { - initIndexingRoutes({ router, logger, dataPlugin }); - } + initIndexingRoutes({ router, logger, dataPlugin }); } diff --git a/x-pack/test/functional/config.js b/x-pack/test/functional/config.js index 2c3a3c93e2a0a..a79e51057c90e 100644 --- a/x-pack/test/functional/config.js +++ b/x-pack/test/functional/config.js @@ -85,7 +85,6 @@ export default async function ({ readConfigFile }) { '--server.uuid=5b2de169-2785-441b-ae8c-186a1936b17d', '--xpack.maps.showMapsInspectorAdapter=true', '--xpack.maps.preserveDrawingBuffer=true', - '--xpack.maps.enableDrawingFeature=true', '--xpack.reporting.roles.enabled=false', // use the non-deprecated access control model for Reporting '--xpack.reporting.queue.pollInterval=3000', // make it explicitly the default '--xpack.reporting.csv.maxSizeBytes=2850', // small-ish limit for cutting off a 1999 byte report From ad3601c260b7e7e683c7976a6734604cec04b39f Mon Sep 17 00:00:00 2001 From: fgierlinger <2966031+fgierlinger@users.noreply.github.com> Date: Tue, 29 Jun 2021 02:17:26 +0200 Subject: [PATCH 46/55] fix: typo in time dropdown list (#103407) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../components/node_details/tabs/metrics/time_dropdown.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/node_details/tabs/metrics/time_dropdown.tsx b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/node_details/tabs/metrics/time_dropdown.tsx index c080cf279478f..28e57992cfafb 100644 --- a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/node_details/tabs/metrics/time_dropdown.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/node_details/tabs/metrics/time_dropdown.tsx @@ -20,7 +20,7 @@ export const TimeDropdown = (props: Props) => ( options={[ { text: i18n.translate('xpack.infra.nodeDetails.metrics.last15Minutes', { - defaultMessage: 'Last 15 mintues', + defaultMessage: 'Last 15 minutes', }), value: 15 * 60 * 1000, }, From 699c875b210c2451959c6b7d41116c83d51284b9 Mon Sep 17 00:00:00 2001 From: Scotty Bollinger Date: Mon, 28 Jun 2021 19:18:14 -0500 Subject: [PATCH 47/55] [Workplace Search] Fix edge case API error (#103574) This PR fixes an edge case where a race condition mught cause the total_results from a federated content source to come back null from the server. This PR tells the server to expect null in those edge cases to prevent browser errors --- .../enterprise_search/server/routes/workplace_search/sources.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.ts b/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.ts index 7e9d7d92742ab..044393f65dc59 100644 --- a/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.ts +++ b/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.ts @@ -25,7 +25,7 @@ const pageSchema = schema.object({ current: schema.nullable(schema.number()), size: schema.nullable(schema.number()), total_pages: schema.nullable(schema.number()), - total_results: schema.number(), + total_results: schema.nullable(schema.number()), }); const oauthConfigSchema = schema.object({ From aafcc473f3a947cc406fb3024767351f9c740ed7 Mon Sep 17 00:00:00 2001 From: Tim Sullivan Date: Mon, 28 Jun 2021 17:23:10 -0700 Subject: [PATCH 48/55] [Reporting] Reorganize UI components (#103571) --- x-pack/plugins/reporting/public/index.ts | 2 +- .../job_queue_client.test.mocks.ts | 2 +- .../reporting/public/lib/reporting_api_client.ts | 2 +- .../plugins/reporting/public/lib/stream_handler.ts | 2 +- .../__snapshots__/report_info_button.test.tsx.snap | 0 .../__snapshots__/report_listing.test.tsx.snap | 0 .../buttons/index.tsx => management/index.ts} | 0 .../{ => management}/mount_management_section.tsx | 14 +++++++------- .../report_delete_button.tsx | 6 +++--- .../report_diagnostic.tsx | 0 .../report_download_button.tsx | 4 ++-- .../buttons => management}/report_error_button.tsx | 6 +++--- .../report_info_button.test.tsx | 4 ++-- .../buttons => management}/report_info_button.tsx | 6 +++--- .../report_listing.test.tsx | 0 .../{components => management}/report_listing.tsx | 7 +------ x-pack/plugins/reporting/public/mocks.ts | 2 +- .../{components => notifier}/general_error.tsx | 0 .../public/{components => notifier}/index.ts | 6 ++---- .../job_completion_notifications.ts | 0 .../job_download_button.tsx | 0 .../{components => notifier}/job_failure.tsx | 0 .../{components => notifier}/job_success.tsx | 0 .../job_warning_formulas.tsx | 0 .../job_warning_max_size.tsx | 0 .../{components => notifier}/report_link.tsx | 0 x-pack/plugins/reporting/public/plugin.ts | 5 +++-- .../screen_capture_panel_content.test.tsx.snap | 0 .../panel_spinner.tsx | 0 .../share_context_menu/register_csv_reporting.tsx | 2 +- .../register_pdf_png_reporting.tsx | 2 +- .../reporting_panel_content.test.tsx | 0 .../reporting_panel_content.tsx | 0 .../reporting_panel_content_lazy.tsx | 2 +- .../screen_capture_panel_content.test.tsx | 0 .../screen_capture_panel_content.tsx | 0 .../screen_capture_panel_content_lazy.tsx | 2 +- .../shared/get_shared_components.tsx | 8 ++++---- .../shared/index.tsx => shared/index.ts} | 0 39 files changed, 39 insertions(+), 45 deletions(-) rename x-pack/plugins/reporting/public/{components => lib}/job_queue_client.test.mocks.ts (87%) rename x-pack/plugins/reporting/public/{components/buttons => management}/__snapshots__/report_info_button.test.tsx.snap (100%) rename x-pack/plugins/reporting/public/{components => management}/__snapshots__/report_listing.test.tsx.snap (100%) rename x-pack/plugins/reporting/public/{components/buttons/index.tsx => management/index.ts} (100%) rename x-pack/plugins/reporting/public/{ => management}/mount_management_section.tsx (78%) rename x-pack/plugins/reporting/public/{components/buttons => management}/report_delete_button.tsx (94%) rename x-pack/plugins/reporting/public/{components => management}/report_diagnostic.tsx (100%) rename x-pack/plugins/reporting/public/{components/buttons => management}/report_download_button.tsx (93%) rename x-pack/plugins/reporting/public/{components/buttons => management}/report_error_button.tsx (93%) rename x-pack/plugins/reporting/public/{components/buttons => management}/report_info_button.test.tsx (94%) rename x-pack/plugins/reporting/public/{components/buttons => management}/report_info_button.tsx (97%) rename x-pack/plugins/reporting/public/{components => management}/report_listing.test.tsx (100%) rename x-pack/plugins/reporting/public/{components => management}/report_listing.tsx (99%) rename x-pack/plugins/reporting/public/{components => notifier}/general_error.tsx (100%) rename x-pack/plugins/reporting/public/{components => notifier}/index.ts (81%) rename x-pack/plugins/reporting/public/{lib => notifier}/job_completion_notifications.ts (100%) rename x-pack/plugins/reporting/public/{components => notifier}/job_download_button.tsx (100%) rename x-pack/plugins/reporting/public/{components => notifier}/job_failure.tsx (100%) rename x-pack/plugins/reporting/public/{components => notifier}/job_success.tsx (100%) rename x-pack/plugins/reporting/public/{components => notifier}/job_warning_formulas.tsx (100%) rename x-pack/plugins/reporting/public/{components => notifier}/job_warning_max_size.tsx (100%) rename x-pack/plugins/reporting/public/{components => notifier}/report_link.tsx (100%) rename x-pack/plugins/reporting/public/{components => share_context_menu}/__snapshots__/screen_capture_panel_content.test.tsx.snap (100%) rename x-pack/plugins/reporting/public/{components => share_context_menu}/panel_spinner.tsx (100%) rename x-pack/plugins/reporting/public/{components => share_context_menu}/reporting_panel_content.test.tsx (100%) rename x-pack/plugins/reporting/public/{components => share_context_menu}/reporting_panel_content.tsx (100%) rename x-pack/plugins/reporting/public/{components => share_context_menu}/reporting_panel_content_lazy.tsx (94%) rename x-pack/plugins/reporting/public/{components => share_context_menu}/screen_capture_panel_content.test.tsx (100%) rename x-pack/plugins/reporting/public/{components => share_context_menu}/screen_capture_panel_content.tsx (100%) rename x-pack/plugins/reporting/public/{components => share_context_menu}/screen_capture_panel_content_lazy.tsx (94%) rename x-pack/plugins/reporting/public/{components => }/shared/get_shared_components.tsx (79%) rename x-pack/plugins/reporting/public/{components/shared/index.tsx => shared/index.ts} (100%) diff --git a/x-pack/plugins/reporting/public/index.ts b/x-pack/plugins/reporting/public/index.ts index 7179a81664b6f..6acdd8fb048e8 100644 --- a/x-pack/plugins/reporting/public/index.ts +++ b/x-pack/plugins/reporting/public/index.ts @@ -7,9 +7,9 @@ import { PluginInitializerContext } from 'src/core/public'; import { getDefaultLayoutSelectors } from '../common'; -import { getSharedComponents } from './components'; import { ReportingAPIClient } from './lib/reporting_api_client'; import { ReportingPublicPlugin } from './plugin'; +import { getSharedComponents } from './shared'; export interface ReportingSetup { getDefaultLayoutSelectors: typeof getDefaultLayoutSelectors; diff --git a/x-pack/plugins/reporting/public/components/job_queue_client.test.mocks.ts b/x-pack/plugins/reporting/public/lib/job_queue_client.test.mocks.ts similarity index 87% rename from x-pack/plugins/reporting/public/components/job_queue_client.test.mocks.ts rename to x-pack/plugins/reporting/public/lib/job_queue_client.test.mocks.ts index 9426ec8d8751d..6c66eada6a825 100644 --- a/x-pack/plugins/reporting/public/components/job_queue_client.test.mocks.ts +++ b/x-pack/plugins/reporting/public/lib/job_queue_client.test.mocks.ts @@ -15,4 +15,4 @@ export const mockAPIClient = { downloadReport: jest.fn(), }; -jest.mock('../lib/reporting_api_client', () => mockAPIClient); +jest.mock('./reporting_api_client', () => mockAPIClient); diff --git a/x-pack/plugins/reporting/public/lib/reporting_api_client.ts b/x-pack/plugins/reporting/public/lib/reporting_api_client.ts index 92604de0f4712..4ce9e8760f21f 100644 --- a/x-pack/plugins/reporting/public/lib/reporting_api_client.ts +++ b/x-pack/plugins/reporting/public/lib/reporting_api_client.ts @@ -22,7 +22,7 @@ import { ReportDocument, ReportSource, } from '../../common/types'; -import { add } from './job_completion_notifications'; +import { add } from '../notifier/job_completion_notifications'; export interface JobQueueEntry { _id: string; diff --git a/x-pack/plugins/reporting/public/lib/stream_handler.ts b/x-pack/plugins/reporting/public/lib/stream_handler.ts index 961345aeb592c..53191cacb5ba1 100644 --- a/x-pack/plugins/reporting/public/lib/stream_handler.ts +++ b/x-pack/plugins/reporting/public/lib/stream_handler.ts @@ -17,7 +17,7 @@ import { getSuccessToast, getWarningFormulasToast, getWarningMaxSizeToast, -} from '../components'; +} from '../notifier'; import { ReportingAPIClient } from './reporting_api_client'; function updateStored(jobIds: JobId[]): void { diff --git a/x-pack/plugins/reporting/public/components/buttons/__snapshots__/report_info_button.test.tsx.snap b/x-pack/plugins/reporting/public/management/__snapshots__/report_info_button.test.tsx.snap similarity index 100% rename from x-pack/plugins/reporting/public/components/buttons/__snapshots__/report_info_button.test.tsx.snap rename to x-pack/plugins/reporting/public/management/__snapshots__/report_info_button.test.tsx.snap diff --git a/x-pack/plugins/reporting/public/components/__snapshots__/report_listing.test.tsx.snap b/x-pack/plugins/reporting/public/management/__snapshots__/report_listing.test.tsx.snap similarity index 100% rename from x-pack/plugins/reporting/public/components/__snapshots__/report_listing.test.tsx.snap rename to x-pack/plugins/reporting/public/management/__snapshots__/report_listing.test.tsx.snap diff --git a/x-pack/plugins/reporting/public/components/buttons/index.tsx b/x-pack/plugins/reporting/public/management/index.ts similarity index 100% rename from x-pack/plugins/reporting/public/components/buttons/index.tsx rename to x-pack/plugins/reporting/public/management/index.ts diff --git a/x-pack/plugins/reporting/public/mount_management_section.tsx b/x-pack/plugins/reporting/public/management/mount_management_section.tsx similarity index 78% rename from x-pack/plugins/reporting/public/mount_management_section.tsx rename to x-pack/plugins/reporting/public/management/mount_management_section.tsx index bc165badae713..eb1057a9bdfc7 100644 --- a/x-pack/plugins/reporting/public/mount_management_section.tsx +++ b/x-pack/plugins/reporting/public/management/mount_management_section.tsx @@ -5,16 +5,16 @@ * 2.0. */ +import { I18nProvider } from '@kbn/i18n/react'; import * as React from 'react'; import { render, unmountComponentAtNode } from 'react-dom'; -import { I18nProvider } from '@kbn/i18n/react'; -import { CoreSetup, CoreStart } from 'src/core/public'; import { Observable } from 'rxjs'; -import { ReportListing } from './components/report_listing'; -import { ManagementAppMountParams } from '../../../../src/plugins/management/public'; -import { ILicense } from '../../licensing/public'; -import { ClientConfigType } from './plugin'; -import { ReportingAPIClient } from './lib/reporting_api_client'; +import { CoreSetup, CoreStart } from 'src/core/public'; +import { ManagementAppMountParams } from '../../../../../src/plugins/management/public'; +import { ILicense } from '../../../licensing/public'; +import { ReportingAPIClient } from '../lib/reporting_api_client'; +import { ClientConfigType } from '../plugin'; +import { ReportListing } from './report_listing'; export async function mountManagementSection( coreSetup: CoreSetup, diff --git a/x-pack/plugins/reporting/public/components/buttons/report_delete_button.tsx b/x-pack/plugins/reporting/public/management/report_delete_button.tsx similarity index 94% rename from x-pack/plugins/reporting/public/components/buttons/report_delete_button.tsx rename to x-pack/plugins/reporting/public/management/report_delete_button.tsx index cd432758fa767..dfb411fc195e8 100644 --- a/x-pack/plugins/reporting/public/components/buttons/report_delete_button.tsx +++ b/x-pack/plugins/reporting/public/management/report_delete_button.tsx @@ -5,9 +5,9 @@ * 2.0. */ -import { EuiConfirmModal, EuiButton } from '@elastic/eui'; -import React, { PureComponent, Fragment } from 'react'; -import { Job, Props as ListingProps } from '../report_listing'; +import { EuiButton, EuiConfirmModal } from '@elastic/eui'; +import React, { Fragment, PureComponent } from 'react'; +import { Job, Props as ListingProps } from './report_listing'; type DeleteFn = () => Promise; type Props = { jobsToDelete: Job[]; performDelete: DeleteFn } & ListingProps; diff --git a/x-pack/plugins/reporting/public/components/report_diagnostic.tsx b/x-pack/plugins/reporting/public/management/report_diagnostic.tsx similarity index 100% rename from x-pack/plugins/reporting/public/components/report_diagnostic.tsx rename to x-pack/plugins/reporting/public/management/report_diagnostic.tsx diff --git a/x-pack/plugins/reporting/public/components/buttons/report_download_button.tsx b/x-pack/plugins/reporting/public/management/report_download_button.tsx similarity index 93% rename from x-pack/plugins/reporting/public/components/buttons/report_download_button.tsx rename to x-pack/plugins/reporting/public/management/report_download_button.tsx index a5e57dafc2867..78022b85e2ff8 100644 --- a/x-pack/plugins/reporting/public/components/buttons/report_download_button.tsx +++ b/x-pack/plugins/reporting/public/management/report_download_button.tsx @@ -7,8 +7,8 @@ import { EuiButtonIcon, EuiToolTip } from '@elastic/eui'; import React, { FunctionComponent } from 'react'; -import { JOB_STATUSES } from '../../../common/constants'; -import { Job as ListingJob, Props as ListingProps } from '../report_listing'; +import { JOB_STATUSES } from '../../common/constants'; +import { Job as ListingJob, Props as ListingProps } from './report_listing'; type Props = { record: ListingJob } & ListingProps; diff --git a/x-pack/plugins/reporting/public/components/buttons/report_error_button.tsx b/x-pack/plugins/reporting/public/management/report_error_button.tsx similarity index 93% rename from x-pack/plugins/reporting/public/components/buttons/report_error_button.tsx rename to x-pack/plugins/reporting/public/management/report_error_button.tsx index b34463b61253b..0ebdf5ca60b5a 100644 --- a/x-pack/plugins/reporting/public/components/buttons/report_error_button.tsx +++ b/x-pack/plugins/reporting/public/management/report_error_button.tsx @@ -8,9 +8,9 @@ import { EuiButtonIcon, EuiCallOut, EuiPopover } from '@elastic/eui'; import { InjectedIntl, injectI18n } from '@kbn/i18n/react'; import React, { Component } from 'react'; -import { JOB_STATUSES } from '../../../common/constants'; -import { JobContent, ReportingAPIClient } from '../../lib/reporting_api_client'; -import { Job as ListingJob } from '../report_listing'; +import { JOB_STATUSES } from '../../common/constants'; +import { JobContent, ReportingAPIClient } from '../lib/reporting_api_client'; +import { Job as ListingJob } from './report_listing'; interface Props { intl: InjectedIntl; diff --git a/x-pack/plugins/reporting/public/components/buttons/report_info_button.test.tsx b/x-pack/plugins/reporting/public/management/report_info_button.test.tsx similarity index 94% rename from x-pack/plugins/reporting/public/components/buttons/report_info_button.test.tsx rename to x-pack/plugins/reporting/public/management/report_info_button.test.tsx index 785f49646110a..119856042a326 100644 --- a/x-pack/plugins/reporting/public/components/buttons/report_info_button.test.tsx +++ b/x-pack/plugins/reporting/public/management/report_info_button.test.tsx @@ -9,9 +9,9 @@ import React from 'react'; import { mountWithIntl } from '@kbn/test/jest'; import { ReportInfoButton } from './report_info_button'; -jest.mock('../../lib/reporting_api_client'); +jest.mock('../lib/reporting_api_client'); -import { ReportingAPIClient } from '../../lib/reporting_api_client'; +import { ReportingAPIClient } from '../lib/reporting_api_client'; const httpSetup = {} as any; const apiClient = new ReportingAPIClient(httpSetup); diff --git a/x-pack/plugins/reporting/public/components/buttons/report_info_button.tsx b/x-pack/plugins/reporting/public/management/report_info_button.tsx similarity index 97% rename from x-pack/plugins/reporting/public/components/buttons/report_info_button.tsx rename to x-pack/plugins/reporting/public/management/report_info_button.tsx index 7f2d5b6adcc33..719f1ff341daf 100644 --- a/x-pack/plugins/reporting/public/components/buttons/report_info_button.tsx +++ b/x-pack/plugins/reporting/public/management/report_info_button.tsx @@ -18,9 +18,9 @@ import { } from '@elastic/eui'; import { get } from 'lodash'; import React, { Component, Fragment } from 'react'; -import { USES_HEADLESS_JOB_TYPES } from '../../../common/constants'; -import { ReportApiJSON } from '../../../common/types'; -import { ReportingAPIClient } from '../../lib/reporting_api_client'; +import { USES_HEADLESS_JOB_TYPES } from '../../common/constants'; +import { ReportApiJSON } from '../../common/types'; +import { ReportingAPIClient } from '../lib/reporting_api_client'; interface Props { jobId: string; diff --git a/x-pack/plugins/reporting/public/components/report_listing.test.tsx b/x-pack/plugins/reporting/public/management/report_listing.test.tsx similarity index 100% rename from x-pack/plugins/reporting/public/components/report_listing.test.tsx rename to x-pack/plugins/reporting/public/management/report_listing.test.tsx diff --git a/x-pack/plugins/reporting/public/components/report_listing.tsx b/x-pack/plugins/reporting/public/management/report_listing.tsx similarity index 99% rename from x-pack/plugins/reporting/public/components/report_listing.tsx rename to x-pack/plugins/reporting/public/management/report_listing.tsx index 618c91fba0715..fffa952be6cb4 100644 --- a/x-pack/plugins/reporting/public/components/report_listing.tsx +++ b/x-pack/plugins/reporting/public/management/report_listing.tsx @@ -28,12 +28,7 @@ import { durationToNumber } from '../../common/schema_utils'; import { checkLicense } from '../lib/license_check'; import { JobQueueEntry, ReportingAPIClient } from '../lib/reporting_api_client'; import { ClientConfigType } from '../plugin'; -import { - ReportDeleteButton, - ReportDownloadButton, - ReportErrorButton, - ReportInfoButton, -} from './buttons'; +import { ReportDeleteButton, ReportDownloadButton, ReportErrorButton, ReportInfoButton } from './'; import { ReportDiagnostic } from './report_diagnostic'; export interface Job { diff --git a/x-pack/plugins/reporting/public/mocks.ts b/x-pack/plugins/reporting/public/mocks.ts index 414d1b0ae70fe..a6b6d835499c6 100644 --- a/x-pack/plugins/reporting/public/mocks.ts +++ b/x-pack/plugins/reporting/public/mocks.ts @@ -8,7 +8,7 @@ import { coreMock } from 'src/core/public/mocks'; import { ReportingSetup } from '.'; import { getDefaultLayoutSelectors } from '../common'; -import { getSharedComponents } from './components/shared'; +import { getSharedComponents } from './shared'; type Setup = jest.Mocked; diff --git a/x-pack/plugins/reporting/public/components/general_error.tsx b/x-pack/plugins/reporting/public/notifier/general_error.tsx similarity index 100% rename from x-pack/plugins/reporting/public/components/general_error.tsx rename to x-pack/plugins/reporting/public/notifier/general_error.tsx diff --git a/x-pack/plugins/reporting/public/components/index.ts b/x-pack/plugins/reporting/public/notifier/index.ts similarity index 81% rename from x-pack/plugins/reporting/public/components/index.ts rename to x-pack/plugins/reporting/public/notifier/index.ts index b8cccda2a6613..b44f1e9169747 100644 --- a/x-pack/plugins/reporting/public/components/index.ts +++ b/x-pack/plugins/reporting/public/notifier/index.ts @@ -5,10 +5,8 @@ * 2.0. */ -export { getSuccessToast } from './job_success'; export { getFailureToast } from './job_failure'; +export { getGeneralErrorToast } from './general_error'; +export { getSuccessToast } from './job_success'; export { getWarningFormulasToast } from './job_warning_formulas'; export { getWarningMaxSizeToast } from './job_warning_max_size'; -export { getGeneralErrorToast } from './general_error'; -export { ScreenCapturePanelContent } from './screen_capture_panel_content'; -export { getSharedComponents } from './shared'; diff --git a/x-pack/plugins/reporting/public/lib/job_completion_notifications.ts b/x-pack/plugins/reporting/public/notifier/job_completion_notifications.ts similarity index 100% rename from x-pack/plugins/reporting/public/lib/job_completion_notifications.ts rename to x-pack/plugins/reporting/public/notifier/job_completion_notifications.ts diff --git a/x-pack/plugins/reporting/public/components/job_download_button.tsx b/x-pack/plugins/reporting/public/notifier/job_download_button.tsx similarity index 100% rename from x-pack/plugins/reporting/public/components/job_download_button.tsx rename to x-pack/plugins/reporting/public/notifier/job_download_button.tsx diff --git a/x-pack/plugins/reporting/public/components/job_failure.tsx b/x-pack/plugins/reporting/public/notifier/job_failure.tsx similarity index 100% rename from x-pack/plugins/reporting/public/components/job_failure.tsx rename to x-pack/plugins/reporting/public/notifier/job_failure.tsx diff --git a/x-pack/plugins/reporting/public/components/job_success.tsx b/x-pack/plugins/reporting/public/notifier/job_success.tsx similarity index 100% rename from x-pack/plugins/reporting/public/components/job_success.tsx rename to x-pack/plugins/reporting/public/notifier/job_success.tsx diff --git a/x-pack/plugins/reporting/public/components/job_warning_formulas.tsx b/x-pack/plugins/reporting/public/notifier/job_warning_formulas.tsx similarity index 100% rename from x-pack/plugins/reporting/public/components/job_warning_formulas.tsx rename to x-pack/plugins/reporting/public/notifier/job_warning_formulas.tsx diff --git a/x-pack/plugins/reporting/public/components/job_warning_max_size.tsx b/x-pack/plugins/reporting/public/notifier/job_warning_max_size.tsx similarity index 100% rename from x-pack/plugins/reporting/public/components/job_warning_max_size.tsx rename to x-pack/plugins/reporting/public/notifier/job_warning_max_size.tsx diff --git a/x-pack/plugins/reporting/public/components/report_link.tsx b/x-pack/plugins/reporting/public/notifier/report_link.tsx similarity index 100% rename from x-pack/plugins/reporting/public/components/report_link.tsx rename to x-pack/plugins/reporting/public/notifier/report_link.tsx diff --git a/x-pack/plugins/reporting/public/plugin.ts b/x-pack/plugins/reporting/public/plugin.ts index 577732fdb1392..a2881af902072 100644 --- a/x-pack/plugins/reporting/public/plugin.ts +++ b/x-pack/plugins/reporting/public/plugin.ts @@ -29,10 +29,11 @@ import { constants, getDefaultLayoutSelectors } from '../common'; import { durationToNumber } from '../common/schema_utils'; import { JobId, JobSummarySet } from '../common/types'; import { ReportingSetup, ReportingStart } from './'; -import { getGeneralErrorToast, getSharedComponents } from './components'; import { ReportingAPIClient } from './lib/reporting_api_client'; import { ReportingNotifierStreamHandler as StreamHandler } from './lib/stream_handler'; +import { getGeneralErrorToast } from './notifier'; import { ReportingCsvPanelAction } from './panel_actions/get_csv_panel_action'; +import { getSharedComponents } from './shared'; import { ReportingCsvShareProvider } from './share_context_menu/register_csv_reporting'; import { reportingScreenshotShareProvider } from './share_context_menu/register_pdf_png_reporting'; @@ -150,7 +151,7 @@ export class ReportingPublicPlugin params.setBreadcrumbs([{ text: this.breadcrumbText }]); const [[start], { mountManagementSection }] = await Promise.all([ getStartServices(), - import('./mount_management_section'), + import('./management/mount_management_section'), ]); return await mountManagementSection( core, diff --git a/x-pack/plugins/reporting/public/components/__snapshots__/screen_capture_panel_content.test.tsx.snap b/x-pack/plugins/reporting/public/share_context_menu/__snapshots__/screen_capture_panel_content.test.tsx.snap similarity index 100% rename from x-pack/plugins/reporting/public/components/__snapshots__/screen_capture_panel_content.test.tsx.snap rename to x-pack/plugins/reporting/public/share_context_menu/__snapshots__/screen_capture_panel_content.test.tsx.snap diff --git a/x-pack/plugins/reporting/public/components/panel_spinner.tsx b/x-pack/plugins/reporting/public/share_context_menu/panel_spinner.tsx similarity index 100% rename from x-pack/plugins/reporting/public/components/panel_spinner.tsx rename to x-pack/plugins/reporting/public/share_context_menu/panel_spinner.tsx diff --git a/x-pack/plugins/reporting/public/share_context_menu/register_csv_reporting.tsx b/x-pack/plugins/reporting/public/share_context_menu/register_csv_reporting.tsx index 7fe5268fc9910..7165fcf6f8681 100644 --- a/x-pack/plugins/reporting/public/share_context_menu/register_csv_reporting.tsx +++ b/x-pack/plugins/reporting/public/share_context_menu/register_csv_reporting.tsx @@ -16,9 +16,9 @@ import type { ShareContext } from '../../../../../src/plugins/share/public'; import type { LicensingPluginSetup } from '../../../licensing/public'; import { CSV_JOB_TYPE } from '../../common/constants'; import type { JobParamsCSV } from '../../server/export_types/csv_searchsource/types'; -import { ReportingPanelContent } from '../components/reporting_panel_content_lazy'; import { checkLicense } from '../lib/license_check'; import type { ReportingAPIClient } from '../lib/reporting_api_client'; +import { ReportingPanelContent } from './reporting_panel_content_lazy'; export const ReportingCsvShareProvider = ({ apiClient, diff --git a/x-pack/plugins/reporting/public/share_context_menu/register_pdf_png_reporting.tsx b/x-pack/plugins/reporting/public/share_context_menu/register_pdf_png_reporting.tsx index 42f6ee5fcb898..eb80f64be55e1 100644 --- a/x-pack/plugins/reporting/public/share_context_menu/register_pdf_png_reporting.tsx +++ b/x-pack/plugins/reporting/public/share_context_menu/register_pdf_png_reporting.tsx @@ -16,9 +16,9 @@ import type { LicensingPluginSetup } from '../../../licensing/public'; import type { LayoutParams } from '../../common/types'; import type { JobParamsPNG } from '../../server/export_types/png/types'; import type { JobParamsPDF } from '../../server/export_types/printable_pdf/types'; -import { ScreenCapturePanelContent } from '../components/screen_capture_panel_content_lazy'; import { checkLicense } from '../lib/license_check'; import type { ReportingAPIClient } from '../lib/reporting_api_client'; +import { ScreenCapturePanelContent } from './screen_capture_panel_content_lazy'; interface JobParamsProviderOptions { shareableUrl: string; diff --git a/x-pack/plugins/reporting/public/components/reporting_panel_content.test.tsx b/x-pack/plugins/reporting/public/share_context_menu/reporting_panel_content.test.tsx similarity index 100% rename from x-pack/plugins/reporting/public/components/reporting_panel_content.test.tsx rename to x-pack/plugins/reporting/public/share_context_menu/reporting_panel_content.test.tsx diff --git a/x-pack/plugins/reporting/public/components/reporting_panel_content.tsx b/x-pack/plugins/reporting/public/share_context_menu/reporting_panel_content.tsx similarity index 100% rename from x-pack/plugins/reporting/public/components/reporting_panel_content.tsx rename to x-pack/plugins/reporting/public/share_context_menu/reporting_panel_content.tsx diff --git a/x-pack/plugins/reporting/public/components/reporting_panel_content_lazy.tsx b/x-pack/plugins/reporting/public/share_context_menu/reporting_panel_content_lazy.tsx similarity index 94% rename from x-pack/plugins/reporting/public/components/reporting_panel_content_lazy.tsx rename to x-pack/plugins/reporting/public/share_context_menu/reporting_panel_content_lazy.tsx index 8a937f6d2eb36..a231d7aa881de 100644 --- a/x-pack/plugins/reporting/public/components/reporting_panel_content_lazy.tsx +++ b/x-pack/plugins/reporting/public/share_context_menu/reporting_panel_content_lazy.tsx @@ -6,7 +6,7 @@ */ import * as React from 'react'; -import { lazy, Suspense, FC } from 'react'; +import { FC, lazy, Suspense } from 'react'; import { PanelSpinner } from './panel_spinner'; import type { Props } from './reporting_panel_content'; diff --git a/x-pack/plugins/reporting/public/components/screen_capture_panel_content.test.tsx b/x-pack/plugins/reporting/public/share_context_menu/screen_capture_panel_content.test.tsx similarity index 100% rename from x-pack/plugins/reporting/public/components/screen_capture_panel_content.test.tsx rename to x-pack/plugins/reporting/public/share_context_menu/screen_capture_panel_content.test.tsx diff --git a/x-pack/plugins/reporting/public/components/screen_capture_panel_content.tsx b/x-pack/plugins/reporting/public/share_context_menu/screen_capture_panel_content.tsx similarity index 100% rename from x-pack/plugins/reporting/public/components/screen_capture_panel_content.tsx rename to x-pack/plugins/reporting/public/share_context_menu/screen_capture_panel_content.tsx diff --git a/x-pack/plugins/reporting/public/components/screen_capture_panel_content_lazy.tsx b/x-pack/plugins/reporting/public/share_context_menu/screen_capture_panel_content_lazy.tsx similarity index 94% rename from x-pack/plugins/reporting/public/components/screen_capture_panel_content_lazy.tsx rename to x-pack/plugins/reporting/public/share_context_menu/screen_capture_panel_content_lazy.tsx index 253b4fb88dcd4..a162dd749ff02 100644 --- a/x-pack/plugins/reporting/public/components/screen_capture_panel_content_lazy.tsx +++ b/x-pack/plugins/reporting/public/share_context_menu/screen_capture_panel_content_lazy.tsx @@ -6,7 +6,7 @@ */ import * as React from 'react'; -import { lazy, Suspense, FC } from 'react'; +import { FC, lazy, Suspense } from 'react'; import { PanelSpinner } from './panel_spinner'; import type { Props } from './screen_capture_panel_content'; diff --git a/x-pack/plugins/reporting/public/components/shared/get_shared_components.tsx b/x-pack/plugins/reporting/public/shared/get_shared_components.tsx similarity index 79% rename from x-pack/plugins/reporting/public/components/shared/get_shared_components.tsx rename to x-pack/plugins/reporting/public/shared/get_shared_components.tsx index 12d70c8701975..87ddf0cfdb389 100644 --- a/x-pack/plugins/reporting/public/components/shared/get_shared_components.tsx +++ b/x-pack/plugins/reporting/public/shared/get_shared_components.tsx @@ -7,10 +7,10 @@ import { CoreSetup } from 'kibana/public'; import React from 'react'; -import { ReportingAPIClient } from '../..'; -import { PDF_REPORT_TYPE } from '../../../common/constants'; -import type { Props as PanelPropsScreenCapture } from '../screen_capture_panel_content'; -import { ScreenCapturePanelContent } from '../screen_capture_panel_content_lazy'; +import { ReportingAPIClient } from '../'; +import { PDF_REPORT_TYPE } from '../../common/constants'; +import type { Props as PanelPropsScreenCapture } from '../share_context_menu/screen_capture_panel_content'; +import { ScreenCapturePanelContent } from '../share_context_menu/screen_capture_panel_content_lazy'; interface IncludeOnCloseFn { onClose: () => void; diff --git a/x-pack/plugins/reporting/public/components/shared/index.tsx b/x-pack/plugins/reporting/public/shared/index.ts similarity index 100% rename from x-pack/plugins/reporting/public/components/shared/index.tsx rename to x-pack/plugins/reporting/public/shared/index.ts From 633649460a59931eeb3f459da5e3eb334afdc83d Mon Sep 17 00:00:00 2001 From: Constance Date: Mon, 28 Jun 2021 17:25:24 -0700 Subject: [PATCH 49/55] [Enterprise Search] Improve flash messages screen reader UX (#103412) * Remove role region on flash messages - just `aria-live` is enough for screen readers to read it out, and `role` was causing "Flash messages" to get read out loud repeatedly between page navigation even when empty which was annoying and not good * Further a11y attribute recommendations from @myasonik --- .../shared/flash_messages/flash_messages.tsx | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/flash_messages/flash_messages.tsx b/x-pack/plugins/enterprise_search/public/applications/shared/flash_messages/flash_messages.tsx index ba42b89d6ab56..a96a179bd58c0 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/flash_messages/flash_messages.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/shared/flash_messages/flash_messages.tsx @@ -10,7 +10,6 @@ import React, { Fragment } from 'react'; import { useValues, useActions } from 'kea'; import { EuiCallOut, EuiSpacer, EuiGlobalToastList } from '@elastic/eui'; -import { i18n } from '@kbn/i18n'; import { FLASH_MESSAGE_TYPES, DEFAULT_TOAST_TIMEOUT } from './constants'; import { FlashMessagesLogic } from './flash_messages_logic'; @@ -19,14 +18,7 @@ export const FlashMessages: React.FC = ({ children }) => { const { messages } = useValues(FlashMessagesLogic); return ( -
+
{messages.map(({ type, message, description }, index) => ( Date: Mon, 28 Jun 2021 20:35:27 -0400 Subject: [PATCH 50/55] [Alerting] Enable rule import/export and allow rule types to exclude themselves from export (#102999) * Removing feature flag changes * Adding isExportable flag to rule type definition * Adding isExportable flag to rule type definition * Adding isExportable flag to rule type definition * Filtering rule on export by rule type isExportable flag * Fixing types * Adding docs * Fix condition when exportCount is 0 * Unit test for fix condition when exportCount is 0 Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- docs/api/alerting/legacy/list.asciidoc | 4 + docs/api/alerting/list_rule_types.asciidoc | 4 + .../alerting/create-and-manage-rules.asciidoc | 19 ++ .../server/saved_objects/routes/utils.test.ts | 16 ++ src/core/server/saved_objects/routes/utils.ts | 2 +- .../server/alert_types/always_firing.ts | 1 + .../server/alert_types/astros.ts | 1 + x-pack/plugins/alerting/README.md | 2 + x-pack/plugins/alerting/common/alert_type.ts | 1 + .../plugins/alerting/public/alert_api.test.ts | 3 + .../alert_navigation_registry.test.ts | 1 + .../server/alert_type_registry.test.ts | 16 ++ .../alerting/server/alert_type_registry.ts | 3 + .../alerts_client/tests/aggregate.test.ts | 2 + .../server/alerts_client/tests/create.test.ts | 1 + .../server/alerts_client/tests/find.test.ts | 2 + .../server/alerts_client/tests/lib.ts | 1 + .../tests/list_alert_types.test.ts | 5 + .../server/alerts_client/tests/update.test.ts | 3 + .../alerts_client_conflict_retries.test.ts | 2 + .../alerting_authorization.test.ts | 18 ++ .../alerting_authorization_kuery.test.ts | 10 + x-pack/plugins/alerting/server/config.test.ts | 1 - x-pack/plugins/alerting/server/config.ts | 1 - .../alerting/server/health/get_state.test.ts | 8 - .../alerting/server/lib/license_state.test.ts | 2 + x-pack/plugins/alerting/server/plugin.test.ts | 72 +----- x-pack/plugins/alerting/server/plugin.ts | 9 +- .../routes/legacy/list_alert_types.test.ts | 4 + .../alerting/server/routes/rule_types.test.ts | 5 + .../alerting/server/routes/rule_types.ts | 2 + .../alerting/server/saved_objects/index.ts | 111 +++++----- .../saved_objects/is_rule_exportable.test.ts | 208 ++++++++++++++++++ .../saved_objects/is_rule_exportable.ts | 33 +++ .../create_execution_handler.test.ts | 1 + .../server/task_runner/task_runner.test.ts | 1 + .../task_runner/task_runner_factory.test.ts | 1 + x-pack/plugins/alerting/server/types.ts | 1 + x-pack/plugins/apm/common/alert_types.ts | 5 + .../alerts/register_error_count_alert_type.ts | 1 + ...egister_transaction_duration_alert_type.ts | 1 + ...transaction_duration_anomaly_alert_type.ts | 1 + ...ister_transaction_error_rate_alert_type.ts | 1 + ...r_inventory_metric_threshold_alert_type.ts | 1 + .../register_log_threshold_alert_type.ts | 1 + .../register_metric_anomaly_alert_type.ts | 1 + .../register_metric_threshold_alert_type.ts | 1 + .../register_anomaly_detection_alert_type.ts | 1 + .../monitoring/server/alerts/base_alert.ts | 1 + .../utils/create_lifecycle_rule_type.test.ts | 1 + .../rules_notification_alert_type.ts | 1 + .../detection_engine/reference_rules/eql.ts | 1 + .../detection_engine/reference_rules/ml.ts | 1 + .../detection_engine/reference_rules/query.ts | 1 + .../reference_rules/threshold.ts | 1 + .../signals/signal_rule_alert_type.ts | 1 + .../server/alert_types/es_query/alert_type.ts | 1 + .../alert_types/geo_containment/alert_type.ts | 1 + .../alert_types/index_threshold/alert_type.ts | 1 + .../server/lib/alerts/duration_anomaly.ts | 1 + .../uptime/server/lib/alerts/status_check.ts | 1 + .../plugins/uptime/server/lib/alerts/tls.ts | 1 + .../uptime/server/lib/alerts/tls_legacy.ts | 1 + .../plugins/alerts/server/alert_types.ts | 13 ++ .../alerts_restricted/server/alert_types.ts | 2 + .../tests/alerting/rule_types.ts | 2 + .../spaces_only/tests/alerting/rule_types.ts | 2 + .../fixtures/plugins/alerts/server/plugin.ts | 3 + 68 files changed, 489 insertions(+), 139 deletions(-) create mode 100644 x-pack/plugins/alerting/server/saved_objects/is_rule_exportable.test.ts create mode 100644 x-pack/plugins/alerting/server/saved_objects/is_rule_exportable.ts diff --git a/docs/api/alerting/legacy/list.asciidoc b/docs/api/alerting/legacy/list.asciidoc index be37be36cd0e8..07307797c4223 100644 --- a/docs/api/alerting/legacy/list.asciidoc +++ b/docs/api/alerting/legacy/list.asciidoc @@ -80,6 +80,7 @@ The API returns the following: }, "producer":"stackAlerts", "minimumLicenseRequired":"basic", + "isExportable":true, "enabledInLicense":true, "authorizedConsumers":{ "alerts":{ @@ -113,6 +114,9 @@ Each alert type contains the following properties: | `minimumLicenseRequired` | The license required to use the alert type. +| `isExportable` +| Whether the rule type is exportable through the Saved Objects Management UI. + | `enabledInLicense` | Whether the alert type is enabled or disabled based on the license. diff --git a/docs/api/alerting/list_rule_types.asciidoc b/docs/api/alerting/list_rule_types.asciidoc index 31c8416e75059..21ace9f3105c0 100644 --- a/docs/api/alerting/list_rule_types.asciidoc +++ b/docs/api/alerting/list_rule_types.asciidoc @@ -82,6 +82,7 @@ The API returns the following: }, "producer":"stackAlerts", "minimum_license_required":"basic", + "is_exportable":true, "enabled_in_license":true, "authorized_consumers":{ "alerts":{ @@ -115,6 +116,9 @@ Each rule type contains the following properties: | `minimum_license_required` | The license required to use the rule type. +| `is_exportable` +| Whether the rule type is exportable through the Saved Objects Management UI. + | `enabled_in_license` | Whether the rule type is enabled or disabled based on the license. diff --git a/docs/user/alerting/create-and-manage-rules.asciidoc b/docs/user/alerting/create-and-manage-rules.asciidoc index af6714aef662f..cc91ebcd99be2 100644 --- a/docs/user/alerting/create-and-manage-rules.asciidoc +++ b/docs/user/alerting/create-and-manage-rules.asciidoc @@ -152,6 +152,25 @@ You can perform these operations in bulk by multi-selecting rules, and then clic [role="screenshot"] image:images/bulk-mute-disable.png[The Manage rules button lets you mute/unmute, enable/disable, and delete in bulk,width=75%] +[float] +[[importing-and-exporting-rules]] +=== Import and export rules + +To import and export rules, use the <>. + +[NOTE] +============================================== +Some rule types cannot be exported through this interface: + +**Security rules** can be imported and exported using the {security-guide}/rules-ui-management.html#import-export-rules-ui[Security UI]. + +**Stack monitoring rules** are <> for you and therefore cannot be managed via the Saved Objects Management UI. +============================================== + +Rules are disabled on export. You are prompted to re-enable rule on successful import. +[role="screenshot"] +image::images/rules-imported-banner.png[Rules import banner, width=50%] + [float] [[rule-details]] === Drilldown to rule details diff --git a/src/core/server/saved_objects/routes/utils.test.ts b/src/core/server/saved_objects/routes/utils.test.ts index 623d2dcc71fac..2127352e4c60e 100644 --- a/src/core/server/saved_objects/routes/utils.test.ts +++ b/src/core/server/saved_objects/routes/utils.test.ts @@ -101,6 +101,22 @@ describe('createSavedObjectsStreamFromNdJson', () => { }, ]); }); + + it('handles an ndjson stream that only contains excluded saved objects', async () => { + const savedObjectsStream = await createSavedObjectsStreamFromNdJson( + new Readable({ + read() { + this.push( + '{"excludedObjects":[{"id":"foo","reason":"excluded","type":"foo-type"}],"excludedObjectsCount":1,"exportedCount":0,"missingRefCount":0,"missingReferences":[]}\n' + ); + this.push(null); + }, + }) + ); + + const result = await readStreamToCompletion(savedObjectsStream); + expect(result).toEqual([]); + }); }); describe('validateTypes', () => { diff --git a/src/core/server/saved_objects/routes/utils.ts b/src/core/server/saved_objects/routes/utils.ts index e933badfe80fe..47996847a8387 100644 --- a/src/core/server/saved_objects/routes/utils.ts +++ b/src/core/server/saved_objects/routes/utils.ts @@ -32,7 +32,7 @@ export async function createSavedObjectsStreamFromNdJson(ndJsonStream: Readable) } }), createFilterStream( - (obj) => !!obj && !(obj as SavedObjectsExportResultDetails).exportedCount + (obj) => !!obj && (obj as SavedObjectsExportResultDetails).exportedCount === undefined ), createConcatStream([]), ]); diff --git a/x-pack/examples/alerting_example/server/alert_types/always_firing.ts b/x-pack/examples/alerting_example/server/alert_types/always_firing.ts index 6e9ec0d367c9a..f056c292b018f 100644 --- a/x-pack/examples/alerting_example/server/alert_types/always_firing.ts +++ b/x-pack/examples/alerting_example/server/alert_types/always_firing.ts @@ -53,6 +53,7 @@ export const alertType: AlertType< ], defaultActionGroupId: DEFAULT_ACTION_GROUP, minimumLicenseRequired: 'basic', + isExportable: true, async executor({ services, params: { instances = DEFAULT_INSTANCES_TO_GENERATE, thresholds }, diff --git a/x-pack/examples/alerting_example/server/alert_types/astros.ts b/x-pack/examples/alerting_example/server/alert_types/astros.ts index 45ea6b48bf6f4..8f9a293518300 100644 --- a/x-pack/examples/alerting_example/server/alert_types/astros.ts +++ b/x-pack/examples/alerting_example/server/alert_types/astros.ts @@ -51,6 +51,7 @@ export const alertType: AlertType< name: 'People In Space Right Now', actionGroups: [{ id: 'default', name: 'default' }], minimumLicenseRequired: 'basic', + isExportable: true, defaultActionGroupId: 'default', recoveryActionGroup: { id: 'hasLandedBackOnEarth', diff --git a/x-pack/plugins/alerting/README.md b/x-pack/plugins/alerting/README.md index 9d314cc048b70..62d2f2b57b8e8 100644 --- a/x-pack/plugins/alerting/README.md +++ b/x-pack/plugins/alerting/README.md @@ -118,6 +118,7 @@ The following table describes the properties of the `options` object. |executor|This is where the code for the rule type lives. This is a function to be called when executing a rule on an interval basis. For full details, see the executor section below.|Function| |producer|The id of the application producing this rule type.|string| |minimumLicenseRequired|The value of a minimum license. Most of the rules are licensed as "basic".|string| +|isExportable|Whether the rule type is exportable from the Saved Objects Management UI.|boolean| ### Executor @@ -262,6 +263,7 @@ const myRuleType: AlertType< ], }, minimumLicenseRequired: 'basic', + isExportable: true, async executor({ alertId, startedAt, diff --git a/x-pack/plugins/alerting/common/alert_type.ts b/x-pack/plugins/alerting/common/alert_type.ts index e39c6d0a66f6c..e56034a4c41f8 100644 --- a/x-pack/plugins/alerting/common/alert_type.ts +++ b/x-pack/plugins/alerting/common/alert_type.ts @@ -20,6 +20,7 @@ export interface AlertType< defaultActionGroupId: ActionGroupIds; producer: string; minimumLicenseRequired: LicenseType; + isExportable: boolean; } export interface ActionGroup { diff --git a/x-pack/plugins/alerting/public/alert_api.test.ts b/x-pack/plugins/alerting/public/alert_api.test.ts index 023ea255e1c42..dd2f7d167c1c3 100644 --- a/x-pack/plugins/alerting/public/alert_api.test.ts +++ b/x-pack/plugins/alerting/public/alert_api.test.ts @@ -24,6 +24,7 @@ describe('loadAlertTypes', () => { actionGroups: [{ id: 'default', name: 'Default' }], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, producer: 'alerts', }, @@ -49,6 +50,7 @@ describe('loadAlertType', () => { actionGroups: [{ id: 'default', name: 'Default' }], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, producer: 'alerts', }; @@ -71,6 +73,7 @@ describe('loadAlertType', () => { actionGroups: [{ id: 'default', name: 'Default' }], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, producer: 'alerts', }; diff --git a/x-pack/plugins/alerting/public/alert_navigation_registry/alert_navigation_registry.test.ts b/x-pack/plugins/alerting/public/alert_navigation_registry/alert_navigation_registry.test.ts index 7eb5996311386..e7e311902d08d 100644 --- a/x-pack/plugins/alerting/public/alert_navigation_registry/alert_navigation_registry.test.ts +++ b/x-pack/plugins/alerting/public/alert_navigation_registry/alert_navigation_registry.test.ts @@ -20,6 +20,7 @@ const mockAlertType = (id: string): AlertType => ({ defaultActionGroupId: 'default', producer: 'alerts', minimumLicenseRequired: 'basic', + isExportable: true, }); describe('AlertNavigationRegistry', () => { diff --git a/x-pack/plugins/alerting/server/alert_type_registry.test.ts b/x-pack/plugins/alerting/server/alert_type_registry.test.ts index 7f34760c73199..63e381bc66c0a 100644 --- a/x-pack/plugins/alerting/server/alert_type_registry.test.ts +++ b/x-pack/plugins/alerting/server/alert_type_registry.test.ts @@ -47,6 +47,7 @@ describe('has()', () => { ], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, executor: jest.fn(), producer: 'alerts', }); @@ -67,6 +68,7 @@ describe('register()', () => { ], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, executor: jest.fn(), producer: 'alerts', }; @@ -99,6 +101,7 @@ describe('register()', () => { ], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, executor: jest.fn(), producer: 'alerts', }; @@ -129,6 +132,7 @@ describe('register()', () => { ], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, executor: jest.fn(), producer: 'alerts', }; @@ -159,6 +163,7 @@ describe('register()', () => { executor: jest.fn(), producer: 'alerts', minimumLicenseRequired: 'basic', + isExportable: true, }; const registry = new AlertTypeRegistry(alertTypeRegistryParams); registry.register(alertType); @@ -203,6 +208,7 @@ describe('register()', () => { }, defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, executor: jest.fn(), producer: 'alerts', }; @@ -227,6 +233,7 @@ describe('register()', () => { ], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, executor: jest.fn(), producer: 'alerts', }; @@ -257,6 +264,7 @@ describe('register()', () => { ], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, executor: jest.fn(), producer: 'alerts', }; @@ -279,6 +287,7 @@ describe('register()', () => { ], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, executor: jest.fn(), producer: 'alerts', }); @@ -294,6 +303,7 @@ describe('register()', () => { ], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, executor: jest.fn(), producer: 'alerts', }) @@ -315,6 +325,7 @@ describe('get()', () => { ], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, executor: jest.fn(), producer: 'alerts', }); @@ -339,6 +350,7 @@ describe('get()', () => { "defaultActionGroupId": "default", "executor": [MockFunction], "id": "test", + "isExportable": true, "minimumLicenseRequired": "basic", "name": "Test", "producer": "alerts", @@ -377,6 +389,7 @@ describe('list()', () => { }, ], defaultActionGroupId: 'testActionGroup', + isExportable: true, minimumLicenseRequired: 'basic', executor: jest.fn(), producer: 'alerts', @@ -403,6 +416,7 @@ describe('list()', () => { "defaultActionGroupId": "testActionGroup", "enabledInLicense": false, "id": "test", + "isExportable": true, "minimumLicenseRequired": "basic", "name": "Test", "producer": "alerts", @@ -467,6 +481,7 @@ describe('ensureAlertTypeEnabled', () => { defaultActionGroupId: 'default', executor: jest.fn(), producer: 'alerts', + isExportable: true, minimumLicenseRequired: 'basic', recoveryActionGroup: { id: 'recovered', name: 'Recovered' }, }); @@ -497,6 +512,7 @@ function alertTypeWithVariables( name: `${id}-name`, actionGroups: [], defaultActionGroupId: id, + isExportable: true, minimumLicenseRequired: 'basic', async executor() {}, producer: 'alerts', diff --git a/x-pack/plugins/alerting/server/alert_type_registry.ts b/x-pack/plugins/alerting/server/alert_type_registry.ts index 21feb76926791..64fca58c25e66 100644 --- a/x-pack/plugins/alerting/server/alert_type_registry.ts +++ b/x-pack/plugins/alerting/server/alert_type_registry.ts @@ -46,6 +46,7 @@ export interface RegistryAlertType | 'actionVariables' | 'producer' | 'minimumLicenseRequired' + | 'isExportable' > { id: string; enabledInLicense: boolean; @@ -250,6 +251,7 @@ export class AlertTypeRegistry { actionVariables, producer, minimumLicenseRequired, + isExportable, }, ]: [string, UntypedNormalizedAlertType]) => ({ id, @@ -260,6 +262,7 @@ export class AlertTypeRegistry { actionVariables, producer, minimumLicenseRequired, + isExportable, enabledInLicense: !!this.licenseState.getLicenseCheckForAlertType( id, name, diff --git a/x-pack/plugins/alerting/server/alerts_client/tests/aggregate.test.ts b/x-pack/plugins/alerting/server/alerts_client/tests/aggregate.test.ts index bf966d38f6bc6..611ff23e46256 100644 --- a/x-pack/plugins/alerting/server/alerts_client/tests/aggregate.test.ts +++ b/x-pack/plugins/alerting/server/alerts_client/tests/aggregate.test.ts @@ -58,6 +58,7 @@ describe('aggregate()', () => { actionVariables: undefined, defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, id: 'myType', name: 'myType', @@ -110,6 +111,7 @@ describe('aggregate()', () => { actionGroups: [{ id: 'default', name: 'Default' }], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, producer: 'alerts', authorizedConsumers: { diff --git a/x-pack/plugins/alerting/server/alerts_client/tests/create.test.ts b/x-pack/plugins/alerting/server/alerts_client/tests/create.test.ts index 793357215d382..e231d1e3c27a2 100644 --- a/x-pack/plugins/alerting/server/alerts_client/tests/create.test.ts +++ b/x-pack/plugins/alerting/server/alerts_client/tests/create.test.ts @@ -1293,6 +1293,7 @@ describe('create()', () => { }), }, minimumLicenseRequired: 'basic', + isExportable: true, async executor() {}, producer: 'alerts', }); diff --git a/x-pack/plugins/alerting/server/alerts_client/tests/find.test.ts b/x-pack/plugins/alerting/server/alerts_client/tests/find.test.ts index fe788cd43bc2b..5ec39681a758b 100644 --- a/x-pack/plugins/alerting/server/alerts_client/tests/find.test.ts +++ b/x-pack/plugins/alerting/server/alerts_client/tests/find.test.ts @@ -67,6 +67,7 @@ describe('find()', () => { actionVariables: undefined, defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, id: 'myType', name: 'myType', producer: 'myApp', @@ -126,6 +127,7 @@ describe('find()', () => { recoveryActionGroup: RecoveredActionGroup, defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, producer: 'alerts', authorizedConsumers: { myApp: { read: true, all: true }, diff --git a/x-pack/plugins/alerting/server/alerts_client/tests/lib.ts b/x-pack/plugins/alerting/server/alerts_client/tests/lib.ts index e4cd24ca7e49a..e0f4f9f6da0f1 100644 --- a/x-pack/plugins/alerting/server/alerts_client/tests/lib.ts +++ b/x-pack/plugins/alerting/server/alerts_client/tests/lib.ts @@ -88,6 +88,7 @@ export function getBeforeSetup( recoveryActionGroup: RecoveredActionGroup, defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, async executor() {}, producer: 'alerts', })); diff --git a/x-pack/plugins/alerting/server/alerts_client/tests/list_alert_types.test.ts b/x-pack/plugins/alerting/server/alerts_client/tests/list_alert_types.test.ts index 9fe33996b9edf..0f849423409d8 100644 --- a/x-pack/plugins/alerting/server/alerts_client/tests/list_alert_types.test.ts +++ b/x-pack/plugins/alerting/server/alerts_client/tests/list_alert_types.test.ts @@ -58,6 +58,7 @@ describe('listAlertTypes', () => { actionVariables: undefined, defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, id: 'alertingAlertType', name: 'alertingAlertType', @@ -69,6 +70,7 @@ describe('listAlertTypes', () => { actionVariables: undefined, defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, id: 'myAppAlertType', name: 'myAppAlertType', @@ -110,6 +112,7 @@ describe('listAlertTypes', () => { actionVariables: undefined, defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, id: 'myType', name: 'myType', @@ -122,6 +125,7 @@ describe('listAlertTypes', () => { actionGroups: [{ id: 'default', name: 'Default' }], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, producer: 'alerts', enabledInLicense: true, @@ -139,6 +143,7 @@ describe('listAlertTypes', () => { actionGroups: [{ id: 'default', name: 'Default' }], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, producer: 'alerts', authorizedConsumers: { diff --git a/x-pack/plugins/alerting/server/alerts_client/tests/update.test.ts b/x-pack/plugins/alerting/server/alerts_client/tests/update.test.ts index 350c9ed31298f..2de56d20702f4 100644 --- a/x-pack/plugins/alerting/server/alerts_client/tests/update.test.ts +++ b/x-pack/plugins/alerting/server/alerts_client/tests/update.test.ts @@ -127,6 +127,7 @@ describe('update()', () => { actionGroups: [{ id: 'default', name: 'Default' }], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, async executor() {}, producer: 'alerts', @@ -773,6 +774,7 @@ describe('update()', () => { actionGroups: [{ id: 'default', name: 'Default' }], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, validate: { params: schema.object({ @@ -1096,6 +1098,7 @@ describe('update()', () => { actionGroups: [{ id: 'default', name: 'Default' }], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, async executor() {}, producer: 'alerts', diff --git a/x-pack/plugins/alerting/server/alerts_client_conflict_retries.test.ts b/x-pack/plugins/alerting/server/alerts_client_conflict_retries.test.ts index 98ad427d0c37b..e45b3513eef26 100644 --- a/x-pack/plugins/alerting/server/alerts_client_conflict_retries.test.ts +++ b/x-pack/plugins/alerting/server/alerts_client_conflict_retries.test.ts @@ -335,6 +335,7 @@ beforeEach(() => { actionGroups: [{ id: 'default', name: 'Default' }], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, async executor() {}, producer: 'alerts', @@ -346,6 +347,7 @@ beforeEach(() => { actionGroups: [{ id: 'default', name: 'Default' }], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, async executor() {}, producer: 'alerts', diff --git a/x-pack/plugins/alerting/server/authorization/alerting_authorization.test.ts b/x-pack/plugins/alerting/server/authorization/alerting_authorization.test.ts index 2227e0cecd0a6..c07148f03c684 100644 --- a/x-pack/plugins/alerting/server/authorization/alerting_authorization.test.ts +++ b/x-pack/plugins/alerting/server/authorization/alerting_authorization.test.ts @@ -203,6 +203,7 @@ beforeEach(() => { actionGroups: [{ id: 'default', name: 'Default' }], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, async executor() {}, producer: 'myApp', @@ -892,6 +893,7 @@ describe('AlertingAuthorization', () => { actionVariables: undefined, defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, id: 'myOtherAppAlertType', name: 'myOtherAppAlertType', @@ -903,6 +905,7 @@ describe('AlertingAuthorization', () => { actionVariables: undefined, defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, id: 'myAppAlertType', name: 'myAppAlertType', @@ -914,6 +917,7 @@ describe('AlertingAuthorization', () => { actionVariables: undefined, defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, id: 'mySecondAppAlertType', name: 'mySecondAppAlertType', @@ -1242,6 +1246,7 @@ describe('AlertingAuthorization', () => { actionVariables: undefined, defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, id: 'myOtherAppAlertType', name: 'myOtherAppAlertType', @@ -1253,6 +1258,7 @@ describe('AlertingAuthorization', () => { actionVariables: undefined, defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, id: 'myAppAlertType', name: 'myAppAlertType', @@ -1300,6 +1306,7 @@ describe('AlertingAuthorization', () => { "defaultActionGroupId": "default", "enabledInLicense": true, "id": "myAppAlertType", + "isExportable": true, "minimumLicenseRequired": "basic", "name": "myAppAlertType", "producer": "myApp", @@ -1328,6 +1335,7 @@ describe('AlertingAuthorization', () => { "defaultActionGroupId": "default", "enabledInLicense": true, "id": "myOtherAppAlertType", + "isExportable": true, "minimumLicenseRequired": "basic", "name": "myOtherAppAlertType", "producer": "myOtherApp", @@ -1387,6 +1395,7 @@ describe('AlertingAuthorization', () => { "defaultActionGroupId": "default", "enabledInLicense": true, "id": "myAppAlertType", + "isExportable": true, "minimumLicenseRequired": "basic", "name": "myAppAlertType", "producer": "myApp", @@ -1423,6 +1432,7 @@ describe('AlertingAuthorization', () => { "defaultActionGroupId": "default", "enabledInLicense": true, "id": "myOtherAppAlertType", + "isExportable": true, "minimumLicenseRequired": "basic", "name": "myOtherAppAlertType", "producer": "myOtherApp", @@ -1502,6 +1512,7 @@ describe('AlertingAuthorization', () => { "defaultActionGroupId": "default", "enabledInLicense": true, "id": "myOtherAppAlertType", + "isExportable": true, "minimumLicenseRequired": "basic", "name": "myOtherAppAlertType", "producer": "myOtherApp", @@ -1526,6 +1537,7 @@ describe('AlertingAuthorization', () => { "defaultActionGroupId": "default", "enabledInLicense": true, "id": "myAppAlertType", + "isExportable": true, "minimumLicenseRequired": "basic", "name": "myAppAlertType", "producer": "myApp", @@ -1605,6 +1617,7 @@ describe('AlertingAuthorization', () => { "defaultActionGroupId": "default", "enabledInLicense": true, "id": "myOtherAppAlertType", + "isExportable": true, "minimumLicenseRequired": "basic", "name": "myOtherAppAlertType", "producer": "myOtherApp", @@ -1633,6 +1646,7 @@ describe('AlertingAuthorization', () => { "defaultActionGroupId": "default", "enabledInLicense": true, "id": "myAppAlertType", + "isExportable": true, "minimumLicenseRequired": "basic", "name": "myAppAlertType", "producer": "myApp", @@ -1703,6 +1717,7 @@ describe('AlertingAuthorization', () => { "defaultActionGroupId": "default", "enabledInLicense": true, "id": "myAppAlertType", + "isExportable": true, "minimumLicenseRequired": "basic", "name": "myAppAlertType", "producer": "myApp", @@ -1807,6 +1822,7 @@ describe('AlertingAuthorization', () => { "defaultActionGroupId": "default", "enabledInLicense": true, "id": "myOtherAppAlertType", + "isExportable": true, "minimumLicenseRequired": "basic", "name": "myOtherAppAlertType", "producer": "myOtherApp", @@ -1831,6 +1847,7 @@ describe('AlertingAuthorization', () => { "defaultActionGroupId": "default", "enabledInLicense": true, "id": "myAppAlertType", + "isExportable": true, "minimumLicenseRequired": "basic", "name": "myAppAlertType", "producer": "myApp", @@ -1914,6 +1931,7 @@ describe('AlertingAuthorization', () => { "defaultActionGroupId": "default", "enabledInLicense": true, "id": "myOtherAppAlertType", + "isExportable": true, "minimumLicenseRequired": "basic", "name": "myOtherAppAlertType", "producer": "myOtherApp", diff --git a/x-pack/plugins/alerting/server/authorization/alerting_authorization_kuery.test.ts b/x-pack/plugins/alerting/server/authorization/alerting_authorization_kuery.test.ts index 8a558b6427383..7d39380f7bd1a 100644 --- a/x-pack/plugins/alerting/server/authorization/alerting_authorization_kuery.test.ts +++ b/x-pack/plugins/alerting/server/authorization/alerting_authorization_kuery.test.ts @@ -26,6 +26,7 @@ describe('asKqlFiltersByRuleTypeAndConsumer', () => { name: 'myAppAlertType', producer: 'myApp', minimumLicenseRequired: 'basic', + isExportable: true, authorizedConsumers: { myApp: { read: true, all: true }, }, @@ -53,6 +54,7 @@ describe('asKqlFiltersByRuleTypeAndConsumer', () => { actionGroups: [], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, id: 'myAppAlertType', name: 'myAppAlertType', @@ -88,6 +90,7 @@ describe('asKqlFiltersByRuleTypeAndConsumer', () => { actionGroups: [], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, id: 'myAppAlertType', name: 'myAppAlertType', @@ -104,6 +107,7 @@ describe('asKqlFiltersByRuleTypeAndConsumer', () => { actionGroups: [], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, id: 'myOtherAppAlertType', name: 'myOtherAppAlertType', @@ -120,6 +124,7 @@ describe('asKqlFiltersByRuleTypeAndConsumer', () => { actionGroups: [], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, id: 'mySecondAppAlertType', name: 'mySecondAppAlertType', @@ -162,6 +167,7 @@ describe('asEsDslFiltersByRuleTypeAndConsumer', () => { name: 'myAppAlertType', producer: 'myApp', minimumLicenseRequired: 'basic', + isExportable: true, authorizedConsumers: { myApp: { read: true, all: true }, }, @@ -216,6 +222,7 @@ describe('asEsDslFiltersByRuleTypeAndConsumer', () => { actionGroups: [], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, id: 'myAppAlertType', name: 'myAppAlertType', @@ -283,6 +290,7 @@ describe('asEsDslFiltersByRuleTypeAndConsumer', () => { actionGroups: [], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, id: 'myAppAlertType', name: 'myAppAlertType', @@ -299,6 +307,7 @@ describe('asEsDslFiltersByRuleTypeAndConsumer', () => { actionGroups: [], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, id: 'myOtherAppAlertType', name: 'myOtherAppAlertType', @@ -315,6 +324,7 @@ describe('asEsDslFiltersByRuleTypeAndConsumer', () => { actionGroups: [], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, id: 'mySecondAppAlertType', name: 'mySecondAppAlertType', diff --git a/x-pack/plugins/alerting/server/config.test.ts b/x-pack/plugins/alerting/server/config.test.ts index a8befe5210752..f7280e05b78f3 100644 --- a/x-pack/plugins/alerting/server/config.test.ts +++ b/x-pack/plugins/alerting/server/config.test.ts @@ -12,7 +12,6 @@ describe('config validation', () => { const config: Record = {}; expect(configSchema.validate(config)).toMatchInlineSnapshot(` Object { - "enableImportExport": false, "healthCheck": Object { "interval": "60m", }, diff --git a/x-pack/plugins/alerting/server/config.ts b/x-pack/plugins/alerting/server/config.ts index d50917fd13578..e42955b385bf1 100644 --- a/x-pack/plugins/alerting/server/config.ts +++ b/x-pack/plugins/alerting/server/config.ts @@ -16,7 +16,6 @@ export const configSchema = schema.object({ interval: schema.string({ validate: validateDurationSchema, defaultValue: '5m' }), removalDelay: schema.string({ validate: validateDurationSchema, defaultValue: '1h' }), }), - enableImportExport: schema.boolean({ defaultValue: false }), }); export type AlertsConfig = TypeOf; diff --git a/x-pack/plugins/alerting/server/health/get_state.test.ts b/x-pack/plugins/alerting/server/health/get_state.test.ts index 2dddf81e3b766..24f3c101b26b6 100644 --- a/x-pack/plugins/alerting/server/health/get_state.test.ts +++ b/x-pack/plugins/alerting/server/health/get_state.test.ts @@ -71,7 +71,6 @@ describe('getHealthServiceStatusWithRetryAndErrorHandling', () => { interval: '5m', removalDelay: '1h', }, - enableImportExport: false, }), pollInterval ).subscribe(); @@ -105,7 +104,6 @@ describe('getHealthServiceStatusWithRetryAndErrorHandling', () => { interval: '5m', removalDelay: '1h', }, - enableImportExport: false, }), pollInterval, retryDelay @@ -150,7 +148,6 @@ describe('getHealthServiceStatusWithRetryAndErrorHandling', () => { interval: '5m', removalDelay: '1h', }, - enableImportExport: false, }) ).toPromise(); @@ -181,7 +178,6 @@ describe('getHealthServiceStatusWithRetryAndErrorHandling', () => { interval: '5m', removalDelay: '1h', }, - enableImportExport: false, }) ).toPromise(); @@ -212,7 +208,6 @@ describe('getHealthServiceStatusWithRetryAndErrorHandling', () => { interval: '5m', removalDelay: '1h', }, - enableImportExport: false, }) ).toPromise(); @@ -240,7 +235,6 @@ describe('getHealthServiceStatusWithRetryAndErrorHandling', () => { interval: '5m', removalDelay: '1h', }, - enableImportExport: false, }), retryDelay ).subscribe((status) => { @@ -271,7 +265,6 @@ describe('getHealthServiceStatusWithRetryAndErrorHandling', () => { interval: '5m', removalDelay: '1h', }, - enableImportExport: false, }), retryDelay ).subscribe((status) => { @@ -308,7 +301,6 @@ describe('getHealthServiceStatusWithRetryAndErrorHandling', () => { interval: '5m', removalDelay: '1h', }, - enableImportExport: false, }) ).toPromise(); diff --git a/x-pack/plugins/alerting/server/lib/license_state.test.ts b/x-pack/plugins/alerting/server/lib/license_state.test.ts index a1c326656f735..e04ce85b35374 100644 --- a/x-pack/plugins/alerting/server/lib/license_state.test.ts +++ b/x-pack/plugins/alerting/server/lib/license_state.test.ts @@ -70,6 +70,7 @@ describe('getLicenseCheckForAlertType', () => { executor: jest.fn(), producer: 'alerts', minimumLicenseRequired: 'gold', + isExportable: true, recoveryActionGroup: { id: 'recovered', name: 'Recovered' }, }; @@ -204,6 +205,7 @@ describe('ensureLicenseForAlertType()', () => { executor: jest.fn(), producer: 'alerts', minimumLicenseRequired: 'gold', + isExportable: true, recoveryActionGroup: { id: 'recovered', name: 'Recovered' }, }; diff --git a/x-pack/plugins/alerting/server/plugin.test.ts b/x-pack/plugins/alerting/server/plugin.test.ts index 4e9249944a6bf..9adc3cc9d6569 100644 --- a/x-pack/plugins/alerting/server/plugin.test.ts +++ b/x-pack/plugins/alerting/server/plugin.test.ts @@ -18,7 +18,6 @@ import { AlertsConfig } from './config'; import { AlertType } from './types'; import { eventLogMock } from '../../event_log/server/mocks'; import { actionsMock } from '../../actions/server/mocks'; -import mappings from './saved_objects/mappings.json'; describe('Alerting Plugin', () => { describe('setup()', () => { @@ -37,7 +36,6 @@ describe('Alerting Plugin', () => { interval: '5m', removalDelay: '1h', }, - enableImportExport: false, }); plugin = new AlertingPlugin(context); @@ -61,78 +59,13 @@ describe('Alerting Plugin', () => { ); }); - it('should register saved object with no management capability if enableImportExport is false', async () => { - const context = coreMock.createPluginInitializerContext({ - healthCheck: { - interval: '5m', - }, - invalidateApiKeysTask: { - interval: '5m', - removalDelay: '1h', - }, - enableImportExport: false, - }); - plugin = new AlertingPlugin(context); - - const setupMocks = coreMock.createSetup(); - await plugin.setup(setupMocks, { - licensing: licensingMock.createSetup(), - encryptedSavedObjects: encryptedSavedObjectsMock.createSetup(), - taskManager: taskManagerMock.createSetup(), - eventLog: eventLogServiceMock.create(), - actions: actionsMock.createSetup(), - statusService: statusServiceMock.createSetupContract(), - }); - - expect(setupMocks.savedObjects.registerType).toHaveBeenCalledTimes(2); - const registerAlertingSavedObject = setupMocks.savedObjects.registerType.mock.calls[0][0]; - expect(registerAlertingSavedObject.name).toEqual('alert'); - expect(registerAlertingSavedObject.hidden).toBe(true); - expect(registerAlertingSavedObject.mappings).toEqual(mappings.alert); - expect(registerAlertingSavedObject.management).toBeUndefined(); - }); - - it('should register saved object with import/export capability if enableImportExport is true', async () => { - const context = coreMock.createPluginInitializerContext({ - healthCheck: { - interval: '5m', - }, - invalidateApiKeysTask: { - interval: '5m', - removalDelay: '1h', - }, - enableImportExport: true, - }); - plugin = new AlertingPlugin(context); - - const setupMocks = coreMock.createSetup(); - await plugin.setup(setupMocks, { - licensing: licensingMock.createSetup(), - encryptedSavedObjects: encryptedSavedObjectsMock.createSetup(), - taskManager: taskManagerMock.createSetup(), - eventLog: eventLogServiceMock.create(), - actions: actionsMock.createSetup(), - statusService: statusServiceMock.createSetupContract(), - }); - - expect(setupMocks.savedObjects.registerType).toHaveBeenCalledTimes(2); - const registerAlertingSavedObject = setupMocks.savedObjects.registerType.mock.calls[0][0]; - expect(registerAlertingSavedObject.name).toEqual('alert'); - expect(registerAlertingSavedObject.hidden).toBe(true); - expect(registerAlertingSavedObject.mappings).toEqual(mappings.alert); - expect(registerAlertingSavedObject.management).not.toBeUndefined(); - expect(registerAlertingSavedObject.management?.importableAndExportable).toBe(true); - expect(registerAlertingSavedObject.management?.getTitle).not.toBeUndefined(); - expect(registerAlertingSavedObject.management?.onImport).not.toBeUndefined(); - expect(registerAlertingSavedObject.management?.onExport).not.toBeUndefined(); - }); - describe('registerType()', () => { let setup: PluginSetupContract; const sampleAlertType: AlertType = { id: 'test', name: 'test', minimumLicenseRequired: 'basic', + isExportable: true, actionGroups: [], defaultActionGroupId: 'default', producer: 'test', @@ -189,7 +122,6 @@ describe('Alerting Plugin', () => { interval: '5m', removalDelay: '1h', }, - enableImportExport: false, }); const plugin = new AlertingPlugin(context); @@ -229,7 +161,6 @@ describe('Alerting Plugin', () => { interval: '5m', removalDelay: '1h', }, - enableImportExport: false, }); const plugin = new AlertingPlugin(context); @@ -283,7 +214,6 @@ describe('Alerting Plugin', () => { interval: '5m', removalDelay: '1h', }, - enableImportExport: false, }); const plugin = new AlertingPlugin(context); diff --git a/x-pack/plugins/alerting/server/plugin.ts b/x-pack/plugins/alerting/server/plugin.ts index 5afa1b235a8c1..df63625bf242d 100644 --- a/x-pack/plugins/alerting/server/plugin.ts +++ b/x-pack/plugins/alerting/server/plugin.ts @@ -192,8 +192,6 @@ export class AlertingPlugin { event: { provider: EVENT_LOG_PROVIDER }, }); - setupSavedObjects(core.savedObjects, plugins.encryptedSavedObjects, this.config); - this.eventLogService = plugins.eventLog; plugins.eventLog.registerProviderActions(EVENT_LOG_PROVIDER, Object.values(EVENT_LOG_ACTIONS)); @@ -221,6 +219,13 @@ export class AlertingPlugin { }); } + setupSavedObjects( + core.savedObjects, + plugins.encryptedSavedObjects, + this.alertTypeRegistry, + this.logger + ); + initializeApiKeyInvalidator( this.logger, core.getStartServices(), diff --git a/x-pack/plugins/alerting/server/routes/legacy/list_alert_types.test.ts b/x-pack/plugins/alerting/server/routes/legacy/list_alert_types.test.ts index 3e6f2f484a6d8..e2bf1afdb0f6e 100644 --- a/x-pack/plugins/alerting/server/routes/legacy/list_alert_types.test.ts +++ b/x-pack/plugins/alerting/server/routes/legacy/list_alert_types.test.ts @@ -47,6 +47,7 @@ describe('listAlertTypesRoute', () => { ], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, authorizedConsumers: {}, actionVariables: { @@ -79,6 +80,7 @@ describe('listAlertTypesRoute', () => { "defaultActionGroupId": "default", "enabledInLicense": true, "id": "1", + "isExportable": true, "minimumLicenseRequired": "basic", "name": "name", "producer": "test", @@ -120,6 +122,7 @@ describe('listAlertTypesRoute', () => { ], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, authorizedConsumers: {}, actionVariables: { @@ -172,6 +175,7 @@ describe('listAlertTypesRoute', () => { ], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, authorizedConsumers: {}, actionVariables: { diff --git a/x-pack/plugins/alerting/server/routes/rule_types.test.ts b/x-pack/plugins/alerting/server/routes/rule_types.test.ts index 58c9a4b4c46fd..4f04f8c7575c5 100644 --- a/x-pack/plugins/alerting/server/routes/rule_types.test.ts +++ b/x-pack/plugins/alerting/server/routes/rule_types.test.ts @@ -48,6 +48,7 @@ describe('ruleTypesRoute', () => { ], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, authorizedConsumers: {}, actionVariables: { @@ -70,6 +71,7 @@ describe('ruleTypesRoute', () => { ], default_action_group_id: 'default', minimum_license_required: 'basic', + is_exportable: true, recovery_action_group: RecoveredActionGroup, authorized_consumers: {}, action_variables: { @@ -102,6 +104,7 @@ describe('ruleTypesRoute', () => { "default_action_group_id": "default", "enabled_in_license": true, "id": "1", + "is_exportable": true, "minimum_license_required": "basic", "name": "name", "producer": "test", @@ -143,6 +146,7 @@ describe('ruleTypesRoute', () => { ], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, authorizedConsumers: {}, actionVariables: { @@ -195,6 +199,7 @@ describe('ruleTypesRoute', () => { ], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, authorizedConsumers: {}, actionVariables: { diff --git a/x-pack/plugins/alerting/server/routes/rule_types.ts b/x-pack/plugins/alerting/server/routes/rule_types.ts index a3a44f9b013cd..f67e07f13feed 100644 --- a/x-pack/plugins/alerting/server/routes/rule_types.ts +++ b/x-pack/plugins/alerting/server/routes/rule_types.ts @@ -19,6 +19,7 @@ const rewriteBodyRes: RewriteResponseCase = (result actionGroups, defaultActionGroupId, minimumLicenseRequired, + isExportable, actionVariables, authorizedConsumers, ...rest @@ -29,6 +30,7 @@ const rewriteBodyRes: RewriteResponseCase = (result action_groups: actionGroups, default_action_group_id: defaultActionGroupId, minimum_license_required: minimumLicenseRequired, + is_exportable: isExportable, action_variables: actionVariables, authorized_consumers: authorizedConsumers, }) diff --git a/x-pack/plugins/alerting/server/saved_objects/index.ts b/x-pack/plugins/alerting/server/saved_objects/index.ts index 1ad0f972b2ec0..88ee3179ab3d8 100644 --- a/x-pack/plugins/alerting/server/saved_objects/index.ts +++ b/x-pack/plugins/alerting/server/saved_objects/index.ts @@ -6,6 +6,7 @@ */ import type { + Logger, SavedObject, SavedObjectsExportTransformContext, SavedObjectsServiceSetup, @@ -17,7 +18,9 @@ import { EncryptedSavedObjectsPluginSetup } from '../../../encrypted_saved_objec import { transformRulesForExport } from './transform_rule_for_export'; import { RawAlert } from '../types'; import { getImportWarnings } from './get_import_warnings'; -import { AlertsConfig } from '../config'; +import { isRuleExportable } from './is_rule_exportable'; +import { AlertTypeRegistry } from '../alert_type_registry'; + export { partiallyUpdateAlert } from './partially_update_alert'; export const AlertAttributesExcludedFromAAD = [ @@ -44,65 +47,63 @@ export type AlertAttributesExcludedFromAADType = export function setupSavedObjects( savedObjects: SavedObjectsServiceSetup, encryptedSavedObjects: EncryptedSavedObjectsPluginSetup, - alertingConfig: Promise + ruleTypeRegistry: AlertTypeRegistry, + logger: Logger ) { - alertingConfig.then((config: AlertsConfig) => { - savedObjects.registerType({ - name: 'alert', - hidden: true, - namespaceType: 'single', - migrations: getMigrations(encryptedSavedObjects), - mappings: mappings.alert as SavedObjectsTypeMappingDefinition, - ...(config.enableImportExport - ? { - management: { - importableAndExportable: true, - getTitle(ruleSavedObject: SavedObject) { - return `Rule: [${ruleSavedObject.attributes.name}]`; - }, - onImport(ruleSavedObjects) { - return { - warnings: getImportWarnings(ruleSavedObjects), - }; - }, - onExport( - context: SavedObjectsExportTransformContext, - objects: Array> - ) { - return transformRulesForExport(objects); - }, - }, - } - : {}), - }); + savedObjects.registerType({ + name: 'alert', + hidden: true, + namespaceType: 'single', + migrations: getMigrations(encryptedSavedObjects), + mappings: mappings.alert as SavedObjectsTypeMappingDefinition, + management: { + importableAndExportable: true, + getTitle(ruleSavedObject: SavedObject) { + return `Rule: [${ruleSavedObject.attributes.name}]`; + }, + onImport(ruleSavedObjects) { + return { + warnings: getImportWarnings(ruleSavedObjects), + }; + }, + onExport( + context: SavedObjectsExportTransformContext, + objects: Array> + ) { + return transformRulesForExport(objects); + }, + isExportable(ruleSavedObject: SavedObject) { + return isRuleExportable(ruleSavedObject, ruleTypeRegistry, logger); + }, + }, + }); - savedObjects.registerType({ - name: 'api_key_pending_invalidation', - hidden: true, - namespaceType: 'agnostic', - mappings: { - properties: { - apiKeyId: { - type: 'keyword', - }, - createdAt: { - type: 'date', - }, + savedObjects.registerType({ + name: 'api_key_pending_invalidation', + hidden: true, + namespaceType: 'agnostic', + mappings: { + properties: { + apiKeyId: { + type: 'keyword', + }, + createdAt: { + type: 'date', }, }, - }); + }, + }); - // Encrypted attributes - encryptedSavedObjects.registerType({ - type: 'alert', - attributesToEncrypt: new Set(['apiKey']), - attributesToExcludeFromAAD: new Set(AlertAttributesExcludedFromAAD), - }); + // Encrypted attributes + encryptedSavedObjects.registerType({ + type: 'alert', + attributesToEncrypt: new Set(['apiKey']), + attributesToExcludeFromAAD: new Set(AlertAttributesExcludedFromAAD), + }); - // Encrypted attributes - encryptedSavedObjects.registerType({ - type: 'api_key_pending_invalidation', - attributesToEncrypt: new Set(['apiKeyId']), - }); + // Encrypted attributes + encryptedSavedObjects.registerType({ + type: 'api_key_pending_invalidation', + attributesToEncrypt: new Set(['apiKeyId']), }); } diff --git a/x-pack/plugins/alerting/server/saved_objects/is_rule_exportable.test.ts b/x-pack/plugins/alerting/server/saved_objects/is_rule_exportable.test.ts new file mode 100644 index 0000000000000..cc2dfbd3e2d2f --- /dev/null +++ b/x-pack/plugins/alerting/server/saved_objects/is_rule_exportable.test.ts @@ -0,0 +1,208 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MockedLogger, loggerMock } from '@kbn/logging/target/mocks'; +import { TaskRunnerFactory } from '../task_runner'; +import { AlertTypeRegistry, ConstructorOptions } from '../alert_type_registry'; +import { taskManagerMock } from '../../../task_manager/server/mocks'; +import { ILicenseState } from '../lib/license_state'; +import { licenseStateMock } from '../lib/license_state.mock'; +import { licensingMock } from '../../../licensing/server/mocks'; +import { isRuleExportable } from './is_rule_exportable'; + +let ruleTypeRegistryParams: ConstructorOptions; +let logger: MockedLogger; +let mockedLicenseState: jest.Mocked; +const taskManager = taskManagerMock.createSetup(); + +beforeEach(() => { + jest.resetAllMocks(); + mockedLicenseState = licenseStateMock.create(); + logger = loggerMock.create(); + ruleTypeRegistryParams = { + taskManager, + taskRunnerFactory: new TaskRunnerFactory(), + licenseState: mockedLicenseState, + licensing: licensingMock.createSetup(), + }; +}); + +describe('isRuleExportable', () => { + it('should return true if rule type isExportable is true', () => { + const registry = new AlertTypeRegistry(ruleTypeRegistryParams); + registry.register({ + id: 'foo', + name: 'Foo', + actionGroups: [ + { + id: 'default', + name: 'Default', + }, + ], + defaultActionGroupId: 'default', + minimumLicenseRequired: 'basic', + isExportable: true, + executor: jest.fn(), + producer: 'alerts', + }); + expect( + isRuleExportable( + { + id: '1', + type: 'alert', + attributes: { + enabled: true, + name: 'rule-name', + tags: ['tag-1', 'tag-2'], + alertTypeId: 'foo', + consumer: 'alert-consumer', + schedule: { interval: '1m' }, + actions: [], + params: {}, + createdBy: 'me', + updatedBy: 'me', + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + apiKey: '4tndskbuhewotw4klrhgjewrt9u', + apiKeyOwner: 'me', + throttle: null, + notifyWhen: 'onActionGroupChange', + muteAll: false, + mutedInstanceIds: [], + executionStatus: { + status: 'active', + lastExecutionDate: '2020-08-20T19:23:38Z', + error: null, + }, + scheduledTaskId: '2q5tjbf3q45twer', + }, + references: [], + }, + registry, + logger + ) + ).toEqual(true); + }); + + it('should return false and log warning if rule type isExportable is false', () => { + const registry = new AlertTypeRegistry(ruleTypeRegistryParams); + registry.register({ + id: 'foo', + name: 'Foo', + actionGroups: [ + { + id: 'default', + name: 'Default', + }, + ], + defaultActionGroupId: 'default', + minimumLicenseRequired: 'basic', + isExportable: false, + executor: jest.fn(), + producer: 'alerts', + }); + expect( + isRuleExportable( + { + id: '1', + type: 'alert', + attributes: { + enabled: true, + name: 'rule-name', + tags: ['tag-1', 'tag-2'], + alertTypeId: 'foo', + consumer: 'alert-consumer', + schedule: { interval: '1m' }, + actions: [], + params: {}, + createdBy: 'me', + updatedBy: 'me', + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + apiKey: '4tndskbuhewotw4klrhgjewrt9u', + apiKeyOwner: 'me', + throttle: null, + notifyWhen: 'onActionGroupChange', + muteAll: false, + mutedInstanceIds: [], + executionStatus: { + status: 'active', + lastExecutionDate: '2020-08-20T19:23:38Z', + error: null, + }, + scheduledTaskId: '2q5tjbf3q45twer', + }, + references: [], + }, + registry, + logger + ) + ).toEqual(false); + expect(logger.warn).toHaveBeenCalledWith( + `Skipping export of rule \"1\" because rule type \"foo\" is not exportable through this interface.` + ); + }); + + it('should return false and log warning if rule type is not registered', () => { + const registry = new AlertTypeRegistry(ruleTypeRegistryParams); + registry.register({ + id: 'foo', + name: 'Foo', + actionGroups: [ + { + id: 'default', + name: 'Default', + }, + ], + defaultActionGroupId: 'default', + minimumLicenseRequired: 'basic', + isExportable: false, + executor: jest.fn(), + producer: 'alerts', + }); + expect( + isRuleExportable( + { + id: '1', + type: 'alert', + attributes: { + enabled: true, + name: 'rule-name', + tags: ['tag-1', 'tag-2'], + alertTypeId: 'bar', + consumer: 'alert-consumer', + schedule: { interval: '1m' }, + actions: [], + params: {}, + createdBy: 'me', + updatedBy: 'me', + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + apiKey: '4tndskbuhewotw4klrhgjewrt9u', + apiKeyOwner: 'me', + throttle: null, + notifyWhen: 'onActionGroupChange', + muteAll: false, + mutedInstanceIds: [], + executionStatus: { + status: 'active', + lastExecutionDate: '2020-08-20T19:23:38Z', + error: null, + }, + scheduledTaskId: '2q5tjbf3q45twer', + }, + references: [], + }, + registry, + logger + ) + ).toEqual(false); + expect(logger.warn).toHaveBeenCalledWith( + `Skipping export of rule \"1\" because rule type \"bar\" is not recognized.` + ); + }); +}); diff --git a/x-pack/plugins/alerting/server/saved_objects/is_rule_exportable.ts b/x-pack/plugins/alerting/server/saved_objects/is_rule_exportable.ts new file mode 100644 index 0000000000000..38290e5f465cc --- /dev/null +++ b/x-pack/plugins/alerting/server/saved_objects/is_rule_exportable.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Logger, SavedObject } from 'kibana/server'; +import { RawAlert } from '../types'; +import { AlertTypeRegistry } from '../alert_type_registry'; + +export function isRuleExportable( + rule: SavedObject, + ruleTypeRegistry: AlertTypeRegistry, + logger: Logger +): boolean { + const ruleSO = rule as SavedObject; + try { + const ruleType = ruleTypeRegistry.get(ruleSO.attributes.alertTypeId); + if (!ruleType.isExportable) { + logger.warn( + `Skipping export of rule "${ruleSO.id}" because rule type "${ruleSO.attributes.alertTypeId}" is not exportable through this interface.` + ); + } + + return ruleType.isExportable; + } catch (err) { + logger.warn( + `Skipping export of rule "${ruleSO.id}" because rule type "${ruleSO.attributes.alertTypeId}" is not recognized.` + ); + return false; + } +} diff --git a/x-pack/plugins/alerting/server/task_runner/create_execution_handler.test.ts b/x-pack/plugins/alerting/server/task_runner/create_execution_handler.test.ts index 1dcd19119b6fd..b264428b4d6f2 100644 --- a/x-pack/plugins/alerting/server/task_runner/create_execution_handler.test.ts +++ b/x-pack/plugins/alerting/server/task_runner/create_execution_handler.test.ts @@ -44,6 +44,7 @@ const alertType: NormalizedAlertType< ], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: { id: 'recovered', name: 'Recovered', diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts index 88d1b1b24a4ec..4f650975f830e 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts @@ -44,6 +44,7 @@ const alertType: jest.Mocked = { actionGroups: [{ id: 'default', name: 'Default' }, RecoveredActionGroup], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: RecoveredActionGroup, executor: jest.fn(), producer: 'alerts', diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner_factory.test.ts b/x-pack/plugins/alerting/server/task_runner/task_runner_factory.test.ts index 343dffa0d5e70..050345f3e617f 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner_factory.test.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner_factory.test.ts @@ -26,6 +26,7 @@ const alertType: UntypedNormalizedAlertType = { actionGroups: [{ id: 'default', name: 'Default' }], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: { id: 'recovered', name: 'Recovered', diff --git a/x-pack/plugins/alerting/server/types.ts b/x-pack/plugins/alerting/server/types.ts index f8846035e6b02..f21e17adc841d 100644 --- a/x-pack/plugins/alerting/server/types.ts +++ b/x-pack/plugins/alerting/server/types.ts @@ -146,6 +146,7 @@ export interface AlertType< params?: ActionVariable[]; }; minimumLicenseRequired: LicenseType; + isExportable: boolean; } export type UntypedAlertType = AlertType< diff --git a/x-pack/plugins/apm/common/alert_types.ts b/x-pack/plugins/apm/common/alert_types.ts index 12df93d54b296..ad233c7f6df92 100644 --- a/x-pack/plugins/apm/common/alert_types.ts +++ b/x-pack/plugins/apm/common/alert_types.ts @@ -33,6 +33,7 @@ export const ALERT_TYPES_CONFIG: Record< actionGroups: Array>; defaultActionGroupId: ThresholdMetActionGroupId; minimumLicenseRequired: string; + isExportable: boolean; producer: string; } > = { @@ -44,6 +45,7 @@ export const ALERT_TYPES_CONFIG: Record< defaultActionGroupId: THRESHOLD_MET_GROUP_ID, minimumLicenseRequired: 'basic', producer: 'apm', + isExportable: true, }, [AlertType.TransactionDuration]: { name: i18n.translate('xpack.apm.transactionDurationAlert.name', { @@ -53,6 +55,7 @@ export const ALERT_TYPES_CONFIG: Record< defaultActionGroupId: THRESHOLD_MET_GROUP_ID, minimumLicenseRequired: 'basic', producer: 'apm', + isExportable: true, }, [AlertType.TransactionDurationAnomaly]: { name: i18n.translate('xpack.apm.transactionDurationAnomalyAlert.name', { @@ -62,6 +65,7 @@ export const ALERT_TYPES_CONFIG: Record< defaultActionGroupId: THRESHOLD_MET_GROUP_ID, minimumLicenseRequired: 'basic', producer: 'apm', + isExportable: true, }, [AlertType.TransactionErrorRate]: { name: i18n.translate('xpack.apm.transactionErrorRateAlert.name', { @@ -71,6 +75,7 @@ export const ALERT_TYPES_CONFIG: Record< defaultActionGroupId: THRESHOLD_MET_GROUP_ID, minimumLicenseRequired: 'basic', producer: 'apm', + isExportable: true, }, }; diff --git a/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts index 2a63c53b626cd..7548d6eba060a 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts @@ -71,6 +71,7 @@ export function registerErrorCountAlertType({ }, producer: 'apm', minimumLicenseRequired: 'basic', + isExportable: true, executor: async ({ services, params }) => { const config = await config$.pipe(take(1)).toPromise(); const alertParams = params; diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_alert_type.ts index 24a6376761a7d..ca7806251f75e 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_alert_type.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_alert_type.ts @@ -79,6 +79,7 @@ export function registerTransactionDurationAlertType({ }, producer: 'apm', minimumLicenseRequired: 'basic', + isExportable: true, executor: async ({ services, params }) => { const config = await config$.pipe(take(1)).toPromise(); const alertParams = params; diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts index f640925b0a0fa..de0657d075d7f 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts @@ -87,6 +87,7 @@ export function registerTransactionDurationAnomalyAlertType({ }, producer: 'apm', minimumLicenseRequired: 'basic', + isExportable: true, executor: async ({ services, params }) => { if (!ml) { return {}; diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts index 47ed5236ce74c..718ffd9c92167 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts @@ -77,6 +77,7 @@ export function registerTransactionErrorRateAlertType({ }, producer: 'apm', minimumLicenseRequired: 'basic', + isExportable: true, executor: async ({ services, params: alertParams }) => { const config = await config$.pipe(take(1)).toPromise(); const indices = await getApmIndices({ diff --git a/x-pack/plugins/infra/server/lib/alerting/inventory_metric_threshold/register_inventory_metric_threshold_alert_type.ts b/x-pack/plugins/infra/server/lib/alerting/inventory_metric_threshold/register_inventory_metric_threshold_alert_type.ts index 81204c7b71a68..a2d8e522c7c8d 100644 --- a/x-pack/plugins/infra/server/lib/alerting/inventory_metric_threshold/register_inventory_metric_threshold_alert_type.ts +++ b/x-pack/plugins/infra/server/lib/alerting/inventory_metric_threshold/register_inventory_metric_threshold_alert_type.ts @@ -89,6 +89,7 @@ export const registerMetricInventoryThresholdAlertType = ( actionGroups: [FIRED_ACTIONS, WARNING_ACTIONS], producer: 'infrastructure', minimumLicenseRequired: 'basic', + isExportable: true, executor: createInventoryMetricThresholdExecutor(libs), actionVariables: { context: [ diff --git a/x-pack/plugins/infra/server/lib/alerting/log_threshold/register_log_threshold_alert_type.ts b/x-pack/plugins/infra/server/lib/alerting/log_threshold/register_log_threshold_alert_type.ts index b7c6881b20390..62d92d0487ff7 100644 --- a/x-pack/plugins/infra/server/lib/alerting/log_threshold/register_log_threshold_alert_type.ts +++ b/x-pack/plugins/infra/server/lib/alerting/log_threshold/register_log_threshold_alert_type.ts @@ -107,6 +107,7 @@ export async function registerLogThresholdAlertType( defaultActionGroupId: FIRED_ACTIONS.id, actionGroups: [FIRED_ACTIONS], minimumLicenseRequired: 'basic', + isExportable: true, executor: createLogThresholdExecutor(libs), actionVariables: { context: [ diff --git a/x-pack/plugins/infra/server/lib/alerting/metric_anomaly/register_metric_anomaly_alert_type.ts b/x-pack/plugins/infra/server/lib/alerting/metric_anomaly/register_metric_anomaly_alert_type.ts index 2adf37c60cf3a..63354111a1a98 100644 --- a/x-pack/plugins/infra/server/lib/alerting/metric_anomaly/register_metric_anomaly_alert_type.ts +++ b/x-pack/plugins/infra/server/lib/alerting/metric_anomaly/register_metric_anomaly_alert_type.ts @@ -64,6 +64,7 @@ export const registerMetricAnomalyAlertType = ( actionGroups: [FIRED_ACTIONS], producer: 'infrastructure', minimumLicenseRequired: 'basic', + isExportable: true, executor: createMetricAnomalyExecutor(libs, ml), actionVariables: { context: [ diff --git a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/register_metric_threshold_alert_type.ts b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/register_metric_threshold_alert_type.ts index 7dbae03f20fbd..e519d67b446a5 100644 --- a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/register_metric_threshold_alert_type.ts +++ b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/register_metric_threshold_alert_type.ts @@ -100,6 +100,7 @@ export function registerMetricThresholdAlertType(libs: InfraBackendLibs): Metric defaultActionGroupId: FIRED_ACTIONS.id, actionGroups: [FIRED_ACTIONS, WARNING_ACTIONS], minimumLicenseRequired: 'basic', + isExportable: true, executor: createMetricThresholdExecutor(libs), actionVariables: { context: [ diff --git a/x-pack/plugins/ml/server/lib/alerts/register_anomaly_detection_alert_type.ts b/x-pack/plugins/ml/server/lib/alerts/register_anomaly_detection_alert_type.ts index f39b3850b71b1..93c627c0f6311 100644 --- a/x-pack/plugins/ml/server/lib/alerts/register_anomaly_detection_alert_type.ts +++ b/x-pack/plugins/ml/server/lib/alerts/register_anomaly_detection_alert_type.ts @@ -119,6 +119,7 @@ export function registerAnomalyDetectionAlertType({ }, producer: PLUGIN_ID, minimumLicenseRequired: MINIMUM_FULL_LICENSE, + isExportable: true, async executor({ services, params, alertId, state, previousStartedAt, startedAt }) { const fakeRequest = {} as KibanaRequest; const { execute } = mlSharedServices.alertingServiceProvider( diff --git a/x-pack/plugins/monitoring/server/alerts/base_alert.ts b/x-pack/plugins/monitoring/server/alerts/base_alert.ts index bb80d84210a48..8954a4ae2486d 100644 --- a/x-pack/plugins/monitoring/server/alerts/base_alert.ts +++ b/x-pack/plugins/monitoring/server/alerts/base_alert.ts @@ -96,6 +96,7 @@ export class BaseAlert { ], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: false, executor: ( options: AlertExecutorOptions & { state: ExecutedState; diff --git a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts index 85e69eb51fd02..a362dcccc2f0f 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts @@ -40,6 +40,7 @@ function createRule() { }, id: 'test_type', minimumLicenseRequired: 'basic', + isExportable: true, name: 'Test type', producer: 'test', actionVariables: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/rules_notification_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/rules_notification_alert_type.ts index a4863e577c6bc..c85848ba6dcfe 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/rules_notification_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/rules_notification_alert_type.ts @@ -37,6 +37,7 @@ export const rulesNotificationAlertType = ({ }), }, minimumLicenseRequired: 'basic', + isExportable: false, async executor({ startedAt, previousStartedAt, alertId, services, params }) { const ruleAlertSavedObject = await services.savedObjectsClient.get( 'alert', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/eql.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/eql.ts index 39d02c808d09e..b98bd9b3551c3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/eql.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/eql.ts @@ -45,6 +45,7 @@ export const createEqlAlertType = (ruleDataClient: RuleDataClient, logger: Logge context: [{ name: 'server', description: 'the server' }], }, minimumLicenseRequired: 'basic', + isExportable: false, producer: 'security-solution', async executor({ startedAt, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/ml.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/ml.ts index c07d0436cc90d..14252bf62ef83 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/ml.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/ml.ts @@ -57,6 +57,7 @@ export const mlAlertType = createSecurityMlRuleType({ context: [{ name: 'server', description: 'the server' }], }, minimumLicenseRequired: 'basic', + isExportable: false, producer: 'security-solution', async executor({ services: { alertWithPersistence, findAlerts }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/query.ts index 39f325fd6cf8f..4ca9448f5e3c7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/query.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/query.ts @@ -43,6 +43,7 @@ export const createQueryAlertType = (ruleDataClient: RuleDataClient, logger: Log context: [{ name: 'server', description: 'the server' }], }, minimumLicenseRequired: 'basic', + isExportable: false, producer: 'security-solution', async executor({ services: { alertWithPersistence, findAlerts }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/threshold.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/threshold.ts index d4721e8bab11d..fa291ef3139cd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/threshold.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/threshold.ts @@ -113,6 +113,7 @@ export const createThresholdAlertType = (ruleDataClient: RuleDataClient, logger: context: [{ name: 'server', description: 'the server' }], }, minimumLicenseRequired: 'basic', + isExportable: false, producer: 'security-solution', async executor({ startedAt, services, params, alertId }) { const fromDate = moment(startedAt).subtract(moment.duration(5, 'm')); // hardcoded 5-minute rule interval diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 32bd6d71bfb1d..ba665fa43e8b8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -103,6 +103,7 @@ export const signalRulesAlertType = ({ }, producer: SERVER_APP_ID, minimumLicenseRequired: 'basic', + isExportable: false, async executor({ previousStartedAt, startedAt, diff --git a/x-pack/plugins/stack_alerts/server/alert_types/es_query/alert_type.ts b/x-pack/plugins/stack_alerts/server/alert_types/es_query/alert_type.ts index b81bc19d5c731..c9f233002d79b 100644 --- a/x-pack/plugins/stack_alerts/server/alert_types/es_query/alert_type.ts +++ b/x-pack/plugins/stack_alerts/server/alert_types/es_query/alert_type.ts @@ -140,6 +140,7 @@ export function getAlertType( ], }, minimumLicenseRequired: 'basic', + isExportable: true, executor, producer: STACK_ALERTS_FEATURE_ID, }; diff --git a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/alert_type.ts b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/alert_type.ts index e3d379f2869b9..43ae726fa2478 100644 --- a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/alert_type.ts +++ b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/alert_type.ts @@ -177,5 +177,6 @@ export function getAlertType(logger: Logger): GeoContainmentAlertType { }, actionVariables, minimumLicenseRequired: 'gold', + isExportable: true, }; } diff --git a/x-pack/plugins/stack_alerts/server/alert_types/index_threshold/alert_type.ts b/x-pack/plugins/stack_alerts/server/alert_types/index_threshold/alert_type.ts index a242c1e0eb29e..aa56951b5dcba 100644 --- a/x-pack/plugins/stack_alerts/server/alert_types/index_threshold/alert_type.ts +++ b/x-pack/plugins/stack_alerts/server/alert_types/index_threshold/alert_type.ts @@ -125,6 +125,7 @@ export function getAlertType( ], }, minimumLicenseRequired: 'basic', + isExportable: true, executor, producer: STACK_ALERTS_FEATURE_ID, }; diff --git a/x-pack/plugins/uptime/server/lib/alerts/duration_anomaly.ts b/x-pack/plugins/uptime/server/lib/alerts/duration_anomaly.ts index b3262976b0cac..981a7e7ca3920 100644 --- a/x-pack/plugins/uptime/server/lib/alerts/duration_anomaly.ts +++ b/x-pack/plugins/uptime/server/lib/alerts/duration_anomaly.ts @@ -89,6 +89,7 @@ export const durationAnomalyAlertFactory: UptimeAlertTypeFactory state: [...durationAnomalyTranslations.actionVariables, ...commonStateTranslations], }, minimumLicenseRequired: 'basic', + isExportable: true, async executor({ options, uptimeEsClient, savedObjectsClient, dynamicSettings }) { const { services: { alertInstanceFactory }, diff --git a/x-pack/plugins/uptime/server/lib/alerts/status_check.ts b/x-pack/plugins/uptime/server/lib/alerts/status_check.ts index c5a6ef877c47a..6f3e3303f6bdc 100644 --- a/x-pack/plugins/uptime/server/lib/alerts/status_check.ts +++ b/x-pack/plugins/uptime/server/lib/alerts/status_check.ts @@ -258,6 +258,7 @@ export const statusCheckAlertFactory: UptimeAlertTypeFactory = ( state: [...commonMonitorStateI18, ...commonStateTranslations], }, minimumLicenseRequired: 'basic', + isExportable: true, async executor({ options: { params: rawParams, diff --git a/x-pack/plugins/uptime/server/lib/alerts/tls.ts b/x-pack/plugins/uptime/server/lib/alerts/tls.ts index f29744fdbb70f..09f5e2fe0f6d5 100644 --- a/x-pack/plugins/uptime/server/lib/alerts/tls.ts +++ b/x-pack/plugins/uptime/server/lib/alerts/tls.ts @@ -112,6 +112,7 @@ export const tlsAlertFactory: UptimeAlertTypeFactory = (_server, state: [...tlsTranslations.actionVariables, ...commonStateTranslations], }, minimumLicenseRequired: 'basic', + isExportable: true, async executor({ options, dynamicSettings, uptimeEsClient }) { const { services: { alertInstanceFactory }, diff --git a/x-pack/plugins/uptime/server/lib/alerts/tls_legacy.ts b/x-pack/plugins/uptime/server/lib/alerts/tls_legacy.ts index 8f1c0093e60ac..5bf91b7c5486d 100644 --- a/x-pack/plugins/uptime/server/lib/alerts/tls_legacy.ts +++ b/x-pack/plugins/uptime/server/lib/alerts/tls_legacy.ts @@ -103,6 +103,7 @@ export const tlsLegacyAlertFactory: UptimeAlertTypeFactory = (_s state: [...tlsTranslations.actionVariables, ...commonStateTranslations], }, minimumLicenseRequired: 'basic', + isExportable: true, async executor({ options, dynamicSettings, uptimeEsClient }) { const { services: { alertInstanceFactory }, diff --git a/x-pack/test/alerting_api_integration/common/fixtures/plugins/alerts/server/alert_types.ts b/x-pack/test/alerting_api_integration/common/fixtures/plugins/alerts/server/alert_types.ts index 3e622b49d03df..3c9d783f5a357 100644 --- a/x-pack/test/alerting_api_integration/common/fixtures/plugins/alerts/server/alert_types.ts +++ b/x-pack/test/alerting_api_integration/common/fixtures/plugins/alerts/server/alert_types.ts @@ -81,6 +81,7 @@ function getAlwaysFiringAlertType() { producer: 'alertsFixture', defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, actionVariables: { state: [{ name: 'instanceStateValue', description: 'the instance state value' }], params: [{ name: 'instanceParamsValue', description: 'the instance params value' }], @@ -167,6 +168,7 @@ function getCumulativeFiringAlertType() { producer: 'alertsFixture', defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, async executor(alertExecutorOptions) { const { services, state } = alertExecutorOptions; const group = 'default'; @@ -212,6 +214,7 @@ function getNeverFiringAlertType() { producer: 'alertsFixture', defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, async executor({ services, params, state }) { await services.scopedClusterClient.asCurrentUser.index({ index: params.index, @@ -252,6 +255,7 @@ function getFailingAlertType() { producer: 'alertsFixture', defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, async executor({ services, params, state }) { await services.scopedClusterClient.asCurrentUser.index({ index: params.index, @@ -290,6 +294,7 @@ function getAuthorizationAlertType(core: CoreSetup) { defaultActionGroupId: 'default', producer: 'alertsFixture', minimumLicenseRequired: 'basic', + isExportable: true, validate: { params: paramsSchema, }, @@ -376,6 +381,7 @@ function getValidationAlertType() { ], producer: 'alertsFixture', minimumLicenseRequired: 'basic', + isExportable: true, defaultActionGroupId: 'default', validate: { params: paramsSchema, @@ -404,6 +410,7 @@ function getPatternFiringAlertType() { producer: 'alertsFixture', defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, async executor(alertExecutorOptions) { const { services, state, params } = alertExecutorOptions; const pattern = params.pattern; @@ -468,6 +475,7 @@ export function defineAlertTypes( producer: 'alertsFixture', defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, async executor() {}, }; const goldNoopAlertType: AlertType<{}, {}, {}, {}, 'default'> = { @@ -477,6 +485,7 @@ export function defineAlertTypes( producer: 'alertsFixture', defaultActionGroupId: 'default', minimumLicenseRequired: 'gold', + isExportable: true, async executor() {}, }; const onlyContextVariablesAlertType: AlertType<{}, {}, {}, {}, 'default'> = { @@ -486,6 +495,7 @@ export function defineAlertTypes( producer: 'alertsFixture', defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, actionVariables: { context: [{ name: 'aContextVariable', description: 'this is a context variable' }], }, @@ -501,6 +511,7 @@ export function defineAlertTypes( state: [{ name: 'aStateVariable', description: 'this is a state variable' }], }, minimumLicenseRequired: 'basic', + isExportable: true, async executor() {}, }; const throwAlertType: AlertType<{}, {}, {}, {}, 'default'> = { @@ -515,6 +526,7 @@ export function defineAlertTypes( producer: 'alertsFixture', defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, async executor() { throw new Error('this alert is intended to fail'); }, @@ -531,6 +543,7 @@ export function defineAlertTypes( producer: 'alertsFixture', defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, async executor() { await new Promise((resolve) => setTimeout(resolve, 5000)); }, diff --git a/x-pack/test/alerting_api_integration/common/fixtures/plugins/alerts_restricted/server/alert_types.ts b/x-pack/test/alerting_api_integration/common/fixtures/plugins/alerts_restricted/server/alert_types.ts index 884af7801855f..2255d1fa95e2d 100644 --- a/x-pack/test/alerting_api_integration/common/fixtures/plugins/alerts_restricted/server/alert_types.ts +++ b/x-pack/test/alerting_api_integration/common/fixtures/plugins/alerts_restricted/server/alert_types.ts @@ -20,6 +20,7 @@ export function defineAlertTypes( producer: 'alertsRestrictedFixture', defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, recoveryActionGroup: { id: 'restrictedRecovered', name: 'Restricted Recovery' }, async executor() {}, }; @@ -30,6 +31,7 @@ export function defineAlertTypes( producer: 'alertsRestrictedFixture', defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, async executor() {}, }; alerting.registerType(noopRestrictedAlertType); diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/rule_types.ts b/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/rule_types.ts index c851aaf2bbc88..f52f0977a630b 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/rule_types.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/rule_types.ts @@ -30,6 +30,7 @@ export default function listAlertTypes({ getService }: FtrProviderContext) { }, producer: 'alertsFixture', minimum_license_required: 'basic', + is_exportable: true, recovery_action_group: { id: 'recovered', name: 'Recovered', @@ -56,6 +57,7 @@ export default function listAlertTypes({ getService }: FtrProviderContext) { }, producer: 'alertsRestrictedFixture', minimum_license_required: 'basic', + is_exportable: true, enabled_in_license: true, }; diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/rule_types.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/rule_types.ts index 3d3cec4c30252..86a0e269b26d6 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/rule_types.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/rule_types.ts @@ -42,6 +42,7 @@ export default function listAlertTypes({ getService }: FtrProviderContext) { }, producer: 'alertsFixture', minimum_license_required: 'basic', + is_exportable: true, enabled_in_license: true, }); expect(Object.keys(authorizedConsumers)).to.contain('alertsFixture'); @@ -126,6 +127,7 @@ export default function listAlertTypes({ getService }: FtrProviderContext) { }, producer: 'alertsFixture', minimumLicenseRequired: 'basic', + isExportable: true, enabledInLicense: true, }); expect(Object.keys(authorizedConsumers)).to.contain('alertsFixture'); diff --git a/x-pack/test/functional_with_es_ssl/fixtures/plugins/alerts/server/plugin.ts b/x-pack/test/functional_with_es_ssl/fixtures/plugins/alerts/server/plugin.ts index 10a81e4309088..8d0d2c4f0be33 100644 --- a/x-pack/test/functional_with_es_ssl/fixtures/plugins/alerts/server/plugin.ts +++ b/x-pack/test/functional_with_es_ssl/fixtures/plugins/alerts/server/plugin.ts @@ -24,6 +24,7 @@ export const noopAlertType: AlertType<{}, {}, {}, {}, 'default'> = { actionGroups: [{ id: 'default', name: 'Default' }], defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', + isExportable: true, async executor() {}, producer: 'alerts', }; @@ -47,6 +48,7 @@ export const alwaysFiringAlertType: AlertType< defaultActionGroupId: 'default', producer: 'alerts', minimumLicenseRequired: 'basic', + isExportable: true, async executor(alertExecutorOptions) { const { services, state, params } = alertExecutorOptions; @@ -76,6 +78,7 @@ export const failingAlertType: AlertType Date: Tue, 29 Jun 2021 04:19:02 +0300 Subject: [PATCH 51/55] [Osquery] Add Saved queries (#100965) --- .../osquery/common/schemas/common/schemas.ts | 19 +- .../create_saved_query_request_schema.ts | 14 +- x-pack/plugins/osquery/kibana.json | 3 +- .../osquery/public/actions/actions_table.tsx | 19 +- .../public/common/hooks/use_breadcrumbs.tsx | 34 +++ .../osquery/public/common/page_paths.ts | 10 +- .../plugins/osquery/public/components/app.tsx | 9 + .../public/components/osquery_schema_link.tsx | 20 ++ .../plugins/osquery/public/editor/index.tsx | 2 +- .../public/editor/osquery_schema/v4.8.0.json | 1 + .../osquery/public/editor/osquery_tables.ts | 2 +- ...managed_policy_create_import_extension.tsx | 3 +- .../public/live_queries/form/index.tsx | 54 +++- .../form/live_query_query_field.tsx | 19 +- .../common/add_new_pack_query_flyout.tsx | 7 +- .../osquery/public/queries/edit/index.tsx | 53 ---- .../osquery/public/queries/form/index.tsx | 72 ------ .../osquery/public/queries/form/schema.ts | 30 --- .../plugins/osquery/public/queries/index.tsx | 36 --- .../osquery/public/queries/new/index.tsx | 32 --- .../osquery/public/queries/queries/index.tsx | 244 ------------------ .../plugins/osquery/public/routes/index.tsx | 4 + .../routes/live_queries/details/index.tsx | 2 +- .../public/routes/saved_queries/edit/form.tsx | 81 ++++++ .../routes/saved_queries/edit/index.tsx | 124 +++++++++ .../public/routes/saved_queries/edit/tabs.tsx | 78 ++++++ .../public/routes/saved_queries/index.tsx | 35 +++ .../routes/saved_queries/list/index.tsx | 217 ++++++++++++++++ .../public/routes/saved_queries/new/form.tsx | 81 ++++++ .../public/routes/saved_queries/new/index.tsx | 62 +++++ .../osquery/public/saved_queries/constants.ts | 8 + .../form/code_editor_field.tsx | 17 +- .../public/saved_queries/form/index.tsx | 74 ++++++ .../form/use_saved_query_form.tsx | 64 +++++ .../osquery/public/saved_queries/index.tsx | 12 + .../saved_queries/saved_queries_dropdown.tsx | 104 ++++++++ .../saved_queries/saved_query_flyout.tsx | 89 +++++++ .../saved_queries/use_create_saved_query.ts | 70 +++++ .../saved_queries/use_delete_saved_query.ts | 46 ++++ .../public/saved_queries/use_saved_queries.ts | 46 ++++ .../public/saved_queries/use_saved_query.ts | 54 ++++ .../use_scheduled_query_group.ts | 38 +++ .../saved_queries/use_update_saved_query.ts | 66 +++++ .../queries/constants.ts | 3 + .../queries/platform_checkbox_group_field.tsx | 11 +- .../queries/query_flyout.tsx | 48 +++- .../scheduled_query_groups/queries/schema.tsx | 10 + .../use_scheduled_query_group_query_form.tsx | 3 + .../scheduled_query_groups_table.tsx | 25 +- x-pack/plugins/osquery/public/types.ts | 4 +- .../scripts/schema_formatter/script.ts | 2 +- x-pack/plugins/osquery/server/config.ts | 2 +- .../lib/osquery_app_context_services.ts | 3 +- .../lib/saved_query/saved_object_mappings.ts | 30 ++- x-pack/plugins/osquery/server/plugin.ts | 1 + .../routes/action/create_action_route.ts | 6 +- .../saved_query/create_saved_query_route.ts | 6 +- .../saved_query/read_saved_query_route.ts | 8 +- .../saved_query/update_saved_query_route.ts | 6 +- x-pack/plugins/osquery/server/types.ts | 2 + .../translations/translations/ja-JP.json | 2 +- .../translations/translations/zh-CN.json | 2 +- 62 files changed, 1678 insertions(+), 551 deletions(-) create mode 100644 x-pack/plugins/osquery/public/components/osquery_schema_link.tsx create mode 100644 x-pack/plugins/osquery/public/editor/osquery_schema/v4.8.0.json delete mode 100644 x-pack/plugins/osquery/public/queries/edit/index.tsx delete mode 100644 x-pack/plugins/osquery/public/queries/form/index.tsx delete mode 100644 x-pack/plugins/osquery/public/queries/form/schema.ts delete mode 100644 x-pack/plugins/osquery/public/queries/index.tsx delete mode 100644 x-pack/plugins/osquery/public/queries/new/index.tsx delete mode 100644 x-pack/plugins/osquery/public/queries/queries/index.tsx create mode 100644 x-pack/plugins/osquery/public/routes/saved_queries/edit/form.tsx create mode 100644 x-pack/plugins/osquery/public/routes/saved_queries/edit/index.tsx create mode 100644 x-pack/plugins/osquery/public/routes/saved_queries/edit/tabs.tsx create mode 100644 x-pack/plugins/osquery/public/routes/saved_queries/index.tsx create mode 100644 x-pack/plugins/osquery/public/routes/saved_queries/list/index.tsx create mode 100644 x-pack/plugins/osquery/public/routes/saved_queries/new/form.tsx create mode 100644 x-pack/plugins/osquery/public/routes/saved_queries/new/index.tsx create mode 100644 x-pack/plugins/osquery/public/saved_queries/constants.ts rename x-pack/plugins/osquery/public/{queries => saved_queries}/form/code_editor_field.tsx (69%) create mode 100644 x-pack/plugins/osquery/public/saved_queries/form/index.tsx create mode 100644 x-pack/plugins/osquery/public/saved_queries/form/use_saved_query_form.tsx create mode 100644 x-pack/plugins/osquery/public/saved_queries/index.tsx create mode 100644 x-pack/plugins/osquery/public/saved_queries/saved_queries_dropdown.tsx create mode 100644 x-pack/plugins/osquery/public/saved_queries/saved_query_flyout.tsx create mode 100644 x-pack/plugins/osquery/public/saved_queries/use_create_saved_query.ts create mode 100644 x-pack/plugins/osquery/public/saved_queries/use_delete_saved_query.ts create mode 100644 x-pack/plugins/osquery/public/saved_queries/use_saved_queries.ts create mode 100644 x-pack/plugins/osquery/public/saved_queries/use_saved_query.ts create mode 100644 x-pack/plugins/osquery/public/saved_queries/use_scheduled_query_group.ts create mode 100644 x-pack/plugins/osquery/public/saved_queries/use_update_saved_query.ts diff --git a/x-pack/plugins/osquery/common/schemas/common/schemas.ts b/x-pack/plugins/osquery/common/schemas/common/schemas.ts index f5d0a357b85b8..1e52080debb9a 100644 --- a/x-pack/plugins/osquery/common/schemas/common/schemas.ts +++ b/x-pack/plugins/osquery/common/schemas/common/schemas.ts @@ -7,10 +7,10 @@ import * as t from 'io-ts'; -export const name = t.string; -export type Name = t.TypeOf; -export const nameOrUndefined = t.union([name, t.undefined]); -export type NameOrUndefined = t.TypeOf; +export const id = t.string; +export type Id = t.TypeOf; +export const idOrUndefined = t.union([id, t.undefined]); +export type IdOrUndefined = t.TypeOf; export const agentSelection = t.type({ agents: t.array(t.string), @@ -18,6 +18,7 @@ export const agentSelection = t.type({ platformsSelected: t.array(t.string), policiesSelected: t.array(t.string), }); + export type AgentSelection = t.TypeOf; export const agentSelectionOrUndefined = t.union([agentSelection, t.undefined]); export type AgentSelectionOrUndefined = t.TypeOf; @@ -36,3 +37,13 @@ export const query = t.string; export type Query = t.TypeOf; export const queryOrUndefined = t.union([query, t.undefined]); export type QueryOrUndefined = t.TypeOf; + +export const version = t.string; +export type Version = t.TypeOf; +export const versionOrUndefined = t.union([version, t.undefined]); +export type VersionOrUndefined = t.TypeOf; + +export const interval = t.string; +export type Interval = t.TypeOf; +export const intervalOrUndefined = t.union([interval, t.undefined]); +export type IntervalOrUndefined = t.TypeOf; diff --git a/x-pack/plugins/osquery/common/schemas/routes/saved_query/create_saved_query_request_schema.ts b/x-pack/plugins/osquery/common/schemas/routes/saved_query/create_saved_query_request_schema.ts index 9e901be1476bc..5aa08d9afde4f 100644 --- a/x-pack/plugins/osquery/common/schemas/routes/saved_query/create_saved_query_request_schema.ts +++ b/x-pack/plugins/osquery/common/schemas/routes/saved_query/create_saved_query_request_schema.ts @@ -7,14 +7,24 @@ import * as t from 'io-ts'; -import { name, description, Description, platform, query } from '../../common/schemas'; +import { + id, + description, + Description, + platform, + query, + version, + interval, +} from '../../common/schemas'; import { RequiredKeepUndefined } from '../../../types'; export const createSavedQueryRequestSchema = t.type({ - name, + id, description, platform, query, + version, + interval, }); export type CreateSavedQueryRequestSchema = t.OutputOf; diff --git a/x-pack/plugins/osquery/kibana.json b/x-pack/plugins/osquery/kibana.json index 86a4d817de40e..a8f3975430e5e 100644 --- a/x-pack/plugins/osquery/kibana.json +++ b/x-pack/plugins/osquery/kibana.json @@ -26,7 +26,8 @@ "features", "fleet", "navigation", - "triggersActionsUi" + "triggersActionsUi", + "security" ], "server": true, "ui": true, diff --git a/x-pack/plugins/osquery/public/actions/actions_table.tsx b/x-pack/plugins/osquery/public/actions/actions_table.tsx index 5d1b9b723d98b..0ee928ad8aa14 100644 --- a/x-pack/plugins/osquery/public/actions/actions_table.tsx +++ b/x-pack/plugins/osquery/public/actions/actions_table.tsx @@ -5,6 +5,7 @@ * 2.0. */ +import { isArray } from 'lodash'; import { i18n } from '@kbn/i18n'; import { EuiBasicTable, EuiButtonIcon, EuiCodeBlock, formatDate } from '@elastic/eui'; import React, { useState, useCallback, useMemo } from 'react'; @@ -54,6 +55,8 @@ const ActionsTableComponent = () => { const renderAgentsColumn = useCallback((_, item) => <>{item.fields.agents?.length ?? 0}, []); + const renderCreatedByColumn = useCallback((userId) => (isArray(userId) ? userId[0] : '-'), []); + const renderTimestampColumn = useCallback( (_, item) => <>{formatDate(item.fields['@timestamp'][0])}, [] @@ -90,6 +93,14 @@ const ActionsTableComponent = () => { width: '200px', render: renderTimestampColumn, }, + { + field: 'fields.user_id', + name: i18n.translate('xpack.osquery.liveQueryActions.table.createdByColumnTitle', { + defaultMessage: 'Run by', + }), + width: '200px', + render: renderCreatedByColumn, + }, { name: i18n.translate('xpack.osquery.liveQueryActions.table.viewDetailsColumnTitle', { defaultMessage: 'View details', @@ -101,7 +112,13 @@ const ActionsTableComponent = () => { ], }, ], - [renderActionsColumn, renderAgentsColumn, renderQueryColumn, renderTimestampColumn] + [ + renderActionsColumn, + renderAgentsColumn, + renderCreatedByColumn, + renderQueryColumn, + renderTimestampColumn, + ] ); const pagination = useMemo( diff --git a/x-pack/plugins/osquery/public/common/hooks/use_breadcrumbs.tsx b/x-pack/plugins/osquery/public/common/hooks/use_breadcrumbs.tsx index 660ef87fb57e3..7b52b330d0148 100644 --- a/x-pack/plugins/osquery/public/common/hooks/use_breadcrumbs.tsx +++ b/x-pack/plugins/osquery/public/common/hooks/use_breadcrumbs.tsx @@ -67,6 +67,40 @@ const breadcrumbGetters: { text: liveQueryId, }, ], + saved_queries: () => [ + BASE_BREADCRUMB, + { + text: i18n.translate('xpack.osquery.breadcrumbs.savedQueriesPageTitle', { + defaultMessage: 'Saved queries', + }), + }, + ], + saved_query_new: () => [ + BASE_BREADCRUMB, + { + href: pagePathGetters.saved_queries(), + text: i18n.translate('xpack.osquery.breadcrumbs.savedQueriesPageTitle', { + defaultMessage: 'Saved queries', + }), + }, + { + text: i18n.translate('xpack.osquery.breadcrumbs.newSavedQueryPageTitle', { + defaultMessage: 'New', + }), + }, + ], + saved_query_edit: ({ savedQueryName }) => [ + BASE_BREADCRUMB, + { + href: pagePathGetters.saved_queries(), + text: i18n.translate('xpack.osquery.breadcrumbs.savedQueriesPageTitle', { + defaultMessage: 'Saved queries', + }), + }, + { + text: savedQueryName, + }, + ], scheduled_query_groups: () => [ BASE_BREADCRUMB, { diff --git a/x-pack/plugins/osquery/public/common/page_paths.ts b/x-pack/plugins/osquery/public/common/page_paths.ts index b4c7963fb9a02..0e0d8310ae8be 100644 --- a/x-pack/plugins/osquery/public/common/page_paths.ts +++ b/x-pack/plugins/osquery/public/common/page_paths.ts @@ -11,12 +11,15 @@ export type StaticPage = | 'live_queries' | 'live_query_new' | 'scheduled_query_groups' - | 'scheduled_query_group_add'; + | 'scheduled_query_group_add' + | 'saved_queries' + | 'saved_query_new'; export type DynamicPage = | 'live_query_details' | 'scheduled_query_group_details' - | 'scheduled_query_group_edit'; + | 'scheduled_query_group_edit' + | 'saved_query_edit'; export type Page = StaticPage | DynamicPage; @@ -50,6 +53,9 @@ export const pagePathGetters: { live_queries: () => '/live_queries', live_query_new: () => '/live_queries/new', live_query_details: ({ liveQueryId }) => `/live_queries/${liveQueryId}`, + saved_queries: () => '/saved_queries', + saved_query_new: () => '/saved_queries/new', + saved_query_edit: ({ savedQueryId }) => `/saved_queries/${savedQueryId}`, scheduled_query_groups: () => '/scheduled_query_groups', scheduled_query_group_add: () => '/scheduled_query_groups/add', scheduled_query_group_details: ({ scheduledQueryGroupId }) => diff --git a/x-pack/plugins/osquery/public/components/app.tsx b/x-pack/plugins/osquery/public/components/app.tsx index d56aacc99ad53..df61b116a5647 100644 --- a/x-pack/plugins/osquery/public/components/app.tsx +++ b/x-pack/plugins/osquery/public/components/app.tsx @@ -50,6 +50,15 @@ const OsqueryAppComponent = () => { defaultMessage="Scheduled query groups" /> + + + diff --git a/x-pack/plugins/osquery/public/components/osquery_schema_link.tsx b/x-pack/plugins/osquery/public/components/osquery_schema_link.tsx new file mode 100644 index 0000000000000..d1f346bad3350 --- /dev/null +++ b/x-pack/plugins/osquery/public/components/osquery_schema_link.tsx @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiText, EuiLink } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; +import React from 'react'; + +export const OsquerySchemaLink = React.memo(() => ( + + + + + +)); + +OsquerySchemaLink.displayName = 'OsquerySchemaLink'; diff --git a/x-pack/plugins/osquery/public/editor/index.tsx b/x-pack/plugins/osquery/public/editor/index.tsx index 70da55ca3f007..5be2b1816ad86 100644 --- a/x-pack/plugins/osquery/public/editor/index.tsx +++ b/x-pack/plugins/osquery/public/editor/index.tsx @@ -40,7 +40,7 @@ const OsqueryEditorComponent: React.FC = ({ name="osquery_editor" setOptions={EDITOR_SET_OPTIONS} editorProps={EDITOR_PROPS} - height="200px" + height="150px" width="100%" /> ); diff --git a/x-pack/plugins/osquery/public/editor/osquery_schema/v4.8.0.json b/x-pack/plugins/osquery/public/editor/osquery_schema/v4.8.0.json new file mode 100644 index 0000000000000..2a15d54e7d323 --- /dev/null +++ b/x-pack/plugins/osquery/public/editor/osquery_schema/v4.8.0.json @@ -0,0 +1 @@ +[{"name":"account_policy_data"},{"name":"acpi_tables"},{"name":"ad_config"},{"name":"alf"},{"name":"alf_exceptions"},{"name":"alf_explicit_auths"},{"name":"app_schemes"},{"name":"apparmor_events"},{"name":"apparmor_profiles"},{"name":"appcompat_shims"},{"name":"apps"},{"name":"apt_sources"},{"name":"arp_cache"},{"name":"asl"},{"name":"atom_packages"},{"name":"augeas"},{"name":"authenticode"},{"name":"authorization_mechanisms"},{"name":"authorizations"},{"name":"authorized_keys"},{"name":"autoexec"},{"name":"azure_instance_metadata"},{"name":"azure_instance_tags"},{"name":"background_activities_moderator"},{"name":"battery"},{"name":"bitlocker_info"},{"name":"block_devices"},{"name":"bpf_process_events"},{"name":"bpf_socket_events"},{"name":"browser_plugins"},{"name":"carbon_black_info"},{"name":"carves"},{"name":"certificates"},{"name":"chassis_info"},{"name":"chocolatey_packages"},{"name":"chrome_extension_content_scripts"},{"name":"chrome_extensions"},{"name":"connectivity"},{"name":"cpu_info"},{"name":"cpu_time"},{"name":"cpuid"},{"name":"crashes"},{"name":"crontab"},{"name":"cups_destinations"},{"name":"cups_jobs"},{"name":"curl"},{"name":"curl_certificate"},{"name":"deb_packages"},{"name":"default_environment"},{"name":"device_file"},{"name":"device_firmware"},{"name":"device_hash"},{"name":"device_partitions"},{"name":"disk_encryption"},{"name":"disk_events"},{"name":"disk_info"},{"name":"dns_cache"},{"name":"dns_resolvers"},{"name":"docker_container_fs_changes"},{"name":"docker_container_labels"},{"name":"docker_container_mounts"},{"name":"docker_container_networks"},{"name":"docker_container_ports"},{"name":"docker_container_processes"},{"name":"docker_container_stats"},{"name":"docker_containers"},{"name":"docker_image_history"},{"name":"docker_image_labels"},{"name":"docker_image_layers"},{"name":"docker_images"},{"name":"docker_info"},{"name":"docker_network_labels"},{"name":"docker_networks"},{"name":"docker_version"},{"name":"docker_volume_labels"},{"name":"docker_volumes"},{"name":"drivers"},{"name":"ec2_instance_metadata"},{"name":"ec2_instance_tags"},{"name":"elf_dynamic"},{"name":"elf_info"},{"name":"elf_sections"},{"name":"elf_segments"},{"name":"elf_symbols"},{"name":"etc_hosts"},{"name":"etc_protocols"},{"name":"etc_services"},{"name":"event_taps"},{"name":"example"},{"name":"extended_attributes"},{"name":"fan_speed_sensors"},{"name":"fbsd_kmods"},{"name":"file"},{"name":"file_events"},{"name":"firefox_addons"},{"name":"gatekeeper"},{"name":"gatekeeper_approved_apps"},{"name":"groups"},{"name":"hardware_events"},{"name":"hash"},{"name":"homebrew_packages"},{"name":"hvci_status"},{"name":"ibridge_info"},{"name":"ie_extensions"},{"name":"intel_me_info"},{"name":"interface_addresses"},{"name":"interface_details"},{"name":"interface_ipv6"},{"name":"iokit_devicetree"},{"name":"iokit_registry"},{"name":"iptables"},{"name":"kernel_extensions"},{"name":"kernel_info"},{"name":"kernel_modules"},{"name":"kernel_panics"},{"name":"keychain_acls"},{"name":"keychain_items"},{"name":"known_hosts"},{"name":"kva_speculative_info"},{"name":"last"},{"name":"launchd"},{"name":"launchd_overrides"},{"name":"listening_ports"},{"name":"lldp_neighbors"},{"name":"load_average"},{"name":"location_services"},{"name":"logged_in_users"},{"name":"logical_drives"},{"name":"logon_sessions"},{"name":"lxd_certificates"},{"name":"lxd_cluster"},{"name":"lxd_cluster_members"},{"name":"lxd_images"},{"name":"lxd_instance_config"},{"name":"lxd_instance_devices"},{"name":"lxd_instances"},{"name":"lxd_networks"},{"name":"lxd_storage_pools"},{"name":"magic"},{"name":"managed_policies"},{"name":"md_devices"},{"name":"md_drives"},{"name":"md_personalities"},{"name":"mdfind"},{"name":"mdls"},{"name":"memory_array_mapped_addresses"},{"name":"memory_arrays"},{"name":"memory_device_mapped_addresses"},{"name":"memory_devices"},{"name":"memory_error_info"},{"name":"memory_info"},{"name":"memory_map"},{"name":"mounts"},{"name":"msr"},{"name":"nfs_shares"},{"name":"npm_packages"},{"name":"ntdomains"},{"name":"ntfs_acl_permissions"},{"name":"ntfs_journal_events"},{"name":"nvram"},{"name":"oem_strings"},{"name":"office_mru"},{"name":"os_version"},{"name":"osquery_events"},{"name":"osquery_extensions"},{"name":"osquery_flags"},{"name":"osquery_info"},{"name":"osquery_packs"},{"name":"osquery_registry"},{"name":"osquery_schedule"},{"name":"package_bom"},{"name":"package_install_history"},{"name":"package_receipts"},{"name":"patches"},{"name":"pci_devices"},{"name":"physical_disk_performance"},{"name":"pipes"},{"name":"pkg_packages"},{"name":"platform_info"},{"name":"plist"},{"name":"portage_keywords"},{"name":"portage_packages"},{"name":"portage_use"},{"name":"power_sensors"},{"name":"powershell_events"},{"name":"preferences"},{"name":"process_envs"},{"name":"process_events"},{"name":"process_file_events"},{"name":"process_memory_map"},{"name":"process_namespaces"},{"name":"process_open_files"},{"name":"process_open_pipes"},{"name":"process_open_sockets"},{"name":"processes"},{"name":"programs"},{"name":"prometheus_metrics"},{"name":"python_packages"},{"name":"quicklook_cache"},{"name":"registry"},{"name":"routes"},{"name":"rpm_package_files"},{"name":"rpm_packages"},{"name":"running_apps"},{"name":"safari_extensions"},{"name":"sandboxes"},{"name":"scheduled_tasks"},{"name":"screenlock"},{"name":"seccomp_events"},{"name":"selinux_events"},{"name":"selinux_settings"},{"name":"services"},{"name":"shadow"},{"name":"shared_folders"},{"name":"shared_memory"},{"name":"shared_resources"},{"name":"sharing_preferences"},{"name":"shell_history"},{"name":"shellbags"},{"name":"shimcache"},{"name":"shortcut_files"},{"name":"signature"},{"name":"sip_config"},{"name":"smart_drive_info"},{"name":"smbios_tables"},{"name":"smc_keys"},{"name":"socket_events"},{"name":"ssh_configs"},{"name":"startup_items"},{"name":"sudoers"},{"name":"suid_bin"},{"name":"syslog_events"},{"name":"system_controls"},{"name":"system_extensions"},{"name":"system_info"},{"name":"systemd_units"},{"name":"temperature_sensors"},{"name":"time"},{"name":"time_machine_backups"},{"name":"time_machine_destinations"},{"name":"ulimit_info"},{"name":"uptime"},{"name":"usb_devices"},{"name":"user_events"},{"name":"user_groups"},{"name":"user_interaction_events"},{"name":"user_ssh_keys"},{"name":"userassist"},{"name":"users"},{"name":"video_info"},{"name":"virtual_memory_info"},{"name":"wifi_networks"},{"name":"wifi_status"},{"name":"wifi_survey"},{"name":"winbaseobj"},{"name":"windows_crashes"},{"name":"windows_eventlog"},{"name":"windows_events"},{"name":"windows_optional_features"},{"name":"windows_security_center"},{"name":"windows_security_products"},{"name":"wmi_bios_info"},{"name":"wmi_cli_event_consumers"},{"name":"wmi_event_filters"},{"name":"wmi_filter_consumer_binding"},{"name":"wmi_script_event_consumers"},{"name":"xprotect_entries"},{"name":"xprotect_meta"},{"name":"xprotect_reports"},{"name":"yara"},{"name":"yara_events"},{"name":"ycloud_instance_metadata"},{"name":"yum_sources"}] \ No newline at end of file diff --git a/x-pack/plugins/osquery/public/editor/osquery_tables.ts b/x-pack/plugins/osquery/public/editor/osquery_tables.ts index d114cda742f9d..d41df4021bae6 100644 --- a/x-pack/plugins/osquery/public/editor/osquery_tables.ts +++ b/x-pack/plugins/osquery/public/editor/osquery_tables.ts @@ -20,7 +20,7 @@ let osqueryTables: TablesJSON | null = null; export const getOsqueryTables = () => { if (!osqueryTables) { // eslint-disable-next-line @typescript-eslint/no-var-requires - osqueryTables = normalizeTables(require('./osquery_schema/v4.7.0.json')); + osqueryTables = normalizeTables(require('./osquery_schema/v4.8.0.json')); } return osqueryTables; }; diff --git a/x-pack/plugins/osquery/public/fleet_integration/osquery_managed_policy_create_import_extension.tsx b/x-pack/plugins/osquery/public/fleet_integration/osquery_managed_policy_create_import_extension.tsx index 2305df807f1c8..28d69a6a7b15a 100644 --- a/x-pack/plugins/osquery/public/fleet_integration/osquery_managed_policy_create_import_extension.tsx +++ b/x-pack/plugins/osquery/public/fleet_integration/osquery_managed_policy_create_import_extension.tsx @@ -207,7 +207,8 @@ export const OsqueryManagedPolicyCreateImportExtension = React.memo< integrationPolicyId={policy?.id} agentPolicyId={policy?.policy_id} /> - + + {editMode && scheduledQueryGroupTableData.inputs[0].streams.length ? ( diff --git a/x-pack/plugins/osquery/public/live_queries/form/index.tsx b/x-pack/plugins/osquery/public/live_queries/form/index.tsx index 6f2d1afec6fe9..9e952810e3352 100644 --- a/x-pack/plugins/osquery/public/live_queries/form/index.tsx +++ b/x-pack/plugins/osquery/public/live_queries/form/index.tsx @@ -5,20 +5,28 @@ * 2.0. */ -import { EuiButton, EuiSteps, EuiSpacer, EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import { + EuiButton, + EuiButtonEmpty, + EuiSteps, + EuiSpacer, + EuiFlexGroup, + EuiFlexItem, +} from '@elastic/eui'; import { EuiContainedStepProps } from '@elastic/eui/src/components/steps/steps'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; -import React, { useMemo } from 'react'; +import React, { useCallback, useMemo, useState } from 'react'; import { useMutation } from 'react-query'; import { UseField, Form, FormData, useForm, useFormData, FIELD_TYPES } from '../../shared_imports'; import { AgentsTableField } from './agents_table_field'; import { LiveQueryQueryField } from './live_query_query_field'; import { useKibana } from '../../common/lib/kibana'; -import { ResultTabs } from '../../queries/edit/tabs'; +import { ResultTabs } from '../../routes/saved_queries/edit/tabs'; import { queryFieldValidation } from '../../common/validations'; import { fieldValidators } from '../../shared_imports'; +import { SavedQueryFlyout } from '../../saved_queries'; import { useErrorToast } from '../../common/hooks/use_error_toast'; const FORM_ID = 'liveQueryForm'; @@ -27,19 +35,17 @@ export const MAX_QUERY_LENGTH = 2000; interface LiveQueryFormProps { defaultValue?: Partial | undefined; - onSubmit?: (payload: Record) => Promise; onSuccess?: () => void; } -const LiveQueryFormComponent: React.FC = ({ - defaultValue, - // onSubmit, - onSuccess, -}) => { +const LiveQueryFormComponent: React.FC = ({ defaultValue, onSuccess }) => { const { http } = useKibana().services; - + const [showSavedQueryFlyout, setShowSavedQueryFlyout] = useState(false); const setErrorToast = useErrorToast(); + const handleShowSaveQueryFlout = useCallback(() => setShowSavedQueryFlyout(true), []); + const handleCloseSaveQueryFlout = useCallback(() => setShowSavedQueryFlyout(false), []); + const { data, isLoading, @@ -139,6 +145,8 @@ const LiveQueryFormComponent: React.FC = ({ [queryStatus] ); + const flyoutFormDefaultValue = useMemo(() => ({ query }), [query]); + const formSteps: EuiContainedStepProps[] = useMemo( () => [ { @@ -161,6 +169,17 @@ const LiveQueryFormComponent: React.FC = ({ /> + + + + + = ({ actionId, agentIds, agentSelected, + handleShowSaveQueryFlout, queryComponentProps, queryStatus, queryValueProvided, @@ -203,9 +223,17 @@ const LiveQueryFormComponent: React.FC = ({ ); return ( -
- - + <> +
+ + + {showSavedQueryFlyout ? ( + + ) : null} + ); }; diff --git a/x-pack/plugins/osquery/public/live_queries/form/live_query_query_field.tsx b/x-pack/plugins/osquery/public/live_queries/form/live_query_query_field.tsx index 07c13b930e143..9f0b5acd8994a 100644 --- a/x-pack/plugins/osquery/public/live_queries/form/live_query_query_field.tsx +++ b/x-pack/plugins/osquery/public/live_queries/form/live_query_query_field.tsx @@ -5,11 +5,13 @@ * 2.0. */ +import { EuiFormRow, EuiSpacer } from '@elastic/eui'; import React, { useCallback } from 'react'; -import { EuiFormRow } from '@elastic/eui'; +import { OsquerySchemaLink } from '../../components/osquery_schema_link'; import { FieldHook } from '../../shared_imports'; import { OsqueryEditor } from '../../editor'; +import { SavedQueriesDropdown } from '../../saved_queries/saved_queries_dropdown'; interface LiveQueryQueryFieldProps { disabled?: boolean; @@ -20,6 +22,13 @@ const LiveQueryQueryFieldComponent: React.FC = ({ disa const { value, setValue, errors } = field; const error = errors[0]?.message; + const handleSavedQueryChange = useCallback( + (savedQuery) => { + setValue(savedQuery.query); + }, + [setValue] + ); + const handleEditorChange = useCallback( (newValue) => { setValue(newValue); @@ -29,7 +38,13 @@ const LiveQueryQueryFieldComponent: React.FC = ({ disa return ( - + <> + + + }> + + + ); }; diff --git a/x-pack/plugins/osquery/public/packs/common/add_new_pack_query_flyout.tsx b/x-pack/plugins/osquery/public/packs/common/add_new_pack_query_flyout.tsx index 2680b5198fadb..85578564b1eb2 100644 --- a/x-pack/plugins/osquery/public/packs/common/add_new_pack_query_flyout.tsx +++ b/x-pack/plugins/osquery/public/packs/common/add_new_pack_query_flyout.tsx @@ -8,7 +8,7 @@ import { EuiFlyout, EuiFlyoutBody, EuiFlyoutHeader, EuiTitle } from '@elastic/eui'; import React from 'react'; -import { SavedQueryForm } from '../../queries/form'; +import { SavedQueryForm } from '../../saved_queries/form'; // @ts-expect-error update types const AddNewPackQueryFlyoutComponent = ({ handleClose, handleSubmit }) => ( @@ -19,7 +19,10 @@ const AddNewPackQueryFlyoutComponent = ({ handleClose, handleSubmit }) => ( - + { + // @ts-expect-error update types + + } ); diff --git a/x-pack/plugins/osquery/public/queries/edit/index.tsx b/x-pack/plugins/osquery/public/queries/edit/index.tsx deleted file mode 100644 index 61094b2d07940..0000000000000 --- a/x-pack/plugins/osquery/public/queries/edit/index.tsx +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { isEmpty } from 'lodash/fp'; -import React from 'react'; -import { useMutation, useQuery } from 'react-query'; - -import { SavedQueryForm } from '../form'; -import { useKibana } from '../../common/lib/kibana'; - -interface EditSavedQueryPageProps { - onSuccess: () => void; - savedQueryId: string; -} - -const EditSavedQueryPageComponent: React.FC = ({ - onSuccess, - savedQueryId, -}) => { - const { http } = useKibana().services; - - const { isLoading, data: savedQueryDetails } = useQuery(['savedQuery', { savedQueryId }], () => - http.get(`/internal/osquery/saved_query/${savedQueryId}`) - ); - const updateSavedQueryMutation = useMutation( - (payload) => - http.put(`/internal/osquery/saved_query/${savedQueryId}`, { body: JSON.stringify(payload) }), - { onSuccess } - ); - - if (isLoading) { - return <>{'Loading...'}; - } - - return ( - <> - {!isEmpty(savedQueryDetails) && ( - - )} - - ); -}; - -export const EditSavedQueryPage = React.memo(EditSavedQueryPageComponent); diff --git a/x-pack/plugins/osquery/public/queries/form/index.tsx b/x-pack/plugins/osquery/public/queries/form/index.tsx deleted file mode 100644 index 02468fbfde228..0000000000000 --- a/x-pack/plugins/osquery/public/queries/form/index.tsx +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { EuiButton, EuiSpacer } from '@elastic/eui'; -import React from 'react'; - -import { Field, getUseField, useForm, UseField, Form } from '../../shared_imports'; -import { CodeEditorField } from './code_editor_field'; -import { formSchema } from './schema'; - -export const CommonUseField = getUseField({ component: Field }); - -const SAVED_QUERY_FORM_ID = 'savedQueryForm'; - -interface SavedQueryFormProps { - defaultValue?: unknown; - handleSubmit: () => Promise; - type?: string; -} - -const SavedQueryFormComponent: React.FC = ({ - defaultValue, - handleSubmit, - type, -}) => { - const { form } = useForm({ - // @ts-expect-error update types - id: defaultValue ? SAVED_QUERY_FORM_ID + defaultValue.id : SAVED_QUERY_FORM_ID, - schema: formSchema, - onSubmit: handleSubmit, - options: { - stripEmptyFields: false, - }, - // @ts-expect-error update types - defaultValue, - }); - - const { submit } = form; - - return ( -
- - - - - - - - - {type === 'edit' ? 'Update' : 'Save'} - - ); -}; - -export const SavedQueryForm = React.memo(SavedQueryFormComponent); diff --git a/x-pack/plugins/osquery/public/queries/form/schema.ts b/x-pack/plugins/osquery/public/queries/form/schema.ts deleted file mode 100644 index 33200e45dc8e3..0000000000000 --- a/x-pack/plugins/osquery/public/queries/form/schema.ts +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { FIELD_TYPES, FormSchema } from '../../shared_imports'; - -export const formSchema: FormSchema = { - name: { - type: FIELD_TYPES.TEXT, - label: 'Query name', - }, - description: { - type: FIELD_TYPES.TEXTAREA, - label: 'Description', - validations: [], - }, - platform: { - type: FIELD_TYPES.SELECT, - label: 'Platform', - defaultValue: 'all', - }, - query: { - label: 'Query', - type: FIELD_TYPES.TEXTAREA, - validations: [], - }, -}; diff --git a/x-pack/plugins/osquery/public/queries/index.tsx b/x-pack/plugins/osquery/public/queries/index.tsx deleted file mode 100644 index 7ecce3cfb22f4..0000000000000 --- a/x-pack/plugins/osquery/public/queries/index.tsx +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import React, { useCallback, useState } from 'react'; - -import { QueriesPage } from './queries'; -import { NewSavedQueryPage } from './new'; -import { EditSavedQueryPage } from './edit'; - -const QueriesComponent = () => { - const [showNewSavedQueryForm, setShowNewSavedQueryForm] = useState(false); - const [editSavedQueryId, setEditSavedQueryId] = useState(null); - - const goBack = useCallback(() => { - setShowNewSavedQueryForm(false); - setEditSavedQueryId(null); - }, []); - - const handleNewQueryClick = useCallback(() => setShowNewSavedQueryForm(true), []); - - if (showNewSavedQueryForm) { - return ; - } - - if (editSavedQueryId?.length) { - return ; - } - - return ; -}; - -export const Queries = React.memo(QueriesComponent); diff --git a/x-pack/plugins/osquery/public/queries/new/index.tsx b/x-pack/plugins/osquery/public/queries/new/index.tsx deleted file mode 100644 index 2682db126ea09..0000000000000 --- a/x-pack/plugins/osquery/public/queries/new/index.tsx +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import React from 'react'; -import { useMutation } from 'react-query'; - -import { useKibana } from '../../common/lib/kibana'; -import { SavedQueryForm } from '../form'; - -interface NewSavedQueryPageProps { - onSuccess: () => void; -} - -const NewSavedQueryPageComponent: React.FC = ({ onSuccess }) => { - const { http } = useKibana().services; - - const createSavedQueryMutation = useMutation( - (payload) => http.post(`/internal/osquery/saved_query`, { body: JSON.stringify(payload) }), - { - onSuccess, - } - ); - - // @ts-expect-error update types - return ; -}; - -export const NewSavedQueryPage = React.memo(NewSavedQueryPageComponent); diff --git a/x-pack/plugins/osquery/public/queries/queries/index.tsx b/x-pack/plugins/osquery/public/queries/queries/index.tsx deleted file mode 100644 index bf766a15a44a3..0000000000000 --- a/x-pack/plugins/osquery/public/queries/queries/index.tsx +++ /dev/null @@ -1,244 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { map } from 'lodash/fp'; -import { - EuiBasicTable, - EuiButton, - EuiButtonIcon, - EuiCodeBlock, - EuiFlexGroup, - EuiFlexItem, - EuiSpacer, - RIGHT_ALIGNMENT, -} from '@elastic/eui'; -import React, { useCallback, useMemo, useState } from 'react'; -import { useQuery, useQueryClient, useMutation } from 'react-query'; -import { useHistory } from 'react-router-dom'; -import qs from 'query-string'; - -import { useKibana } from '../../common/lib/kibana'; - -interface QueriesPageProps { - onEditClick: (savedQueryId: string) => void; - onNewClick: () => void; -} - -const QueriesPageComponent: React.FC = ({ onEditClick, onNewClick }) => { - const { push } = useHistory(); - const queryClient = useQueryClient(); - const [pageIndex, setPageIndex] = useState(0); - const [pageSize, setPageSize] = useState(5); - const [sortField, setSortField] = useState('updated_at'); - const [sortDirection, setSortDirection] = useState('desc'); - const [selectedItems, setSelectedItems] = useState([]); - const [itemIdToExpandedRowMap, setItemIdToExpandedRowMap] = useState>({}); - const { http } = useKibana().services; - - const deleteSavedQueriesMutation = useMutation( - (payload) => http.delete(`/internal/osquery/saved_query`, { body: JSON.stringify(payload) }), - { - onSuccess: () => queryClient.invalidateQueries('savedQueryList'), - } - ); - - const { data = {} } = useQuery( - ['savedQueryList', { pageIndex, pageSize, sortField, sortDirection }], - () => - http.get('/internal/osquery/saved_query', { - query: { - pageIndex, - pageSize, - sortField, - sortDirection, - }, - }), - { - keepPreviousData: true, - // Refetch the data every 10 seconds - refetchInterval: 5000, - } - ); - const { total = 0, saved_objects: savedQueries } = data; - - const toggleDetails = useCallback( - (item) => () => { - const itemIdToExpandedRowMapValues = { ...itemIdToExpandedRowMap }; - if (itemIdToExpandedRowMapValues[item.id]) { - delete itemIdToExpandedRowMapValues[item.id]; - } else { - itemIdToExpandedRowMapValues[item.id] = ( - - {item.attributes.query} - - ); - } - setItemIdToExpandedRowMap(itemIdToExpandedRowMapValues); - }, - [itemIdToExpandedRowMap] - ); - - const renderExtendedItemToggle = useCallback( - (item) => ( - - ), - [itemIdToExpandedRowMap, toggleDetails] - ); - - const handleEditClick = useCallback((item) => onEditClick(item.id), [onEditClick]); - - const handlePlayClick = useCallback( - (item) => - push({ - search: qs.stringify({ - tab: 'live_query', - }), - state: { - query: { - id: item.id, - query: item.attributes.query, - }, - }, - }), - [push] - ); - - const columns = useMemo( - () => [ - { - field: 'attributes.name', - name: 'Query name', - sortable: true, - truncateText: true, - }, - { - field: 'attributes.description', - name: 'Description', - sortable: true, - truncateText: true, - }, - { - field: 'updated_at', - name: 'Last updated at', - sortable: true, - truncateText: true, - }, - { - name: 'Actions', - actions: [ - { - name: 'Live query', - description: 'Run live query', - type: 'icon', - icon: 'play', - onClick: handlePlayClick, - }, - { - name: 'Edit', - description: 'Edit or run this query', - type: 'icon', - icon: 'documentEdit', - onClick: handleEditClick, - }, - ], - }, - { - align: RIGHT_ALIGNMENT, - width: '40px', - isExpander: true, - render: renderExtendedItemToggle, - }, - ], - [handleEditClick, handlePlayClick, renderExtendedItemToggle] - ); - - const onTableChange = useCallback(({ page = {}, sort = {} }) => { - setPageIndex(page.index); - setPageSize(page.size); - setSortField(sort.field); - setSortDirection(sort.direction); - }, []); - - const pagination = useMemo( - () => ({ - pageIndex, - pageSize, - totalItemCount: total, - pageSizeOptions: [3, 5, 8], - }), - [total, pageIndex, pageSize] - ); - - const sorting = useMemo( - () => ({ - sort: { - field: sortField, - direction: sortDirection, - }, - }), - [sortDirection, sortField] - ); - - const selection = useMemo( - () => ({ - selectable: () => true, - onSelectionChange: setSelectedItems, - initialSelected: [], - }), - [] - ); - - const handleDeleteClick = useCallback(() => { - const selectedItemsIds = map('id', selectedItems); - // @ts-expect-error update types - deleteSavedQueriesMutation.mutate({ savedQueryIds: selectedItemsIds }); - }, [deleteSavedQueriesMutation, selectedItems]); - - return ( -
- - - {!selectedItems.length ? ( - - {'New query'} - - ) : ( - - {`Delete ${selectedItems.length} Queries`} - - )} - - - - - - {savedQueries && ( - - )} -
- ); -}; - -export const QueriesPage = React.memo(QueriesPageComponent); diff --git a/x-pack/plugins/osquery/public/routes/index.tsx b/x-pack/plugins/osquery/public/routes/index.tsx index 7007feb19d663..a858a51aad64e 100644 --- a/x-pack/plugins/osquery/public/routes/index.tsx +++ b/x-pack/plugins/osquery/public/routes/index.tsx @@ -11,12 +11,16 @@ import { Switch, Redirect, Route } from 'react-router-dom'; import { useBreadcrumbs } from '../common/hooks/use_breadcrumbs'; import { LiveQueries } from './live_queries'; import { ScheduledQueryGroups } from './scheduled_query_groups'; +import { SavedQueries } from './saved_queries'; const OsqueryAppRoutesComponent = () => { useBreadcrumbs('base'); return ( + + + diff --git a/x-pack/plugins/osquery/public/routes/live_queries/details/index.tsx b/x-pack/plugins/osquery/public/routes/live_queries/details/index.tsx index 64a1fb0791e83..e4f1bb447a15a 100644 --- a/x-pack/plugins/osquery/public/routes/live_queries/details/index.tsx +++ b/x-pack/plugins/osquery/public/routes/live_queries/details/index.tsx @@ -27,7 +27,7 @@ import { useRouterNavigate } from '../../../common/lib/kibana'; import { WithHeaderLayout } from '../../../components/layouts'; import { useActionResults } from '../../../action_results/use_action_results'; import { useActionDetails } from '../../../actions/use_action_details'; -import { ResultTabs } from '../../../queries/edit/tabs'; +import { ResultTabs } from '../../saved_queries/edit/tabs'; import { useBreadcrumbs } from '../../../common/hooks/use_breadcrumbs'; import { BetaBadge, BetaBadgeRowWrapper } from '../../../components/beta_badge'; diff --git a/x-pack/plugins/osquery/public/routes/saved_queries/edit/form.tsx b/x-pack/plugins/osquery/public/routes/saved_queries/edit/form.tsx new file mode 100644 index 0000000000000..8d77b7819bd3e --- /dev/null +++ b/x-pack/plugins/osquery/public/routes/saved_queries/edit/form.tsx @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + EuiBottomBar, + EuiButtonEmpty, + EuiButton, + EuiFlexGroup, + EuiFlexItem, + EuiSpacer, +} from '@elastic/eui'; +import React from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; + +import { useRouterNavigate } from '../../../common/lib/kibana'; +import { Form } from '../../../shared_imports'; +import { SavedQueryForm } from '../../../saved_queries/form'; +import { useSavedQueryForm } from '../../../saved_queries/form/use_saved_query_form'; + +interface EditSavedQueryFormProps { + defaultValue?: unknown; + handleSubmit: () => Promise; +} + +const EditSavedQueryFormComponent: React.FC = ({ + defaultValue, + handleSubmit, +}) => { + const savedQueryListProps = useRouterNavigate('saved_queries'); + + const { form } = useSavedQueryForm({ + defaultValue, + handleSubmit, + }); + + return ( +
+ + + + + + + + + + + + + + + + + + + + + + + + ); +}; + +export const EditSavedQueryForm = React.memo(EditSavedQueryFormComponent); diff --git a/x-pack/plugins/osquery/public/routes/saved_queries/edit/index.tsx b/x-pack/plugins/osquery/public/routes/saved_queries/edit/index.tsx new file mode 100644 index 0000000000000..4aaf8e4fc4fc3 --- /dev/null +++ b/x-pack/plugins/osquery/public/routes/saved_queries/edit/index.tsx @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + EuiButtonEmpty, + EuiButton, + EuiFlexGroup, + EuiFlexItem, + EuiConfirmModal, +} from '@elastic/eui'; +import { isEmpty } from 'lodash/fp'; +import React, { useCallback, useMemo, useState } from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { useParams } from 'react-router-dom'; + +import { useRouterNavigate } from '../../../common/lib/kibana'; +import { WithHeaderLayout } from '../../../components/layouts'; +import { useBreadcrumbs } from '../../../common/hooks/use_breadcrumbs'; +import { BetaBadge, BetaBadgeRowWrapper } from '../../../components/beta_badge'; +import { EditSavedQueryForm } from './form'; +import { useDeleteSavedQuery, useUpdateSavedQuery, useSavedQuery } from '../../../saved_queries'; + +const EditSavedQueryPageComponent = () => { + const [isDeleteModalVisible, setIsDeleteModalVisible] = useState(false); + const { savedQueryId } = useParams<{ savedQueryId: string }>(); + const savedQueryListProps = useRouterNavigate('saved_queries'); + + const { isLoading, data: savedQueryDetails } = useSavedQuery({ savedQueryId }); + const updateSavedQueryMutation = useUpdateSavedQuery({ savedQueryId }); + const deleteSavedQueryMutation = useDeleteSavedQuery({ savedQueryId }); + + useBreadcrumbs('saved_query_edit', { savedQueryId: savedQueryDetails?.attributes?.id ?? '' }); + + const handleCloseDeleteConfirmationModal = useCallback(() => { + setIsDeleteModalVisible(false); + }, []); + + const handleDeleteClick = useCallback(() => { + setIsDeleteModalVisible(true); + }, []); + + const handleDeleteConfirmClick = useCallback(() => { + deleteSavedQueryMutation.mutateAsync().then(() => { + handleCloseDeleteConfirmationModal(); + }); + }, [deleteSavedQueryMutation, handleCloseDeleteConfirmationModal]); + + const LeftColumn = useMemo( + () => ( + + + + + + + + +

+ +

+ + + + + ), + [savedQueryDetails?.attributes?.id, savedQueryListProps] + ); + + const RightColumn = useMemo( + () => ( + + + + ), + [handleDeleteClick] + ); + + if (isLoading) return null; + + return ( + + {!isLoading && !isEmpty(savedQueryDetails) && ( + + )} + {isDeleteModalVisible ? ( + +

You’re about to delete this query.

+

Are you sure you want to do this?

+ + ) : null} + + ); +}; + +export const EditSavedQueryPage = React.memo(EditSavedQueryPageComponent); diff --git a/x-pack/plugins/osquery/public/routes/saved_queries/edit/tabs.tsx b/x-pack/plugins/osquery/public/routes/saved_queries/edit/tabs.tsx new file mode 100644 index 0000000000000..1946cd6dd3450 --- /dev/null +++ b/x-pack/plugins/osquery/public/routes/saved_queries/edit/tabs.tsx @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiTabbedContent, EuiSpacer } from '@elastic/eui'; +import React, { useMemo } from 'react'; + +import { ResultsTable } from '../../../results/results_table'; +import { ActionResultsSummary } from '../../../action_results/action_results_summary'; + +interface ResultTabsProps { + actionId: string; + agentIds?: string[]; + expirationDate: Date; + isLive?: boolean; + startDate?: string; + endDate?: string; +} + +const ResultTabsComponent: React.FC = ({ + actionId, + agentIds, + endDate, + expirationDate, + isLive, + startDate, +}) => { + const tabs = useMemo( + () => [ + { + id: 'results', + name: 'Results', + content: ( + <> + + + + ), + }, + { + id: 'status', + name: 'Status', + content: ( + <> + + + + ), + }, + ], + [actionId, agentIds, endDate, expirationDate, isLive, startDate] + ); + + return ( + + ); +}; + +export const ResultTabs = React.memo(ResultTabsComponent); diff --git a/x-pack/plugins/osquery/public/routes/saved_queries/index.tsx b/x-pack/plugins/osquery/public/routes/saved_queries/index.tsx new file mode 100644 index 0000000000000..f986129bdfefc --- /dev/null +++ b/x-pack/plugins/osquery/public/routes/saved_queries/index.tsx @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { Switch, Route, useRouteMatch } from 'react-router-dom'; + +import { QueriesPage } from './list'; +import { NewSavedQueryPage } from './new'; +import { EditSavedQueryPage } from './edit'; +import { useBreadcrumbs } from '../../common/hooks/use_breadcrumbs'; + +const SavedQueriesComponent = () => { + useBreadcrumbs('saved_queries'); + const match = useRouteMatch(); + + return ( + + + + + + + + + + + + ); +}; + +export const SavedQueries = React.memo(SavedQueriesComponent); diff --git a/x-pack/plugins/osquery/public/routes/saved_queries/list/index.tsx b/x-pack/plugins/osquery/public/routes/saved_queries/list/index.tsx new file mode 100644 index 0000000000000..7e8e8e543dfab --- /dev/null +++ b/x-pack/plugins/osquery/public/routes/saved_queries/list/index.tsx @@ -0,0 +1,217 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; +import { + EuiInMemoryTable, + EuiButton, + EuiButtonIcon, + EuiFlexGroup, + EuiFlexItem, +} from '@elastic/eui'; +import React, { useCallback, useMemo, useState } from 'react'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; + +import { SavedObject } from 'kibana/public'; +import { WithHeaderLayout } from '../../../components/layouts'; +import { useBreadcrumbs } from '../../../common/hooks/use_breadcrumbs'; +import { useRouterNavigate } from '../../../common/lib/kibana'; +import { BetaBadge, BetaBadgeRowWrapper } from '../../../components/beta_badge'; +import { useSavedQueries } from '../../../saved_queries/use_saved_queries'; + +interface EditButtonProps { + savedQueryId: string; + savedQueryName: string; +} + +const EditButtonComponent: React.FC = ({ savedQueryId, savedQueryName }) => { + const buttonProps = useRouterNavigate(`saved_queries/${savedQueryId}`); + + return ( + + ); +}; + +const EditButton = React.memo(EditButtonComponent); + +const SavedQueriesPageComponent = () => { + useBreadcrumbs('saved_queries'); + const newQueryLinkProps = useRouterNavigate('saved_queries/new'); + const [pageIndex, setPageIndex] = useState(0); + const [pageSize, setPageSize] = useState(10); + const [sortField, setSortField] = useState('updated_at'); + const [sortDirection, setSortDirection] = useState('desc'); + + const { data } = useSavedQueries({ isLive: true }); + + // const handlePlayClick = useCallback( + // (item) => + // push({ + // search: qs.stringify({ + // tab: 'live_query', + // }), + // state: { + // query: { + // id: item.id, + // query: item.attributes.query, + // }, + // }, + // }), + // [push] + // ); + + const renderEditAction = useCallback( + (item: SavedObject<{ name: string }>) => ( + + ), + [] + ); + + const renderUpdatedAt = useCallback((updatedAt, item) => { + if (!updatedAt) return '-'; + + const updatedBy = + item.attributes.updated_by !== item.attributes.created_by + ? ` @ ${item.attributes.updated_by}` + : ''; + return updatedAt ? `${moment(updatedAt).fromNow()}${updatedBy}` : '-'; + }, []); + + const columns = useMemo( + () => [ + { + field: 'attributes.id', + name: 'Query ID', + sortable: true, + truncateText: true, + }, + { + field: 'attributes.description', + name: 'Description', + sortable: true, + truncateText: true, + }, + { + field: 'attributes.created_by', + name: 'Created by', + sortable: true, + truncateText: true, + }, + { + field: 'attributes.updated_at', + name: 'Last updated at', + sortable: (item: SavedObject<{ updated_at: string }>) => + item.attributes.updated_at ? Date.parse(item.attributes.updated_at) : 0, + truncateText: true, + render: renderUpdatedAt, + }, + { + name: 'Actions', + actions: [ + // { + // name: 'Live query', + // description: 'Run live query', + // type: 'icon', + // icon: 'play', + // onClick: handlePlayClick, + // }, + { render: renderEditAction }, + ], + }, + ], + [renderEditAction, renderUpdatedAt] + ); + + const onTableChange = useCallback(({ page = {}, sort = {} }) => { + setPageIndex(page.index); + setPageSize(page.size); + setSortField(sort.field); + setSortDirection(sort.direction); + }, []); + + const pagination = useMemo( + () => ({ + pageIndex, + pageSize, + totalItemCount: data?.total ?? 0, + pageSizeOptions: [10, 20, 50, 100], + }), + [pageIndex, pageSize, data?.total] + ); + + const sorting = useMemo( + () => ({ + sort: { + field: sortField, + direction: sortDirection, + }, + }), + [sortDirection, sortField] + ); + + const LeftColumn = useMemo( + () => ( + + + +

+ +

+ + + + + ), + [] + ); + + const RightColumn = useMemo( + () => ( + + + + ), + [newQueryLinkProps] + ); + + return ( + + {data?.savedObjects && ( + + )} + + ); +}; + +export const QueriesPage = React.memo(SavedQueriesPageComponent); diff --git a/x-pack/plugins/osquery/public/routes/saved_queries/new/form.tsx b/x-pack/plugins/osquery/public/routes/saved_queries/new/form.tsx new file mode 100644 index 0000000000000..31d0c5637cc3e --- /dev/null +++ b/x-pack/plugins/osquery/public/routes/saved_queries/new/form.tsx @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + EuiBottomBar, + EuiButtonEmpty, + EuiButton, + EuiFlexGroup, + EuiFlexItem, + EuiSpacer, +} from '@elastic/eui'; +import React from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; + +import { useRouterNavigate } from '../../../common/lib/kibana'; +import { Form } from '../../../shared_imports'; +import { SavedQueryForm } from '../../../saved_queries/form'; +import { useSavedQueryForm } from '../../../saved_queries/form/use_saved_query_form'; + +interface NewSavedQueryFormProps { + defaultValue?: unknown; + handleSubmit: () => Promise; +} + +const NewSavedQueryFormComponent: React.FC = ({ + defaultValue, + handleSubmit, +}) => { + const savedQueryListProps = useRouterNavigate('saved_queries'); + + const { form } = useSavedQueryForm({ + defaultValue, + handleSubmit, + }); + + return ( +
+ + + + + + + + + + + + + + + + + + + + + + + + ); +}; + +export const NewSavedQueryForm = React.memo(NewSavedQueryFormComponent); diff --git a/x-pack/plugins/osquery/public/routes/saved_queries/new/index.tsx b/x-pack/plugins/osquery/public/routes/saved_queries/new/index.tsx new file mode 100644 index 0000000000000..3f5a1af64fe34 --- /dev/null +++ b/x-pack/plugins/osquery/public/routes/saved_queries/new/index.tsx @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiButtonEmpty, EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import React, { useMemo } from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; + +import { useRouterNavigate } from '../../../common/lib/kibana'; +import { WithHeaderLayout } from '../../../components/layouts'; +import { useBreadcrumbs } from '../../../common/hooks/use_breadcrumbs'; +import { BetaBadge, BetaBadgeRowWrapper } from '../../../components/beta_badge'; +import { NewSavedQueryForm } from './form'; +import { useCreateSavedQuery } from '../../../saved_queries/use_create_saved_query'; + +const NewSavedQueryPageComponent = () => { + useBreadcrumbs('saved_query_new'); + const savedQueryListProps = useRouterNavigate('saved_queries'); + + const createSavedQueryMutation = useCreateSavedQuery({ withRedirect: true }); + + const LeftColumn = useMemo( + () => ( + + + + + + + + +

+ +

+ + + + + ), + [savedQueryListProps] + ); + + return ( + + { + // @ts-expect-error update types + + } + + ); +}; + +export const NewSavedQueryPage = React.memo(NewSavedQueryPageComponent); diff --git a/x-pack/plugins/osquery/public/saved_queries/constants.ts b/x-pack/plugins/osquery/public/saved_queries/constants.ts new file mode 100644 index 0000000000000..69ca805e3e8fa --- /dev/null +++ b/x-pack/plugins/osquery/public/saved_queries/constants.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const SAVED_QUERIES_ID = 'savedQueryList'; diff --git a/x-pack/plugins/osquery/public/queries/form/code_editor_field.tsx b/x-pack/plugins/osquery/public/saved_queries/form/code_editor_field.tsx similarity index 69% rename from x-pack/plugins/osquery/public/queries/form/code_editor_field.tsx rename to x-pack/plugins/osquery/public/saved_queries/form/code_editor_field.tsx index 77ffdc4457d3d..c70aeae66396e 100644 --- a/x-pack/plugins/osquery/public/queries/form/code_editor_field.tsx +++ b/x-pack/plugins/osquery/public/saved_queries/form/code_editor_field.tsx @@ -5,11 +5,11 @@ * 2.0. */ -import { FormattedMessage } from '@kbn/i18n/react'; import { isEmpty } from 'lodash/fp'; -import { EuiFormRow, EuiLink, EuiText } from '@elastic/eui'; +import { EuiFormRow } from '@elastic/eui'; import React from 'react'; +import { OsquerySchemaLink } from '../../components/osquery_schema_link'; import { OsqueryEditor } from '../../editor'; import { FieldHook } from '../../shared_imports'; @@ -17,19 +17,6 @@ interface CodeEditorFieldProps { field: FieldHook; } -const OsquerySchemaLink = React.memo(() => ( - - - - - -)); - -OsquerySchemaLink.displayName = 'OsquerySchemaLink'; - const CodeEditorFieldComponent: React.FC = ({ field }) => { const { value, label, labelAppend, helpText, setValue, errors } = field; const error = errors[0]?.message; diff --git a/x-pack/plugins/osquery/public/saved_queries/form/index.tsx b/x-pack/plugins/osquery/public/saved_queries/form/index.tsx new file mode 100644 index 0000000000000..174227eb5e6e5 --- /dev/null +++ b/x-pack/plugins/osquery/public/saved_queries/form/index.tsx @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiFlexGroup, EuiFlexItem, EuiSpacer, EuiTitle, EuiText } from '@elastic/eui'; +import React from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; + +import { ALL_OSQUERY_VERSIONS_OPTIONS } from '../../scheduled_query_groups/queries/constants'; +import { PlatformCheckBoxGroupField } from '../../scheduled_query_groups/queries/platform_checkbox_group_field'; +import { Field, getUseField, UseField } from '../../shared_imports'; +import { CodeEditorField } from './code_editor_field'; + +export const CommonUseField = getUseField({ component: Field }); + +const SavedQueryFormComponent = () => ( + <> + + + + + + + + + +
+ +
+ + + + + + + + + + + + + + + + + + + +); + +export const SavedQueryForm = React.memo(SavedQueryFormComponent); diff --git a/x-pack/plugins/osquery/public/saved_queries/form/use_saved_query_form.tsx b/x-pack/plugins/osquery/public/saved_queries/form/use_saved_query_form.tsx new file mode 100644 index 0000000000000..6417b40747e0f --- /dev/null +++ b/x-pack/plugins/osquery/public/saved_queries/form/use_saved_query_form.tsx @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isArray } from 'lodash'; +import uuid from 'uuid'; +import { produce } from 'immer'; + +import { useForm } from '../../shared_imports'; +import { formSchema } from '../../scheduled_query_groups/queries/schema'; +import { ScheduledQueryGroupFormData } from '../../scheduled_query_groups/queries/use_scheduled_query_group_query_form'; + +const SAVED_QUERY_FORM_ID = 'savedQueryForm'; + +interface UseSavedQueryFormProps { + defaultValue?: unknown; + handleSubmit: (payload: unknown) => Promise; +} + +export const useSavedQueryForm = ({ defaultValue, handleSubmit }: UseSavedQueryFormProps) => + useForm({ + id: SAVED_QUERY_FORM_ID + uuid.v4(), + schema: formSchema, + onSubmit: handleSubmit, + options: { + stripEmptyFields: false, + }, + // @ts-expect-error update types + defaultValue, + serializer: (payload) => + produce(payload, (draft) => { + // @ts-expect-error update types + if (draft.platform?.split(',').length === 3) { + // if all platforms are checked then use undefined + // @ts-expect-error update types + delete draft.platform; + } + if (isArray(draft.version)) { + if (!draft.version.length) { + // @ts-expect-error update types + delete draft.version; + } else { + draft.version = draft.version[0]; + } + } + return draft; + }), + // @ts-expect-error update types + deserializer: (payload) => { + if (!payload) return {} as ScheduledQueryGroupFormData; + + return { + id: payload.id, + description: payload.description, + query: payload.query, + interval: payload.interval ? parseInt(payload.interval, 10) : undefined, + platform: payload.platform, + version: payload.version ? [payload.version] : [], + }; + }, + }); diff --git a/x-pack/plugins/osquery/public/saved_queries/index.tsx b/x-pack/plugins/osquery/public/saved_queries/index.tsx new file mode 100644 index 0000000000000..405af5638c868 --- /dev/null +++ b/x-pack/plugins/osquery/public/saved_queries/index.tsx @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './saved_query_flyout'; +export * from './use_saved_query'; +export * from './use_saved_queries'; +export * from './use_update_saved_query'; +export * from './use_delete_saved_query'; diff --git a/x-pack/plugins/osquery/public/saved_queries/saved_queries_dropdown.tsx b/x-pack/plugins/osquery/public/saved_queries/saved_queries_dropdown.tsx new file mode 100644 index 0000000000000..e30954a695b2d --- /dev/null +++ b/x-pack/plugins/osquery/public/saved_queries/saved_queries_dropdown.tsx @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { find } from 'lodash/fp'; +import { EuiCodeBlock, EuiFormRow, EuiComboBox, EuiText } from '@elastic/eui'; +import React, { useCallback, useState } from 'react'; +import { SimpleSavedObject } from 'kibana/public'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; + +import { useSavedQueries } from './use_saved_queries'; + +interface SavedQueriesDropdownProps { + disabled?: boolean; + onChange: ( + value: SimpleSavedObject<{ + id: string; + description?: string | undefined; + query: string; + }>['attributes'] + ) => void; +} + +const SavedQueriesDropdownComponent: React.FC = ({ + disabled, + onChange, +}) => { + const [selectedOptions, setSelectedOptions] = useState([]); + + const { data } = useSavedQueries({}); + + const queryOptions = + data?.savedObjects?.map((savedQuery) => ({ + label: savedQuery.attributes.id ?? '', + value: { + id: savedQuery.attributes.id, + description: savedQuery.attributes.description, + query: savedQuery.attributes.query, + }, + })) ?? []; + + const handleSavedQueryChange = useCallback( + (newSelectedOptions) => { + const selectedSavedQuery = find( + ['attributes.id', newSelectedOptions[0].value.id], + data?.savedObjects + ); + + if (selectedSavedQuery) { + onChange(selectedSavedQuery.attributes); + } + setSelectedOptions(newSelectedOptions); + }, + [data?.savedObjects, onChange] + ); + + const renderOption = useCallback( + ({ value }) => ( + <> + {value.id} + +

{value.description}

+ + + {value.query} + + + ), + [] + ); + + return ( + + } + fullWidth + > + + + ); +}; + +export const SavedQueriesDropdown = React.memo(SavedQueriesDropdownComponent); diff --git a/x-pack/plugins/osquery/public/saved_queries/saved_query_flyout.tsx b/x-pack/plugins/osquery/public/saved_queries/saved_query_flyout.tsx new file mode 100644 index 0000000000000..6d14943a6bc84 --- /dev/null +++ b/x-pack/plugins/osquery/public/saved_queries/saved_query_flyout.tsx @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + EuiFlyout, + EuiTitle, + EuiFlyoutBody, + EuiFlyoutHeader, + EuiFlyoutFooter, + EuiPortal, + EuiFlexGroup, + EuiFlexItem, + EuiButtonEmpty, + EuiButton, +} from '@elastic/eui'; +import React, { useCallback } from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; + +import { Form } from '../shared_imports'; +import { useSavedQueryForm } from './form/use_saved_query_form'; +import { SavedQueryForm } from './form'; +import { useCreateSavedQuery } from './use_create_saved_query'; + +interface AddQueryFlyoutProps { + defaultValue: unknown; + onClose: () => void; +} + +const SavedQueryFlyoutComponent: React.FC = ({ defaultValue, onClose }) => { + const createSavedQueryMutation = useCreateSavedQuery({ withRedirect: false }); + + const handleSubmit = useCallback( + (payload) => createSavedQueryMutation.mutateAsync(payload).then(() => onClose()), + [createSavedQueryMutation, onClose] + ); + + const { form } = useSavedQueryForm({ + defaultValue, + handleSubmit, + }); + + return ( + + + + +

+ +

+ + + +
+ + +
+ + + + + + + + + + + + + + + + + ); +}; + +export const SavedQueryFlyout = React.memo(SavedQueryFlyoutComponent); diff --git a/x-pack/plugins/osquery/public/saved_queries/use_create_saved_query.ts b/x-pack/plugins/osquery/public/saved_queries/use_create_saved_query.ts new file mode 100644 index 0000000000000..cc5c33c6e4280 --- /dev/null +++ b/x-pack/plugins/osquery/public/saved_queries/use_create_saved_query.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useMutation, useQueryClient } from 'react-query'; +import { i18n } from '@kbn/i18n'; + +import { useKibana } from '../common/lib/kibana'; +import { savedQuerySavedObjectType } from '../../common/types'; +import { PLUGIN_ID } from '../../common'; +import { pagePathGetters } from '../common/page_paths'; +import { SAVED_QUERIES_ID } from './constants'; +import { useErrorToast } from '../common/hooks/use_error_toast'; + +interface UseCreateSavedQueryProps { + withRedirect?: boolean; +} + +export const useCreateSavedQuery = ({ withRedirect }: UseCreateSavedQueryProps) => { + const queryClient = useQueryClient(); + const { + application: { navigateToApp }, + savedObjects, + security, + notifications: { toasts }, + } = useKibana().services; + const setErrorToast = useErrorToast(); + + return useMutation( + async (payload) => { + const currentUser = await security.authc.getCurrentUser(); + + if (!currentUser) { + throw new Error('CurrentUser is missing'); + } + + return savedObjects.client.create(savedQuerySavedObjectType, { + // @ts-expect-error update types + ...payload, + created_by: currentUser.username, + created_at: new Date(Date.now()).toISOString(), + updated_by: currentUser.username, + updated_at: new Date(Date.now()).toISOString(), + }); + }, + { + onError: (error) => { + // @ts-expect-error update types + setErrorToast(error, { title: error.body.error, toastMessage: error.body.message }); + }, + onSuccess: (payload) => { + queryClient.invalidateQueries(SAVED_QUERIES_ID); + if (withRedirect) { + navigateToApp(PLUGIN_ID, { path: pagePathGetters.saved_queries() }); + } + toasts.addSuccess( + i18n.translate('xpack.osquery.newSavedQuery.successToastMessageText', { + defaultMessage: 'Successfully saved "{savedQueryId}" query', + values: { + savedQueryId: payload.attributes?.id ?? '', + }, + }) + ); + }, + } + ); +}; diff --git a/x-pack/plugins/osquery/public/saved_queries/use_delete_saved_query.ts b/x-pack/plugins/osquery/public/saved_queries/use_delete_saved_query.ts new file mode 100644 index 0000000000000..b2fee8b25f7a4 --- /dev/null +++ b/x-pack/plugins/osquery/public/saved_queries/use_delete_saved_query.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useMutation, useQueryClient } from 'react-query'; +import { i18n } from '@kbn/i18n'; + +import { useKibana } from '../common/lib/kibana'; +import { savedQuerySavedObjectType } from '../../common/types'; +import { PLUGIN_ID } from '../../common'; +import { pagePathGetters } from '../common/page_paths'; +import { SAVED_QUERIES_ID } from './constants'; +import { useErrorToast } from '../common/hooks/use_error_toast'; + +interface UseDeleteSavedQueryProps { + savedQueryId: string; +} + +export const useDeleteSavedQuery = ({ savedQueryId }: UseDeleteSavedQueryProps) => { + const queryClient = useQueryClient(); + const { + application: { navigateToApp }, + savedObjects, + notifications: { toasts }, + } = useKibana().services; + const setErrorToast = useErrorToast(); + + return useMutation(() => savedObjects.client.delete(savedQuerySavedObjectType, savedQueryId), { + onError: (error) => { + // @ts-expect-error update types + setErrorToast(error, { title: error.body.error, toastMessage: error.body.message }); + }, + onSuccess: () => { + queryClient.invalidateQueries(SAVED_QUERIES_ID); + navigateToApp(PLUGIN_ID, { path: pagePathGetters.saved_queries() }); + toasts.addSuccess( + i18n.translate('xpack.osquery.editSavedQuery.deleteSuccessToastMessageText', { + defaultMessage: 'Successfully deleted saved query', + }) + ); + }, + }); +}; diff --git a/x-pack/plugins/osquery/public/saved_queries/use_saved_queries.ts b/x-pack/plugins/osquery/public/saved_queries/use_saved_queries.ts new file mode 100644 index 0000000000000..324d4aace1647 --- /dev/null +++ b/x-pack/plugins/osquery/public/saved_queries/use_saved_queries.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useQuery } from 'react-query'; + +import { useKibana } from '../common/lib/kibana'; +import { savedQuerySavedObjectType } from '../../common/types'; +import { SAVED_QUERIES_ID } from './constants'; + +export const useSavedQueries = ({ + isLive = false, + pageIndex = 0, + pageSize = 10000, + sortField = 'updated_at', + sortDirection = 'desc', +}) => { + const { savedObjects } = useKibana().services; + + return useQuery( + [SAVED_QUERIES_ID, { pageIndex, pageSize, sortField, sortDirection }], + async () => + savedObjects.client.find<{ + id: string; + description?: string; + query: string; + updated_at: string; + updated_by: string; + created_at: string; + created_by: string; + }>({ + type: savedQuerySavedObjectType, + page: pageIndex + 1, + perPage: pageSize, + sortField, + }), + { + keepPreviousData: true, + // Refetch the data every 10 seconds + refetchInterval: isLive ? 10000 : false, + } + ); +}; diff --git a/x-pack/plugins/osquery/public/saved_queries/use_saved_query.ts b/x-pack/plugins/osquery/public/saved_queries/use_saved_query.ts new file mode 100644 index 0000000000000..92662cd24fd20 --- /dev/null +++ b/x-pack/plugins/osquery/public/saved_queries/use_saved_query.ts @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useQuery } from 'react-query'; + +import { PLUGIN_ID } from '../../common'; +import { useKibana } from '../common/lib/kibana'; +import { savedQuerySavedObjectType } from '../../common/types'; +import { pagePathGetters } from '../common/page_paths'; +import { useErrorToast } from '../common/hooks/use_error_toast'; + +export const SAVED_QUERY_ID = 'savedQuery'; + +interface UseSavedQueryProps { + savedQueryId: string; +} + +export const useSavedQuery = ({ savedQueryId }: UseSavedQueryProps) => { + const { + application: { navigateToApp }, + savedObjects, + } = useKibana().services; + const setErrorToast = useErrorToast(); + + return useQuery( + [SAVED_QUERY_ID, { savedQueryId }], + async () => + savedObjects.client.get<{ + id: string; + description?: string; + query: string; + }>(savedQuerySavedObjectType, savedQueryId), + { + keepPreviousData: true, + onSuccess: (data) => { + if (data.error) { + setErrorToast(data.error, { + title: data.error.error, + toastMessage: data.error.message, + }); + navigateToApp(PLUGIN_ID, { path: pagePathGetters.saved_queries() }); + } + }, + onError: (error) => { + // @ts-expect-error update types + setErrorToast(error, { title: error.body.error, toastMessage: error.body.message }); + }, + } + ); +}; diff --git a/x-pack/plugins/osquery/public/saved_queries/use_scheduled_query_group.ts b/x-pack/plugins/osquery/public/saved_queries/use_scheduled_query_group.ts new file mode 100644 index 0000000000000..93d552b3f71f3 --- /dev/null +++ b/x-pack/plugins/osquery/public/saved_queries/use_scheduled_query_group.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useQuery } from 'react-query'; + +import { useKibana } from '../common/lib/kibana'; +import { GetOnePackagePolicyResponse, packagePolicyRouteService } from '../../../fleet/common'; +import { OsqueryManagerPackagePolicy } from '../../common/types'; + +interface UseScheduledQueryGroup { + scheduledQueryGroupId: string; + skip?: boolean; +} + +export const useScheduledQueryGroup = ({ + scheduledQueryGroupId, + skip = false, +}: UseScheduledQueryGroup) => { + const { http } = useKibana().services; + + return useQuery< + Omit & { item: OsqueryManagerPackagePolicy }, + unknown, + OsqueryManagerPackagePolicy + >( + ['scheduledQueryGroup', { scheduledQueryGroupId }], + () => http.get(packagePolicyRouteService.getInfoPath(scheduledQueryGroupId)), + { + keepPreviousData: true, + enabled: !skip, + select: (response) => response.item, + } + ); +}; diff --git a/x-pack/plugins/osquery/public/saved_queries/use_update_saved_query.ts b/x-pack/plugins/osquery/public/saved_queries/use_update_saved_query.ts new file mode 100644 index 0000000000000..1260413676a4e --- /dev/null +++ b/x-pack/plugins/osquery/public/saved_queries/use_update_saved_query.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useMutation, useQueryClient } from 'react-query'; +import { i18n } from '@kbn/i18n'; + +import { useKibana } from '../common/lib/kibana'; +import { savedQuerySavedObjectType } from '../../common/types'; +import { PLUGIN_ID } from '../../common'; +import { pagePathGetters } from '../common/page_paths'; +import { SAVED_QUERIES_ID } from './constants'; +import { useErrorToast } from '../common/hooks/use_error_toast'; + +interface UseUpdateSavedQueryProps { + savedQueryId: string; +} + +export const useUpdateSavedQuery = ({ savedQueryId }: UseUpdateSavedQueryProps) => { + const queryClient = useQueryClient(); + const { + application: { navigateToApp }, + savedObjects, + security, + notifications: { toasts }, + } = useKibana().services; + const setErrorToast = useErrorToast(); + + return useMutation( + async (payload) => { + const currentUser = await security.authc.getCurrentUser(); + + if (!currentUser) { + throw new Error('CurrentUser is missing'); + } + + return savedObjects.client.update(savedQuerySavedObjectType, savedQueryId, { + // @ts-expect-error update types + ...payload, + updated_by: currentUser.username, + updated_at: new Date(Date.now()).toISOString(), + }); + }, + { + onError: (error) => { + // @ts-expect-error update types + setErrorToast(error, { title: error.body.error, toastMessage: error.body.message }); + }, + onSuccess: (payload) => { + queryClient.invalidateQueries(SAVED_QUERIES_ID); + navigateToApp(PLUGIN_ID, { path: pagePathGetters.saved_queries() }); + toasts.addSuccess( + i18n.translate('xpack.osquery.editSavedQuery.successToastMessageText', { + defaultMessage: 'Successfully updated "{savedQueryName}" query', + values: { + savedQueryName: payload.attributes?.name ?? '', + }, + }) + ); + }, + } + ); +}; diff --git a/x-pack/plugins/osquery/public/scheduled_query_groups/queries/constants.ts b/x-pack/plugins/osquery/public/scheduled_query_groups/queries/constants.ts index 3345c18d07b2c..bedca9d5ef8d7 100644 --- a/x-pack/plugins/osquery/public/scheduled_query_groups/queries/constants.ts +++ b/x-pack/plugins/osquery/public/scheduled_query_groups/queries/constants.ts @@ -6,6 +6,9 @@ */ export const ALL_OSQUERY_VERSIONS_OPTIONS = [ + { + label: '4.8.0', + }, { label: '4.7.0', }, diff --git a/x-pack/plugins/osquery/public/scheduled_query_groups/queries/platform_checkbox_group_field.tsx b/x-pack/plugins/osquery/public/scheduled_query_groups/queries/platform_checkbox_group_field.tsx index 4e433e9e240b1..0d455486bfa25 100644 --- a/x-pack/plugins/osquery/public/scheduled_query_groups/queries/platform_checkbox_group_field.tsx +++ b/x-pack/plugins/osquery/public/scheduled_query_groups/queries/platform_checkbox_group_field.tsx @@ -6,7 +6,7 @@ */ import { isEmpty, pickBy } from 'lodash'; -import React, { useCallback, useMemo, useState } from 'react'; +import React, { useCallback, useEffect, useMemo, useState } from 'react'; import { EuiFlexGroup, EuiFlexItem, @@ -112,6 +112,15 @@ export const PlatformCheckBoxGroupField = ({ const describedByIds = useMemo(() => (idAria ? [idAria] : []), [idAria]); + useEffect(() => { + setCheckboxIdToSelectedMap(() => + (options as EuiCheckboxGroupOption[]).reduce((acc, option) => { + acc[option.id] = isEmpty(field.value) ? true : field.value?.includes(option.id) ?? false; + return acc; + }, {} as Record) + ); + }, [field.value, options]); + return ( = ({ onSave, onClose, }) => { + const [isEditMode] = useState(!!defaultValue); const { form } = useScheduledQueryGroupQueryForm({ defaultValue, handleSubmit: (payload, isValid) => @@ -67,7 +70,31 @@ const QueryFlyoutComponent: React.FC = ({ [integrationPackageVersion] ); - const { submit } = form; + const { submit, setFieldValue } = form; + + const handleSetQueryValue = useCallback( + (savedQuery) => { + setFieldValue('id', savedQuery.id); + setFieldValue('query', savedQuery.query); + + if (savedQuery.description) { + setFieldValue('description', savedQuery.description); + } + + if (savedQuery.interval) { + setFieldValue('interval', savedQuery.interval); + } + + if (isFieldSupported && savedQuery.platform) { + setFieldValue('platform', savedQuery.platform); + } + + if (isFieldSupported && savedQuery.version) { + setFieldValue('version', [savedQuery.version]); + } + }, + [isFieldSupported, setFieldValue] + ); return ( @@ -75,7 +102,7 @@ const QueryFlyoutComponent: React.FC = ({

- {defaultValue ? ( + {isEditMode ? ( = ({
+ {!isEditMode ? ( + <> + + + + ) : null} - + Set heading level based on context

} + description={'Will be wrapped in a small, subdued EuiText block.'} + > = ({ euiFieldProps={{ disabled: !isFieldSupported }} /> -
+ {!isFieldSupported ? ( diff --git a/x-pack/plugins/osquery/public/scheduled_query_groups/queries/schema.tsx b/x-pack/plugins/osquery/public/scheduled_query_groups/queries/schema.tsx index 344c33b419dd6..d8dbaad2f17e8 100644 --- a/x-pack/plugins/osquery/public/scheduled_query_groups/queries/schema.tsx +++ b/x-pack/plugins/osquery/public/scheduled_query_groups/queries/schema.tsx @@ -22,6 +22,16 @@ export const formSchema = { }), validations: idFieldValidations.map((validator) => ({ validator })), }, + description: { + type: FIELD_TYPES.TEXT, + label: i18n.translate( + 'xpack.osquery.scheduledQueryGroup.queryFlyoutForm.descriptionFieldLabel', + { + defaultMessage: 'Description', + } + ), + validations: [], + }, query: { type: FIELD_TYPES.TEXT, label: i18n.translate('xpack.osquery.scheduledQueryGroup.queryFlyoutForm.queryFieldLabel', { diff --git a/x-pack/plugins/osquery/public/scheduled_query_groups/queries/use_scheduled_query_group_query_form.tsx b/x-pack/plugins/osquery/public/scheduled_query_groups/queries/use_scheduled_query_group_query_form.tsx index bcde5f4b970d4..fdf781c6d6f7a 100644 --- a/x-pack/plugins/osquery/public/scheduled_query_groups/queries/use_scheduled_query_group_query_form.tsx +++ b/x-pack/plugins/osquery/public/scheduled_query_groups/queries/use_scheduled_query_group_query_form.tsx @@ -45,6 +45,9 @@ export const useScheduledQueryGroupQueryForm = ({ // @ts-expect-error update types serializer: (payload) => produce(payload, (draft) => { + if (isArray(draft.platform)) { + draft.platform.join(','); + } if (draft.platform?.split(',').length === 3) { // if all platforms are checked then use undefined delete draft.platform; diff --git a/x-pack/plugins/osquery/public/scheduled_query_groups/scheduled_query_groups_table.tsx b/x-pack/plugins/osquery/public/scheduled_query_groups/scheduled_query_groups_table.tsx index 7b5f91157132e..391e20c63653f 100644 --- a/x-pack/plugins/osquery/public/scheduled_query_groups/scheduled_query_groups_table.tsx +++ b/x-pack/plugins/osquery/public/scheduled_query_groups/scheduled_query_groups_table.tsx @@ -6,6 +6,7 @@ */ import { EuiInMemoryTable, EuiBasicTableColumn, EuiLink } from '@elastic/eui'; +import moment from 'moment'; import React, { useCallback, useMemo } from 'react'; import { i18n } from '@kbn/i18n'; @@ -37,6 +38,13 @@ const ScheduledQueryGroupsTableComponent = () => { const renderActive = useCallback((_, item) => , []); + const renderUpdatedAt = useCallback((updatedAt, item) => { + if (!updatedAt) return '-'; + + const updatedBy = item.updated_by !== item.created_by ? ` @ ${item.updated_by}` : ''; + return updatedAt ? `${moment(updatedAt).fromNow()}${updatedBy}` : '-'; + }, []); + const columns: Array> = useMemo( () => [ { @@ -66,6 +74,21 @@ const ScheduledQueryGroupsTableComponent = () => { render: renderQueries, width: '150px', }, + { + field: 'created_by', + name: i18n.translate('xpack.osquery.scheduledQueryGroups.table.createdByColumnTitle', { + defaultMessage: 'Created by', + }), + sortable: true, + truncateText: true, + }, + { + field: 'updated_at', + name: 'Last updated', + sortable: (item) => (item.updated_at ? Date.parse(item.updated_at) : 0), + truncateText: true, + render: renderUpdatedAt, + }, { field: 'enabled', name: i18n.translate('xpack.osquery.scheduledQueryGroups.table.activeColumnTitle', { @@ -77,7 +100,7 @@ const ScheduledQueryGroupsTableComponent = () => { render: renderActive, }, ], - [renderActive, renderAgentPolicy, renderQueries] + [renderActive, renderAgentPolicy, renderQueries, renderUpdatedAt] ); const sorting = useMemo( diff --git a/x-pack/plugins/osquery/public/types.ts b/x-pack/plugins/osquery/public/types.ts index 441c00f2d0968..9a466dfc619b6 100644 --- a/x-pack/plugins/osquery/public/types.ts +++ b/x-pack/plugins/osquery/public/types.ts @@ -8,7 +8,8 @@ import { DiscoverStart } from '../../../../src/plugins/discover/public'; import { DataPublicPluginStart } from '../../../../src/plugins/data/public'; import { FleetStart } from '../../fleet/public'; -import { LensPublicStart } from '../../../plugins/lens/public'; +import { LensPublicStart } from '../../lens/public'; +import { SecurityPluginStart } from '../../security/public'; import { CoreStart } from '../../../../src/core/public'; import { NavigationPublicPluginStart } from '../../../../src/plugins/navigation/public'; import { @@ -30,6 +31,7 @@ export interface StartPlugins { data: DataPublicPluginStart; fleet: FleetStart; lens?: LensPublicStart; + security: SecurityPluginStart; triggersActionsUi: TriggersAndActionsUIPublicPluginStart; } diff --git a/x-pack/plugins/osquery/scripts/schema_formatter/script.ts b/x-pack/plugins/osquery/scripts/schema_formatter/script.ts index 578c4a1120962..5cdf11d48c696 100644 --- a/x-pack/plugins/osquery/scripts/schema_formatter/script.ts +++ b/x-pack/plugins/osquery/scripts/schema_formatter/script.ts @@ -37,7 +37,7 @@ run( const mapFunc = pullFields.bind(null, { name: true }); const formattedSchema = schemaData.map(mapFunc); await fs.writeFile( - path.join(schemaPath, `${flags.schema_version}-formatted`), + path.join(schemaPath, `${flags.schema_version}-formatted.json`), JSON.stringify(formattedSchema) ); }, diff --git a/x-pack/plugins/osquery/server/config.ts b/x-pack/plugins/osquery/server/config.ts index 56d67400a47d9..3ec9213ae6d60 100644 --- a/x-pack/plugins/osquery/server/config.ts +++ b/x-pack/plugins/osquery/server/config.ts @@ -10,7 +10,7 @@ import { TypeOf, schema } from '@kbn/config-schema'; export const ConfigSchema = schema.object({ enabled: schema.boolean({ defaultValue: true }), actionEnabled: schema.boolean({ defaultValue: false }), - savedQueries: schema.boolean({ defaultValue: false }), + savedQueries: schema.boolean({ defaultValue: true }), packs: schema.boolean({ defaultValue: false }), }); diff --git a/x-pack/plugins/osquery/server/lib/osquery_app_context_services.ts b/x-pack/plugins/osquery/server/lib/osquery_app_context_services.ts index 5b1f8e780494d..6ebf469b8fb29 100644 --- a/x-pack/plugins/osquery/server/lib/osquery_app_context_services.ts +++ b/x-pack/plugins/osquery/server/lib/osquery_app_context_services.ts @@ -6,6 +6,7 @@ */ import { Logger, LoggerFactory } from 'src/core/server'; +import { SecurityPluginStart } from '../../../security/server'; import { AgentService, FleetStartContract, @@ -69,7 +70,7 @@ export class OsqueryAppContextService { export interface OsqueryAppContext { logFactory: LoggerFactory; config(): ConfigType; - + security: SecurityPluginStart; /** * Object readiness is tied to plugin start method */ diff --git a/x-pack/plugins/osquery/server/lib/saved_query/saved_object_mappings.ts b/x-pack/plugins/osquery/server/lib/saved_query/saved_object_mappings.ts index dadcea6e2fd4d..537b6d7874ab8 100644 --- a/x-pack/plugins/osquery/server/lib/saved_query/saved_object_mappings.ts +++ b/x-pack/plugins/osquery/server/lib/saved_query/saved_object_mappings.ts @@ -14,34 +14,40 @@ export const savedQuerySavedObjectMappings: SavedObjectsType['mappings'] = { description: { type: 'text', }, - name: { - type: 'text', + id: { + type: 'keyword', }, query: { type: 'text', }, - created: { + created_at: { type: 'date', }, - createdBy: { + created_by: { type: 'text', }, platform: { type: 'keyword', }, - updated: { + version: { + type: 'keyword', + }, + updated_at: { type: 'date', }, - updatedBy: { + updated_by: { type: 'text', }, + interval: { + type: 'keyword', + }, }, }; export const savedQueryType: SavedObjectsType = { name: savedQuerySavedObjectType, hidden: false, - namespaceType: 'single', + namespaceType: 'multiple-isolated', mappings: savedQuerySavedObjectMappings, }; @@ -53,16 +59,16 @@ export const packSavedObjectMappings: SavedObjectsType['mappings'] = { name: { type: 'text', }, - created: { + created_at: { type: 'date', }, - createdBy: { + created_by: { type: 'text', }, - updated: { + updated_at: { type: 'date', }, - updatedBy: { + updated_by: { type: 'text', }, queries: { @@ -81,6 +87,6 @@ export const packSavedObjectMappings: SavedObjectsType['mappings'] = { export const packType: SavedObjectsType = { name: packSavedObjectType, hidden: false, - namespaceType: 'single', + namespaceType: 'multiple-isolated', mappings: packSavedObjectMappings, }; diff --git a/x-pack/plugins/osquery/server/plugin.ts b/x-pack/plugins/osquery/server/plugin.ts index ae779a9788238..6bc12f5736e5e 100644 --- a/x-pack/plugins/osquery/server/plugin.ts +++ b/x-pack/plugins/osquery/server/plugin.ts @@ -46,6 +46,7 @@ export class OsqueryPlugin implements Plugin config, + security: plugins.security, }; initSavedObjects(core.savedObjects, osqueryContext); diff --git a/x-pack/plugins/osquery/server/routes/action/create_action_route.ts b/x-pack/plugins/osquery/server/routes/action/create_action_route.ts index 86e871f041160..478bfc1053bdf 100644 --- a/x-pack/plugins/osquery/server/routes/action/create_action_route.ts +++ b/x-pack/plugins/osquery/server/routes/action/create_action_route.ts @@ -48,13 +48,15 @@ export const createActionRoute = (router: IRouter, osqueryContext: OsqueryAppCon } try { + const currentUser = await osqueryContext.security.authc.getCurrentUser(request)?.username; const action = { action_id: uuid.v4(), '@timestamp': moment().toISOString(), - expiration: moment().add(1, 'days').toISOString(), + expiration: moment().add(5, 'minutes').toISOString(), type: 'INPUT_ACTION', input_type: 'osquery', agents: selectedAgents, + user_id: currentUser, data: { id: uuid.v4(), query: request.body.query, @@ -75,7 +77,7 @@ export const createActionRoute = (router: IRouter, osqueryContext: OsqueryAppCon incrementCount(soClient, 'live_query', 'errors'); return response.customError({ statusCode: 500, - body: new Error(`Error occurred whlie processing ${error}`), + body: new Error(`Error occurred while processing ${error}`), }); } } diff --git a/x-pack/plugins/osquery/server/routes/saved_query/create_saved_query_route.ts b/x-pack/plugins/osquery/server/routes/saved_query/create_saved_query_route.ts index 5eb7147d46607..a41cb7cc39b40 100644 --- a/x-pack/plugins/osquery/server/routes/saved_query/create_saved_query_route.ts +++ b/x-pack/plugins/osquery/server/routes/saved_query/create_saved_query_route.ts @@ -28,13 +28,15 @@ export const createSavedQueryRoute = (router: IRouter) => { async (context, request, response) => { const savedObjectsClient = context.core.savedObjects.client; - const { name, description, platform, query } = request.body; + const { id, description, platform, query, version, interval } = request.body; const savedQuerySO = await savedObjectsClient.create(savedQuerySavedObjectType, { - name, + id, description, query, platform, + version, + interval, }); return response.ok({ diff --git a/x-pack/plugins/osquery/server/routes/saved_query/read_saved_query_route.ts b/x-pack/plugins/osquery/server/routes/saved_query/read_saved_query_route.ts index 8be4c6c50d821..2d399648df4cc 100644 --- a/x-pack/plugins/osquery/server/routes/saved_query/read_saved_query_route.ts +++ b/x-pack/plugins/osquery/server/routes/saved_query/read_saved_query_route.ts @@ -21,18 +21,14 @@ export const readSavedQueryRoute = (router: IRouter) => { async (context, request, response) => { const savedObjectsClient = context.core.savedObjects.client; - const { attributes, ...savedQuery } = await savedObjectsClient.get( + const savedQuery = await savedObjectsClient.get( savedQuerySavedObjectType, // @ts-expect-error update types request.params.id ); return response.ok({ - body: { - ...savedQuery, - // @ts-expect-error update types - ...attributes, - }, + body: savedQuery, }); } ); diff --git a/x-pack/plugins/osquery/server/routes/saved_query/update_saved_query_route.ts b/x-pack/plugins/osquery/server/routes/saved_query/update_saved_query_route.ts index 579cd9b654cc0..f9ecf675489dc 100644 --- a/x-pack/plugins/osquery/server/routes/saved_query/update_saved_query_route.ts +++ b/x-pack/plugins/osquery/server/routes/saved_query/update_saved_query_route.ts @@ -23,17 +23,19 @@ export const updateSavedQueryRoute = (router: IRouter) => { const savedObjectsClient = context.core.savedObjects.client; // @ts-expect-error update types - const { name, description, platform, query } = request.body; + const { id, description, platform, query, version, interval } = request.body; const savedQuerySO = await savedObjectsClient.update( savedQuerySavedObjectType, // @ts-expect-error update types request.params.id, { - name, + id, description, platform, query, + version, + interval, } ); diff --git a/x-pack/plugins/osquery/server/types.ts b/x-pack/plugins/osquery/server/types.ts index 667fba2bc98e2..84b2ff41dc1cf 100644 --- a/x-pack/plugins/osquery/server/types.ts +++ b/x-pack/plugins/osquery/server/types.ts @@ -13,6 +13,7 @@ import { import { FleetStartContract } from '../../fleet/server'; import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/server'; import { PluginSetupContract } from '../../features/server'; +import { SecurityPluginStart } from '../../security/server'; // eslint-disable-next-line @typescript-eslint/no-empty-interface export interface OsqueryPluginSetup {} @@ -24,6 +25,7 @@ export interface SetupPlugins { actions: ActionsPlugin['setup']; data: DataPluginSetup; features: PluginSetupContract; + security: SecurityPluginStart; } export interface StartPlugins { diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index ca2be10624e66..7ffd5d8f20ffd 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -17336,7 +17336,7 @@ "xpack.osquery.breadcrumbs.newLiveQueryPageTitle": "新規", "xpack.osquery.breadcrumbs.overviewPageTitle": "概要", "xpack.osquery.breadcrumbs.scheduledQueryGroupsPageTitle": "スケジュールされたクエリグループ", - "xpack.osquery.codeEditorField.osquerySchemaLinkLabel": "Osqueryスキーマ", + "xpack.osquery.osquerySchemaLinkLabel": "Osqueryスキーマ", "xpack.osquery.common.tabBetaBadgeLabel": "ベータ", "xpack.osquery.common.tabBetaBadgeTooltipContent": "この機能は現在開発中です。他にも機能が追加され、機能によっては変更されるものもあります。", "xpack.osquery.editScheduledQuery.pageTitle": "{queryName}を編集", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 341741dd4046a..4a964fc5e2fd0 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -17573,7 +17573,7 @@ "xpack.osquery.breadcrumbs.newLiveQueryPageTitle": "新建", "xpack.osquery.breadcrumbs.overviewPageTitle": "概览", "xpack.osquery.breadcrumbs.scheduledQueryGroupsPageTitle": "已计划查询组", - "xpack.osquery.codeEditorField.osquerySchemaLinkLabel": "Osquery 架构", + "xpack.osquery.osquerySchemaLinkLabel": "Osquery 架构", "xpack.osquery.common.tabBetaBadgeLabel": "公测版", "xpack.osquery.common.tabBetaBadgeTooltipContent": "我们正在开发此功能。将会有更多的功能，某些功能可能有变更。", "xpack.osquery.createScheduledQuery.agentPolicyAgentsCountText": "{count, plural, other {# 个代理}}已注册", From da13795ed4e3c511ce44b87da7a5d146971f3866 Mon Sep 17 00:00:00 2001 From: Candace Park <56409205+parkiino@users.noreply.github.com> Date: Mon, 28 Jun 2021 22:03:08 -0400 Subject: [PATCH 52/55] Task/host isolation status pending (#103549) --- .../event_details/alert_summary_view.tsx | 4 ++-- .../detection_engine/alerts/translations.ts | 5 +++++ .../alerts/use_host_isolation_status.tsx | 22 +++++++++++++++++-- .../body/renderers/agent_statuses.tsx | 13 +++++++++-- .../body/renderers/formatted_field.tsx | 2 +- 5 files changed, 39 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/alert_summary_view.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/alert_summary_view.tsx index 59acb16c028d8..d89f44542318e 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/alert_summary_view.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/alert_summary_view.tsx @@ -183,7 +183,7 @@ const AlertSummaryViewComponent: React.FC<{ return endpointAlertCheck({ data }); }, [data]); - const agentId = useMemo(() => { + const endpointId = useMemo(() => { const findAgentId = find({ category: 'agent', field: 'agent.id' }, data)?.values; return findAgentId ? findAgentId[0] : ''; }, [data]); @@ -194,7 +194,7 @@ const AlertSummaryViewComponent: React.FC<{ contextId: timelineId, eventId, fieldName: 'agent.status', - value: agentId, + value: endpointId, linkValue: undefined, }, }; diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/translations.ts b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/translations.ts index 9e4497f2f096b..d5234e719b869 100644 --- a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/translations.ts +++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/translations.ts @@ -42,3 +42,8 @@ export const ISOLATION_STATUS_FAILURE = i18n.translate( 'xpack.securitySolution.endpoint.hostIsolation.isolationStatus.title', { defaultMessage: 'Failed to retrieve current isolation status' } ); + +export const ISOLATION_PENDING_FAILURE = i18n.translate( + 'xpack.securitySolution.endpoint.hostIsolation.isolationPending.title', + { defaultMessage: 'Failed to retrieve isolation pending statuses' } +); diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_host_isolation_status.tsx b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_host_isolation_status.tsx index 7419727fff6a2..3bdd8c9813785 100644 --- a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_host_isolation_status.tsx +++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_host_isolation_status.tsx @@ -9,7 +9,8 @@ import { isEmpty } from 'lodash'; import { useEffect, useState } from 'react'; import { useAppToasts } from '../../../../common/hooks/use_app_toasts'; import { getHostMetadata } from './api'; -import { ISOLATION_STATUS_FAILURE } from './translations'; +import { ISOLATION_STATUS_FAILURE, ISOLATION_PENDING_FAILURE } from './translations'; +import { fetchPendingActionsByAgentId } from '../../../../common/lib/endpoint_pending_actions'; import { isEndpointHostIsolated } from '../../../../common/utils/validators'; import { HostStatus } from '../../../../../common/endpoint/types'; @@ -17,6 +18,8 @@ interface HostIsolationStatusResponse { loading: boolean; isIsolated: boolean; agentStatus: HostStatus; + pendingIsolation: number; + pendingUnisolation: number; } /* @@ -28,6 +31,8 @@ export const useHostIsolationStatus = ({ }): HostIsolationStatusResponse => { const [isIsolated, setIsIsolated] = useState(false); const [agentStatus, setAgentStatus] = useState(HostStatus.UNHEALTHY); + const [pendingIsolation, setPendingIsolation] = useState(0); + const [pendingUnisolation, setPendingUnisolation] = useState(0); const [loading, setLoading] = useState(false); const { addError } = useAppToasts(); @@ -35,16 +40,29 @@ export const useHostIsolationStatus = ({ useEffect(() => { // isMounted tracks if a component is mounted before changing state let isMounted = true; + let fleetAgentId: string; const fetchData = async () => { try { const metadataResponse = await getHostMetadata({ agentId }); if (isMounted) { setIsIsolated(isEndpointHostIsolated(metadataResponse.metadata)); setAgentStatus(metadataResponse.host_status); + fleetAgentId = metadataResponse.metadata.elastic.agent.id; } } catch (error) { addError(error.message, { title: ISOLATION_STATUS_FAILURE }); } + + try { + const { data } = await fetchPendingActionsByAgentId(fleetAgentId); + if (isMounted) { + setPendingIsolation(data[0].pending_actions?.isolate ?? 0); + setPendingUnisolation(data[0].pending_actions?.unisolate ?? 0); + } + } catch (error) { + addError(error.message, { title: ISOLATION_PENDING_FAILURE }); + } + if (isMounted) { setLoading(false); } @@ -64,5 +82,5 @@ export const useHostIsolationStatus = ({ isMounted = false; }; }, [addError, agentId]); - return { loading, isIsolated, agentStatus }; + return { loading, isIsolated, agentStatus, pendingIsolation, pendingUnisolation }; }; diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/agent_statuses.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/agent_statuses.tsx index 16f11809dd72b..2c88b305c7d05 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/agent_statuses.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/agent_statuses.tsx @@ -24,7 +24,12 @@ export const AgentStatuses = React.memo( eventId: string; value: string; }) => { - const { isIsolated, agentStatus } = useHostIsolationStatus({ agentId: value }); + const { + isIsolated, + agentStatus, + pendingIsolation, + pendingUnisolation, + } = useHostIsolationStatus({ agentId: value }); const isolationFieldName = 'host.isolation'; return ( @@ -45,7 +50,11 @@ export const AgentStatuses = React.memo( tooltipContent={isolationFieldName} value={`${isIsolated}`} > - + diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/formatted_field.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/formatted_field.tsx index 3d5d410abb87e..1d04849b198ad 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/formatted_field.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/formatted_field.tsx @@ -126,7 +126,7 @@ const FormattedFieldValueComponent: React.FC<{ contextId={contextId} eventId={eventId} fieldName={fieldName} - value={value as string} + value={typeof value === 'string' ? value : ''} /> ); } else if ( From 1dce600efecf22138ddd225c10b06c6464515a45 Mon Sep 17 00:00:00 2001 From: Nathan L Smith Date: Mon, 28 Jun 2021 21:27:10 -0500 Subject: [PATCH 53/55] Collect `host.os.platform` telemetry for APM (#103520) Fixes #97958. --- .../__snapshots__/apm_telemetry.test.ts.snap | 22 +++++++++ .../collect_data_telemetry/tasks.test.ts | 38 +++++++++++++++ .../collect_data_telemetry/tasks.ts | 48 +++++++++++++++++++ .../apm/server/lib/apm_telemetry/schema.ts | 2 + .../apm/server/lib/apm_telemetry/types.ts | 2 + .../schema/xpack_plugins.json | 33 +++++++++++-- 6 files changed, 141 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap index d7fc8e6442f12..71b0929164705 100644 --- a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap +++ b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap @@ -675,6 +675,17 @@ exports[`APM telemetry helpers getApmTelemetry generates a JSON object with the } } }, + "host": { + "properties": { + "os": { + "properties": { + "platform": { + "type": "keyword" + } + } + } + } + }, "counts": { "properties": { "transaction": { @@ -967,6 +978,17 @@ exports[`APM telemetry helpers getApmTelemetry generates a JSON object with the } } }, + "host": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, "processor_events": { "properties": { "took": { diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts index 129da71097863..4bfac442b4a3c 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts @@ -209,6 +209,44 @@ describe('data telemetry collection tasks', () => { }); }); + describe('host', () => { + const task = tasks.find((t) => t.name === 'host'); + + it('returns a map of host provider data', async () => { + const search = jest.fn().mockResolvedValueOnce({ + aggregations: { + platform: { + buckets: [ + { doc_count: 1, key: 'linux' }, + { doc_count: 1, key: 'windows' }, + { doc_count: 1, key: 'macos' }, + ], + }, + }, + }); + + expect(await task?.executor({ indices, search } as any)).toEqual({ + host: { + os: { platform: ['linux', 'windows', 'macos'] }, + }, + }); + }); + + describe('with no results', () => { + it('returns an empty map', async () => { + const search = jest.fn().mockResolvedValueOnce({}); + + expect(await task?.executor({ indices, search } as any)).toEqual({ + host: { + os: { + platform: [], + }, + }, + }); + }); + }); + }); + describe('processor_events', () => { const task = tasks.find((t) => t.name === 'processor_events'); diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts index 3d5b4b754e4aa..fd341565c235b 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts @@ -20,6 +20,7 @@ import { CONTAINER_ID, ERROR_GROUP_ID, HOST_NAME, + HOST_OS_PLATFORM, OBSERVER_HOSTNAME, PARENT_ID, POD_NAME, @@ -293,6 +294,53 @@ export const tasks: TelemetryTask[] = [ return { cloud }; }, }, + { + name: 'host', + executor: async ({ indices, search }) => { + function getBucketKeys({ + buckets, + }: { + buckets: Array<{ + doc_count: number; + key: string | number; + }>; + }) { + return buckets.map((bucket) => bucket.key as string); + } + + const response = await search({ + index: [ + indices['apm_oss.errorIndices'], + indices['apm_oss.metricsIndices'], + indices['apm_oss.spanIndices'], + indices['apm_oss.transactionIndices'], + ], + body: { + size: 0, + timeout, + aggs: { + platform: { + terms: { + field: HOST_OS_PLATFORM, + }, + }, + }, + }, + }); + + const { aggregations } = response; + + if (!aggregations) { + return { host: { os: { platform: [] } } }; + } + const host = { + os: { + platform: getBucketKeys(aggregations.platform), + }, + }; + return { host }; + }, + }, { name: 'environments', executor: async ({ indices, search }) => { diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/schema.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/schema.ts index 0b1bc3d50d4c1..b04f64c6bccff 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/schema.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/schema.ts @@ -135,6 +135,7 @@ export const apmSchema: MakeSchemaFrom = { provider: { type: 'array', items: { type: 'keyword' } }, region: { type: 'array', items: { type: 'keyword' } }, }, + host: { os: { platform: { type: 'array', items: { type: 'keyword' } } } }, counts: { transaction: timeframeMapSchema, span: timeframeMapSchema, @@ -185,6 +186,7 @@ export const apmSchema: MakeSchemaFrom = { tasks: { aggregated_transactions: { took: { ms: long } }, cloud: { took: { ms: long } }, + host: { took: { ms: long } }, processor_events: { took: { ms: long } }, agent_configuration: { took: { ms: long } }, services: { took: { ms: long } }, diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts index 6dc829425eada..cd4e80ff6bf6b 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts @@ -52,6 +52,7 @@ export interface APMUsage { provider: string[]; region: string[]; }; + host: { os: { platform: string[] } }; counts: { transaction: TimeframeMap; span: TimeframeMap; @@ -132,6 +133,7 @@ export interface APMUsage { tasks: Record< | 'aggregated_transactions' | 'cloud' + | 'host' | 'processor_events' | 'agent_configuration' | 'services' diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index bab4244139df0..5a8b30c5c0f26 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -31,10 +31,10 @@ "__index": { "type": "long" }, - "__swimlane": { + "__pagerduty": { "type": "long" }, - "__pagerduty": { + "__swimlane": { "type": "long" }, "__server-log": { @@ -71,10 +71,10 @@ "__index": { "type": "long" }, - "__swimlane": { + "__pagerduty": { "type": "long" }, - "__pagerduty": { + "__swimlane": { "type": "long" }, "__server-log": { @@ -1260,6 +1260,20 @@ } } }, + "host": { + "properties": { + "os": { + "properties": { + "platform": { + "type": "array", + "items": { + "type": "keyword" + } + } + } + } + } + }, "counts": { "properties": { "transaction": { @@ -1552,6 +1566,17 @@ } } }, + "host": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, "processor_events": { "properties": { "took": { From 09bd6301d63803afb503360bdb6c82ffcc497d08 Mon Sep 17 00:00:00 2001 From: Ignacio Rivas Date: Tue, 29 Jun 2021 08:57:30 +0200 Subject: [PATCH 54/55] [CCR & Snapshot+Restore] Center align states under tabs (#103237) * fix up CCR centered sates in tabs content * update snapshots list * fix lint errors * Change tab states for all pages in snapshot+restore * Remove unnecessary variables * Seems we dont need the class wrapper * fix broken type * Fix bug in ILM table when filtering it down * center align search box * fix linter errors * fix prettier warnings * revert content var refactor and just focus on ux * add breakword class to paragraph so we avoid text overflowing * fix prettier errors Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../authorization/components/page_error.tsx | 6 +- .../auto_follow_pattern_list.test.js | 2 +- .../follower_indices_list.test.js | 2 +- .../auto_follow_pattern_list.js | 47 +-- .../follower_indices_list.js | 47 +-- .../public/shared_imports.ts | 3 + .../sections/policy_table/policy_table.tsx | 6 +- .../helpers/home.helpers.ts | 1 + .../__jest__/client_integration/home.test.ts | 2 +- .../sections/home/policy_list/policy_list.tsx | 93 ++--- .../home/repository_list/repository_list.tsx | 104 ++--- .../home/restore_list/restore_list.tsx | 84 +++-- .../home/snapshot_list/snapshot_list.tsx | 356 ++++++++++-------- .../snapshot_restore/public/shared_imports.ts | 3 + 14 files changed, 415 insertions(+), 341 deletions(-) diff --git a/src/plugins/es_ui_shared/__packages_do_not_import__/authorization/components/page_error.tsx b/src/plugins/es_ui_shared/__packages_do_not_import__/authorization/components/page_error.tsx index 732aa35b05237..9953a8fedb39b 100644 --- a/src/plugins/es_ui_shared/__packages_do_not_import__/authorization/components/page_error.tsx +++ b/src/plugins/es_ui_shared/__packages_do_not_import__/authorization/components/page_error.tsx @@ -43,7 +43,11 @@ export const PageError: React.FunctionComponent = ({ body={ error && ( <> - {cause ? message || errorString :

{message || errorString}

} + {cause ? ( + message || errorString + ) : ( +

{message || errorString}

+ )} {cause && ( <> diff --git a/x-pack/plugins/cross_cluster_replication/public/__jest__/client_integration/auto_follow_pattern_list.test.js b/x-pack/plugins/cross_cluster_replication/public/__jest__/client_integration/auto_follow_pattern_list.test.js index 1e7eb9562313d..7f7a61a6f0177 100644 --- a/x-pack/plugins/cross_cluster_replication/public/__jest__/client_integration/auto_follow_pattern_list.test.js +++ b/x-pack/plugins/cross_cluster_replication/public/__jest__/client_integration/auto_follow_pattern_list.test.js @@ -38,7 +38,7 @@ describe('', () => { }); test('should show a loading indicator on component', async () => { - expect(exists('autoFollowPatternLoading')).toBe(true); + expect(exists('sectionLoading')).toBe(true); }); }); diff --git a/x-pack/plugins/cross_cluster_replication/public/__jest__/client_integration/follower_indices_list.test.js b/x-pack/plugins/cross_cluster_replication/public/__jest__/client_integration/follower_indices_list.test.js index f004617a099d3..b9e47b029e302 100644 --- a/x-pack/plugins/cross_cluster_replication/public/__jest__/client_integration/follower_indices_list.test.js +++ b/x-pack/plugins/cross_cluster_replication/public/__jest__/client_integration/follower_indices_list.test.js @@ -45,7 +45,7 @@ describe('', () => { }); test('should show a loading indicator on component', async () => { - expect(exists('followerIndexLoading')).toBe(true); + expect(exists('sectionLoading')).toBe(true); }); }); diff --git a/x-pack/plugins/cross_cluster_replication/public/app/sections/home/auto_follow_pattern_list/auto_follow_pattern_list.js b/x-pack/plugins/cross_cluster_replication/public/app/sections/home/auto_follow_pattern_list/auto_follow_pattern_list.js index 1ab4e1a3e003a..3a4273c1ed0e3 100644 --- a/x-pack/plugins/cross_cluster_replication/public/app/sections/home/auto_follow_pattern_list/auto_follow_pattern_list.js +++ b/x-pack/plugins/cross_cluster_replication/public/app/sections/home/auto_follow_pattern_list/auto_follow_pattern_list.js @@ -9,13 +9,12 @@ import React, { PureComponent } from 'react'; import PropTypes from 'prop-types'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; -import { EuiButton, EuiEmptyPrompt, EuiText, EuiSpacer } from '@elastic/eui'; +import { EuiPageContent, EuiButton, EuiEmptyPrompt, EuiText, EuiSpacer } from '@elastic/eui'; import { reactRouterNavigate } from '../../../../../../../../src/plugins/kibana_react/public'; -import { extractQueryParams, SectionLoading } from '../../../../shared_imports'; +import { extractQueryParams, PageError, PageLoading } from '../../../../shared_imports'; import { trackUiMetric, METRIC_TYPE } from '../../../services/track_ui_metric'; import { API_STATUS, UIM_AUTO_FOLLOW_PATTERN_LIST_LOAD } from '../../../constants'; -import { SectionError, SectionUnauthorized } from '../../../components'; import { AutoFollowPatternTable, DetailPanel } from './components'; const REFRESH_RATE_MS = 30000; @@ -98,7 +97,12 @@ export class AutoFollowPatternList extends PureComponent { renderEmpty() { return ( -
+ } /> -
+ ); } @@ -170,19 +174,22 @@ export class AutoFollowPatternList extends PureComponent { if (!isAuthorized) { return ( - } - > - - + error={{ + error: ( + + ), + }} + /> ); } @@ -194,7 +201,7 @@ export class AutoFollowPatternList extends PureComponent { } ); - return ; + return ; } if (isEmpty) { @@ -203,14 +210,12 @@ export class AutoFollowPatternList extends PureComponent { if (apiStatus === API_STATUS.LOADING) { return ( -
- - - -
+ + + ); } diff --git a/x-pack/plugins/cross_cluster_replication/public/app/sections/home/follower_indices_list/follower_indices_list.js b/x-pack/plugins/cross_cluster_replication/public/app/sections/home/follower_indices_list/follower_indices_list.js index a52ba0e613ca9..4b593799f5933 100644 --- a/x-pack/plugins/cross_cluster_replication/public/app/sections/home/follower_indices_list/follower_indices_list.js +++ b/x-pack/plugins/cross_cluster_replication/public/app/sections/home/follower_indices_list/follower_indices_list.js @@ -9,13 +9,12 @@ import React, { PureComponent } from 'react'; import PropTypes from 'prop-types'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; -import { EuiButton, EuiEmptyPrompt, EuiText, EuiSpacer } from '@elastic/eui'; +import { EuiPageContent, EuiButton, EuiEmptyPrompt, EuiText, EuiSpacer } from '@elastic/eui'; import { reactRouterNavigate } from '../../../../../../../../src/plugins/kibana_react/public'; -import { extractQueryParams, SectionLoading } from '../../../../shared_imports'; +import { extractQueryParams, PageLoading, PageError } from '../../../../shared_imports'; import { trackUiMetric, METRIC_TYPE } from '../../../services/track_ui_metric'; import { API_STATUS, UIM_FOLLOWER_INDEX_LIST_LOAD } from '../../../constants'; -import { SectionError, SectionUnauthorized } from '../../../components'; import { FollowerIndicesTable, DetailPanel } from './components'; const REFRESH_RATE_MS = 30000; @@ -89,7 +88,12 @@ export class FollowerIndicesList extends PureComponent { renderEmpty() { return ( -
+ } /> -
+ ); } renderLoading() { return ( -
- - - -
+ + + ); } @@ -171,19 +173,22 @@ export class FollowerIndicesList extends PureComponent { if (!isAuthorized) { return ( - } - > - - + error={{ + error: ( + + ), + }} + /> ); } @@ -195,7 +200,7 @@ export class FollowerIndicesList extends PureComponent { } ); - return ; + return ; } if (isEmpty) { diff --git a/x-pack/plugins/cross_cluster_replication/public/shared_imports.ts b/x-pack/plugins/cross_cluster_replication/public/shared_imports.ts index 55a10749230c7..38838968ad212 100644 --- a/x-pack/plugins/cross_cluster_replication/public/shared_imports.ts +++ b/x-pack/plugins/cross_cluster_replication/public/shared_imports.ts @@ -10,4 +10,7 @@ export { indices, SectionLoading, PageError, + PageLoading, } from '../../../../src/plugins/es_ui_shared/public'; + +export { APP_WRAPPER_CLASS } from '../../../../src/core/public'; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/policy_table/policy_table.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/policy_table/policy_table.tsx index ba89d6c895d93..6ed7d42a694cc 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/policy_table/policy_table.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/policy_table/policy_table.tsx @@ -85,8 +85,10 @@ export const PolicyTable: React.FunctionComponent = ({ ); } + // Wrapping the actual contents inside a div, otherwise the table layout will get messed up + // if the table size changes (ie: applying a filter) due to the flex props of the page wrapper. content = ( - +
= ({ {tableContent} - +
); } else { return ( diff --git a/x-pack/plugins/snapshot_restore/__jest__/client_integration/helpers/home.helpers.ts b/x-pack/plugins/snapshot_restore/__jest__/client_integration/helpers/home.helpers.ts index 8a790342134cc..00cec284f3747 100644 --- a/x-pack/plugins/snapshot_restore/__jest__/client_integration/helpers/home.helpers.ts +++ b/x-pack/plugins/snapshot_restore/__jest__/client_integration/helpers/home.helpers.ts @@ -255,6 +255,7 @@ export type TestSubjects = | 'snapshotDetail.version' | 'snapshotLink' | 'snapshotList' + | 'snapshotListEmpty' | 'snapshotList.cell' | 'snapshotList.closeButton' | 'snapshotList.content' diff --git a/x-pack/plugins/snapshot_restore/__jest__/client_integration/home.test.ts b/x-pack/plugins/snapshot_restore/__jest__/client_integration/home.test.ts index c48d6d78d08f6..9f334ce4d49c8 100644 --- a/x-pack/plugins/snapshot_restore/__jest__/client_integration/home.test.ts +++ b/x-pack/plugins/snapshot_restore/__jest__/client_integration/home.test.ts @@ -121,7 +121,7 @@ describe('', () => { }); expect(exists('repositoryList')).toBe(false); - expect(exists('snapshotList')).toBe(true); + expect(exists('snapshotListEmpty')).toBe(true); }); }); }); diff --git a/x-pack/plugins/snapshot_restore/public/application/sections/home/policy_list/policy_list.tsx b/x-pack/plugins/snapshot_restore/public/application/sections/home/policy_list/policy_list.tsx index ca1f28c586522..63aae47e63837 100644 --- a/x-pack/plugins/snapshot_restore/public/application/sections/home/policy_list/policy_list.tsx +++ b/x-pack/plugins/snapshot_restore/public/application/sections/home/policy_list/policy_list.tsx @@ -8,12 +8,13 @@ import React, { Fragment, useEffect } from 'react'; import { FormattedMessage } from '@kbn/i18n/react'; import { RouteComponentProps } from 'react-router-dom'; -import { EuiEmptyPrompt, EuiButton, EuiCallOut, EuiSpacer } from '@elastic/eui'; +import { EuiPageContent, EuiEmptyPrompt, EuiButton, EuiCallOut, EuiSpacer } from '@elastic/eui'; import { reactRouterNavigate } from '../../../../../../../../src/plugins/kibana_react/public'; import { - SectionError, + PageLoading, + PageError, Error, WithPrivileges, NotAuthorizedSection, @@ -21,7 +22,6 @@ import { import { SlmPolicy } from '../../../../../common/types'; import { APP_SLM_CLUSTER_PRIVILEGES } from '../../../../../common'; -import { SectionLoading } from '../../../components'; import { BASE_PATH, UIM_POLICY_LIST_LOAD } from '../../../constants'; import { useDecodedParams } from '../../../lib'; import { useLoadPolicies, useLoadRetentionSettings } from '../../../services/http'; @@ -89,16 +89,16 @@ export const PolicyList: React.FunctionComponent + - + ); } else if (error) { content = ( - - - - } - body={ - -

+ + -

- - } - actions={ - - - - } - data-test-subj="emptyPrompt" - /> + + } + body={ + +

+ +

+ + } + actions={ + + + + } + data-test-subj="emptyPrompt" + /> + ); } else { const policySchedules = policies.map((policy: SlmPolicy) => policy.schedule); @@ -152,7 +159,7 @@ export const PolicyList: React.FunctionComponent policy.retention)); content = ( - +
{hasDuplicateSchedules ? ( - +
); } @@ -198,7 +205,7 @@ export const PolicyList: React.FunctionComponent `cluster.${name}`)}> {({ hasPrivileges, privilegesMissing }) => hasPrivileges ? ( -
+ <> {policyName ? ( ) : null} {content} -
+ ) : ( + - + ); } else if (error) { content = ( - - - - } - body={ - -

+ + -

- - } - actions={ - - - - } - data-test-subj="emptyPrompt" - /> + + } + body={ + +

+ +

+ + } + actions={ + + + + } + data-test-subj="emptyPrompt" + /> + ); } else { content = ( - +
+ +
); } return ( -
+ <> {repositoryName ? ( ) : null} {content} -
+ ); }; diff --git a/x-pack/plugins/snapshot_restore/public/application/sections/home/restore_list/restore_list.tsx b/x-pack/plugins/snapshot_restore/public/application/sections/home/restore_list/restore_list.tsx index 8dabaf4f29847..14044d3aaa161 100644 --- a/x-pack/plugins/snapshot_restore/public/application/sections/home/restore_list/restore_list.tsx +++ b/x-pack/plugins/snapshot_restore/public/application/sections/home/restore_list/restore_list.tsx @@ -8,6 +8,7 @@ import React, { useEffect, useState, Fragment } from 'react'; import { FormattedMessage } from '@kbn/i18n/react'; import { + EuiPageContent, EuiEmptyPrompt, EuiPopover, EuiButtonEmpty, @@ -23,10 +24,10 @@ import { APP_RESTORE_INDEX_PRIVILEGES } from '../../../../../common'; import { WithPrivileges, NotAuthorizedSection, - SectionError, + PageError, + PageLoading, Error, } from '../../../../shared_imports'; -import { SectionLoading } from '../../../components'; import { UIM_RESTORE_LIST_LOAD } from '../../../constants'; import { useLoadRestores } from '../../../services/http'; import { linkToSnapshots } from '../../../services/navigation'; @@ -74,18 +75,18 @@ export const RestoreList: React.FunctionComponent = () => { if (isLoading) { // Because we're polling for new data, we only want to hide the list during the initial fetch. content = ( - + - + ); } else if (error) { // If we get an error while polling we don't need to show it to the user because they can still // work with the table. content = ( - { } else { if (restores && restores.length === 0) { content = ( - - - - } - body={ - -

+ + - - - ), - }} + id="xpack.snapshotRestore.restoreList.emptyPromptTitle" + defaultMessage="No restored snapshots" /> -

- - } - data-test-subj="emptyPrompt" - /> + + } + body={ + +

+ + + + ), + }} + /> +

+ + } + data-test-subj="emptyPrompt" + /> + ); } else { content = ( - +
{ - +
); } } @@ -217,7 +225,7 @@ export const RestoreList: React.FunctionComponent = () => { `index.${name}`)}> {({ hasPrivileges, privilegesMissing }) => hasPrivileges ? ( -
{content}
+ content ) : ( - - + + + + + ); } else if (error) { content = ( - - - - } - body={ -

- - - - ), - }} - /> -

- } - /> + + + + + } + body={ +

+ + + + ), + }} + /> +

+ } + /> + ); } else if (repositories.length === 0) { content = ( - - - - } - body={ - <> -

+ + -

-

- + + } + body={ + <> +

- -

- - } - data-test-subj="emptyPrompt" - /> +

+

+ + + +

+ + } + data-test-subj="emptyPrompt" + /> + ); } else if (snapshots.length === 0) { content = ( - - - - } - body={ - `cluster.${name}`)}> - {({ hasPrivileges }) => - hasPrivileges ? ( - -

- - - - ), - }} - /> -

-

- {policies.length === 0 ? ( - - - - ) : ( - + + + + } + body={ + `cluster.${name}`)} + > + {({ hasPrivileges }) => + hasPrivileges ? ( + +

+ + + + ), + }} + /> +

+

+ {policies.length === 0 ? ( + + + + ) : ( + + + + )} +

+ + ) : ( + +

+ +

+

+ - - )} -

- - ) : ( - -

- -

-

- - {' '} - - -

- - ) - } - - } - data-test-subj="emptyPrompt" - /> + id="xpack.snapshotRestore.emptyPrompt.noSnapshotsDocLinkText" + defaultMessage="Learn how to create a snapshot" + />{' '} + + +

+ + ) + } + + } + data-test-subj="emptyPrompt" + /> + ); } else { const repositoryErrorsWarning = Object.keys(errors).length ? ( @@ -322,7 +350,7 @@ export const SnapshotList: React.FunctionComponent +
{repositoryErrorsWarning} - +
); } return ( -
+ <> {repositoryName && snapshotId ? ( ) : null} {content} -
+ ); }; diff --git a/x-pack/plugins/snapshot_restore/public/shared_imports.ts b/x-pack/plugins/snapshot_restore/public/shared_imports.ts index 759453edaba5d..84c195a51950b 100644 --- a/x-pack/plugins/snapshot_restore/public/shared_imports.ts +++ b/x-pack/plugins/snapshot_restore/public/shared_imports.ts @@ -13,6 +13,7 @@ export { NotAuthorizedSection, SectionError, PageError, + PageLoading, sendRequest, SendRequestConfig, SendRequestResponse, @@ -22,3 +23,5 @@ export { UseRequestConfig, WithPrivileges, } from '../../../../src/plugins/es_ui_shared/public'; + +export { APP_WRAPPER_CLASS } from '../../../../src/core/public'; From ac17ab1436986b37f9d0a3969aff70e728481cf7 Mon Sep 17 00:00:00 2001 From: Esteban Beltran Date: Tue, 29 Jun 2021 08:58:37 +0200 Subject: [PATCH 55/55] Add signal and abort controller to agent metadata and TakeAction button (#103217) --- .../containers/detection_engine/alerts/api.ts | 10 ++++++++-- .../alerts/use_host_isolation_status.tsx | 8 +++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/api.ts b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/api.ts index 05706981a681d..72a9bf6e84441 100644 --- a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/api.ts +++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/api.ts @@ -178,8 +178,14 @@ export const getCaseIdsFromAlertId = async ({ * * @param host id */ -export const getHostMetadata = async ({ agentId }: { agentId: string }): Promise => +export const getHostMetadata = async ({ + agentId, + signal, +}: { + agentId: string; + signal?: AbortSignal; +}): Promise => KibanaServices.get().http.fetch( resolvePathVariables(HOST_METADATA_GET_ROUTE, { id: agentId }), - { method: 'get' } + { method: 'GET', signal } ); diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_host_isolation_status.tsx b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_host_isolation_status.tsx index 3bdd8c9813785..259a377b10b79 100644 --- a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_host_isolation_status.tsx +++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_host_isolation_status.tsx @@ -38,18 +38,23 @@ export const useHostIsolationStatus = ({ const { addError } = useAppToasts(); useEffect(() => { + const abortCtrl = new AbortController(); // isMounted tracks if a component is mounted before changing state let isMounted = true; let fleetAgentId: string; const fetchData = async () => { try { - const metadataResponse = await getHostMetadata({ agentId }); + const metadataResponse = await getHostMetadata({ agentId, signal: abortCtrl.signal }); if (isMounted) { setIsIsolated(isEndpointHostIsolated(metadataResponse.metadata)); setAgentStatus(metadataResponse.host_status); fleetAgentId = metadataResponse.metadata.elastic.agent.id; } } catch (error) { + // don't show self-aborted requests errors to the user + if (error.name === 'AbortError') { + return; + } addError(error.message, { title: ISOLATION_STATUS_FAILURE }); } @@ -80,6 +85,7 @@ export const useHostIsolationStatus = ({ return () => { // updates to show component is unmounted isMounted = false; + abortCtrl.abort(); }; }, [addError, agentId]); return { loading, isIsolated, agentStatus, pendingIsolation, pendingUnisolation };