From dfea0fee21d93e73926092e32105d9e66fcc8c6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20Haro?= Date: Fri, 3 Sep 2021 12:32:59 +0100 Subject: [PATCH 01/20] [GET /api/status] Default to v8format and allow v7format=true (#110830) --- ...in-core-server.deprecationsservicesetup.md | 4 +- .../src/kbn_client/kbn_client_plugins.ts | 14 +- .../src/kbn_client/kbn_client_status.ts | 17 +- .../public/core_app/status/lib/load_status.ts | 2 +- .../core_usage_data_service.mock.ts | 6 +- .../core_usage_data_service.test.ts | 44 +++ .../core_usage_data_service.ts | 26 +- src/core/server/core_usage_data/index.ts | 10 +- src/core/server/core_usage_data/types.ts | 49 ++- .../integration_tests/client.test.ts | 4 +- src/core/server/index.ts | 12 +- src/core/server/internal_types.ts | 3 +- src/core/server/mocks.ts | 4 + src/core/server/plugins/plugin_context.ts | 3 + .../saved_objects/routes/bulk_create.ts | 4 +- .../server/saved_objects/routes/bulk_get.ts | 4 +- .../saved_objects/routes/bulk_update.ts | 4 +- .../server/saved_objects/routes/create.ts | 4 +- .../server/saved_objects/routes/delete.ts | 4 +- .../server/saved_objects/routes/export.ts | 4 +- src/core/server/saved_objects/routes/find.ts | 4 +- src/core/server/saved_objects/routes/get.ts | 4 +- .../server/saved_objects/routes/import.ts | 4 +- src/core/server/saved_objects/routes/index.ts | 4 +- .../server/saved_objects/routes/resolve.ts | 4 +- .../routes/resolve_import_errors.ts | 4 +- .../server/saved_objects/routes/update.ts | 4 +- .../saved_objects/saved_objects_service.ts | 4 +- src/core/server/server.api.md | 23 ++ src/core/server/server.ts | 2 + .../routes/integration_tests/status.test.ts | 310 +++++++++--------- src/core/server/status/routes/status.ts | 31 +- src/core/server/status/status_service.test.ts | 2 + src/core/server/status/status_service.ts | 4 + .../server/plugin.test.ts | 2 + .../kibana_usage_collection/server/plugin.ts | 1 + test/api_integration/apis/status/status.js | 5 +- .../test_suites/core_plugins/status.ts | 2 +- .../http/platform/status.ts | 2 +- 39 files changed, 412 insertions(+), 226 deletions(-) diff --git a/docs/development/core/server/kibana-plugin-core-server.deprecationsservicesetup.md b/docs/development/core/server/kibana-plugin-core-server.deprecationsservicesetup.md index 75732f59f1b3f..eb0dbb59e6c12 100644 --- a/docs/development/core/server/kibana-plugin-core-server.deprecationsservicesetup.md +++ b/docs/development/core/server/kibana-plugin-core-server.deprecationsservicesetup.md @@ -31,10 +31,10 @@ async function getDeprecations({ esClient, savedObjectsClient }: GetDeprecations // Example of a manual correctiveAction deprecations.push({ title: i18n.translate('xpack.timelion.deprecations.worksheetsTitle', { - defaultMessage: 'Found Timelion worksheets.' + defaultMessage: 'Timelion worksheets are deprecated' }), message: i18n.translate('xpack.timelion.deprecations.worksheetsMessage', { - defaultMessage: 'You have {count} Timelion worksheets. The Timelion app will be removed in 8.0. To continue using your Timelion worksheets, migrate them to a dashboard.', + defaultMessage: 'You have {count} Timelion worksheets. Migrate your Timelion worksheets to a dashboard to continue using them.', values: { count }, }), documentationUrl: diff --git a/packages/kbn-test/src/kbn_client/kbn_client_plugins.ts b/packages/kbn-test/src/kbn_client/kbn_client_plugins.ts index c730091c1478b..25c3d7e156e91 100644 --- a/packages/kbn-test/src/kbn_client/kbn_client_plugins.ts +++ b/packages/kbn-test/src/kbn_client/kbn_client_plugins.ts @@ -8,26 +8,14 @@ import { KbnClientStatus } from './kbn_client_status'; -const PLUGIN_STATUS_ID = /^plugin:(.+?)@/; - export class KbnClientPlugins { constructor(private readonly status: KbnClientStatus) {} /** * Get a list of plugin ids that are enabled on the server */ public async getEnabledIds() { - const pluginIds: string[] = []; const apiResp = await this.status.get(); - for (const status of apiResp.status.statuses) { - if (status.id) { - const match = status.id.match(PLUGIN_STATUS_ID); - if (match) { - pluginIds.push(match[1]); - } - } - } - - return pluginIds; + return Object.keys(apiResp.status.plugins); } } diff --git a/packages/kbn-test/src/kbn_client/kbn_client_status.ts b/packages/kbn-test/src/kbn_client/kbn_client_status.ts index 26c46917ae8dd..ed08b6b8cea15 100644 --- a/packages/kbn-test/src/kbn_client/kbn_client_status.ts +++ b/packages/kbn-test/src/kbn_client/kbn_client_status.ts @@ -9,13 +9,11 @@ import { KbnClientRequester } from './kbn_client_requester'; interface Status { - state: 'green' | 'red' | 'yellow'; - title?: string; - id?: string; - icon: string; - message: string; - uiColor: string; - since: string; + level: 'available' | 'degraded' | 'unavailable' | 'critical'; + summary: string; + detail?: string; + documentationUrl?: string; + meta?: Record; } interface ApiResponseStatus { @@ -29,7 +27,8 @@ interface ApiResponseStatus { }; status: { overall: Status; - statuses: Status[]; + core: Record; + plugins: Record; }; metrics: unknown; } @@ -55,6 +54,6 @@ export class KbnClientStatus { */ public async getOverallState() { const status = await this.get(); - return status.status.overall.state; + return status.status.overall.level; } } diff --git a/src/core/public/core_app/status/lib/load_status.ts b/src/core/public/core_app/status/lib/load_status.ts index a5cc18ffd6c16..e65764771f0fc 100644 --- a/src/core/public/core_app/status/lib/load_status.ts +++ b/src/core/public/core_app/status/lib/load_status.ts @@ -145,7 +145,7 @@ export async function loadStatus({ let response: StatusResponse; try { - response = await http.get('/api/status', { query: { v8format: true } }); + response = await http.get('/api/status'); } catch (e) { // API returns a 503 response if not all services are available. // In this case, we want to treat this as a successful API call, so that we can diff --git a/src/core/server/core_usage_data/core_usage_data_service.mock.ts b/src/core/server/core_usage_data/core_usage_data_service.mock.ts index 941ac5afacb40..331a3bbb9c028 100644 --- a/src/core/server/core_usage_data/core_usage_data_service.mock.ts +++ b/src/core/server/core_usage_data/core_usage_data_service.mock.ts @@ -10,12 +10,14 @@ import { PublicMethodsOf } from '@kbn/utility-types'; import { BehaviorSubject } from 'rxjs'; import { CoreUsageDataService } from './core_usage_data_service'; import { coreUsageStatsClientMock } from './core_usage_stats_client.mock'; -import { CoreUsageData, CoreUsageDataSetup, CoreUsageDataStart } from './types'; +import { CoreUsageData, InternalCoreUsageDataSetup, CoreUsageDataStart } from './types'; const createSetupContractMock = (usageStatsClient = coreUsageStatsClientMock.create()) => { - const setupContract: jest.Mocked = { + const setupContract: jest.Mocked = { registerType: jest.fn(), getClient: jest.fn().mockReturnValue(usageStatsClient), + registerUsageCounter: jest.fn(), + incrementUsageCounter: jest.fn(), }; return setupContract; }; diff --git a/src/core/server/core_usage_data/core_usage_data_service.test.ts b/src/core/server/core_usage_data/core_usage_data_service.test.ts index 478cfe5daff46..3c05069d3cd07 100644 --- a/src/core/server/core_usage_data/core_usage_data_service.test.ts +++ b/src/core/server/core_usage_data/core_usage_data_service.test.ts @@ -150,6 +150,50 @@ describe('CoreUsageDataService', () => { expect(usageStatsClient).toBeInstanceOf(CoreUsageStatsClient); }); }); + + describe('Usage Counter', () => { + it('registers a usage counter and uses it to increment the counters', async () => { + const http = httpServiceMock.createInternalSetupContract(); + const metrics = metricsServiceMock.createInternalSetupContract(); + const savedObjectsStartPromise = Promise.resolve( + savedObjectsServiceMock.createStartContract() + ); + const changedDeprecatedConfigPath$ = configServiceMock.create().getDeprecatedConfigPath$(); + const coreUsageData = service.setup({ + http, + metrics, + savedObjectsStartPromise, + changedDeprecatedConfigPath$, + }); + const myUsageCounter = { incrementCounter: jest.fn() }; + coreUsageData.registerUsageCounter(myUsageCounter); + coreUsageData.incrementUsageCounter({ counterName: 'test' }); + expect(myUsageCounter.incrementCounter).toHaveBeenCalledWith({ counterName: 'test' }); + }); + + it('swallows errors when provided increment counter fails', async () => { + const http = httpServiceMock.createInternalSetupContract(); + const metrics = metricsServiceMock.createInternalSetupContract(); + const savedObjectsStartPromise = Promise.resolve( + savedObjectsServiceMock.createStartContract() + ); + const changedDeprecatedConfigPath$ = configServiceMock.create().getDeprecatedConfigPath$(); + const coreUsageData = service.setup({ + http, + metrics, + savedObjectsStartPromise, + changedDeprecatedConfigPath$, + }); + const myUsageCounter = { + incrementCounter: jest.fn(() => { + throw new Error('Something is really wrong'); + }), + }; + coreUsageData.registerUsageCounter(myUsageCounter); + expect(() => coreUsageData.incrementUsageCounter({ counterName: 'test' })).not.toThrow(); + expect(myUsageCounter.incrementCounter).toHaveBeenCalledWith({ counterName: 'test' }); + }); + }); }); describe('start', () => { diff --git a/src/core/server/core_usage_data/core_usage_data_service.ts b/src/core/server/core_usage_data/core_usage_data_service.ts index 73f63d4d634df..ce9013d9437d6 100644 --- a/src/core/server/core_usage_data/core_usage_data_service.ts +++ b/src/core/server/core_usage_data/core_usage_data_service.ts @@ -27,7 +27,7 @@ import type { CoreServicesUsageData, CoreUsageData, CoreUsageDataStart, - CoreUsageDataSetup, + InternalCoreUsageDataSetup, ConfigUsageData, CoreConfigUsageData, } from './types'; @@ -39,6 +39,7 @@ import { LEGACY_URL_ALIAS_TYPE } from '../saved_objects/object_types'; import { CORE_USAGE_STATS_TYPE } from './constants'; import { CoreUsageStatsClient } from './core_usage_stats_client'; import { MetricsServiceSetup, OpsMetrics } from '..'; +import { CoreIncrementUsageCounter } from './types'; export type ExposedConfigsToUsage = Map>; @@ -86,7 +87,8 @@ const isCustomIndex = (index: string) => { return index !== '.kibana'; }; -export class CoreUsageDataService implements CoreService { +export class CoreUsageDataService + implements CoreService { private logger: Logger; private elasticsearchConfig?: ElasticsearchConfigType; private configService: CoreContext['configService']; @@ -98,6 +100,7 @@ export class CoreUsageDataService implements CoreService {}; // Initially set to noop constructor(core: CoreContext) { this.logger = core.logger.get('core-usage-stats-service'); @@ -495,7 +498,24 @@ export class CoreUsageDataService implements CoreService { + this.incrementUsageCounter = (params) => usageCounter.incrementCounter(params); + }, + incrementUsageCounter: (params) => { + try { + this.incrementUsageCounter(params); + } catch (e) { + // Self-defense mechanism since the handler is externally registered + this.logger.debug('Failed to increase the usage counter'); + this.logger.debug(e); + } + }, + }; + + return contract; } start({ savedObjects, elasticsearch, exposedConfigsToUsage }: StartDeps) { diff --git a/src/core/server/core_usage_data/index.ts b/src/core/server/core_usage_data/index.ts index a5c62c75f62d5..4687446bdb3a3 100644 --- a/src/core/server/core_usage_data/index.ts +++ b/src/core/server/core_usage_data/index.ts @@ -7,7 +7,15 @@ */ export { CORE_USAGE_STATS_TYPE, CORE_USAGE_STATS_ID } from './constants'; -export type { CoreUsageDataSetup, ConfigUsageData, CoreUsageDataStart } from './types'; +export type { + InternalCoreUsageDataSetup, + ConfigUsageData, + CoreUsageDataStart, + CoreUsageDataSetup, + CoreUsageCounter, + CoreIncrementUsageCounter, + CoreIncrementCounterParams, +} from './types'; export { CoreUsageDataService } from './core_usage_data_service'; export { CoreUsageStatsClient, REPOSITORY_RESOLVE_OUTCOME_STATS } from './core_usage_stats_client'; diff --git a/src/core/server/core_usage_data/types.ts b/src/core/server/core_usage_data/types.ts index 563a2a337cc8d..68e0b56c56db4 100644 --- a/src/core/server/core_usage_data/types.ts +++ b/src/core/server/core_usage_data/types.ts @@ -280,12 +280,59 @@ export interface CoreConfigUsageData { }; } +/** + * @internal Details about the counter to be incremented + */ +export interface CoreIncrementCounterParams { + /** The name of the counter **/ + counterName: string; + /** The counter type ("count" by default) **/ + counterType?: string; + /** Increment the counter by this number (1 if not specified) **/ + incrementBy?: number; +} + +/** + * @internal + * Method to call whenever an event occurs, so the counter can be increased. + */ +export type CoreIncrementUsageCounter = (params: CoreIncrementCounterParams) => void; + +/** + * @internal + * API to track whenever an event occurs, so the core can report them. + */ +export interface CoreUsageCounter { + /** @internal {@link CoreIncrementUsageCounter} **/ + incrementCounter: CoreIncrementUsageCounter; +} + /** @internal */ -export interface CoreUsageDataSetup { +export interface InternalCoreUsageDataSetup extends CoreUsageDataSetup { registerType( typeRegistry: ISavedObjectTypeRegistry & Pick ): void; getClient(): CoreUsageStatsClient; + + /** @internal {@link CoreIncrementUsageCounter} **/ + incrementUsageCounter: CoreIncrementUsageCounter; +} + +/** + * Internal API for registering the Usage Tracker used for Core's usage data payload. + * + * @note This API should never be used to drive application logic and is only + * intended for telemetry purposes. + * + * @internal + */ +export interface CoreUsageDataSetup { + /** + * @internal + * API for a usage tracker plugin to inject the {@link CoreUsageCounter} to use + * when tracking events. + */ + registerUsageCounter: (usageCounter: CoreUsageCounter) => void; } /** diff --git a/src/core/server/elasticsearch/integration_tests/client.test.ts b/src/core/server/elasticsearch/integration_tests/client.test.ts index 6e40c638614bd..83b20761df1ae 100644 --- a/src/core/server/elasticsearch/integration_tests/client.test.ts +++ b/src/core/server/elasticsearch/integration_tests/client.test.ts @@ -96,8 +96,8 @@ describe('fake elasticsearch', () => { test('should return unknown product when it cannot perform the Product check (503 response)', async () => { const resp = await supertest(kibanaHttpServer).get('/api/status').expect(503); - expect(resp.body.status.overall.state).toBe('red'); - expect(resp.body.status.statuses[0].message).toBe( + expect(resp.body.status.overall.level).toBe('critical'); + expect(resp.body.status.core.elasticsearch.summary).toBe( 'Unable to retrieve version information from Elasticsearch nodes. The client noticed that the server is not Elasticsearch and we do not support this unknown product.' ); }); diff --git a/src/core/server/index.ts b/src/core/server/index.ts index 1c3a0850d3b79..3a55d70109b8c 100644 --- a/src/core/server/index.ts +++ b/src/core/server/index.ts @@ -55,7 +55,7 @@ import { CapabilitiesSetup, CapabilitiesStart } from './capabilities'; import { MetricsServiceSetup, MetricsServiceStart } from './metrics'; import { StatusServiceSetup } from './status'; import { AppenderConfigType, appendersSchema, LoggingServiceSetup } from './logging'; -import { CoreUsageDataStart } from './core_usage_data'; +import { CoreUsageDataStart, CoreUsageDataSetup } from './core_usage_data'; import { I18nServiceSetup } from './i18n'; import { DeprecationsServiceSetup, DeprecationsClient } from './deprecations'; // Because of #79265 we need to explicitly import, then export these types for @@ -410,7 +410,13 @@ export type { export { ServiceStatusLevels } from './status'; export type { CoreStatus, ServiceStatus, ServiceStatusLevel, StatusServiceSetup } from './status'; -export type { CoreUsageDataStart } from './core_usage_data'; +export type { + CoreUsageDataSetup, + CoreUsageDataStart, + CoreUsageCounter, + CoreIncrementUsageCounter, + CoreIncrementCounterParams, +} from './core_usage_data'; /** * Plugin specific context passed to a route handler. @@ -500,6 +506,8 @@ export interface CoreSetup; + /** @internal {@link CoreUsageDataSetup} */ + coreUsageData: CoreUsageDataSetup; } /** diff --git a/src/core/server/internal_types.ts b/src/core/server/internal_types.ts index 8fc76e8b95743..29187c3963add 100644 --- a/src/core/server/internal_types.ts +++ b/src/core/server/internal_types.ts @@ -36,7 +36,7 @@ import { InternalRenderingServiceSetup } from './rendering'; import { InternalHttpResourcesPreboot, InternalHttpResourcesSetup } from './http_resources'; import { InternalStatusServiceSetup } from './status'; import { InternalLoggingServicePreboot, InternalLoggingServiceSetup } from './logging'; -import { CoreUsageDataStart } from './core_usage_data'; +import { CoreUsageDataStart, InternalCoreUsageDataSetup } from './core_usage_data'; import { I18nServiceSetup } from './i18n'; import { InternalDeprecationsServiceSetup, InternalDeprecationsServiceStart } from './deprecations'; import type { @@ -73,6 +73,7 @@ export interface InternalCoreSetup { logging: InternalLoggingServiceSetup; metrics: InternalMetricsServiceSetup; deprecations: InternalDeprecationsServiceSetup; + coreUsageData: InternalCoreUsageDataSetup; } /** diff --git a/src/core/server/mocks.ts b/src/core/server/mocks.ts index b53658b574939..f8b56e81ab188 100644 --- a/src/core/server/mocks.ts +++ b/src/core/server/mocks.ts @@ -169,6 +169,9 @@ function createCoreSetupMock({ metrics: metricsServiceMock.createSetupContract(), deprecations: deprecationsServiceMock.createSetupContract(), executionContext: executionContextServiceMock.createInternalSetupContract(), + coreUsageData: { + registerUsageCounter: coreUsageDataServiceMock.createSetupContract().registerUsageCounter, + }, getStartServices: jest .fn, object, any]>, []>() .mockResolvedValue([createCoreStartMock(), pluginStartDeps, pluginStartContract]), @@ -222,6 +225,7 @@ function createInternalCoreSetupMock() { metrics: metricsServiceMock.createInternalSetupContract(), deprecations: deprecationsServiceMock.createInternalSetupContract(), executionContext: executionContextServiceMock.createInternalSetupContract(), + coreUsageData: coreUsageDataServiceMock.createSetupContract(), }; return setupDeps; } diff --git a/src/core/server/plugins/plugin_context.ts b/src/core/server/plugins/plugin_context.ts index cbefdae525180..bdb4efde9b1fb 100644 --- a/src/core/server/plugins/plugin_context.ts +++ b/src/core/server/plugins/plugin_context.ts @@ -211,6 +211,9 @@ export function createPluginSetupContext( }, getStartServices: () => plugin.startDependencies, deprecations: deps.deprecations.getRegistry(plugin.name), + coreUsageData: { + registerUsageCounter: deps.coreUsageData.registerUsageCounter, + }, }; } diff --git a/src/core/server/saved_objects/routes/bulk_create.ts b/src/core/server/saved_objects/routes/bulk_create.ts index 344a0d151cfb9..f8438a70d0418 100644 --- a/src/core/server/saved_objects/routes/bulk_create.ts +++ b/src/core/server/saved_objects/routes/bulk_create.ts @@ -8,11 +8,11 @@ import { schema } from '@kbn/config-schema'; import { IRouter } from '../../http'; -import { CoreUsageDataSetup } from '../../core_usage_data'; +import { InternalCoreUsageDataSetup } from '../../core_usage_data'; import { catchAndReturnBoomErrors } from './utils'; interface RouteDependencies { - coreUsageData: CoreUsageDataSetup; + coreUsageData: InternalCoreUsageDataSetup; } export const registerBulkCreateRoute = (router: IRouter, { coreUsageData }: RouteDependencies) => { diff --git a/src/core/server/saved_objects/routes/bulk_get.ts b/src/core/server/saved_objects/routes/bulk_get.ts index cf051d6cd25cc..cffa69b06f4e4 100644 --- a/src/core/server/saved_objects/routes/bulk_get.ts +++ b/src/core/server/saved_objects/routes/bulk_get.ts @@ -8,11 +8,11 @@ import { schema } from '@kbn/config-schema'; import { IRouter } from '../../http'; -import { CoreUsageDataSetup } from '../../core_usage_data'; +import { InternalCoreUsageDataSetup } from '../../core_usage_data'; import { catchAndReturnBoomErrors } from './utils'; interface RouteDependencies { - coreUsageData: CoreUsageDataSetup; + coreUsageData: InternalCoreUsageDataSetup; } export const registerBulkGetRoute = (router: IRouter, { coreUsageData }: RouteDependencies) => { diff --git a/src/core/server/saved_objects/routes/bulk_update.ts b/src/core/server/saved_objects/routes/bulk_update.ts index de47ab9c59611..277673971dabe 100644 --- a/src/core/server/saved_objects/routes/bulk_update.ts +++ b/src/core/server/saved_objects/routes/bulk_update.ts @@ -8,11 +8,11 @@ import { schema } from '@kbn/config-schema'; import { IRouter } from '../../http'; -import { CoreUsageDataSetup } from '../../core_usage_data'; +import { InternalCoreUsageDataSetup } from '../../core_usage_data'; import { catchAndReturnBoomErrors } from './utils'; interface RouteDependencies { - coreUsageData: CoreUsageDataSetup; + coreUsageData: InternalCoreUsageDataSetup; } export const registerBulkUpdateRoute = (router: IRouter, { coreUsageData }: RouteDependencies) => { diff --git a/src/core/server/saved_objects/routes/create.ts b/src/core/server/saved_objects/routes/create.ts index 2fa7acfb6cab6..0e321aa7031f2 100644 --- a/src/core/server/saved_objects/routes/create.ts +++ b/src/core/server/saved_objects/routes/create.ts @@ -8,11 +8,11 @@ import { schema } from '@kbn/config-schema'; import { IRouter } from '../../http'; -import { CoreUsageDataSetup } from '../../core_usage_data'; +import { InternalCoreUsageDataSetup } from '../../core_usage_data'; import { catchAndReturnBoomErrors } from './utils'; interface RouteDependencies { - coreUsageData: CoreUsageDataSetup; + coreUsageData: InternalCoreUsageDataSetup; } export const registerCreateRoute = (router: IRouter, { coreUsageData }: RouteDependencies) => { diff --git a/src/core/server/saved_objects/routes/delete.ts b/src/core/server/saved_objects/routes/delete.ts index fe08acf23fd23..e8404ba7fc8cf 100644 --- a/src/core/server/saved_objects/routes/delete.ts +++ b/src/core/server/saved_objects/routes/delete.ts @@ -8,11 +8,11 @@ import { schema } from '@kbn/config-schema'; import { IRouter } from '../../http'; -import { CoreUsageDataSetup } from '../../core_usage_data'; +import { InternalCoreUsageDataSetup } from '../../core_usage_data'; import { catchAndReturnBoomErrors } from './utils'; interface RouteDependencies { - coreUsageData: CoreUsageDataSetup; + coreUsageData: InternalCoreUsageDataSetup; } export const registerDeleteRoute = (router: IRouter, { coreUsageData }: RouteDependencies) => { diff --git a/src/core/server/saved_objects/routes/export.ts b/src/core/server/saved_objects/routes/export.ts index e0293a4522fc1..e224f30a1bb02 100644 --- a/src/core/server/saved_objects/routes/export.ts +++ b/src/core/server/saved_objects/routes/export.ts @@ -11,7 +11,7 @@ import stringify from 'json-stable-stringify'; import { createPromiseFromStreams, createMapStream, createConcatStream } from '@kbn/utils'; import { IRouter, KibanaRequest } from '../../http'; -import { CoreUsageDataSetup } from '../../core_usage_data'; +import { InternalCoreUsageDataSetup } from '../../core_usage_data'; import { SavedObjectConfig } from '../saved_objects_config'; import { SavedObjectsExportByTypeOptions, @@ -22,7 +22,7 @@ import { validateTypes, validateObjects, catchAndReturnBoomErrors } from './util interface RouteDependencies { config: SavedObjectConfig; - coreUsageData: CoreUsageDataSetup; + coreUsageData: InternalCoreUsageDataSetup; } type EitherExportOptions = SavedObjectsExportByTypeOptions | SavedObjectsExportByObjectOptions; diff --git a/src/core/server/saved_objects/routes/find.ts b/src/core/server/saved_objects/routes/find.ts index d21039db30e5f..6e009f80bda7d 100644 --- a/src/core/server/saved_objects/routes/find.ts +++ b/src/core/server/saved_objects/routes/find.ts @@ -8,11 +8,11 @@ import { schema } from '@kbn/config-schema'; import { IRouter } from '../../http'; -import { CoreUsageDataSetup } from '../../core_usage_data'; +import { InternalCoreUsageDataSetup } from '../../core_usage_data'; import { catchAndReturnBoomErrors } from './utils'; interface RouteDependencies { - coreUsageData: CoreUsageDataSetup; + coreUsageData: InternalCoreUsageDataSetup; } export const registerFindRoute = (router: IRouter, { coreUsageData }: RouteDependencies) => { diff --git a/src/core/server/saved_objects/routes/get.ts b/src/core/server/saved_objects/routes/get.ts index f28822d95d814..ae0656599a1e2 100644 --- a/src/core/server/saved_objects/routes/get.ts +++ b/src/core/server/saved_objects/routes/get.ts @@ -8,11 +8,11 @@ import { schema } from '@kbn/config-schema'; import { IRouter } from '../../http'; -import { CoreUsageDataSetup } from '../../core_usage_data'; +import { InternalCoreUsageDataSetup } from '../../core_usage_data'; import { catchAndReturnBoomErrors } from './utils'; interface RouteDependencies { - coreUsageData: CoreUsageDataSetup; + coreUsageData: InternalCoreUsageDataSetup; } export const registerGetRoute = (router: IRouter, { coreUsageData }: RouteDependencies) => { diff --git a/src/core/server/saved_objects/routes/import.ts b/src/core/server/saved_objects/routes/import.ts index 6f75bcf9fd5bf..d373dd5e63bc6 100644 --- a/src/core/server/saved_objects/routes/import.ts +++ b/src/core/server/saved_objects/routes/import.ts @@ -10,14 +10,14 @@ import { Readable } from 'stream'; import { extname } from 'path'; import { schema } from '@kbn/config-schema'; import { IRouter } from '../../http'; -import { CoreUsageDataSetup } from '../../core_usage_data'; +import { InternalCoreUsageDataSetup } from '../../core_usage_data'; import { SavedObjectConfig } from '../saved_objects_config'; import { SavedObjectsImportError } from '../import'; import { catchAndReturnBoomErrors, createSavedObjectsStreamFromNdJson } from './utils'; interface RouteDependencies { config: SavedObjectConfig; - coreUsageData: CoreUsageDataSetup; + coreUsageData: InternalCoreUsageDataSetup; } interface FileStream extends Readable { diff --git a/src/core/server/saved_objects/routes/index.ts b/src/core/server/saved_objects/routes/index.ts index 930e02de7657a..889edfb66a20f 100644 --- a/src/core/server/saved_objects/routes/index.ts +++ b/src/core/server/saved_objects/routes/index.ts @@ -7,7 +7,7 @@ */ import { InternalHttpServiceSetup } from '../../http'; -import { CoreUsageDataSetup } from '../../core_usage_data'; +import { InternalCoreUsageDataSetup } from '../../core_usage_data'; import { Logger } from '../../logging'; import { SavedObjectConfig } from '../saved_objects_config'; import { IKibanaMigrator } from '../migrations'; @@ -34,7 +34,7 @@ export function registerRoutes({ migratorPromise, }: { http: InternalHttpServiceSetup; - coreUsageData: CoreUsageDataSetup; + coreUsageData: InternalCoreUsageDataSetup; logger: Logger; config: SavedObjectConfig; migratorPromise: Promise; diff --git a/src/core/server/saved_objects/routes/resolve.ts b/src/core/server/saved_objects/routes/resolve.ts index ba409f7db7b67..78e85d17fe1fa 100644 --- a/src/core/server/saved_objects/routes/resolve.ts +++ b/src/core/server/saved_objects/routes/resolve.ts @@ -8,10 +8,10 @@ import { schema } from '@kbn/config-schema'; import { IRouter } from '../../http'; -import { CoreUsageDataSetup } from '../../core_usage_data'; +import { InternalCoreUsageDataSetup } from '../../core_usage_data'; interface RouteDependencies { - coreUsageData: CoreUsageDataSetup; + coreUsageData: InternalCoreUsageDataSetup; } export const registerResolveRoute = (router: IRouter, { coreUsageData }: RouteDependencies) => { diff --git a/src/core/server/saved_objects/routes/resolve_import_errors.ts b/src/core/server/saved_objects/routes/resolve_import_errors.ts index a05c7d30b91fd..f1fe2e9cfe431 100644 --- a/src/core/server/saved_objects/routes/resolve_import_errors.ts +++ b/src/core/server/saved_objects/routes/resolve_import_errors.ts @@ -11,13 +11,13 @@ import { Readable } from 'stream'; import { schema } from '@kbn/config-schema'; import { chain } from 'lodash'; import { IRouter } from '../../http'; -import { CoreUsageDataSetup } from '../../core_usage_data'; +import { InternalCoreUsageDataSetup } from '../../core_usage_data'; import { SavedObjectConfig } from '../saved_objects_config'; import { SavedObjectsImportError } from '../import'; import { catchAndReturnBoomErrors, createSavedObjectsStreamFromNdJson } from './utils'; interface RouteDependencies { config: SavedObjectConfig; - coreUsageData: CoreUsageDataSetup; + coreUsageData: InternalCoreUsageDataSetup; } interface FileStream extends Readable { diff --git a/src/core/server/saved_objects/routes/update.ts b/src/core/server/saved_objects/routes/update.ts index b6dd9dc8e9ace..f21fc183cdade 100644 --- a/src/core/server/saved_objects/routes/update.ts +++ b/src/core/server/saved_objects/routes/update.ts @@ -8,12 +8,12 @@ import { schema } from '@kbn/config-schema'; import { IRouter } from '../../http'; -import { CoreUsageDataSetup } from '../../core_usage_data'; +import { InternalCoreUsageDataSetup } from '../../core_usage_data'; import type { SavedObjectsUpdateOptions } from '../service/saved_objects_client'; import { catchAndReturnBoomErrors } from './utils'; interface RouteDependencies { - coreUsageData: CoreUsageDataSetup; + coreUsageData: InternalCoreUsageDataSetup; } export const registerUpdateRoute = (router: IRouter, { coreUsageData }: RouteDependencies) => { diff --git a/src/core/server/saved_objects/saved_objects_service.ts b/src/core/server/saved_objects/saved_objects_service.ts index b25e51da3a749..074eae55acaea 100644 --- a/src/core/server/saved_objects/saved_objects_service.ts +++ b/src/core/server/saved_objects/saved_objects_service.ts @@ -16,7 +16,7 @@ import { } from './'; import { KibanaMigrator, IKibanaMigrator } from './migrations'; import { CoreContext } from '../core_context'; -import { CoreUsageDataSetup } from '../core_usage_data'; +import { InternalCoreUsageDataSetup } from '../core_usage_data'; import { ElasticsearchClient, InternalElasticsearchServiceSetup, @@ -250,7 +250,7 @@ export interface SavedObjectsRepositoryFactory { export interface SavedObjectsSetupDeps { http: InternalHttpServiceSetup; elasticsearch: InternalElasticsearchServiceSetup; - coreUsageData: CoreUsageDataSetup; + coreUsageData: InternalCoreUsageDataSetup; } interface WrappedClientFactoryWrapper { diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md index aa421fe393059..5abd1171a1936 100644 --- a/src/core/server/server.api.md +++ b/src/core/server/server.api.md @@ -359,6 +359,16 @@ export interface CoreEnvironmentUsageData { // @internal (undocumented) export type CoreId = symbol; +// @internal +export interface CoreIncrementCounterParams { + counterName: string; + counterType?: string; + incrementBy?: number; +} + +// @internal +export type CoreIncrementUsageCounter = (params: CoreIncrementCounterParams) => void; + // @public export interface CorePreboot { // (undocumented) @@ -395,6 +405,8 @@ export interface CoreSetup void; +} + // @internal export interface CoreUsageDataStart { // (undocumented) diff --git a/src/core/server/server.ts b/src/core/server/server.ts index 27c35031db46f..865cc71a7e26b 100644 --- a/src/core/server/server.ts +++ b/src/core/server/server.ts @@ -243,6 +243,7 @@ export class Server { environment: environmentSetup, http: httpSetup, metrics: metricsSetup, + coreUsageData: coreUsageDataSetup, }); const renderingSetup = await this.rendering.setup({ @@ -278,6 +279,7 @@ export class Server { logging: loggingSetup, metrics: metricsSetup, deprecations: deprecationsSetup, + coreUsageData: coreUsageDataSetup, }; const pluginsSetup = await this.plugins.setup(coreSetup); diff --git a/src/core/server/status/routes/integration_tests/status.test.ts b/src/core/server/status/routes/integration_tests/status.test.ts index 645ce0b241612..082be62f8dc09 100644 --- a/src/core/server/status/routes/integration_tests/status.test.ts +++ b/src/core/server/status/routes/integration_tests/status.test.ts @@ -29,6 +29,7 @@ describe('GET /api/status', () => { let server: HttpService; let httpSetup: InternalHttpServiceSetup; let metrics: jest.Mocked; + let incrementUsageCounter: jest.Mock; const setupServer = async ({ allowAnonymous = true }: { allowAnonymous?: boolean } = {}) => { const coreContext = createCoreContext({ coreId }); @@ -50,6 +51,8 @@ describe('GET /api/status', () => { d: { level: ServiceStatusLevels.critical, summary: 'd is critical' }, }); + incrementUsageCounter = jest.fn(); + const router = httpSetup.createRouter(''); registerStatusRoute({ router, @@ -71,6 +74,7 @@ describe('GET /api/status', () => { core$: status.core$, plugins$: pluginsStatus$, }, + incrementUsageCounter, }); // Register dummy auth provider for testing auth @@ -137,69 +141,75 @@ describe('GET /api/status', () => { }); describe('legacy status format', () => { - it('returns legacy status format when no query params provided', async () => { - await setupServer(); - const result = await supertest(httpSetup.server.listener).get('/api/status').expect(200); - expect(result.body.status).toEqual({ - overall: { + const legacyFormat = { + overall: { + icon: 'success', + nickname: 'Looking good', + since: expect.any(String), + state: 'green', + title: 'Green', + uiColor: 'secondary', + }, + statuses: [ + { + icon: 'success', + id: 'core:elasticsearch@9.9.9', + message: 'Service is working', + since: expect.any(String), + state: 'green', + uiColor: 'secondary', + }, + { icon: 'success', - nickname: 'Looking good', + id: 'core:savedObjects@9.9.9', + message: 'Service is working', since: expect.any(String), state: 'green', - title: 'Green', uiColor: 'secondary', }, - statuses: [ - { - icon: 'success', - id: 'core:elasticsearch@9.9.9', - message: 'Service is working', - since: expect.any(String), - state: 'green', - uiColor: 'secondary', - }, - { - icon: 'success', - id: 'core:savedObjects@9.9.9', - message: 'Service is working', - since: expect.any(String), - state: 'green', - uiColor: 'secondary', - }, - { - icon: 'success', - id: 'plugin:a@9.9.9', - message: 'a is available', - since: expect.any(String), - state: 'green', - uiColor: 'secondary', - }, - { - icon: 'warning', - id: 'plugin:b@9.9.9', - message: 'b is degraded', - since: expect.any(String), - state: 'yellow', - uiColor: 'warning', - }, - { - icon: 'danger', - id: 'plugin:c@9.9.9', - message: 'c is unavailable', - since: expect.any(String), - state: 'red', - uiColor: 'danger', - }, - { - icon: 'danger', - id: 'plugin:d@9.9.9', - message: 'd is critical', - since: expect.any(String), - state: 'red', - uiColor: 'danger', - }, - ], - }); + { + icon: 'success', + id: 'plugin:a@9.9.9', + message: 'a is available', + since: expect.any(String), + state: 'green', + uiColor: 'secondary', + }, + { + icon: 'warning', + id: 'plugin:b@9.9.9', + message: 'b is degraded', + since: expect.any(String), + state: 'yellow', + uiColor: 'warning', + }, + { + icon: 'danger', + id: 'plugin:c@9.9.9', + message: 'c is unavailable', + since: expect.any(String), + state: 'red', + uiColor: 'danger', + }, + { + icon: 'danger', + id: 'plugin:d@9.9.9', + message: 'd is critical', + since: expect.any(String), + state: 'red', + uiColor: 'danger', + }, + ], + }; + + it('returns legacy status format when v7format=true is provided', async () => { + await setupServer(); + const result = await supertest(httpSetup.server.listener) + .get('/api/status?v7format=true') + .expect(200); + expect(result.body.status).toEqual(legacyFormat); + expect(incrementUsageCounter).toHaveBeenCalledTimes(1); + expect(incrementUsageCounter).toHaveBeenCalledWith({ counterName: 'status_v7format' }); }); it('returns legacy status format when v8format=false is provided', async () => { @@ -207,109 +217,105 @@ describe('GET /api/status', () => { const result = await supertest(httpSetup.server.listener) .get('/api/status?v8format=false') .expect(200); - expect(result.body.status).toEqual({ - overall: { - icon: 'success', - nickname: 'Looking good', - since: expect.any(String), - state: 'green', - title: 'Green', - uiColor: 'secondary', - }, - statuses: [ - { - icon: 'success', - id: 'core:elasticsearch@9.9.9', - message: 'Service is working', - since: expect.any(String), - state: 'green', - uiColor: 'secondary', - }, - { - icon: 'success', - id: 'core:savedObjects@9.9.9', - message: 'Service is working', - since: expect.any(String), - state: 'green', - uiColor: 'secondary', - }, - { - icon: 'success', - id: 'plugin:a@9.9.9', - message: 'a is available', - since: expect.any(String), - state: 'green', - uiColor: 'secondary', - }, - { - icon: 'warning', - id: 'plugin:b@9.9.9', - message: 'b is degraded', - since: expect.any(String), - state: 'yellow', - uiColor: 'warning', - }, - { - icon: 'danger', - id: 'plugin:c@9.9.9', - message: 'c is unavailable', - since: expect.any(String), - state: 'red', - uiColor: 'danger', - }, - { - icon: 'danger', - id: 'plugin:d@9.9.9', - message: 'd is critical', - since: expect.any(String), - state: 'red', - uiColor: 'danger', - }, - ], - }); + expect(result.body.status).toEqual(legacyFormat); + expect(incrementUsageCounter).toHaveBeenCalledTimes(1); + expect(incrementUsageCounter).toHaveBeenCalledWith({ counterName: 'status_v7format' }); }); }); describe('v8format', () => { - it('returns new status format when v8format=true is provided', async () => { - await setupServer(); - const result = await supertest(httpSetup.server.listener) - .get('/api/status?v8format=true') - .expect(200); - expect(result.body.status).toEqual({ - core: { - elasticsearch: { - level: 'available', - summary: 'Service is working', - }, - savedObjects: { - level: 'available', - summary: 'Service is working', - }, + const newFormat = { + core: { + elasticsearch: { + level: 'available', + summary: 'Service is working', }, - overall: { + savedObjects: { level: 'available', summary: 'Service is working', }, - plugins: { - a: { - level: 'available', - summary: 'a is available', - }, - b: { - level: 'degraded', - summary: 'b is degraded', - }, - c: { - level: 'unavailable', - summary: 'c is unavailable', - }, - d: { - level: 'critical', - summary: 'd is critical', - }, + }, + overall: { + level: 'available', + summary: 'Service is working', + }, + plugins: { + a: { + level: 'available', + summary: 'a is available', + }, + b: { + level: 'degraded', + summary: 'b is degraded', + }, + c: { + level: 'unavailable', + summary: 'c is unavailable', }, - }); + d: { + level: 'critical', + summary: 'd is critical', + }, + }, + }; + + it('returns new status format when no query params are provided', async () => { + await setupServer(); + const result = await supertest(httpSetup.server.listener).get('/api/status').expect(200); + expect(result.body.status).toEqual(newFormat); + expect(incrementUsageCounter).not.toHaveBeenCalled(); + }); + + it('returns new status format when v8format=true is provided', async () => { + await setupServer(); + const result = await supertest(httpSetup.server.listener) + .get('/api/status?v8format=true') + .expect(200); + expect(result.body.status).toEqual(newFormat); + expect(incrementUsageCounter).not.toHaveBeenCalled(); + }); + + it('returns new status format when v7format=false is provided', async () => { + await setupServer(); + const result = await supertest(httpSetup.server.listener) + .get('/api/status?v7format=false') + .expect(200); + expect(result.body.status).toEqual(newFormat); + expect(incrementUsageCounter).not.toHaveBeenCalled(); + }); + }); + + describe('invalid query parameters', () => { + it('v8format=true and v7format=true', async () => { + await setupServer(); + await supertest(httpSetup.server.listener) + .get('/api/status?v8format=true&v7format=true') + .expect(400); + expect(incrementUsageCounter).not.toHaveBeenCalled(); + }); + + it('v8format=true and v7format=false', async () => { + await setupServer(); + await supertest(httpSetup.server.listener) + .get('/api/status?v8format=true&v7format=false') + .expect(400); + expect(incrementUsageCounter).not.toHaveBeenCalled(); + }); + + it('v8format=false and v7format=false', async () => { + await setupServer(); + await supertest(httpSetup.server.listener) + .get('/api/status?v8format=false&v7format=false') + .expect(400); + expect(incrementUsageCounter).not.toHaveBeenCalled(); + }); + + it('v8format=false and v7format=true', async () => { + await setupServer(); + await supertest(httpSetup.server.listener) + .get('/api/status?v8format=false&v7format=true') + .expect(400); + expect(incrementUsageCounter).not.toHaveBeenCalled(); }); }); }); diff --git a/src/core/server/status/routes/status.ts b/src/core/server/status/routes/status.ts index 43a596bd1e0ec..861b41c58a893 100644 --- a/src/core/server/status/routes/status.ts +++ b/src/core/server/status/routes/status.ts @@ -12,6 +12,7 @@ import { schema } from '@kbn/config-schema'; import { IRouter } from '../../http'; import { MetricsServiceSetup } from '../../metrics'; +import type { CoreIncrementUsageCounter } from '../../core_usage_data/types'; import { ServiceStatus, CoreStatus, ServiceStatusLevels } from '../types'; import { PluginName } from '../../plugins'; import { calculateLegacyStatus, LegacyStatusInfo } from '../legacy_status'; @@ -34,6 +35,7 @@ interface Deps { core$: Observable; plugins$: Observable>; }; + incrementUsageCounter: CoreIncrementUsageCounter; } interface StatusInfo { @@ -47,7 +49,13 @@ interface StatusHttpBody extends Omit { status: StatusInfo | LegacyStatusInfo; } -export const registerStatusRoute = ({ router, config, metrics, status }: Deps) => { +export const registerStatusRoute = ({ + router, + config, + metrics, + status, + incrementUsageCounter, +}: Deps) => { // Since the status.plugins$ observable is not subscribed to elsewhere, we need to subscribe it here to eagerly load // the plugins status when Kibana starts up so this endpoint responds quickly on first boot. const combinedStatus$ = new ReplaySubject< @@ -63,9 +71,19 @@ export const registerStatusRoute = ({ router, config, metrics, status }: Deps) = tags: ['api'], // ensures that unauthenticated calls receive a 401 rather than a 302 redirect to login page }, validate: { - query: schema.object({ - v8format: schema.boolean({ defaultValue: false }), - }), + query: schema.object( + { + v7format: schema.maybe(schema.boolean()), + v8format: schema.maybe(schema.boolean()), + }, + { + validate: ({ v7format, v8format }) => { + if (typeof v7format === 'boolean' && typeof v8format === 'boolean') { + return `provide only one format option: v7format or v8format`; + } + }, + } + ), }, }, async (context, req, res) => { @@ -73,14 +91,17 @@ export const registerStatusRoute = ({ router, config, metrics, status }: Deps) = const versionWithoutSnapshot = version.replace(SNAPSHOT_POSTFIX, ''); const [overall, core, plugins] = await combinedStatus$.pipe(first()).toPromise(); + const { v8format = true, v7format = false } = req.query ?? {}; + let statusInfo: StatusInfo | LegacyStatusInfo; - if (req.query?.v8format) { + if (!v7format && v8format) { statusInfo = { overall, core, plugins, }; } else { + incrementUsageCounter({ counterName: 'status_v7format' }); statusInfo = calculateLegacyStatus({ overall, core, diff --git a/src/core/server/status/status_service.test.ts b/src/core/server/status/status_service.test.ts index 4ead81a6638dd..9148f69e079aa 100644 --- a/src/core/server/status/status_service.test.ts +++ b/src/core/server/status/status_service.test.ts @@ -18,6 +18,7 @@ import { httpServiceMock } from '../http/http_service.mock'; import { mockRouter, RouterMock } from '../http/router/router.mock'; import { metricsServiceMock } from '../metrics/metrics_service.mock'; import { configServiceMock } from '../config/mocks'; +import { coreUsageDataServiceMock } from '../core_usage_data/core_usage_data_service.mock'; expect.addSnapshotSerializer(ServiceStatusLevelSnapshotSerializer); @@ -51,6 +52,7 @@ describe('StatusService', () => { environment: environmentServiceMock.createSetupContract(), http: httpServiceMock.createInternalSetupContract(), metrics: metricsServiceMock.createInternalSetupContract(), + coreUsageData: coreUsageDataServiceMock.createSetupContract(), ...overrides, }; }; diff --git a/src/core/server/status/status_service.ts b/src/core/server/status/status_service.ts index 8e9db30bbebd3..107074bdb98b1 100644 --- a/src/core/server/status/status_service.ts +++ b/src/core/server/status/status_service.ts @@ -20,6 +20,7 @@ import { PluginName } from '../plugins'; import { InternalMetricsServiceSetup } from '../metrics'; import { registerStatusRoute } from './routes'; import { InternalEnvironmentServiceSetup } from '../environment'; +import type { InternalCoreUsageDataSetup } from '../core_usage_data'; import { config, StatusConfigType } from './status_config'; import { ServiceStatus, CoreStatus, InternalStatusServiceSetup } from './types'; @@ -38,6 +39,7 @@ interface SetupDeps { http: InternalHttpServiceSetup; metrics: InternalMetricsServiceSetup; savedObjects: Pick; + coreUsageData: Pick; } export class StatusService implements CoreService { @@ -61,6 +63,7 @@ export class StatusService implements CoreService { metrics, savedObjects, environment, + coreUsageData, }: SetupDeps) { const statusConfig = await this.config$.pipe(take(1)).toPromise(); const core$ = this.setupCoreStatus({ elasticsearch, savedObjects }); @@ -101,6 +104,7 @@ export class StatusService implements CoreService { plugins$: this.pluginsStatus.getAll$(), core$, }, + incrementUsageCounter: coreUsageData.incrementUsageCounter, }; const router = http.createRouter(''); diff --git a/src/plugins/kibana_usage_collection/server/plugin.test.ts b/src/plugins/kibana_usage_collection/server/plugin.test.ts index 1584366a42dc1..75c323ba0332f 100644 --- a/src/plugins/kibana_usage_collection/server/plugin.test.ts +++ b/src/plugins/kibana_usage_collection/server/plugin.test.ts @@ -42,6 +42,8 @@ describe('kibana_usage_collection', () => { expect(pluginInstance.setup(coreSetup, { usageCollection })).toBe(undefined); + expect(coreSetup.coreUsageData.registerUsageCounter).toHaveBeenCalled(); + await expect( Promise.all( usageCollectors.map(async (usageCollector) => { diff --git a/src/plugins/kibana_usage_collection/server/plugin.ts b/src/plugins/kibana_usage_collection/server/plugin.ts index dadb4283e84a7..275dcc761125e 100644 --- a/src/plugins/kibana_usage_collection/server/plugin.ts +++ b/src/plugins/kibana_usage_collection/server/plugin.ts @@ -73,6 +73,7 @@ export class KibanaUsageCollectionPlugin implements Plugin { public setup(coreSetup: CoreSetup, { usageCollection }: KibanaUsageCollectionPluginsDepsSetup) { usageCollection.createUsageCounter('uiCounters'); this.eventLoopUsageCounter = usageCollection.createUsageCounter('eventLoop'); + coreSetup.coreUsageData.registerUsageCounter(usageCollection.createUsageCounter('core')); this.registerUsageCollectors( usageCollection, coreSetup, diff --git a/test/api_integration/apis/status/status.js b/test/api_integration/apis/status/status.js index 22076b2cddbc5..e1545c448fce8 100644 --- a/test/api_integration/apis/status/status.js +++ b/test/api_integration/apis/status/status.js @@ -25,9 +25,10 @@ export default function ({ getService }) { expect(body.version.build_number).to.be.a('number'); expect(body.status.overall).to.be.an('object'); - expect(body.status.overall.state).to.be('green'); + expect(body.status.overall.level).to.be('available'); - expect(body.status.statuses).to.be.an('array'); + expect(body.status.core).to.be.an('object'); + expect(body.status.plugins).to.be.an('object'); expect(body.metrics.collection_interval_in_millis).to.be.a('number'); diff --git a/test/plugin_functional/test_suites/core_plugins/status.ts b/test/plugin_functional/test_suites/core_plugins/status.ts index 2b0f15cb39273..10ca8c6722046 100644 --- a/test/plugin_functional/test_suites/core_plugins/status.ts +++ b/test/plugin_functional/test_suites/core_plugins/status.ts @@ -16,7 +16,7 @@ export default function ({ getService, getPageObjects }: PluginFunctionalProvide const delay = (ms: number) => new Promise((r) => setTimeout(r, ms)); const getStatus = async (pluginName?: string) => { - const resp = await supertest.get('/api/status?v8format=true'); + const resp = await supertest.get('/api/status'); if (pluginName) { return resp.body.status.plugins[pluginName]; diff --git a/test/server_integration/http/platform/status.ts b/test/server_integration/http/platform/status.ts index 0dcf82c9bea9e..e443ce3f31cbf 100644 --- a/test/server_integration/http/platform/status.ts +++ b/test/server_integration/http/platform/status.ts @@ -18,7 +18,7 @@ export default function ({ getService }: FtrProviderContext) { const retry = getService('retry'); const getStatus = async (pluginName: string): Promise => { - const resp = await supertest.get('/api/status?v8format=true'); + const resp = await supertest.get('/api/status'); return resp.body.status.plugins[pluginName]; }; From df8ed81195f4772f203538ab1c5e4f5bd9d60871 Mon Sep 17 00:00:00 2001 From: ymao1 Date: Fri, 3 Sep 2021 07:35:17 -0400 Subject: [PATCH 02/20] Adding experimental to event log mentions in the docs (#110876) --- .../alerting/troubleshooting/alerting-common-issues.asciidoc | 2 +- docs/user/alerting/troubleshooting/event-log-index.asciidoc | 2 ++ .../alerting-production-considerations.asciidoc | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/user/alerting/troubleshooting/alerting-common-issues.asciidoc b/docs/user/alerting/troubleshooting/alerting-common-issues.asciidoc index c57e9876a4118..408b18143f27f 100644 --- a/docs/user/alerting/troubleshooting/alerting-common-issues.asciidoc +++ b/docs/user/alerting/troubleshooting/alerting-common-issues.asciidoc @@ -68,7 +68,7 @@ Rules are taking a long time to execute and are impacting the overall health of [IMPORTANT] ============================================== -By default, only users with a `superuser` role can query the {kib} event log because it is a system index. To enable additional users to execute this query, assign `read` privileges to the `.kibana-event-log*` index. +By default, only users with a `superuser` role can query the experimental[] {kib} event log because it is a system index. To enable additional users to execute this query, assign `read` privileges to the `.kibana-event-log*` index. ============================================== *Solution* diff --git a/docs/user/alerting/troubleshooting/event-log-index.asciidoc b/docs/user/alerting/troubleshooting/event-log-index.asciidoc index fa5b5831c04ee..393b982b279f5 100644 --- a/docs/user/alerting/troubleshooting/event-log-index.asciidoc +++ b/docs/user/alerting/troubleshooting/event-log-index.asciidoc @@ -2,6 +2,8 @@ [[event-log-index]] === Event log index +experimental[] + Use the event log index to determine: * Whether a rule successfully ran but its associated actions did not diff --git a/docs/user/production-considerations/alerting-production-considerations.asciidoc b/docs/user/production-considerations/alerting-production-considerations.asciidoc index 57cc2a72a8895..cd8a60a1d5fe3 100644 --- a/docs/user/production-considerations/alerting-production-considerations.asciidoc +++ b/docs/user/production-considerations/alerting-production-considerations.asciidoc @@ -54,6 +54,8 @@ Predicting the buffer required to account for actions depends heavily on the rul [[event-log-ilm]] === Event log index lifecycle managment +experimental[] + Alerts and actions log activity in a set of "event log" indices. These indices are configured with an index lifecycle management (ILM) policy, which you can customize. The default policy rolls over the index when it reaches 50GB, or after 30 days. Indices over 90 days old are deleted. The name of the index policy is `kibana-event-log-policy`. {kib} creates the index policy on startup, if it doesn't already exist. The index policy can be customized for your environment, but {kib} never modifies the index policy after creating it. From 5b4d26557179665130b9f1c61cc9a07138666325 Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Fri, 3 Sep 2021 13:47:36 +0200 Subject: [PATCH 03/20] [Security Solution][Endpoint] Use super date picker instead of date range picker (#108722) * Use super date picker instead of date range picker fixes elastic/security-team/issues/1571 * fix test target Super date picker's `data-test-subj` prop gets garbled and doesn't show up in rendered DOM. In other words, the component is entirely void of a data-test-subj attribute. * make auto refresh work!! fixes https://github.com/elastic/security-team/issues/1571 * set max width as per mock fixes elastic/security-team/issues/1571 * show a callout to inform users to select different date ranges fixes elastic/security-team/issues/1571 * persist recently used date ranges on the component only fixes elastic/security-team/issues/1571 * use commonly used ranges from default common security solution ranges fixes elastic/security-team/issues/1571 * Better align date picker * full width panel for date picker so content flows below it review comments * mock time picker settings for tests * use eui token for bg color review comment * persist recently used dates fixes elastic/security-team/issues/1571 * persist date range selection over new endpoint selection review comments * remove obsolete local state since update button is not visible. review comments * fix bg color for dark mode and relative path * update relative path review comments * cleanup - the action doesn't allow for undefined start and end dates anyway refs 28a859ab3a5fbbd9a14f4009ded671b969f2dc09 * fix types after sync * update test title * add a test for callout when empty data * fix lint * show update button when dates are changed Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../common/endpoint/schema/actions.ts | 4 +- .../common/endpoint/types/actions.ts | 4 +- .../management/pages/endpoint_hosts/mocks.ts | 2 + .../pages/endpoint_hosts/store/action.ts | 19 ++- .../pages/endpoint_hosts/store/builders.ts | 9 +- .../pages/endpoint_hosts/store/index.test.ts | 7 + .../endpoint_hosts/store/middleware.test.ts | 6 + .../pages/endpoint_hosts/store/middleware.ts | 4 +- .../pages/endpoint_hosts/store/reducer.ts | 63 +++++++- .../management/pages/endpoint_hosts/types.ts | 10 +- .../pages/endpoint_hosts/utils.test.ts | 14 +- .../management/pages/endpoint_hosts/utils.ts | 11 +- .../activity_log_date_range_picker/index.tsx | 153 +++++++++++------- .../view/details/endpoint_activity_log.tsx | 13 ++ .../view/details/endpoints.stories.tsx | 2 + .../pages/endpoint_hosts/view/index.test.tsx | 124 +++++++++++--- .../pages/endpoint_hosts/view/translations.ts | 21 +-- .../endpoint/routes/actions/audit_log.test.ts | 20 +-- .../server/endpoint/services/actions.ts | 19 +-- 19 files changed, 360 insertions(+), 145 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/schema/actions.ts b/x-pack/plugins/security_solution/common/endpoint/schema/actions.ts index 98cb7729c9440..69fce914cb1d5 100644 --- a/x-pack/plugins/security_solution/common/endpoint/schema/actions.ts +++ b/x-pack/plugins/security_solution/common/endpoint/schema/actions.ts @@ -23,8 +23,8 @@ export const EndpointActionLogRequestSchema = { query: schema.object({ page: schema.number({ defaultValue: 1, min: 1 }), page_size: schema.number({ defaultValue: 10, min: 1, max: 100 }), - start_date: schema.maybe(schema.string()), - end_date: schema.maybe(schema.string()), + start_date: schema.string(), + end_date: schema.string(), }), params: schema.object({ agent_id: schema.string(), diff --git a/x-pack/plugins/security_solution/common/endpoint/types/actions.ts b/x-pack/plugins/security_solution/common/endpoint/types/actions.ts index d49868aae9227..c6d30825c21c9 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types/actions.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types/actions.ts @@ -65,8 +65,8 @@ export type ActivityLogEntry = ActivityLogAction | ActivityLogActionResponse; export interface ActivityLog { page: number; pageSize: number; - startDate?: string; - endDate?: string; + startDate: string; + endDate: string; data: ActivityLogEntry[]; } diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/mocks.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/mocks.ts index 0ac73df6704c8..9c557f83012bf 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/mocks.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/mocks.ts @@ -126,6 +126,8 @@ export const endpointActivityLogHttpMock = httpHandlerMockFactory => { disabled: false, page: 1, pageSize: 50, - startDate: undefined, - endDate: undefined, + startDate: 'now-1d', + endDate: 'now', isInvalidDateRange: false, + autoRefreshOptions: { + enabled: false, + duration: DEFAULT_POLL_INTERVAL, + }, + recentlyUsedDateRanges: [], }, logData: createUninitialisedResourceState(), }, diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/index.test.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/index.test.ts index 7fbe2dfc0a099..49ba88fd47717 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/index.test.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/index.test.ts @@ -48,7 +48,14 @@ describe('EndpointList store concerns', () => { disabled: false, page: 1, pageSize: 50, + startDate: 'now-1d', + endDate: 'now', isInvalidDateRange: false, + autoRefreshOptions: { + enabled: false, + duration: DEFAULT_POLL_INTERVAL, + }, + recentlyUsedDateRanges: [], }, logData: { type: 'UninitialisedResourceState' }, }, diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.test.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.test.ts index e51fe15e7130f..83d3e62cf98f2 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.test.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.test.ts @@ -267,6 +267,8 @@ describe('endpoint list middleware', () => { payload: { page, pageSize: 50, + startDate: 'now-1d', + endDate: 'now', }, }); }; @@ -311,6 +313,8 @@ describe('endpoint list middleware', () => { expect(mockedApis.responseProvider.activityLogResponse).toHaveBeenCalledWith({ path: expect.any(String), query: { + end_date: 'now', + start_date: 'now-1d', page: 1, page_size: 50, }, @@ -396,6 +400,8 @@ describe('endpoint list middleware', () => { query: { page: 3, page_size: 50, + start_date: 'now-1d', + end_date: 'now', }, }); }); diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.ts index df4361a6048a8..6b88183db6841 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.ts @@ -640,12 +640,12 @@ async function endpointDetailsActivityLogChangedMiddleware({ }); try { - const { page, pageSize } = getActivityLogDataPaging(getState()); + const { page, pageSize, startDate, endDate } = getActivityLogDataPaging(getState()); const route = resolvePathVariables(ENDPOINT_ACTION_LOG_ROUTE, { agent_id: selectedAgent(getState()), }); const activityLog = await coreStart.http.get(route, { - query: { page, page_size: pageSize }, + query: { page, page_size: pageSize, start_date: startDate, end_date: endDate }, }); dispatch({ type: 'endpointDetailsActivityLogChanged', diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/reducer.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/reducer.ts index 02d2adce833cf..b16caf00b4e28 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/reducer.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/reducer.ts @@ -24,6 +24,7 @@ import { AppAction } from '../../../../common/store/actions'; import { ImmutableReducer } from '../../../../common/store'; import { Immutable } from '../../../../../common/endpoint/types'; import { createUninitialisedResourceState, isUninitialisedResourceState } from '../../../state'; +import { DEFAULT_POLL_INTERVAL } from '../../../common/constants'; type StateReducer = ImmutableReducer; type CaseReducer = ( @@ -172,7 +173,11 @@ export const endpointListReducer: StateReducer = (state = initialEndpointPageSta }, }, }; - } else if (action.type === 'endpointDetailsActivityLogUpdatePaging') { + } else if ( + action.type === 'endpointDetailsActivityLogUpdatePaging' || + action.type === 'endpointDetailsActivityLogUpdateIsInvalidDateRange' || + action.type === 'userUpdatedActivityLogRefreshOptions' + ) { return { ...state, endpointDetails: { @@ -186,7 +191,7 @@ export const endpointListReducer: StateReducer = (state = initialEndpointPageSta }, }, }; - } else if (action.type === 'endpointDetailsActivityLogUpdateIsInvalidDateRange') { + } else if (action.type === 'userUpdatedActivityLogRecentlyUsedDateRanges') { return { ...state, endpointDetails: { @@ -195,7 +200,7 @@ export const endpointListReducer: StateReducer = (state = initialEndpointPageSta ...state.endpointDetails.activityLog, paging: { ...state.endpointDetails.activityLog.paging, - ...action.payload, + recentlyUsedDateRanges: action.payload, }, }, }, @@ -315,9 +320,16 @@ export const endpointListReducer: StateReducer = (state = initialEndpointPageSta logData: createUninitialisedResourceState(), paging: { disabled: false, + isInvalidDateRange: false, page: 1, pageSize: 50, - isInvalidDateRange: false, + startDate: 'now-1d', + endDate: 'now', + autoRefreshOptions: { + enabled: false, + duration: DEFAULT_POLL_INTERVAL, + }, + recentlyUsedDateRanges: [], }, }; @@ -337,7 +349,16 @@ export const endpointListReducer: StateReducer = (state = initialEndpointPageSta ...stateUpdates, endpointDetails: { ...state.endpointDetails, - activityLog, + activityLog: { + ...activityLog, + paging: { + ...activityLog.paging, + startDate: state.endpointDetails.activityLog.paging.startDate, + endDate: state.endpointDetails.activityLog.paging.endDate, + recentlyUsedDateRanges: + state.endpointDetails.activityLog.paging.recentlyUsedDateRanges, + }, + }, hostDetails: { ...state.endpointDetails.hostDetails, detailsError: undefined, @@ -355,7 +376,16 @@ export const endpointListReducer: StateReducer = (state = initialEndpointPageSta ...stateUpdates, endpointDetails: { ...state.endpointDetails, - activityLog, + activityLog: { + ...activityLog, + paging: { + ...activityLog.paging, + startDate: state.endpointDetails.activityLog.paging.startDate, + endDate: state.endpointDetails.activityLog.paging.endDate, + recentlyUsedDateRanges: + state.endpointDetails.activityLog.paging.recentlyUsedDateRanges, + }, + }, hostDetails: { ...state.endpointDetails.hostDetails, detailsLoading: !isNotLoadingDetails, @@ -372,7 +402,16 @@ export const endpointListReducer: StateReducer = (state = initialEndpointPageSta ...stateUpdates, endpointDetails: { ...state.endpointDetails, - activityLog, + activityLog: { + ...activityLog, + paging: { + ...activityLog.paging, + startDate: state.endpointDetails.activityLog.paging.startDate, + endDate: state.endpointDetails.activityLog.paging.endDate, + recentlyUsedDateRanges: + state.endpointDetails.activityLog.paging.recentlyUsedDateRanges, + }, + }, hostDetails: { ...state.endpointDetails.hostDetails, detailsLoading: true, @@ -391,7 +430,15 @@ export const endpointListReducer: StateReducer = (state = initialEndpointPageSta ...stateUpdates, endpointDetails: { ...state.endpointDetails, - activityLog, + activityLog: { + ...activityLog, + paging: { + ...activityLog.paging, + startDate: state.endpointDetails.activityLog.paging.startDate, + endDate: state.endpointDetails.activityLog.paging.endDate, + recentlyUsedDateRanges: state.endpointDetails.activityLog.paging.recentlyUsedDateRanges, + }, + }, hostDetails: { ...state.endpointDetails.hostDetails, detailsError: undefined, diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/types.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/types.ts index 82057af233e43..dd0bc79f1ba52 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/types.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/types.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { EuiSuperDatePickerRecentRange } from '@elastic/eui'; import { ActivityLog, HostInfo, @@ -41,9 +42,14 @@ export interface EndpointState { disabled?: boolean; page: number; pageSize: number; - startDate?: string; - endDate?: string; + startDate: string; + endDate: string; isInvalidDateRange: boolean; + autoRefreshOptions: { + enabled: boolean; + duration: number; + }; + recentlyUsedDateRanges: EuiSuperDatePickerRecentRange[]; }; logData: AsyncResourceState; }; diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/utils.test.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/utils.test.ts index fa2aaaa16ae37..ee723bd0bf0f5 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/utils.test.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/utils.test.ts @@ -10,12 +10,14 @@ import { getIsInvalidDateRange } from './utils'; describe('utils', () => { describe('getIsInvalidDateRange', () => { - it('should return FALSE when either dates are undefined', () => { - expect(getIsInvalidDateRange({})).toBe(false); - expect(getIsInvalidDateRange({ startDate: moment().subtract(1, 'd').toISOString() })).toBe( - false - ); - expect(getIsInvalidDateRange({ endDate: moment().toISOString() })).toBe(false); + it('should return FALSE when startDate is before endDate', () => { + expect(getIsInvalidDateRange({ startDate: 'now-1d', endDate: 'now' })).toBe(false); + expect( + getIsInvalidDateRange({ + startDate: moment().subtract(1, 'd').toISOString(), + endDate: moment().toISOString(), + }) + ).toBe(false); }); it('should return TRUE when startDate is after endDate', () => { diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/utils.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/utils.ts index e2d619743c83b..1bfb99c68ef66 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/utils.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/utils.ts @@ -5,6 +5,7 @@ * 2.0. */ +import dateMath from '@elastic/datemath'; import moment from 'moment'; import { HostInfo, HostMetadata } from '../../../../common/endpoint/types'; @@ -29,12 +30,12 @@ export const getIsInvalidDateRange = ({ startDate, endDate, }: { - startDate?: string; - endDate?: string; + startDate: string; + endDate: string; }) => { - if (startDate && endDate) { - const start = moment(startDate); - const end = moment(endDate); + const start = moment(dateMath.parse(startDate)); + const end = moment(dateMath.parse(endDate)); + if (start.isValid() && end.isValid()) { return start.isAfter(end); } return false; diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/components/activity_log_date_range_picker/index.tsx b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/components/activity_log_date_range_picker/index.tsx index e921078539303..30ab082559c7b 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/components/activity_log_date_range_picker/index.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/components/activity_log_date_range_picker/index.tsx @@ -8,95 +8,140 @@ import { useDispatch } from 'react-redux'; import React, { memo, useCallback } from 'react'; import styled from 'styled-components'; -import moment from 'moment'; -import { EuiFlexGroup, EuiFlexItem, EuiDatePicker, EuiDatePickerRange } from '@elastic/eui'; +import dateMath from '@elastic/datemath'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiSuperDatePicker, + EuiSuperDatePickerRecentRange, +} from '@elastic/eui'; -import * as i18 from '../../../translations'; import { useEndpointSelector } from '../../../hooks'; -import { getActivityLogDataPaging } from '../../../../store/selectors'; +import { + getActivityLogDataPaging, + getActivityLogRequestLoading, +} from '../../../../store/selectors'; +import { DEFAULT_TIMEPICKER_QUICK_RANGES } from '../../../../../../../../common/constants'; +import { useUiSetting$ } from '../../../../../../../common/lib/kibana'; + +interface Range { + from: string; + to: string; + display: string; +} const DatePickerWrapper = styled.div` width: ${(props) => props.theme.eui.fractions.single.percentage}; - background: white; + max-width: 350px; `; const StickyFlexItem = styled(EuiFlexItem)` - max-width: 350px; + background: ${(props) => `${props.theme.eui.euiHeaderBackgroundColor}`}; position: sticky; - top: ${(props) => props.theme.eui.euiSizeM}; + top: 0; z-index: 1; - padding: ${(props) => `0 ${props.theme.eui.paddingSizes.m}`}; + padding: ${(props) => `${props.theme.eui.paddingSizes.m}`}; `; export const DateRangePicker = memo(() => { const dispatch = useDispatch(); - const { page, pageSize, startDate, endDate, isInvalidDateRange } = useEndpointSelector( - getActivityLogDataPaging - ); + const { + page, + pageSize, + startDate, + endDate, + autoRefreshOptions, + recentlyUsedDateRanges, + } = useEndpointSelector(getActivityLogDataPaging); + + const activityLogLoading = useEndpointSelector(getActivityLogRequestLoading); - const onChangeStartDate = useCallback( - (date) => { + const dispatchActionUpdateActivityLogPaging = useCallback( + async ({ start, end }) => { dispatch({ type: 'endpointDetailsActivityLogUpdatePaging', payload: { disabled: false, page, pageSize, - startDate: date ? date?.toISOString() : undefined, - endDate: endDate ? endDate : undefined, + startDate: dateMath.parse(start)?.toISOString(), + endDate: dateMath.parse(end)?.toISOString(), }, }); }, - [dispatch, endDate, page, pageSize] + [dispatch, page, pageSize] ); - const onChangeEndDate = useCallback( - (date) => { + const onRefreshChange = useCallback( + (evt) => { dispatch({ - type: 'endpointDetailsActivityLogUpdatePaging', + type: 'userUpdatedActivityLogRefreshOptions', payload: { - disabled: false, - page, - pageSize, - startDate: startDate ? startDate : undefined, - endDate: date ? date.toISOString() : undefined, + autoRefreshOptions: { enabled: !evt.isPaused, duration: evt.refreshInterval }, }, }); }, - [dispatch, startDate, page, pageSize] + [dispatch] ); + const onRefresh = useCallback(() => { + dispatch({ + type: 'endpointDetailsActivityLogUpdatePaging', + payload: { + disabled: false, + page, + pageSize, + startDate, + endDate, + }, + }); + }, [dispatch, page, pageSize, startDate, endDate]); + + const onTimeChange = useCallback( + ({ start: newStart, end: newEnd }) => { + const newRecentlyUsedDateRanges = [ + { start: newStart, end: newEnd }, + ...recentlyUsedDateRanges + .filter( + (recentlyUsedRange) => + !(recentlyUsedRange.start === newStart && recentlyUsedRange.end === newEnd) + ) + .slice(0, 9), + ]; + dispatch({ + type: 'userUpdatedActivityLogRecentlyUsedDateRanges', + payload: newRecentlyUsedDateRanges, + }); + + dispatchActionUpdateActivityLogPaging({ start: newStart, end: newEnd }); + }, + [dispatch, recentlyUsedDateRanges, dispatchActionUpdateActivityLogPaging] + ); + + const [quickRanges] = useUiSetting$(DEFAULT_TIMEPICKER_QUICK_RANGES); + const commonlyUsedRanges = !quickRanges.length + ? [] + : quickRanges.map(({ from, to, display }) => ({ + start: from, + end: to, + label: display, + })); + return ( - - + + - - } - endDateControl={ - - } + diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/endpoint_activity_log.tsx b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/endpoint_activity_log.tsx index 5172b59450e03..f0b6b5fbc8962 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/endpoint_activity_log.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/endpoint_activity_log.tsx @@ -9,11 +9,13 @@ import React, { memo, useCallback, useEffect, useMemo, useRef } from 'react'; import styled from 'styled-components'; import { + EuiCallOut, EuiText, EuiFlexGroup, EuiFlexItem, EuiLoadingContent, EuiEmptyPrompt, + EuiSpacer, } from '@elastic/eui'; import { useDispatch } from 'react-redux'; import { LogEntry } from './components/log_entry'; @@ -114,6 +116,17 @@ export const EndpointActivityLog = memo( <> + {!isPagingDisabled && activityLogLoaded && !activityLogData.length && ( + <> + + + + )} {activityLogLoaded && activityLogData.map((logEntry) => ( diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/endpoints.stories.tsx b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/endpoints.stories.tsx index 372bd4491d7d4..123a51e5a52bd 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/endpoints.stories.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/endpoints.stories.tsx @@ -22,6 +22,8 @@ export const dummyEndpointActivityLog = ( data: { page: 1, pageSize: 50, + startDate: moment().subtract(5, 'day').fromNow().toString(), + endDate: moment().toString(), data: [ { type: 'action', diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/index.test.tsx b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/index.test.tsx index 996198568ad27..ea999334ee771 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/index.test.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/index.test.tsx @@ -10,6 +10,8 @@ import * as reactTestingLibrary from '@testing-library/react'; import { EndpointList } from './index'; import '../../../../common/mock/match_media'; +import { createUseUiSetting$Mock } from '../../../../../public/common/lib/kibana/kibana_react.mock'; + import { mockEndpointDetailsApiResult, mockEndpointResultList, @@ -28,7 +30,7 @@ import { EndpointDocGenerator } from '../../../../../common/endpoint/generate_da import { POLICY_STATUS_TO_HEALTH_COLOR, POLICY_STATUS_TO_TEXT } from './host_constants'; import { mockPolicyResultList } from '../../policy/store/test_mock_utils'; import { getEndpointDetailsPath } from '../../../common/routing'; -import { KibanaServices, useKibana, useToasts } from '../../../../common/lib/kibana'; +import { KibanaServices, useKibana, useToasts, useUiSetting$ } from '../../../../common/lib/kibana'; import { hostIsolationHttpMocks } from '../../../../common/lib/endpoint_isolation/mocks'; import { createFailedResourceState, @@ -40,7 +42,11 @@ import { import { getCurrentIsolationRequestState } from '../store/selectors'; import { licenseService } from '../../../../common/hooks/use_license'; import { FleetActionGenerator } from '../../../../../common/endpoint/data_generators/fleet_action_generator'; -import { APP_PATH, MANAGEMENT_PATH } from '../../../../../common/constants'; +import { + APP_PATH, + MANAGEMENT_PATH, + DEFAULT_TIMEPICKER_QUICK_RANGES, +} from '../../../../../common/constants'; import { TransformStats, TRANSFORM_STATE } from '../types'; import { metadataTransformPrefix } from '../../../../../common/endpoint/constants'; @@ -63,6 +69,59 @@ jest.mock('../../policy/store/services/ingest', () => { sendGetEndpointSecurityPackage: () => Promise.resolve({}), }; }); +const mockUseUiSetting$ = useUiSetting$ as jest.Mock; +const timepickerRanges = [ + { + from: 'now/d', + to: 'now/d', + display: 'Today', + }, + { + from: 'now/w', + to: 'now/w', + display: 'This week', + }, + { + from: 'now-15m', + to: 'now', + display: 'Last 15 minutes', + }, + { + from: 'now-30m', + to: 'now', + display: 'Last 30 minutes', + }, + { + from: 'now-1h', + to: 'now', + display: 'Last 1 hour', + }, + { + from: 'now-24h', + to: 'now', + display: 'Last 24 hours', + }, + { + from: 'now-7d', + to: 'now', + display: 'Last 7 days', + }, + { + from: 'now-30d', + to: 'now', + display: 'Last 30 days', + }, + { + from: 'now-90d', + to: 'now', + display: 'Last 90 days', + }, + { + from: 'now-1y', + to: 'now', + display: 'Last 1 year', + }, +]; jest.mock('../../../../common/lib/kibana'); jest.mock('../../../../common/hooks/use_license'); @@ -759,6 +818,14 @@ describe('when on the endpoint list page', () => { disconnect: jest.fn(), })); + mockUseUiSetting$.mockImplementation((key, defaultValue) => { + const useUiSetting$Mock = createUseUiSetting$Mock(); + + return key === DEFAULT_TIMEPICKER_QUICK_RANGES + ? [timepickerRanges, jest.fn()] + : useUiSetting$Mock(key, defaultValue); + }); + const fleetActionGenerator = new FleetActionGenerator('seed'); const responseData = fleetActionGenerator.generateResponse({ agent_id: agentId, @@ -766,9 +833,12 @@ describe('when on the endpoint list page', () => { const actionData = fleetActionGenerator.generate({ agents: [agentId], }); + getMockData = () => ({ page: 1, pageSize: 50, + startDate: 'now-1d', + endDate: 'now', data: [ { type: 'response', @@ -838,7 +908,7 @@ describe('when on the endpoint list page', () => { expect(emptyState).not.toBe(null); }); - it('should display empty state when no log data', async () => { + it('should not display empty state when no log data', async () => { const activityLogTab = await renderResult.findByTestId('activity_log'); reactTestingLibrary.act(() => { reactTestingLibrary.fireEvent.click(activityLogTab); @@ -848,36 +918,39 @@ describe('when on the endpoint list page', () => { dispatchEndpointDetailsActivityLogChanged('success', { page: 1, pageSize: 50, + startDate: 'now-1d', + endDate: 'now', data: [], }); }); const emptyState = await renderResult.queryByTestId('activityLogEmpty'); - expect(emptyState).not.toBe(null); + expect(emptyState).toBe(null); + + const superDatePicker = await renderResult.queryByTestId('activityLogSuperDatePicker'); + expect(superDatePicker).not.toBe(null); }); - it('should not display empty state with no log data while date range filter is active', async () => { - const activityLogTab = await renderResult.findByTestId('activity_log'); + it('should display activity log when tab is loaded using the URL', async () => { + const userChangedUrlChecker = middlewareSpy.waitForAction('userChangedUrl'); reactTestingLibrary.act(() => { - reactTestingLibrary.fireEvent.click(activityLogTab); + history.push( + `${MANAGEMENT_PATH}/endpoints?page_index=0&page_size=10&selected_endpoint=1&show=activity_log` + ); }); + const changedUrlAction = await userChangedUrlChecker; + expect(changedUrlAction.payload.search).toEqual( + '?page_index=0&page_size=10&selected_endpoint=1&show=activity_log' + ); await middlewareSpy.waitForAction('endpointDetailsActivityLogChanged'); reactTestingLibrary.act(() => { - dispatchEndpointDetailsActivityLogChanged('success', { - page: 1, - pageSize: 50, - startDate: new Date().toISOString(), - data: [], - }); + dispatchEndpointDetailsActivityLogChanged('success', getMockData()); }); - - const emptyState = await renderResult.queryByTestId('activityLogEmpty'); - const dateRangePicker = await renderResult.queryByTestId('activityLogDateRangePicker'); - expect(emptyState).toBe(null); - expect(dateRangePicker).not.toBe(null); + const logEntries = await renderResult.queryAllByTestId('timelineEntry'); + expect(logEntries.length).toEqual(2); }); - it('should display activity log when tab is loaded using the URL', async () => { + it('should display a callout message if no log data', async () => { const userChangedUrlChecker = middlewareSpy.waitForAction('userChangedUrl'); reactTestingLibrary.act(() => { history.push( @@ -890,10 +963,17 @@ describe('when on the endpoint list page', () => { ); await middlewareSpy.waitForAction('endpointDetailsActivityLogChanged'); reactTestingLibrary.act(() => { - dispatchEndpointDetailsActivityLogChanged('success', getMockData()); + dispatchEndpointDetailsActivityLogChanged('success', { + page: 1, + pageSize: 50, + startDate: 'now-1d', + endDate: 'now', + data: [], + }); }); - const logEntries = await renderResult.queryAllByTestId('timelineEntry'); - expect(logEntries.length).toEqual(2); + + const activityLogCallout = await renderResult.findByTestId('activityLogNoDataCallout'); + expect(activityLogCallout).not.toBeNull(); }); }); diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/translations.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/translations.ts index 57ad3e4808bd5..c8a29eed3fda7 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/translations.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/translations.ts @@ -15,20 +15,6 @@ export const ACTIVITY_LOG = { tabTitle: i18n.translate('xpack.securitySolution.endpointDetails.activityLog', { defaultMessage: 'Activity Log', }), - datePicker: { - startDate: i18n.translate( - 'xpack.securitySolution.endpointDetails.activityLog.datePicker.startDate', - { - defaultMessage: 'Pick a start date', - } - ), - endDate: i18n.translate( - 'xpack.securitySolution.endpointDetails.activityLog.datePicker.endDate', - { - defaultMessage: 'Pick an end date', - } - ), - }, LogEntry: { endOfLog: i18n.translate( 'xpack.securitySolution.endpointDetails.activityLog.logEntry.action.endOfLog', @@ -36,6 +22,13 @@ export const ACTIVITY_LOG = { defaultMessage: 'Nothing more to show', } ), + dateRangeMessage: i18n.translate( + 'xpack.securitySolution.endpointDetails.activityLog.logEntry.dateRangeMessage.title', + { + defaultMessage: + 'Nothing to show for selected date range, please select another and try again.', + } + ), emptyState: { title: i18n.translate( 'xpack.securitySolution.endpointDetails.activityLog.logEntry.emptyState.title', diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/actions/audit_log.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/actions/audit_log.test.ts index 83f38bc904576..4bd63c83169e5 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/actions/audit_log.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/actions/audit_log.test.ts @@ -48,19 +48,13 @@ describe('Action Log API', () => { }).not.toThrow(); }); - it('should work without query params', () => { + it('should not work when no params while requesting with query params', () => { expect(() => { EndpointActionLogRequestSchema.query.validate({}); - }).not.toThrow(); - }); - - it('should work with query params', () => { - expect(() => { - EndpointActionLogRequestSchema.query.validate({ page: 10, page_size: 100 }); - }).not.toThrow(); + }).toThrow(); }); - it('should work with all query params', () => { + it('should work with all required query params', () => { expect(() => { EndpointActionLogRequestSchema.query.validate({ page: 10, @@ -71,24 +65,24 @@ describe('Action Log API', () => { }).not.toThrow(); }); - it('should work with just startDate', () => { + it('should not work without endDate', () => { expect(() => { EndpointActionLogRequestSchema.query.validate({ page: 1, page_size: 100, start_date: new Date(new Date().setDate(new Date().getDate() - 1)).toISOString(), // yesterday }); - }).not.toThrow(); + }).toThrow(); }); - it('should work with just endDate', () => { + it('should not work without startDate', () => { expect(() => { EndpointActionLogRequestSchema.query.validate({ page: 1, page_size: 100, end_date: new Date().toISOString(), // today }); - }).not.toThrow(); + }).toThrow(); }); it('should not work without allowed page and page_size params', () => { diff --git a/x-pack/plugins/security_solution/server/endpoint/services/actions.ts b/x-pack/plugins/security_solution/server/endpoint/services/actions.ts index 80fb1c5d9c7b0..a04a6eea5ab65 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/actions.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/actions.ts @@ -31,8 +31,8 @@ export const getAuditLogResponse = async ({ elasticAgentId: string; page: number; pageSize: number; - startDate?: string; - endDate?: string; + startDate: string; + endDate: string; context: SecuritySolutionRequestHandlerContext; logger: Logger; }): Promise => { @@ -71,8 +71,8 @@ const getActivityLog = async ({ elasticAgentId: string; size: number; from: number; - startDate?: string; - endDate?: string; + startDate: string; + endDate: string; logger: Logger; }) => { const options = { @@ -84,13 +84,10 @@ const getActivityLog = async ({ let actionsResult; let responsesResult; - const dateFilters = []; - if (startDate) { - dateFilters.push({ range: { '@timestamp': { gte: startDate } } }); - } - if (endDate) { - dateFilters.push({ range: { '@timestamp': { lte: endDate } } }); - } + const dateFilters = [ + { range: { '@timestamp': { gte: startDate } } }, + { range: { '@timestamp': { lte: endDate } } }, + ]; try { // fetch actions with matching agent_id From b4f5877ff8e1ac5a01053a5615526a9bdc202244 Mon Sep 17 00:00:00 2001 From: Pierre Gayvallet Date: Fri, 3 Sep 2021 14:38:19 +0200 Subject: [PATCH 04/20] catch errors from providers (#111093) --- .../public/services/search_service.test.ts | 64 +++++++++++++++++++ .../public/services/search_service.ts | 8 +-- .../server/services/search_service.test.ts | 38 +++++++++++ .../server/services/search_service.ts | 5 +- 4 files changed, 109 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/global_search/public/services/search_service.test.ts b/x-pack/plugins/global_search/public/services/search_service.test.ts index 4b3c06f03dcc8..b0e6a72290438 100644 --- a/x-pack/plugins/global_search/public/services/search_service.test.ts +++ b/x-pack/plugins/global_search/public/services/search_service.test.ts @@ -272,6 +272,43 @@ describe('SearchService', () => { }); }); + it('catches errors from providers', async () => { + const { registerResultProvider } = service.setup({ + config: createConfig(), + }); + + getTestScheduler().run(({ expectObservable, hot }) => { + registerResultProvider( + createProvider('A', { + source: hot('a---c-|', { + a: [providerResult('A1'), providerResult('A2')], + c: [providerResult('A3')], + }), + }) + ); + registerResultProvider( + createProvider('B', { + source: hot( + '-b-# ', + { + b: [providerResult('B1')], + }, + new Error('something went bad') + ), + }) + ); + + const { find } = service.start(startDeps()); + const results = find({ term: 'foobar' }, {}); + + expectObservable(results).toBe('ab--c-|', { + a: expectedBatch('A1', 'A2'), + b: expectedBatch('B1'), + c: expectedBatch('A3'), + }); + }); + }); + it('return mixed server/client providers results', async () => { const { registerResultProvider } = service.setup({ config: createConfig(), @@ -304,6 +341,33 @@ describe('SearchService', () => { }); }); + it('catches errors from the server', async () => { + const { registerResultProvider } = service.setup({ + config: createConfig(), + }); + + getTestScheduler().run(({ expectObservable, hot }) => { + fetchServerResultsMock.mockReturnValue(hot('#', {}, new Error('fetch error'))); + + registerResultProvider( + createProvider('A', { + source: hot('a-b-|', { + a: [providerResult('P1')], + b: [providerResult('P2')], + }), + }) + ); + + const { find } = service.start(startDeps()); + const results = find({ term: 'foobar' }, {}); + + expectObservable(results).toBe('a-b-|', { + a: expectedBatch('P1'), + b: expectedBatch('P2'), + }); + }); + }); + it('handles the `aborted$` option', async () => { const { registerResultProvider } = service.setup({ config: createConfig(), diff --git a/x-pack/plugins/global_search/public/services/search_service.ts b/x-pack/plugins/global_search/public/services/search_service.ts index bf06aa04061ed..85f4d4143a609 100644 --- a/x-pack/plugins/global_search/public/services/search_service.ts +++ b/x-pack/plugins/global_search/public/services/search_service.ts @@ -5,8 +5,8 @@ * 2.0. */ -import { merge, Observable, timer, throwError } from 'rxjs'; -import { map, takeUntil } from 'rxjs/operators'; +import { merge, Observable, timer, throwError, EMPTY } from 'rxjs'; +import { map, takeUntil, catchError } from 'rxjs/operators'; import { uniq } from 'lodash'; import { duration } from 'moment'; import { i18n } from '@kbn/i18n'; @@ -177,16 +177,16 @@ export class SearchService { const serverResults$ = fetchServerResults(this.http!, params, { preference, aborted$, - }); + }).pipe(catchError(() => EMPTY)); const providersResults$ = [...this.providers.values()].map((provider) => provider.find(params, providerOptions).pipe( + catchError(() => EMPTY), takeInArray(this.maxProviderResults), takeUntil(aborted$), map((results) => results.map((r) => processResult(r))) ) ); - return merge(...providersResults$, serverResults$).pipe( map((results) => ({ results, diff --git a/x-pack/plugins/global_search/server/services/search_service.test.ts b/x-pack/plugins/global_search/server/services/search_service.test.ts index 246fbd675aba2..45824fde26afe 100644 --- a/x-pack/plugins/global_search/server/services/search_service.test.ts +++ b/x-pack/plugins/global_search/server/services/search_service.test.ts @@ -178,6 +178,44 @@ describe('SearchService', () => { }); }); + it('catches errors from providers', async () => { + const { registerResultProvider } = service.setup({ + config: createConfig(), + basePath, + }); + + getTestScheduler().run(({ expectObservable, hot }) => { + registerResultProvider( + createProvider('A', { + source: hot('a---c-|', { + a: [result('A1'), result('A2')], + c: [result('A3')], + }), + }) + ); + registerResultProvider( + createProvider('B', { + source: hot( + '-b-# ', + { + b: [result('B1')], + }, + new Error('something went bad') + ), + }) + ); + + const { find } = service.start({ core: coreStart, licenseChecker }); + const results = find({ term: 'foobar' }, {}, request); + + expectObservable(results).toBe('ab--c-|', { + a: expectedBatch('A1', 'A2'), + b: expectedBatch('B1'), + c: expectedBatch('A3'), + }); + }); + }); + it('handles the `aborted$` option', async () => { const { registerResultProvider } = service.setup({ config: createConfig(), diff --git a/x-pack/plugins/global_search/server/services/search_service.ts b/x-pack/plugins/global_search/server/services/search_service.ts index a6c2a7ee234d6..22bac036544ab 100644 --- a/x-pack/plugins/global_search/server/services/search_service.ts +++ b/x-pack/plugins/global_search/server/services/search_service.ts @@ -5,8 +5,8 @@ * 2.0. */ -import { Observable, timer, merge, throwError } from 'rxjs'; -import { map, takeUntil } from 'rxjs/operators'; +import { Observable, timer, merge, throwError, EMPTY } from 'rxjs'; +import { map, takeUntil, catchError } from 'rxjs/operators'; import { uniq } from 'lodash'; import { i18n } from '@kbn/i18n'; import { KibanaRequest, CoreStart, IBasePath } from 'src/core/server'; @@ -174,6 +174,7 @@ export class SearchService { const providersResults$ = [...this.providers.values()].map((provider) => provider.find(params, findOptions, context).pipe( + catchError(() => EMPTY), takeInArray(this.maxProviderResults), takeUntil(aborted$), map((results) => results.map((r) => processResult(r))) From e2ee2637e2e8d18455bebc758333401a9a88f197 Mon Sep 17 00:00:00 2001 From: mgiota Date: Fri, 3 Sep 2021 14:39:52 +0200 Subject: [PATCH 05/20] Update alert documents when the write index changes (#110788) * first draft(work in progress) * add back missing await * disable require_alias flag only when we update * cleanup --- .../server/utils/create_lifecycle_executor.ts | 109 ++++++++++-------- 1 file changed, 63 insertions(+), 46 deletions(-) diff --git a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts index 259a9e9e8de38..48f3a81a00af2 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts @@ -162,7 +162,6 @@ export const createLifecycleExecutor = ( > = { alertWithLifecycle: ({ id, fields }) => { currentAlerts[id] = fields; - return alertInstanceFactory(id); }, }; @@ -179,7 +178,6 @@ export const createLifecycleExecutor = ( const currentAlertIds = Object.keys(currentAlerts); const trackedAlertIds = Object.keys(state.trackedAlerts); const newAlertIds = currentAlertIds.filter((alertId) => !trackedAlertIds.includes(alertId)); - const allAlertIds = [...new Set(currentAlertIds.concat(trackedAlertIds))]; const trackedAlertStates = Object.values(state.trackedAlerts); @@ -188,9 +186,10 @@ export const createLifecycleExecutor = ( `Tracking ${allAlertIds.length} alerts (${newAlertIds.length} new, ${trackedAlertStates.length} previous)` ); - const alertsDataMap: Record> = { - ...currentAlerts, - }; + const trackedAlertsDataMap: Record< + string, + { indexName: string; fields: Partial } + > = {}; if (trackedAlertStates.length) { const { hits } = await ruleDataClient.getReader().search({ @@ -228,59 +227,77 @@ export const createLifecycleExecutor = ( hits.hits.forEach((hit) => { const fields = parseTechnicalFields(hit.fields); + const indexName = hit._index; const alertId = fields[ALERT_INSTANCE_ID]; - alertsDataMap[alertId] = { - ...commonRuleFields, - ...fields, + trackedAlertsDataMap[alertId] = { + indexName, + fields, }; }); } - const eventsToIndex = allAlertIds.map((alertId) => { - const alertData = alertsDataMap[alertId]; - - if (!alertData) { - logger.warn(`Could not find alert data for ${alertId}`); - } - - const isNew = !state.trackedAlerts[alertId]; - const isRecovered = !currentAlerts[alertId]; - const isActive = !isRecovered; - - const { alertUuid, started } = state.trackedAlerts[alertId] ?? { - alertUuid: v4(), - started: commonRuleFields[TIMESTAMP], - }; - const event: ParsedTechnicalFields = { - ...alertData, - ...commonRuleFields, - [ALERT_DURATION]: (options.startedAt.getTime() - new Date(started).getTime()) * 1000, - [ALERT_INSTANCE_ID]: alertId, - [ALERT_START]: started, - [ALERT_STATUS]: isActive ? ALERT_STATUS_ACTIVE : ALERT_STATUS_RECOVERED, - [ALERT_WORKFLOW_STATUS]: alertData[ALERT_WORKFLOW_STATUS] ?? 'open', - [ALERT_UUID]: alertUuid, - [EVENT_KIND]: 'signal', - [EVENT_ACTION]: isNew ? 'open' : isActive ? 'active' : 'close', - [VERSION]: ruleDataClient.kibanaVersion, - ...(isRecovered ? { [ALERT_END]: commonRuleFields[TIMESTAMP] } : {}), - }; - - return event; - }); + const makeEventsDataMapFor = (alertIds: string[]) => + alertIds.map((alertId) => { + const alertData = trackedAlertsDataMap[alertId]; + const currentAlertData = currentAlerts[alertId]; + + if (!alertData) { + logger.warn(`Could not find alert data for ${alertId}`); + } + + const isNew = !state.trackedAlerts[alertId]; + const isRecovered = !currentAlerts[alertId]; + const isActive = !isRecovered; + + const { alertUuid, started } = state.trackedAlerts[alertId] ?? { + alertUuid: v4(), + started: commonRuleFields[TIMESTAMP], + }; + + const event: ParsedTechnicalFields = { + ...alertData?.fields, + ...commonRuleFields, + ...currentAlertData, + [ALERT_DURATION]: (options.startedAt.getTime() - new Date(started).getTime()) * 1000, + + [ALERT_INSTANCE_ID]: alertId, + [ALERT_START]: started, + [ALERT_UUID]: alertUuid, + [ALERT_STATUS]: isRecovered ? ALERT_STATUS_RECOVERED : ALERT_STATUS_ACTIVE, + [ALERT_WORKFLOW_STATUS]: alertData?.fields[ALERT_WORKFLOW_STATUS] ?? 'open', + [EVENT_KIND]: 'signal', + [EVENT_ACTION]: isNew ? 'open' : isActive ? 'active' : 'close', + [VERSION]: ruleDataClient.kibanaVersion, + ...(isRecovered ? { [ALERT_END]: commonRuleFields[TIMESTAMP] } : {}), + }; + + return { + indexName: alertData?.indexName, + event, + }; + }); + + const trackedEventsToIndex = makeEventsDataMapFor(trackedAlertIds); + const newEventsToIndex = makeEventsDataMapFor(newAlertIds); + const allEventsToIndex = [...trackedEventsToIndex, ...newEventsToIndex]; - if (eventsToIndex.length > 0 && ruleDataClient.isWriteEnabled()) { - logger.debug(`Preparing to index ${eventsToIndex.length} alerts.`); + if (allEventsToIndex.length > 0 && ruleDataClient.isWriteEnabled()) { + logger.debug(`Preparing to index ${allEventsToIndex.length} alerts.`); await ruleDataClient.getWriter().bulk({ - body: eventsToIndex.flatMap((event) => [{ index: { _id: event[ALERT_UUID]! } }, event]), + body: allEventsToIndex.flatMap(({ event, indexName }) => [ + indexName + ? { index: { _id: event[ALERT_UUID]!, _index: indexName, require_alias: false } } + : { index: { _id: event[ALERT_UUID]! } }, + event, + ]), }); } const nextTrackedAlerts = Object.fromEntries( - eventsToIndex - .filter((event) => event[ALERT_STATUS] !== 'closed') - .map((event) => { + allEventsToIndex + .filter(({ event }) => event[ALERT_STATUS] !== 'closed') + .map(({ event }) => { const alertId = event[ALERT_INSTANCE_ID]!; const alertUuid = event[ALERT_UUID]!; const started = new Date(event[ALERT_START]!).toISOString(); From d83c8244a296a1397c87ccdbd63db66784bceeca Mon Sep 17 00:00:00 2001 From: Dominique Clarke Date: Fri, 3 Sep 2021 08:47:26 -0400 Subject: [PATCH 06/20] [Uptime] [Synthetics Integration] fix content typo (#110088) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../public/components/fleet_package/browser/simple_fields.tsx | 2 +- .../public/components/fleet_package/http/simple_fields.tsx | 2 +- .../public/components/fleet_package/icmp/simple_fields.tsx | 2 +- .../public/components/fleet_package/tcp/simple_fields.tsx | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/uptime/public/components/fleet_package/browser/simple_fields.tsx b/x-pack/plugins/uptime/public/components/fleet_package/browser/simple_fields.tsx index 34f56a65df3e8..0e2f10b96fe6d 100644 --- a/x-pack/plugins/uptime/public/components/fleet_package/browser/simple_fields.tsx +++ b/x-pack/plugins/uptime/public/components/fleet_package/browser/simple_fields.tsx @@ -168,7 +168,7 @@ export const BrowserSimpleFields = memo(({ validate }) => { helpText={ } > diff --git a/x-pack/plugins/uptime/public/components/fleet_package/http/simple_fields.tsx b/x-pack/plugins/uptime/public/components/fleet_package/http/simple_fields.tsx index 8eb81eb92f7b4..c4de1d53fe998 100644 --- a/x-pack/plugins/uptime/public/components/fleet_package/http/simple_fields.tsx +++ b/x-pack/plugins/uptime/public/components/fleet_package/http/simple_fields.tsx @@ -186,7 +186,7 @@ export const HTTPSimpleFields = memo(({ validate }) => { helpText={ } > diff --git a/x-pack/plugins/uptime/public/components/fleet_package/icmp/simple_fields.tsx b/x-pack/plugins/uptime/public/components/fleet_package/icmp/simple_fields.tsx index 420f218429e40..92afe4c5072e1 100644 --- a/x-pack/plugins/uptime/public/components/fleet_package/icmp/simple_fields.tsx +++ b/x-pack/plugins/uptime/public/components/fleet_package/icmp/simple_fields.tsx @@ -190,7 +190,7 @@ export const ICMPSimpleFields = memo(({ validate }) => { helpText={ } > diff --git a/x-pack/plugins/uptime/public/components/fleet_package/tcp/simple_fields.tsx b/x-pack/plugins/uptime/public/components/fleet_package/tcp/simple_fields.tsx index 8bc017a51cfa9..37f0c82595e02 100644 --- a/x-pack/plugins/uptime/public/components/fleet_package/tcp/simple_fields.tsx +++ b/x-pack/plugins/uptime/public/components/fleet_package/tcp/simple_fields.tsx @@ -157,7 +157,7 @@ export const TCPSimpleFields = memo(({ validate }) => { helpText={ } > From 71571c5b60df320de9f36b2322c27bb96af41357 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Fri, 3 Sep 2021 14:05:53 +0100 Subject: [PATCH 07/20] [ML] Job import and export functional tests (#110578) * [ML] Job import export functional tests * adding title check * adding dfa tests * removing export file * adds bad data test * commented code * adding export job tests * adds version to file names * improving tests * removing comment Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../export_jobs_flyout/export_jobs_flyout.tsx | 140 +++++--- .../cannot_import_jobs_callout.tsx | 1 + .../cannot_read_file_callout.tsx | 10 +- .../import_jobs_flyout/import_jobs_flyout.tsx | 43 ++- .../ml/stack_management_jobs/export_jobs.ts | 314 ++++++++++++++++++ .../anomaly_detection_jobs_7.16.json | 213 ++++++++++++ .../files_to_import/bad_data.json | 1 + .../data_frame_analytics_jobs_7.16.json | 60 ++++ .../ml/stack_management_jobs/import_jobs.ts | 107 ++++++ .../apps/ml/stack_management_jobs/index.ts | 2 + .../services/ml/stack_management_jobs.ts | 224 ++++++++++++- 11 files changed, 1043 insertions(+), 72 deletions(-) create mode 100644 x-pack/test/functional/apps/ml/stack_management_jobs/export_jobs.ts create mode 100644 x-pack/test/functional/apps/ml/stack_management_jobs/files_to_import/anomaly_detection_jobs_7.16.json create mode 100644 x-pack/test/functional/apps/ml/stack_management_jobs/files_to_import/bad_data.json create mode 100644 x-pack/test/functional/apps/ml/stack_management_jobs/files_to_import/data_frame_analytics_jobs_7.16.json create mode 100644 x-pack/test/functional/apps/ml/stack_management_jobs/import_jobs.ts diff --git a/x-pack/plugins/ml/public/application/components/import_export_jobs/export_jobs_flyout/export_jobs_flyout.tsx b/x-pack/plugins/ml/public/application/components/import_export_jobs/export_jobs_flyout/export_jobs_flyout.tsx index bd4b805baa186..509c74c359657 100644 --- a/x-pack/plugins/ml/public/application/components/import_export_jobs/export_jobs_flyout/export_jobs_flyout.tsx +++ b/x-pack/plugins/ml/public/application/components/import_export_jobs/export_jobs_flyout/export_jobs_flyout.tsx @@ -63,6 +63,7 @@ export const ExportJobsFlyout: FC = ({ isDisabled, currentTab }) => { const [exporting, setExporting] = useState(false); const [selectedJobType, setSelectedJobType] = useState(currentTab); const [switchTabConfirmVisible, setSwitchTabConfirmVisible] = useState(false); + const [switchTabNextTab, setSwitchTabNextTab] = useState(currentTab); const { displayErrorToast, displaySuccessToast } = useMemo( () => toastNotificationServiceProvider(toasts), [toasts] @@ -170,16 +171,23 @@ export const ExportJobsFlyout: FC = ({ isDisabled, currentTab }) => { } } - const attemptTabSwitch = useCallback(() => { - // if the user has already selected some jobs, open a confirm modal - // rather than changing tabs - if (selectedJobIds.length > 0) { - setSwitchTabConfirmVisible(true); - return; - } + const attemptTabSwitch = useCallback( + (jobType: JobType) => { + if (jobType === selectedJobType) { + return; + } + // if the user has already selected some jobs, open a confirm modal + // rather than changing tabs + if (selectedJobIds.length > 0) { + setSwitchTabNextTab(jobType); + setSwitchTabConfirmVisible(true); + return; + } - switchTab(); - }, [selectedJobIds]); + switchTab(jobType); + }, + [selectedJobIds] + ); useEffect(() => { setSelectedJobDependencies( @@ -187,10 +195,7 @@ export const ExportJobsFlyout: FC = ({ isDisabled, currentTab }) => { ); }, [selectedJobIds]); - function switchTab() { - const jobType = - selectedJobType === 'anomaly-detector' ? 'data-frame-analytics' : 'anomaly-detector'; - + function switchTab(jobType: JobType) { setSwitchTabConfirmVisible(false); setSelectedJobIds([]); setSelectedJobType(jobType); @@ -211,7 +216,12 @@ export const ExportJobsFlyout: FC = ({ isDisabled, currentTab }) => { {showFlyout === true && isDisabled === false && ( <> - setShowFlyout(false)} hideCloseButton size="s"> + setShowFlyout(false)} + hideCloseButton + size="s" + data-test-subj="mlJobMgmtExportJobsFlyout" + >

@@ -227,8 +237,9 @@ export const ExportJobsFlyout: FC = ({ isDisabled, currentTab }) => { attemptTabSwitch('anomaly-detector')} disabled={exporting} + data-test-subj="mlJobMgmtExportJobsADTab" > = ({ isDisabled, currentTab }) => { attemptTabSwitch('data-frame-analytics')} disabled={exporting} + data-test-subj="mlJobMgmtExportJobsDFATab" > = ({ isDisabled, currentTab }) => { ) : ( <> - - + + {selectedJobIds.length === adJobIds.length ? ( + + ) : ( + + )} - {adJobIds.map((id) => ( -
- toggleSelectedJob(e.target.checked, id)} - /> - -
- ))} +
+ {adJobIds.map((id) => ( +
+ toggleSelectedJob(e.target.checked, id)} + /> + +
+ ))} +
)} @@ -284,26 +310,39 @@ export const ExportJobsFlyout: FC = ({ isDisabled, currentTab }) => { ) : ( <> - - + + {selectedJobIds.length === dfaJobIds.length ? ( + + ) : ( + + )} - - {dfaJobIds.map((id) => ( -
- toggleSelectedJob(e.target.checked, id)} - /> - -
- ))} +
+ {dfaJobIds.map((id) => ( +
+ toggleSelectedJob(e.target.checked, id)} + /> + +
+ ))} +
)} @@ -329,6 +368,7 @@ export const ExportJobsFlyout: FC = ({ isDisabled, currentTab }) => { disabled={selectedJobIds.length === 0 || exporting === true} onClick={onExport} fill + data-test-subj="mlJobMgmtExportExportButton" > = ({ isDisabled, currentTab }) => { {switchTabConfirmVisible === true ? ( switchTab(switchTabNextTab)} /> ) : null} diff --git a/x-pack/plugins/ml/public/application/components/import_export_jobs/import_jobs_flyout/cannot_import_jobs_callout.tsx b/x-pack/plugins/ml/public/application/components/import_export_jobs/import_jobs_flyout/cannot_import_jobs_callout.tsx index 732be345a1ee4..565ded9c6f6c3 100644 --- a/x-pack/plugins/ml/public/application/components/import_export_jobs/import_jobs_flyout/cannot_import_jobs_callout.tsx +++ b/x-pack/plugins/ml/public/application/components/import_export_jobs/import_jobs_flyout/cannot_import_jobs_callout.tsx @@ -30,6 +30,7 @@ export const CannotImportJobsCallout: FC = ({ jobs, autoExpand = false }) values: { num: jobs.length }, })} color="warning" + data-test-subj="mlJobMgmtImportJobsCannotBeImportedCallout" > {autoExpand ? ( diff --git a/x-pack/plugins/ml/public/application/components/import_export_jobs/import_jobs_flyout/cannot_read_file_callout.tsx b/x-pack/plugins/ml/public/application/components/import_export_jobs/import_jobs_flyout/cannot_read_file_callout.tsx index 4c7a2471db9d6..70f94d1e03155 100644 --- a/x-pack/plugins/ml/public/application/components/import_export_jobs/import_jobs_flyout/cannot_read_file_callout.tsx +++ b/x-pack/plugins/ml/public/application/components/import_export_jobs/import_jobs_flyout/cannot_read_file_callout.tsx @@ -21,10 +21,12 @@ export const CannotReadFileCallout: FC = () => { })} color="warning" > - +
+ +
); diff --git a/x-pack/plugins/ml/public/application/components/import_export_jobs/import_jobs_flyout/import_jobs_flyout.tsx b/x-pack/plugins/ml/public/application/components/import_export_jobs/import_jobs_flyout/import_jobs_flyout.tsx index 68db42cdbf0eb..dfe07b1984e11 100644 --- a/x-pack/plugins/ml/public/application/components/import_export_jobs/import_jobs_flyout/import_jobs_flyout.tsx +++ b/x-pack/plugins/ml/public/application/components/import_export_jobs/import_jobs_flyout/import_jobs_flyout.tsx @@ -341,7 +341,12 @@ export const ImportJobsFlyout: FC = ({ isDisabled }) => { {showFlyout === true && isDisabled === false && ( - +

@@ -373,22 +378,26 @@ export const ImportJobsFlyout: FC = ({ isDisabled }) => { {showFileReadError ? : null} {totalJobsRead > 0 && jobType !== null && ( - <> +
{jobType === 'anomaly-detector' && ( - +
+ +
)} {jobType === 'data-frame-analytics' && ( - +
+ +
)} @@ -426,6 +435,7 @@ export const ImportJobsFlyout: FC = ({ isDisabled }) => { value={jobId.jobId} onChange={(e) => renameJob(e.target.value, i)} isInvalid={jobId.jobIdValid === false} + data-test-subj="mlJobMgmtImportJobIdInput" /> @@ -465,7 +475,7 @@ export const ImportJobsFlyout: FC = ({ isDisabled }) => {
))} - + )} @@ -484,7 +494,12 @@ export const ImportJobsFlyout: FC = ({ isDisabled }) => { - + = [ + { + // @ts-expect-error not full interface + job: { + job_id: 'fq_single_1_smv', + groups: ['farequote', 'automated', 'single-metric'], + description: 'mean(responsetime) on farequote dataset with 15m bucket span', + analysis_config: { + bucket_span: '15m', + detectors: [ + { + detector_description: 'mean(responsetime)', + function: 'mean', + field_name: 'responsetime', + }, + ], + influencers: [], + }, + analysis_limits: { + model_memory_limit: '10mb', + categorization_examples_limit: 4, + }, + data_description: { + time_field: '@timestamp', + time_format: 'epoch_ms', + }, + model_plot_config: { + enabled: true, + annotations_enabled: true, + }, + model_snapshot_retention_days: 10, + daily_model_snapshot_retention_after_days: 1, + results_index_name: 'shared', + allow_lazy_open: false, + }, + datafeed: { + datafeed_id: 'datafeed-fq_single_1_smv', + job_id: 'fq_single_1_smv', + query: { + bool: { + must: [ + { + match_all: {}, + }, + ], + }, + }, + indices: ['ft_farequote'], + scroll_size: 1000, + delayed_data_check_config: { + enabled: true, + }, + }, + }, + { + // @ts-expect-error not full interface + job: { + job_id: 'fq_single_2_smv', + groups: ['farequote', 'automated', 'single-metric'], + description: 'low_mean(responsetime) on farequote dataset with 15m bucket span', + analysis_config: { + bucket_span: '15m', + detectors: [ + { + detector_description: 'low_mean(responsetime)', + function: 'low_mean', + field_name: 'responsetime', + }, + ], + influencers: ['responsetime'], + }, + analysis_limits: { + model_memory_limit: '11mb', + categorization_examples_limit: 4, + }, + data_description: { + time_field: '@timestamp', + time_format: 'epoch_ms', + }, + model_plot_config: { + enabled: true, + annotations_enabled: true, + }, + model_snapshot_retention_days: 10, + daily_model_snapshot_retention_after_days: 1, + results_index_name: 'shared', + allow_lazy_open: false, + }, + datafeed: { + datafeed_id: 'datafeed-fq_single_2_smv', + job_id: 'fq_single_2_smv', + query: { + bool: { + must: [ + { + match_all: {}, + }, + ], + }, + }, + indices: ['ft_farequote'], + scroll_size: 1000, + delayed_data_check_config: { + enabled: true, + }, + }, + }, + { + // @ts-expect-error not full interface + job: { + job_id: 'fq_single_3_smv', + groups: ['farequote', 'automated', 'single-metric'], + description: 'high_mean(responsetime) on farequote dataset with 15m bucket span', + analysis_config: { + bucket_span: '15m', + detectors: [ + { + detector_description: 'high_mean(responsetime)', + function: 'high_mean', + field_name: 'responsetime', + }, + ], + influencers: ['responsetime'], + }, + analysis_limits: { + model_memory_limit: '11mb', + categorization_examples_limit: 4, + }, + data_description: { + time_field: '@timestamp', + time_format: 'epoch_ms', + }, + model_plot_config: { + enabled: true, + annotations_enabled: true, + }, + model_snapshot_retention_days: 10, + daily_model_snapshot_retention_after_days: 1, + results_index_name: 'shared', + allow_lazy_open: false, + }, + datafeed: { + datafeed_id: 'datafeed-fq_single_3_smv', + job_id: 'fq_single_3_smv', + query: { + bool: { + must: [ + { + match_all: {}, + }, + ], + }, + }, + indices: ['ft_farequote'], + scroll_size: 1000, + delayed_data_check_config: { + enabled: true, + }, + }, + }, +]; + +const testDFAJobs: DataFrameAnalyticsConfig[] = [ + // @ts-expect-error not full interface + { + id: `bm_1_1`, + description: + "Classification job based on 'ft_bank_marketing' dataset with dependentVariable 'y' and trainingPercent '20'", + source: { + index: ['ft_bank_marketing'], + query: { + match_all: {}, + }, + }, + dest: { + index: 'user-bm_1_1', + results_field: 'ml', + }, + analysis: { + classification: { + prediction_field_name: 'test', + dependent_variable: 'y', + training_percent: 20, + }, + }, + analyzed_fields: { + includes: [], + excludes: [], + }, + model_memory_limit: '60mb', + allow_lazy_start: false, + }, + // @ts-expect-error not full interface + { + id: `ihp_1_2`, + description: 'This is the job description', + source: { + index: ['ft_ihp_outlier'], + query: { + match_all: {}, + }, + }, + dest: { + index: 'user-ihp_1_2', + results_field: 'ml', + }, + analysis: { + outlier_detection: {}, + }, + analyzed_fields: { + includes: [], + excludes: [], + }, + model_memory_limit: '5mb', + }, + // @ts-expect-error not full interface + { + id: `egs_1_3`, + description: 'This is the job description', + source: { + index: ['ft_egs_regression'], + query: { + match_all: {}, + }, + }, + dest: { + index: 'user-egs_1_3', + results_field: 'ml', + }, + analysis: { + regression: { + prediction_field_name: 'test', + dependent_variable: 'stab', + training_percent: 20, + }, + }, + analyzed_fields: { + includes: [], + excludes: [], + }, + model_memory_limit: '20mb', + }, +]; + +export default function ({ getService }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const ml = getService('ml'); + + describe('export jobs', function () { + this.tags(['mlqa']); + before(async () => { + await ml.api.cleanMlIndices(); + await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote'); + await ml.testResources.createIndexPatternIfNeeded('ft_farequote', '@timestamp'); + + await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/bm_classification'); + await ml.testResources.createIndexPatternIfNeeded('ft_bank_marketing', '@timestamp'); + + await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/ihp_outlier'); + await ml.testResources.createIndexPatternIfNeeded('ft_ihp_outlier', '@timestamp'); + + await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/egs_regression'); + await ml.testResources.createIndexPatternIfNeeded('ft_egs_regression', '@timestamp'); + + await ml.testResources.setKibanaTimeZoneToUTC(); + + for (const { job, datafeed } of testADJobs) { + await ml.api.createAnomalyDetectionJob(job); + await ml.api.createDatafeed(datafeed); + } + for (const job of testDFAJobs) { + await ml.api.createDataFrameAnalyticsJob(job); + } + + await ml.securityUI.loginAsMlPowerUser(); + await ml.navigation.navigateToStackManagement(); + await ml.navigation.navigateToStackManagementJobsListPage(); + }); + after(async () => { + await ml.api.cleanMlIndices(); + ml.stackManagementJobs.deleteExportedFiles([ + 'anomaly_detection_jobs', + 'data_frame_analytics_jobs', + ]); + }); + + it('opens export flyout and exports anomaly detector jobs', async () => { + await ml.stackManagementJobs.openExportFlyout(); + await ml.stackManagementJobs.selectExportJobType('anomaly-detector'); + await ml.stackManagementJobs.selectExportJobSelectAll('anomaly-detector'); + await ml.stackManagementJobs.selectExportJobs(); + await ml.stackManagementJobs.assertExportedADJobsAreCorrect(testADJobs); + }); + + it('opens export flyout and exports data frame analytics jobs', async () => { + await ml.stackManagementJobs.openExportFlyout(); + await ml.stackManagementJobs.selectExportJobType('data-frame-analytics'); + await ml.stackManagementJobs.selectExportJobSelectAll('data-frame-analytics'); + await ml.stackManagementJobs.selectExportJobs(); + await ml.stackManagementJobs.assertExportedDFAJobsAreCorrect(testDFAJobs); + }); + }); +} diff --git a/x-pack/test/functional/apps/ml/stack_management_jobs/files_to_import/anomaly_detection_jobs_7.16.json b/x-pack/test/functional/apps/ml/stack_management_jobs/files_to_import/anomaly_detection_jobs_7.16.json new file mode 100644 index 0000000000000..1bc51d433858e --- /dev/null +++ b/x-pack/test/functional/apps/ml/stack_management_jobs/files_to_import/anomaly_detection_jobs_7.16.json @@ -0,0 +1,213 @@ +[ + { + "job": { + "job_id": "ad-test1", + "description": "", + "analysis_config": { + "bucket_span": "15m", + "summary_count_field_name": "doc_count", + "detectors": [ + { + "detector_description": "mean(responsetime)", + "function": "mean", + "field_name": "responsetime", + "detector_index": 0 + } + ], + "influencers": [] + }, + "analysis_limits": { + "model_memory_limit": "11mb", + "categorization_examples_limit": 4 + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "model_plot_config": { + "enabled": true, + "annotations_enabled": true + }, + "model_snapshot_retention_days": 10, + "daily_model_snapshot_retention_after_days": 1, + "results_index_name": "shared", + "allow_lazy_open": false + }, + "datafeed": { + "datafeed_id": "datafeed-ad-test1", + "job_id": "ad-test1", + "query": { + "bool": { + "must": [ + { + "match_all": {} + } + ] + } + }, + "indices": [ + "ft_farequote" + ], + "aggregations": { + "buckets": { + "date_histogram": { + "field": "@timestamp", + "fixed_interval": "90000ms" + }, + "aggregations": { + "responsetime": { + "avg": { + "field": "responsetime" + } + }, + "@timestamp": { + "max": { + "field": "@timestamp" + } + } + } + } + }, + "scroll_size": 1000, + "delayed_data_check_config": { + "enabled": true + } + } + }, + { + "job": { + "job_id": "ad-test2", + "groups": [ + "newgroup" + ], + "description": "", + "analysis_config": { + "bucket_span": "15m", + "summary_count_field_name": "doc_count", + "detectors": [ + { + "detector_description": "mean(responsetime)", + "function": "mean", + "field_name": "responsetime", + "detector_index": 0 + } + ], + "influencers": [] + }, + "analysis_limits": { + "model_memory_limit": "11mb", + "categorization_examples_limit": 4 + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "model_plot_config": { + "enabled": true, + "annotations_enabled": true + }, + "model_snapshot_retention_days": 10, + "daily_model_snapshot_retention_after_days": 1, + "results_index_name": "shared", + "allow_lazy_open": false + }, + "datafeed": { + "datafeed_id": "datafeed-ad-test2", + "job_id": "ad-test2", + "query": { + "bool": { + "must": [ + { + "match_all": {} + } + ] + } + }, + "indices": [ + "missing" + ], + "aggregations": { + "buckets": { + "date_histogram": { + "field": "@timestamp", + "fixed_interval": "90000ms" + }, + "aggregations": { + "responsetime": { + "avg": { + "field": "responsetime" + } + }, + "@timestamp": { + "max": { + "field": "@timestamp" + } + } + } + } + }, + "scroll_size": 1000, + "delayed_data_check_config": { + "enabled": true + } + } + }, + { + "job": { + "job_id": "ad-test3", + "custom_settings": {}, + "description": "", + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "mean(responsetime) partitionfield=airline", + "function": "mean", + "field_name": "responsetime", + "partition_field_name": "airline", + "detector_index": 0 + } + ], + "influencers": [ + "airline" + ] + }, + "analysis_limits": { + "model_memory_limit": "11mb", + "categorization_examples_limit": 4 + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "model_plot_config": { + "enabled": false, + "annotations_enabled": false + }, + "model_snapshot_retention_days": 10, + "daily_model_snapshot_retention_after_days": 1, + "results_index_name": "shared", + "allow_lazy_open": false + }, + "datafeed": { + "datafeed_id": "datafeed-ad-test3", + "job_id": "ad-test3", + "query": { + "bool": { + "must": [ + { + "match_all": {} + } + ] + } + }, + "indices": [ + "ft_farequote" + ], + "scroll_size": 1000, + "delayed_data_check_config": { + "enabled": true + } + } + } +] diff --git a/x-pack/test/functional/apps/ml/stack_management_jobs/files_to_import/bad_data.json b/x-pack/test/functional/apps/ml/stack_management_jobs/files_to_import/bad_data.json new file mode 100644 index 0000000000000..5c40480832c00 --- /dev/null +++ b/x-pack/test/functional/apps/ml/stack_management_jobs/files_to_import/bad_data.json @@ -0,0 +1 @@ +Hey! this isn't JSON. diff --git a/x-pack/test/functional/apps/ml/stack_management_jobs/files_to_import/data_frame_analytics_jobs_7.16.json b/x-pack/test/functional/apps/ml/stack_management_jobs/files_to_import/data_frame_analytics_jobs_7.16.json new file mode 100644 index 0000000000000..cb93aa9e24c5f --- /dev/null +++ b/x-pack/test/functional/apps/ml/stack_management_jobs/files_to_import/data_frame_analytics_jobs_7.16.json @@ -0,0 +1,60 @@ +[ + { + "id": "dfa-test1", + "description": "Classification job based on 'ft_bank_marketing' dataset with dependentVariable 'y' and trainingPercent '20'", + "source": { + "index": [ + "ft_bank_marketing" + ], + "query": { + "match_all": {} + } + }, + "dest": { + "index": "user-dfa-test1", + "results_field": "ml" + }, + "analysis": { + "classification": { + "prediction_field_name": "user-test", + "dependent_variable": "y", + "training_percent": 20 + } + }, + "analyzed_fields": { + "includes": [], + "excludes": [] + }, + "model_memory_limit": "60mb", + "allow_lazy_start": false + }, + { + "id": "dfa-test2", + "description": "Classification job based on 'ft_bank_marketing' dataset with dependentVariable 'y' and trainingPercent '20'", + "source": { + "index": [ + "missing-index" + ], + "query": { + "match_all": {} + } + }, + "dest": { + "index": "user-dfa-test2", + "results_field": "ml" + }, + "analysis": { + "classification": { + "prediction_field_name": "test", + "dependent_variable": "y", + "training_percent": 20 + } + }, + "analyzed_fields": { + "includes": [], + "excludes": [] + }, + "model_memory_limit": "60mb", + "allow_lazy_start": false + } +] diff --git a/x-pack/test/functional/apps/ml/stack_management_jobs/import_jobs.ts b/x-pack/test/functional/apps/ml/stack_management_jobs/import_jobs.ts new file mode 100644 index 0000000000000..6211885af0a2a --- /dev/null +++ b/x-pack/test/functional/apps/ml/stack_management_jobs/import_jobs.ts @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import path from 'path'; + +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { JobType } from '../../../../../plugins/ml/common/types/saved_objects'; + +export default function ({ getService }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const ml = getService('ml'); + const testDataListPositive = [ + { + filePath: path.join(__dirname, 'files_to_import', 'anomaly_detection_jobs_7.16.json'), + expected: { + jobType: 'anomaly-detector' as JobType, + jobIds: ['ad-test1', 'ad-test3'], + skippedJobIds: ['ad-test2'], + }, + }, + { + filePath: path.join(__dirname, 'files_to_import', 'data_frame_analytics_jobs_7.16.json'), + expected: { + jobType: 'data-frame-analytics' as JobType, + jobIds: ['dfa-test1'], + skippedJobIds: ['dfa-test2'], + }, + }, + ]; + + describe('import jobs', function () { + this.tags(['mlqa']); + before(async () => { + await ml.api.cleanMlIndices(); + await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote'); + await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/bm_classification'); + await ml.testResources.createIndexPatternIfNeeded('ft_farequote', '@timestamp'); + await ml.testResources.createIndexPatternIfNeeded('ft_bank_marketing', '@timestamp'); + await ml.testResources.setKibanaTimeZoneToUTC(); + + await ml.securityUI.loginAsMlPowerUser(); + await ml.navigation.navigateToStackManagement(); + await ml.navigation.navigateToStackManagementJobsListPage(); + }); + + after(async () => { + await ml.api.cleanMlIndices(); + }); + + for (const testData of testDataListPositive) { + it('selects and reads file', async () => { + await ml.testExecution.logTestStep('selects job import'); + await ml.stackManagementJobs.openImportFlyout(); + await ml.stackManagementJobs.selectFileToImport(testData.filePath); + }); + it('has the correct importable jobs', async () => { + await ml.stackManagementJobs.assertCorrectTitle( + [...testData.expected.jobIds, ...testData.expected.skippedJobIds].length, + testData.expected.jobType + ); + await ml.stackManagementJobs.assertJobIdsExist(testData.expected.jobIds); + await ml.stackManagementJobs.assertJobIdsSkipped(testData.expected.skippedJobIds); + }); + + it('imports jobs', async () => { + await ml.stackManagementJobs.importJobs(); + }); + + it('ensures jobs have been imported', async () => { + if (testData.expected.jobType === 'anomaly-detector') { + await ml.navigation.navigateToStackManagementJobsListPageAnomalyDetectionTab(); + await ml.jobTable.refreshJobList(); + for (const id of testData.expected.jobIds) { + await ml.jobTable.filterWithSearchString(id); + } + for (const id of testData.expected.skippedJobIds) { + await ml.jobTable.filterWithSearchString(id, 0); + } + } else { + await ml.navigation.navigateToStackManagementJobsListPageAnalyticsTab(); + await ml.dataFrameAnalyticsTable.refreshAnalyticsTable(); + for (const id of testData.expected.jobIds) { + await ml.dataFrameAnalyticsTable.assertAnalyticsJobDisplayedInTable(id, true); + } + for (const id of testData.expected.skippedJobIds) { + await ml.dataFrameAnalyticsTable.assertAnalyticsJobDisplayedInTable(id, false); + } + } + }); + } + + describe('correctly fails to import bad data', async () => { + it('selects and reads file', async () => { + await ml.testExecution.logTestStep('selects job import'); + await ml.stackManagementJobs.openImportFlyout(); + await ml.stackManagementJobs.selectFileToImport( + path.join(__dirname, 'files_to_import', 'bad_data.json'), + true + ); + }); + }); + }); +} diff --git a/x-pack/test/functional/apps/ml/stack_management_jobs/index.ts b/x-pack/test/functional/apps/ml/stack_management_jobs/index.ts index f120ab0b450dc..c5e0728266bab 100644 --- a/x-pack/test/functional/apps/ml/stack_management_jobs/index.ts +++ b/x-pack/test/functional/apps/ml/stack_management_jobs/index.ts @@ -13,5 +13,7 @@ export default function ({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./synchronize')); loadTestFile(require.resolve('./manage_spaces')); + loadTestFile(require.resolve('./import_jobs')); + loadTestFile(require.resolve('./export_jobs')); }); } diff --git a/x-pack/test/functional/services/ml/stack_management_jobs.ts b/x-pack/test/functional/services/ml/stack_management_jobs.ts index 48fb89e51ff11..45b9fa2f29ccd 100644 --- a/x-pack/test/functional/services/ml/stack_management_jobs.ts +++ b/x-pack/test/functional/services/ml/stack_management_jobs.ts @@ -6,10 +6,16 @@ */ import expect from '@kbn/expect'; +import { REPO_ROOT } from '@kbn/utils'; +import fs from 'fs'; +import path from 'path'; -import { FtrProviderContext } from '../../ftr_provider_context'; -import { MlADJobTable } from './job_table'; -import { MlDFAJobTable } from './data_frame_analytics_table'; +import type { FtrProviderContext } from '../../ftr_provider_context'; +import type { MlADJobTable } from './job_table'; +import type { MlDFAJobTable } from './data_frame_analytics_table'; +import type { JobType } from '../../../../plugins/ml/common/types/saved_objects'; +import type { Job, Datafeed } from '../../../../plugins/ml/common/types/anomaly_detection_jobs'; +import type { DataFrameAnalyticsConfig } from '../../../../plugins/ml/public/application/data_frame_analytics/common'; type SyncFlyoutObjectType = | 'MissingObjects' @@ -18,7 +24,7 @@ type SyncFlyoutObjectType = | 'ObjectsUnmatchedDatafeed'; export function MachineLearningStackManagementJobsProvider( - { getService }: FtrProviderContext, + { getService, getPageObjects }: FtrProviderContext, mlADJobTable: MlADJobTable, mlDFAJobTable: MlDFAJobTable ) { @@ -26,6 +32,9 @@ export function MachineLearningStackManagementJobsProvider( const retry = getService('retry'); const testSubjects = getService('testSubjects'); const toasts = getService('toasts'); + const log = getService('log'); + + const PageObjects = getPageObjects(['common']); return { async openSyncFlyout() { @@ -194,5 +203,212 @@ export function MachineLearningStackManagementJobsProvider( } await this.assertSpaceSelectionRowSelected(spaceId, shouldSelect); }, + + async openImportFlyout() { + await retry.tryForTime(5000, async () => { + await testSubjects.click('mlJobsImportButton', 1000); + await testSubjects.existOrFail('mlJobMgmtImportJobsFlyout'); + }); + }, + + async openExportFlyout() { + await retry.tryForTime(5000, async () => { + await testSubjects.click('mlJobsExportButton', 1000); + await testSubjects.existOrFail('mlJobMgmtExportJobsFlyout'); + }); + }, + + async selectFileToImport(filePath: string, expectError: boolean = false) { + log.debug(`Importing file '${filePath}' ...`); + await PageObjects.common.setFileInputPath(filePath); + + if (expectError) { + await testSubjects.existOrFail('~mlJobMgmtImportJobsFileReadErrorCallout'); + } else { + await testSubjects.missingOrFail('~mlJobMgmtImportJobsFileReadErrorCallout'); + await testSubjects.existOrFail('mlJobMgmtImportJobsFileRead'); + } + }, + + async assertJobIdsExist(expectedJobIds: string[]) { + const inputs = await testSubjects.findAll('mlJobMgmtImportJobIdInput'); + const actualJobIds = await Promise.all(inputs.map((i) => i.getAttribute('value'))); + + expect(actualJobIds.sort()).to.eql( + expectedJobIds.sort(), + `Expected job ids to be '${JSON.stringify(expectedJobIds)}' (got '${JSON.stringify( + actualJobIds + )}')` + ); + }, + + async assertCorrectTitle(jobCount: number, jobType: JobType) { + const dataTestSubj = + jobType === 'anomaly-detector' + ? 'mlJobMgmtImportJobsADTitle' + : 'mlJobMgmtImportJobsDFATitle'; + const subj = await testSubjects.find(dataTestSubj); + const title = (await subj.parseDomContent()).html(); + + const jobTypeString = + jobType === 'anomaly-detector' ? 'anomaly detection' : 'data frame analytics'; + + const results = title.match( + /(\d) (anomaly detection|data frame analytics) job[s]? read from file$/ + ); + expect(results).to.not.eql(null, `Expected regex results to not be null`); + const foundCount = results![1]; + const foundJobTypeString = results![2]; + expect(foundCount).to.eql( + jobCount, + `Expected job count to be '${jobCount}' (got '${foundCount}')` + ); + expect(foundJobTypeString).to.eql( + jobTypeString, + `Expected job count to be '${jobTypeString}' (got '${foundJobTypeString}')` + ); + }, + + async assertJobIdsSkipped(expectedJobIds: string[]) { + const subj = await testSubjects.find('mlJobMgmtImportJobsCannotBeImportedCallout'); + const skippedJobTitles = await subj.findAllByTagName('h5'); + const actualJobIds = ( + await Promise.all(skippedJobTitles.map((i) => i.parseDomContent())) + ).map((t) => t.html()); + + expect(actualJobIds.sort()).to.eql( + expectedJobIds.sort(), + `Expected job ids to be '${JSON.stringify(expectedJobIds)}' (got '${JSON.stringify( + actualJobIds + )}')` + ); + }, + + async importJobs() { + await testSubjects.click('mlJobMgmtImportImportButton', 1000); + await testSubjects.missingOrFail('mlJobMgmtImportJobsFlyout', { timeout: 60 * 1000 }); + }, + + async assertReadErrorCalloutExists() { + await testSubjects.existOrFail('~mlJobMgmtImportJobsFileReadErrorCallout'); + }, + + async selectExportJobType(jobType: JobType) { + if (jobType === 'anomaly-detector') { + await testSubjects.click('mlJobMgmtExportJobsADTab'); + await testSubjects.existOrFail('mlJobMgmtExportJobsADJobList'); + } else { + await testSubjects.click('mlJobMgmtExportJobsDFATab'); + await testSubjects.existOrFail('mlJobMgmtExportJobsDFAJobList'); + } + }, + + async selectExportJobSelectAll(jobType: JobType) { + await testSubjects.click('mlJobMgmtExportJobsSelectAllButton'); + const subjLabel = + jobType === 'anomaly-detector' + ? 'mlJobMgmtExportJobsADJobList' + : 'mlJobMgmtExportJobsDFAJobList'; + const subj = await testSubjects.find(subjLabel); + const inputs = await subj.findAllByTagName('input'); + const allInputValues = await Promise.all(inputs.map((input) => input.getAttribute('value'))); + expect(allInputValues.every((i) => i === 'on')).to.eql( + true, + `Expected all inputs to be checked` + ); + }, + + async getDownload(filePath: string) { + return retry.tryForTime(5000, async () => { + expect(fs.existsSync(filePath)).to.be(true); + return fs.readFileSync(filePath).toString(); + }); + }, + + getExportedFile(fileName: string) { + return path.resolve(REPO_ROOT, `target/functional-tests/downloads/${fileName}.json`); + }, + + deleteExportedFiles(fileNames: string[]) { + fileNames.forEach((file) => { + try { + fs.unlinkSync(this.getExportedFile(file)); + } catch (e) { + // it might not have been there to begin with + } + }); + }, + + async selectExportJobs() { + await testSubjects.click('mlJobMgmtExportExportButton'); + await testSubjects.missingOrFail('mlJobMgmtExportJobsFlyout', { timeout: 60 * 1000 }); + }, + + async assertExportedADJobsAreCorrect(expectedJobs: Array<{ job: Job; datafeed: Datafeed }>) { + const file = JSON.parse( + await this.getDownload(this.getExportedFile('anomaly_detection_jobs')) + ); + const loadedFile = Array.isArray(file) ? file : [file]; + const sortedActualJobs = loadedFile.sort((a, b) => a.job.job_id.localeCompare(b.job.job_id)); + + const sortedExpectedJobs = expectedJobs.sort((a, b) => + a.job.job_id.localeCompare(b.job.job_id) + ); + expect(sortedActualJobs.length).to.eql( + sortedExpectedJobs.length, + `Expected length of exported jobs to be '${sortedExpectedJobs.length}' (got '${sortedActualJobs.length}')` + ); + + sortedExpectedJobs.forEach((expectedJob, i) => { + expect(sortedActualJobs[i].job.job_id).to.eql( + expectedJob.job.job_id, + `Expected job id to be '${expectedJob.job.job_id}' (got '${sortedActualJobs[i].job.job_id}')` + ); + expect(sortedActualJobs[i].job.analysis_config.detectors.length).to.eql( + expectedJob.job.analysis_config.detectors.length, + `Expected detectors length to be '${expectedJob.job.analysis_config.detectors.length}' (got '${sortedActualJobs[i].job.analysis_config.detectors.length}')` + ); + expect(sortedActualJobs[i].job.analysis_config.detectors[0].function).to.eql( + expectedJob.job.analysis_config.detectors[0].function, + `Expected first detector function to be '${expectedJob.job.analysis_config.detectors[0].function}' (got '${sortedActualJobs[i].job.analysis_config.detectors[0].function}')` + ); + expect(sortedActualJobs[i].datafeed.datafeed_id).to.eql( + expectedJob.datafeed.datafeed_id, + `Expected job id to be '${expectedJob.datafeed.datafeed_id}' (got '${sortedActualJobs[i].datafeed.datafeed_id}')` + ); + }); + }, + + async assertExportedDFAJobsAreCorrect(expectedJobs: DataFrameAnalyticsConfig[]) { + const file = JSON.parse( + await this.getDownload(this.getExportedFile('data_frame_analytics_jobs')) + ); + const loadedFile = Array.isArray(file) ? file : [file]; + const sortedActualJobs = loadedFile.sort((a, b) => a.id.localeCompare(b.id)); + + const sortedExpectedJobs = expectedJobs.sort((a, b) => a.id.localeCompare(b.id)); + + expect(sortedActualJobs.length).to.eql( + sortedExpectedJobs.length, + `Expected length of exported jobs to be '${sortedExpectedJobs.length}' (got '${sortedActualJobs.length}')` + ); + + sortedExpectedJobs.forEach((expectedJob, i) => { + expect(sortedActualJobs[i].id).to.eql( + expectedJob.id, + `Expected job id to be '${expectedJob.id}' (got '${sortedActualJobs[i].id}')` + ); + const expectedType = Object.keys(expectedJob.analysis)[0]; + const actualType = Object.keys(sortedActualJobs[i].analysis)[0]; + expect(actualType).to.eql( + expectedType, + `Expected job type to be '${expectedType}' (got '${actualType}')` + ); + expect(sortedActualJobs[i].dest.index).to.eql( + expectedJob.dest.index, + `Expected destination index to be '${expectedJob.dest.index}' (got '${sortedActualJobs[i].dest.index}')` + ); + }); + }, }; } From 6f357e043331e0291f16750dc1cb907c2d4ce86a Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Fri, 3 Sep 2021 16:10:29 +0300 Subject: [PATCH 08/20] [Cases] Do not show status dropdown on modal cases selector (#111101) --- .../all_cases/all_cases_generic.tsx | 3 +- .../public/components/all_cases/columns.tsx | 68 ++++++++++--------- .../components/all_cases/index.test.tsx | 27 +++++++- 3 files changed, 63 insertions(+), 35 deletions(-) diff --git a/x-pack/plugins/cases/public/components/all_cases/all_cases_generic.tsx b/x-pack/plugins/cases/public/components/all_cases/all_cases_generic.tsx index 72491a2bc1e31..9cbb13f7227a3 100644 --- a/x-pack/plugins/cases/public/components/all_cases/all_cases_generic.tsx +++ b/x-pack/plugins/cases/public/components/all_cases/all_cases_generic.tsx @@ -203,7 +203,8 @@ export const AllCasesGeneric = React.memo( handleIsLoading, isLoadingCases: loading, refreshCases, - showActions, + // isSelectorView is boolean | undefined. We need to convert it to a boolean. + isSelectorView: !!isSelectorView, userCanCrud, connectors, }); diff --git a/x-pack/plugins/cases/public/components/all_cases/columns.tsx b/x-pack/plugins/cases/public/components/all_cases/columns.tsx index 8b755b0c60968..c0bd6536f1b73 100644 --- a/x-pack/plugins/cases/public/components/all_cases/columns.tsx +++ b/x-pack/plugins/cases/public/components/all_cases/columns.tsx @@ -72,7 +72,7 @@ export interface GetCasesColumn { handleIsLoading: (a: boolean) => void; isLoadingCases: string[]; refreshCases?: (a?: boolean) => void; - showActions: boolean; + isSelectorView: boolean; userCanCrud: boolean; connectors?: ActionConnector[]; } @@ -84,7 +84,7 @@ export const useCasesColumns = ({ handleIsLoading, isLoadingCases, refreshCases, - showActions, + isSelectorView, userCanCrud, connectors = [], }: GetCasesColumn): CasesColumns[] => { @@ -281,38 +281,42 @@ export const useCasesColumns = ({ return getEmptyTagValue(); }, }, - { - name: i18n.STATUS, - render: (theCase: Case) => { - if (theCase?.subCases == null || theCase.subCases.length === 0) { - if (theCase.status == null || theCase.type === CaseType.collection) { - return getEmptyTagValue(); - } - return ( - 0} - onStatusChanged={(status) => - handleDispatchUpdate({ - updateKey: 'status', - updateValue: status, - caseId: theCase.id, - version: theCase.version, - }) + ...(!isSelectorView + ? [ + { + name: i18n.STATUS, + render: (theCase: Case) => { + if (theCase?.subCases == null || theCase.subCases.length === 0) { + if (theCase.status == null || theCase.type === CaseType.collection) { + return getEmptyTagValue(); + } + return ( + 0} + onStatusChanged={(status) => + handleDispatchUpdate({ + updateKey: 'status', + updateValue: status, + caseId: theCase.id, + version: theCase.version, + }) + } + /> + ); } - /> - ); - } - const badges = getSubCasesStatusCountsBadges(theCase.subCases); - return badges.map(({ color, count }, index) => ( - - {count} - - )); - }, - }, - ...(showActions + const badges = getSubCasesStatusCountsBadges(theCase.subCases); + return badges.map(({ color, count }, index) => ( + + {count} + + )); + }, + }, + ] + : []), + ...(userCanCrud && !isSelectorView ? [ { name: ( diff --git a/x-pack/plugins/cases/public/components/all_cases/index.test.tsx b/x-pack/plugins/cases/public/components/all_cases/index.test.tsx index 9e6928d43c862..3fff43108772d 100644 --- a/x-pack/plugins/cases/public/components/all_cases/index.test.tsx +++ b/x-pack/plugins/cases/public/components/all_cases/index.test.tsx @@ -144,7 +144,7 @@ describe('AllCasesGeneric', () => { filterStatus: CaseStatuses.open, handleIsLoading: jest.fn(), isLoadingCases: [], - showActions: true, + isSelectorView: false, userCanCrud: true, }; @@ -377,7 +377,7 @@ describe('AllCasesGeneric', () => { isLoadingCases: [], filterStatus: CaseStatuses.open, handleIsLoading: jest.fn(), - showActions: false, + isSelectorView: true, userCanCrud: true, }) ); @@ -926,4 +926,27 @@ describe('AllCasesGeneric', () => { ).toBeFalsy(); }); }); + + it('should not render status when isSelectorView=true', async () => { + const wrapper = mount( + + + + ); + + const { result } = renderHook(() => + useCasesColumns({ + ...defaultColumnArgs, + isSelectorView: true, + }) + ); + + expect(result.current.find((i) => i.name === 'Status')).toBeFalsy(); + + await waitFor(() => { + expect(wrapper.find('[data-test-subj="cases-table"]').exists()).toBeTruthy(); + }); + + expect(wrapper.find('[data-test-subj="case-view-status-dropdown"]').exists()).toBeFalsy(); + }); }); From 75486ecd1228825cafca280dde998220b24fbb64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ester=20Mart=C3=AD=20Vilaseca?= Date: Fri, 3 Sep 2021 15:15:53 +0200 Subject: [PATCH 09/20] [Stack Monitoring] Add setup mode to react app (#110670) * Show setup mode button and setup bottom bar * Adapt setup mode in react components to work without angular * Add setup mode data update to react app * Add missing functions from setup mode * Revert setup mode changes from react components * remove some empty lines * Add setup button to monitoring toolbar * Fix types Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../application/global_state_context.tsx | 4 +- .../pages/cluster/overview_page.tsx | 26 ++- .../application/pages/page_template.tsx | 31 +-- .../application/setup_mode/setup_mode.tsx | 200 ++++++++++++++++ .../setup_mode/setup_mode_renderer.d.ts | 8 + .../setup_mode/setup_mode_renderer.js | 217 ++++++++++++++++++ .../public/components/shared/toolbar.tsx | 54 +++-- .../monitoring/public/external_config.ts | 16 ++ .../monitoring/public/lib/setup_mode.tsx | 4 + x-pack/plugins/monitoring/public/plugin.ts | 2 + 10 files changed, 514 insertions(+), 48 deletions(-) create mode 100644 x-pack/plugins/monitoring/public/application/setup_mode/setup_mode.tsx create mode 100644 x-pack/plugins/monitoring/public/application/setup_mode/setup_mode_renderer.d.ts create mode 100644 x-pack/plugins/monitoring/public/application/setup_mode/setup_mode_renderer.js create mode 100644 x-pack/plugins/monitoring/public/external_config.ts diff --git a/x-pack/plugins/monitoring/public/application/global_state_context.tsx b/x-pack/plugins/monitoring/public/application/global_state_context.tsx index dc33316dbd9d9..57bb638651d05 100644 --- a/x-pack/plugins/monitoring/public/application/global_state_context.tsx +++ b/x-pack/plugins/monitoring/public/application/global_state_context.tsx @@ -13,9 +13,11 @@ interface GlobalStateProviderProps { toasts: MonitoringStartPluginDependencies['core']['notifications']['toasts']; } -interface State { +export interface State { cluster_uuid?: string; ccs?: any; + inSetupMode?: boolean; + save?: () => void; } export const GlobalStateContext = createContext({} as State); diff --git a/x-pack/plugins/monitoring/public/application/pages/cluster/overview_page.tsx b/x-pack/plugins/monitoring/public/application/pages/cluster/overview_page.tsx index ddc097caea575..f329323bafda8 100644 --- a/x-pack/plugins/monitoring/public/application/pages/cluster/overview_page.tsx +++ b/x-pack/plugins/monitoring/public/application/pages/cluster/overview_page.tsx @@ -15,8 +15,15 @@ import { TabMenuItem } from '../page_template'; import { PageLoading } from '../../../components'; import { Overview } from '../../../components/cluster/overview'; import { ExternalConfigContext } from '../../external_config_context'; +import { SetupModeRenderer } from '../../setup_mode/setup_mode_renderer'; +import { SetupModeContext } from '../../../components/setup_mode/setup_mode_context'; const CODE_PATHS = [CODE_PATH_ALL]; +interface SetupModeProps { + setupMode: any; + flyoutComponent: any; + bottomBarComponent: any; +} export const ClusterOverview: React.FC<{}> = () => { // TODO: check how many requests with useClusters @@ -49,11 +56,20 @@ export const ClusterOverview: React.FC<{}> = () => { return ( {loaded ? ( - ( + + {flyoutComponent} + + {/* */} + {bottomBarComponent} + + )} /> ) : ( diff --git a/x-pack/plugins/monitoring/public/application/pages/page_template.tsx b/x-pack/plugins/monitoring/public/application/pages/page_template.tsx index f40c2d3ec5e50..29aafa09814fb 100644 --- a/x-pack/plugins/monitoring/public/application/pages/page_template.tsx +++ b/x-pack/plugins/monitoring/public/application/pages/page_template.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import { EuiFlexGroup, EuiFlexItem, EuiTab, EuiTabs, EuiTitle } from '@elastic/eui'; +import { EuiTab, EuiTabs } from '@elastic/eui'; import React from 'react'; import { useTitle } from '../hooks/use_title'; import { MonitoringToolbar } from '../../components/shared/toolbar'; @@ -29,34 +29,7 @@ export const PageTemplate: React.FC = ({ title, pageTitle, ta return (
- - - - -
{/* HERE GOES THE SETUP BUTTON */}
- - - {pageTitle && ( -
- -

{pageTitle}

- -
- )} - - - - - - - - - + {tabs && ( {tabs.map((item, idx) => { diff --git a/x-pack/plugins/monitoring/public/application/setup_mode/setup_mode.tsx b/x-pack/plugins/monitoring/public/application/setup_mode/setup_mode.tsx new file mode 100644 index 0000000000000..70932e5177337 --- /dev/null +++ b/x-pack/plugins/monitoring/public/application/setup_mode/setup_mode.tsx @@ -0,0 +1,200 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { render } from 'react-dom'; +import { get, includes } from 'lodash'; +import { i18n } from '@kbn/i18n'; +import { HttpStart } from 'kibana/public'; +import { KibanaContextProvider } from '../../../../../../src/plugins/kibana_react/public'; +import { Legacy } from '../../legacy_shims'; +import { SetupModeEnterButton } from '../../components/setup_mode/enter_button'; +import { SetupModeFeature } from '../../../common/enums'; +import { ISetupModeContext } from '../../components/setup_mode/setup_mode_context'; +import { State as GlobalState } from '../../application/global_state_context'; + +function isOnPage(hash: string) { + return includes(window.location.hash, hash); +} + +let globalState: GlobalState; +let httpService: HttpStart; + +interface ISetupModeState { + enabled: boolean; + data: any; + callback?: (() => void) | null; + hideBottomBar: boolean; +} +const setupModeState: ISetupModeState = { + enabled: false, + data: null, + callback: null, + hideBottomBar: false, +}; + +export const getSetupModeState = () => setupModeState; + +export const setNewlyDiscoveredClusterUuid = (clusterUuid: string) => { + globalState.cluster_uuid = clusterUuid; + globalState.save?.(); +}; + +export const fetchCollectionData = async (uuid?: string, fetchWithoutClusterUuid = false) => { + const clusterUuid = globalState.cluster_uuid; + const ccs = globalState.ccs; + + let url = '../api/monitoring/v1/setup/collection'; + if (uuid) { + url += `/node/${uuid}`; + } else if (!fetchWithoutClusterUuid && clusterUuid) { + url += `/cluster/${clusterUuid}`; + } else { + url += '/cluster'; + } + + try { + const response = await httpService.post(url, { + body: JSON.stringify({ + ccs, + }), + }); + return response; + } catch (err) { + // TODO: handle errors + throw new Error(err); + } +}; + +const notifySetupModeDataChange = () => setupModeState.callback && setupModeState.callback(); + +export const updateSetupModeData = async (uuid?: string, fetchWithoutClusterUuid = false) => { + const data = await fetchCollectionData(uuid, fetchWithoutClusterUuid); + setupModeState.data = data; + const hasPermissions = get(data, '_meta.hasPermissions', false); + if (!hasPermissions) { + let text: string = ''; + if (!hasPermissions) { + text = i18n.translate('xpack.monitoring.setupMode.notAvailablePermissions', { + defaultMessage: 'You do not have the necessary permissions to do this.', + }); + } + + Legacy.shims.toastNotifications.addDanger({ + title: i18n.translate('xpack.monitoring.setupMode.notAvailableTitle', { + defaultMessage: 'Setup mode is not available', + }), + text, + }); + return toggleSetupMode(false); + } + notifySetupModeDataChange(); + + const clusterUuid = globalState.cluster_uuid; + if (!clusterUuid) { + const liveClusterUuid: string = get(data, '_meta.liveClusterUuid'); + const migratedEsNodes = Object.values(get(data, 'elasticsearch.byUuid', {})).filter( + (node: any) => node.isPartiallyMigrated || node.isFullyMigrated + ); + if (liveClusterUuid && migratedEsNodes.length > 0) { + setNewlyDiscoveredClusterUuid(liveClusterUuid); + } + } +}; + +export const hideBottomBar = () => { + setupModeState.hideBottomBar = true; + notifySetupModeDataChange(); +}; +export const showBottomBar = () => { + setupModeState.hideBottomBar = false; + notifySetupModeDataChange(); +}; + +export const disableElasticsearchInternalCollection = async () => { + const clusterUuid = globalState.cluster_uuid; + const url = `../api/monitoring/v1/setup/collection/${clusterUuid}/disable_internal_collection`; + try { + const response = await httpService.post(url); + return response; + } catch (err) { + // TODO: handle errors + throw new Error(err); + } +}; + +export const toggleSetupMode = (inSetupMode: boolean) => { + setupModeState.enabled = inSetupMode; + globalState.inSetupMode = inSetupMode; + globalState.save?.(); + setSetupModeMenuItem(); + notifySetupModeDataChange(); + + if (inSetupMode) { + // Intentionally do not await this so we don't block UI operations + updateSetupModeData(); + } +}; + +export const setSetupModeMenuItem = () => { + if (isOnPage('no-data')) { + return; + } + + const enabled = !globalState.inSetupMode; + const I18nContext = Legacy.shims.I18nContext; + + render( + + + + + , + document.getElementById('setupModeNav') + ); +}; + +export const initSetupModeState = async ( + state: GlobalState, + http: HttpStart, + callback?: () => void +) => { + globalState = state; + httpService = http; + if (callback) { + setupModeState.callback = callback; + } + + if (globalState.inSetupMode) { + toggleSetupMode(true); + } +}; + +export const isInSetupMode = (context?: ISetupModeContext) => { + if (context?.setupModeSupported === false) { + return false; + } + if (setupModeState.enabled) { + return true; + } + + return globalState.inSetupMode; +}; + +export const isSetupModeFeatureEnabled = (feature: SetupModeFeature) => { + if (!setupModeState.enabled) { + return false; + } + + if (feature === SetupModeFeature.MetricbeatMigration) { + if (Legacy.shims.isCloud) { + return false; + } + } + + return true; +}; diff --git a/x-pack/plugins/monitoring/public/application/setup_mode/setup_mode_renderer.d.ts b/x-pack/plugins/monitoring/public/application/setup_mode/setup_mode_renderer.d.ts new file mode 100644 index 0000000000000..27462f07c07be --- /dev/null +++ b/x-pack/plugins/monitoring/public/application/setup_mode/setup_mode_renderer.d.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const SetupModeRenderer: FunctionComponent; diff --git a/x-pack/plugins/monitoring/public/application/setup_mode/setup_mode_renderer.js b/x-pack/plugins/monitoring/public/application/setup_mode/setup_mode_renderer.js new file mode 100644 index 0000000000000..337dacd4ecae9 --- /dev/null +++ b/x-pack/plugins/monitoring/public/application/setup_mode/setup_mode_renderer.js @@ -0,0 +1,217 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { Fragment } from 'react'; +import { + getSetupModeState, + initSetupModeState, + updateSetupModeData, + disableElasticsearchInternalCollection, + toggleSetupMode, + setSetupModeMenuItem, +} from './setup_mode'; +import { Flyout } from '../../components/metricbeat_migration/flyout'; +import { + EuiBottomBar, + EuiButton, + EuiFlexGroup, + EuiFlexItem, + EuiTextColor, + EuiIcon, + EuiSpacer, +} from '@elastic/eui'; +import { findNewUuid } from '../../components/renderers/lib/find_new_uuid'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { GlobalStateContext } from '../../application/global_state_context'; +import { withKibana } from '../../../../../../src/plugins/kibana_react/public'; + +class WrappedSetupModeRenderer extends React.Component { + globalState; + state = { + renderState: false, + isFlyoutOpen: false, + instance: null, + newProduct: null, + isSettingUpNew: false, + }; + + UNSAFE_componentWillMount() { + this.globalState = this.context; + const { kibana } = this.props; + initSetupModeState(this.globalState, kibana.services.http, (_oldData) => { + const newState = { renderState: true }; + + const { productName } = this.props; + if (!productName) { + this.setState(newState); + return; + } + + const setupModeState = getSetupModeState(); + if (!setupModeState.enabled || !setupModeState.data) { + this.setState(newState); + return; + } + + const data = setupModeState.data[productName]; + const oldData = _oldData ? _oldData[productName] : null; + if (data && oldData) { + const newUuid = findNewUuid(Object.keys(oldData.byUuid), Object.keys(data.byUuid)); + if (newUuid) { + newState.newProduct = data.byUuid[newUuid]; + } + } + + this.setState(newState); + }); + setSetupModeMenuItem(); + } + + reset() { + this.setState({ + renderState: false, + isFlyoutOpen: false, + instance: null, + newProduct: null, + isSettingUpNew: false, + }); + } + + getFlyout(data, meta) { + const { productName } = this.props; + const { isFlyoutOpen, instance, isSettingUpNew, newProduct } = this.state; + if (!data || !isFlyoutOpen) { + return null; + } + + let product = null; + if (newProduct) { + product = newProduct; + } + // For new instance discovery flow, we pass in empty instance object + else if (instance && Object.keys(instance).length) { + product = data.byUuid[instance.uuid]; + } + + if (!product) { + const uuids = Object.values(data.byUuid); + if (uuids.length && !isSettingUpNew) { + product = uuids[0]; + } else { + product = { + isNetNewUser: true, + }; + } + } + + return ( + this.reset()} + productName={productName} + product={product} + meta={meta} + instance={instance} + updateProduct={updateSetupModeData} + isSettingUpNew={isSettingUpNew} + /> + ); + } + + getBottomBar(setupModeState) { + if (!setupModeState.enabled || setupModeState.hideBottomBar) { + return null; + } + + return ( + + + + + + + + + , + }} + /> + + + + + + + + toggleSetupMode(false)} + > + {i18n.translate('xpack.monitoring.setupMode.exit', { + defaultMessage: `Exit setup mode`, + })} + + + + + + + + ); + } + + async shortcutToFinishMigration() { + await disableElasticsearchInternalCollection(); + await updateSetupModeData(); + } + + render() { + const { render, productName } = this.props; + const setupModeState = getSetupModeState(); + + let data = { byUuid: {} }; + if (setupModeState.data) { + if (productName && setupModeState.data[productName]) { + data = setupModeState.data[productName]; + } else if (setupModeState.data) { + data = setupModeState.data; + } + } + + const meta = setupModeState.data ? setupModeState.data._meta : null; + + return render({ + setupMode: { + data, + meta, + enabled: setupModeState.enabled, + productName, + updateSetupModeData, + shortcutToFinishMigration: () => this.shortcutToFinishMigration(), + openFlyout: (instance, isSettingUpNew) => + this.setState({ isFlyoutOpen: true, instance, isSettingUpNew }), + closeFlyout: () => this.setState({ isFlyoutOpen: false }), + }, + flyoutComponent: this.getFlyout(data, meta), + bottomBarComponent: this.getBottomBar(setupModeState), + }); + } +} + +WrappedSetupModeRenderer.contextType = GlobalStateContext; +export const SetupModeRenderer = withKibana(WrappedSetupModeRenderer); diff --git a/x-pack/plugins/monitoring/public/components/shared/toolbar.tsx b/x-pack/plugins/monitoring/public/components/shared/toolbar.tsx index 6e45d4d831ec9..e5962b7f80876 100644 --- a/x-pack/plugins/monitoring/public/components/shared/toolbar.tsx +++ b/x-pack/plugins/monitoring/public/components/shared/toolbar.tsx @@ -5,11 +5,21 @@ * 2.0. */ -import { EuiFlexGroup, EuiFlexItem, EuiSuperDatePicker, OnRefreshChangeProps } from '@elastic/eui'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiSuperDatePicker, + EuiTitle, + OnRefreshChangeProps, +} from '@elastic/eui'; import React, { useContext, useCallback } from 'react'; import { MonitoringTimeContainer } from '../../application/pages/use_monitoring_time'; -export const MonitoringToolbar = () => { +interface MonitoringToolbarProps { + pageTitle?: string; +} + +export const MonitoringToolbar: React.FC = ({ pageTitle }) => { const { currentTimerange, handleTimeChange, @@ -38,18 +48,36 @@ export const MonitoringToolbar = () => { ); return ( - - Setup Button + + + + +
{/* HERE GOES THE SETUP BUTTON */}
+ + + {pageTitle && ( +
+ +

{pageTitle}

+ +
+ )} + + + + - {}} - isPaused={isPaused} - refreshInterval={refreshInterval} - onRefreshChange={onRefreshChange} - /> +
+ {}} + isPaused={isPaused} + refreshInterval={refreshInterval} + onRefreshChange={onRefreshChange} + /> +
); diff --git a/x-pack/plugins/monitoring/public/external_config.ts b/x-pack/plugins/monitoring/public/external_config.ts new file mode 100644 index 0000000000000..29ce410a5a9dc --- /dev/null +++ b/x-pack/plugins/monitoring/public/external_config.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +let config: { [key: string]: unknown } = {}; + +export const setConfig = (externalConfig: { [key: string]: unknown }) => { + config = externalConfig; +}; + +export const isReactMigrationEnabled = () => { + return config.renderReactApp; +}; diff --git a/x-pack/plugins/monitoring/public/lib/setup_mode.tsx b/x-pack/plugins/monitoring/public/lib/setup_mode.tsx index 28fd7494b1d10..f622f2944a31a 100644 --- a/x-pack/plugins/monitoring/public/lib/setup_mode.tsx +++ b/x-pack/plugins/monitoring/public/lib/setup_mode.tsx @@ -15,6 +15,8 @@ import { ajaxErrorHandlersProvider } from './ajax_error_handler'; import { SetupModeEnterButton } from '../components/setup_mode/enter_button'; import { SetupModeFeature } from '../../common/enums'; import { ISetupModeContext } from '../components/setup_mode/setup_mode_context'; +import * as setupModeReact from '../application/setup_mode/setup_mode'; +import { isReactMigrationEnabled } from '../external_config'; function isOnPage(hash: string) { return includes(window.location.hash, hash); @@ -209,6 +211,7 @@ export const initSetupModeState = async ($scope: any, $injector: any, callback?: }; export const isInSetupMode = (context?: ISetupModeContext) => { + if (isReactMigrationEnabled()) return setupModeReact.isInSetupMode(context); if (context?.setupModeSupported === false) { return false; } @@ -222,6 +225,7 @@ export const isInSetupMode = (context?: ISetupModeContext) => { }; export const isSetupModeFeatureEnabled = (feature: SetupModeFeature) => { + if (isReactMigrationEnabled()) return setupModeReact.isSetupModeFeatureEnabled(feature); if (!setupModeState.enabled) { return false; } diff --git a/x-pack/plugins/monitoring/public/plugin.ts b/x-pack/plugins/monitoring/public/plugin.ts index 6884dba760fcd..6f625194287ba 100644 --- a/x-pack/plugins/monitoring/public/plugin.ts +++ b/x-pack/plugins/monitoring/public/plugin.ts @@ -36,6 +36,7 @@ import { createThreadPoolRejectionsAlertType } from './alerts/thread_pool_reject import { createMemoryUsageAlertType } from './alerts/memory_usage_alert'; import { createCCRReadExceptionsAlertType } from './alerts/ccr_read_exceptions_alert'; import { createLargeShardSizeAlertType } from './alerts/large_shard_size_alert'; +import { setConfig } from './external_config'; interface MonitoringSetupPluginDependencies { home?: HomePublicPluginSetup; @@ -125,6 +126,7 @@ export class MonitoringPlugin }); const config = Object.fromEntries(externalConfig); + setConfig(config); if (config.renderReactApp) { const { renderApp } = await import('./application'); return renderApp(coreStart, pluginsStart, params, config); From 9ba00ee594ee7dc8411127d983195efe71051ac1 Mon Sep 17 00:00:00 2001 From: ymao1 Date: Fri, 3 Sep 2021 09:49:00 -0400 Subject: [PATCH 10/20] [Actions] Allowing `service` specification in email connector config (#110458) * Initial commit of serverType in email connector config * Fleshing in route to get well known email service configs from nodemailer * Adding elastic cloud to well known server type * Cleaning up email constants and allowing for empty selection * Showing error if user doesn't select server type * Adding hook for setting email config based on server type * Adding tests and making sure settings are not overwritten on edit * Fixing functional test * Adding migration * Adding functional test for migration * Repurposing service instead of adding serverType * Cleanup * Disabling host/port/secure form fields when settings retrieved from API * Updating docs for service * Filtering options based on whether cloud is enabled * Initialize as disabled * Fixing types * Update docs/management/connectors/action-types/email.asciidoc Co-authored-by: David Kilfoyle <41695641+kilfoyle@users.noreply.github.com> Co-authored-by: David Kilfoyle <41695641+kilfoyle@users.noreply.github.com> --- .../connectors/action-types/email.asciidoc | 2 +- x-pack/plugins/actions/common/index.ts | 1 + .../server/builtin_action_types/email.test.ts | 109 ++++++++++- .../server/builtin_action_types/email.ts | 43 ++++- .../get_well_known_email_service.test.ts | 175 ++++++++++++++++++ .../routes/get_well_known_email_service.ts | 57 ++++++ x-pack/plugins/actions/server/routes/index.ts | 3 + .../saved_objects/actions_migrations.test.ts | 48 +++++ .../saved_objects/actions_migrations.ts | 25 +++ .../plugins/triggers_actions_ui/kibana.json | 2 +- .../public/application/app.tsx | 1 + .../builtin_action_types/email/api.ts | 20 ++ .../builtin_action_types/email/email.test.tsx | 61 ++++++ .../builtin_action_types/email/email.tsx | 68 +++++++ .../email/email_connector.test.tsx | 137 +++++++++++++- .../email/email_connector.tsx | 46 ++++- .../email/translations.ts | 7 + .../email/use_email_config.test.ts | 118 ++++++++++++ .../email/use_email_config.ts | 66 +++++++ .../components/builtin_action_types/types.ts | 1 + .../public/application/constants/index.ts | 2 +- .../common/lib/kibana/kibana_react.mock.ts | 1 + .../triggers_actions_ui/public/plugin.ts | 2 + .../alerting_api_integration/common/config.ts | 6 +- .../actions/builtin_action_types/email.ts | 117 ++++++++++++ .../spaces_only/tests/actions/migrations.ts | 18 ++ .../functional/es_archives/actions/data.json | 64 +++++++ .../es_archives/actions/mappings.json | 3 + 28 files changed, 1181 insertions(+), 22 deletions(-) create mode 100644 x-pack/plugins/actions/server/routes/get_well_known_email_service.test.ts create mode 100644 x-pack/plugins/actions/server/routes/get_well_known_email_service.ts create mode 100644 x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/email/api.ts create mode 100644 x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/email/use_email_config.test.ts create mode 100644 x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/email/use_email_config.ts diff --git a/docs/management/connectors/action-types/email.asciidoc b/docs/management/connectors/action-types/email.asciidoc index bab04b8052674..98d7b2591a572 100644 --- a/docs/management/connectors/action-types/email.asciidoc +++ b/docs/management/connectors/action-types/email.asciidoc @@ -51,7 +51,7 @@ Use the <> to customize connecto Config defines information for the connector type. -`service`:: The name of a https://nodemailer.com/smtp/well-known/[well-known email service provider]. If `service` is provided, `host`, `port`, and `secure` properties are ignored. For more information on the `gmail` service value, see the https://nodemailer.com/usage/using-gmail/[Nodemailer Gmail documentation]. +`service`:: The name of the email service. If `service` is `elastic_cloud` (for Elastic Cloud notifications) or one of Nodemailer's https://nodemailer.com/smtp/well-known/[well-known email service providers], `host`, `port`, and `secure` properties are ignored. If `service` is `other`, `host` and `port` properties must be defined. For more information on the `gmail` service value, see the https://nodemailer.com/usage/using-gmail/[Nodemailer Gmail documentation]. `from`:: An email address that corresponds to *Sender*. `host`:: A string that corresponds to *Host*. `port`:: A number that corresponds to *Port*. diff --git a/x-pack/plugins/actions/common/index.ts b/x-pack/plugins/actions/common/index.ts index 7825cbfb45f37..d3abfca83c8e8 100644 --- a/x-pack/plugins/actions/common/index.ts +++ b/x-pack/plugins/actions/common/index.ts @@ -13,3 +13,4 @@ export * from './alert_history_schema'; export * from './rewrite_request_case'; export const BASE_ACTION_API_PATH = '/api/actions'; +export const INTERNAL_BASE_ACTION_API_PATH = '/internal/actions'; diff --git a/x-pack/plugins/actions/server/builtin_action_types/email.test.ts b/x-pack/plugins/actions/server/builtin_action_types/email.test.ts index 8e9ea1c5e4aa9..450bf1744150d 100644 --- a/x-pack/plugins/actions/server/builtin_action_types/email.test.ts +++ b/x-pack/plugins/actions/server/builtin_action_types/email.test.ts @@ -52,7 +52,7 @@ describe('actionTypeRegistry.get() works', () => { }); describe('config validation', () => { - test('config validation succeeds when config is valid', () => { + test('config validation succeeds when config is valid for nodemailer well known service', () => { const config: Record = { service: 'gmail', from: 'bob@example.com', @@ -64,14 +64,46 @@ describe('config validation', () => { port: null, secure: null, }); + }); + + test(`config validation succeeds when config is valid and defaults to 'other' when service is undefined`, () => { + const config: Record = { + from: 'bob@example.com', + host: 'elastic.co', + port: 8080, + hasAuth: true, + }; + expect(validateConfig(actionType, config)).toEqual({ + ...config, + service: 'other', + secure: null, + }); + }); + + test(`config validation succeeds when config is valid and service requires custom host/port value`, () => { + const config: Record = { + service: 'exchange_server', + from: 'bob@example.com', + host: 'elastic.co', + port: 8080, + hasAuth: true, + }; + expect(validateConfig(actionType, config)).toEqual({ + ...config, + secure: null, + }); + }); - delete config.service; - config.host = 'elastic.co'; - config.port = 8080; - config.hasAuth = true; + test(`config validation succeeds when config is valid and service is elastic_cloud`, () => { + const config: Record = { + service: 'elastic_cloud', + from: 'bob@example.com', + hasAuth: true, + }; expect(validateConfig(actionType, config)).toEqual({ ...config, - service: null, + host: null, + port: null, secure: null, }); }); @@ -325,7 +357,7 @@ describe('execute()', () => { ...executorOptions, config: { ...config, - service: null, + service: 'other', hasAuth: false, }, secrets: { @@ -381,12 +413,73 @@ describe('execute()', () => { `); }); + test('parameters are as expected when using elastic_cloud service', async () => { + const customExecutorOptions: EmailActionTypeExecutorOptions = { + ...executorOptions, + config: { + ...config, + service: 'elastic_cloud', + hasAuth: false, + }, + secrets: { + ...secrets, + user: null, + password: null, + }, + }; + + sendEmailMock.mockReset(); + await actionType.executor(customExecutorOptions); + expect(sendEmailMock.mock.calls[0][1]).toMatchInlineSnapshot(` + Object { + "configurationUtilities": Object { + "ensureActionTypeEnabled": [MockFunction], + "ensureHostnameAllowed": [MockFunction], + "ensureUriAllowed": [MockFunction], + "getCustomHostSettings": [MockFunction], + "getProxySettings": [MockFunction], + "getResponseSettings": [MockFunction], + "getSSLSettings": [MockFunction], + "isActionTypeEnabled": [MockFunction], + "isHostnameAllowed": [MockFunction], + "isUriAllowed": [MockFunction], + }, + "content": Object { + "message": "a message to you + + -- + + This message was sent by Kibana.", + "subject": "the subject", + }, + "hasAuth": false, + "routing": Object { + "bcc": Array [ + "jimmy@example.com", + ], + "cc": Array [ + "james@example.com", + ], + "from": "bob@example.com", + "to": Array [ + "jim@example.com", + ], + }, + "transport": Object { + "host": "dockerhost", + "port": 10025, + "secure": false, + }, + } + `); + }); + test('returns expected result when an error is thrown', async () => { const customExecutorOptions: EmailActionTypeExecutorOptions = { ...executorOptions, config: { ...config, - service: null, + service: 'other', hasAuth: false, }, secrets: { diff --git a/x-pack/plugins/actions/server/builtin_action_types/email.ts b/x-pack/plugins/actions/server/builtin_action_types/email.ts index 47748f0f13722..9b11aec6251f6 100644 --- a/x-pack/plugins/actions/server/builtin_action_types/email.ts +++ b/x-pack/plugins/actions/server/builtin_action_types/email.ts @@ -9,6 +9,7 @@ import { curry } from 'lodash'; import { i18n } from '@kbn/i18n'; import { schema, TypeOf } from '@kbn/config-schema'; import nodemailerGetService from 'nodemailer/lib/well-known'; +import SMTPConnection from 'nodemailer/lib/smtp-connection'; import { sendEmail, JSON_TRANSPORT_SERVICE, SendEmailOptions, Transport } from './lib/send_email'; import { portSchema } from './lib/schemas'; @@ -32,10 +33,29 @@ export type EmailActionTypeExecutorOptions = ActionTypeExecutorOptions< // config definition export type ActionTypeConfigType = TypeOf; +// supported values for `service` in addition to nodemailer's list of well-known services +export enum AdditionalEmailServices { + ELASTIC_CLOUD = 'elastic_cloud', + EXCHANGE = 'exchange_server', + OTHER = 'other', +} + +// these values for `service` require users to fill in host/port/secure +export const CUSTOM_CONFIG_SERVICES: string[] = [ + AdditionalEmailServices.EXCHANGE, + AdditionalEmailServices.OTHER, +]; + +export const ELASTIC_CLOUD_SERVICE: SMTPConnection.Options = { + host: 'dockerhost', + port: 10025, + secure: false, +}; + const EMAIL_FOOTER_DIVIDER = '\n\n--\n\n'; const ConfigSchemaProps = { - service: schema.nullable(schema.string()), + service: schema.string({ defaultValue: 'other' }), host: schema.nullable(schema.string()), port: schema.nullable(portSchema()), secure: schema.nullable(schema.boolean()), @@ -58,7 +78,8 @@ function validateConfig( // translate messages. if (config.service === JSON_TRANSPORT_SERVICE) { return; - } else if (config.service == null) { + } else if (CUSTOM_CONFIG_SERVICES.indexOf(config.service) >= 0) { + // If configured `service` requires custom host/port/secure settings, validate that they are set if (config.host == null && config.port == null) { return 'either [service] or [host]/[port] is required'; } @@ -75,6 +96,7 @@ function validateConfig( return `[host] value '${config.host}' is not in the allowedHosts configuration`; } } else { + // Check configured `service` against nodemailer list of well known services + any custom ones allowed by Kibana const host = getServiceNameHost(config.service); if (host == null) { return `[service] value '${config.service}' is not valid`; @@ -201,13 +223,20 @@ async function executor( transport.password = secrets.password; } - if (config.service !== null) { - transport.service = config.service; - } else { + if (CUSTOM_CONFIG_SERVICES.indexOf(config.service) >= 0) { + // use configured host/port/secure values // already validated service or host/port is not null ... transport.host = config.host!; transport.port = config.port!; transport.secure = getSecureValue(config.secure, config.port); + } else if (config.service === AdditionalEmailServices.ELASTIC_CLOUD) { + // use custom elastic cloud settings + transport.host = ELASTIC_CLOUD_SERVICE.host!; + transport.port = ELASTIC_CLOUD_SERVICE.port!; + transport.secure = ELASTIC_CLOUD_SERVICE.secure!; + } else { + // use nodemailer's well known service config + transport.service = config.service; } const footerMessage = getFooterMessage({ @@ -253,6 +282,10 @@ async function executor( // utilities function getServiceNameHost(service: string): string | null { + if (service === AdditionalEmailServices.ELASTIC_CLOUD) { + return ELASTIC_CLOUD_SERVICE.host!; + } + const serviceEntry = nodemailerGetService(service); if (serviceEntry === false) return null; diff --git a/x-pack/plugins/actions/server/routes/get_well_known_email_service.test.ts b/x-pack/plugins/actions/server/routes/get_well_known_email_service.test.ts new file mode 100644 index 0000000000000..bbcedf18142ef --- /dev/null +++ b/x-pack/plugins/actions/server/routes/get_well_known_email_service.test.ts @@ -0,0 +1,175 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getWellKnownEmailServiceRoute } from './get_well_known_email_service'; +import { httpServiceMock } from 'src/core/server/mocks'; +import { licenseStateMock } from '../lib/license_state.mock'; +import { mockHandlerArguments } from './legacy/_mock_handler_arguments'; +import { verifyAccessAndContext } from './verify_access_and_context'; + +jest.mock('./verify_access_and_context.ts', () => ({ + verifyAccessAndContext: jest.fn(), +})); + +beforeEach(() => { + jest.resetAllMocks(); + (verifyAccessAndContext as jest.Mock).mockImplementation((license, handler) => handler); +}); + +describe('getWellKnownEmailServiceRoute', () => { + it('returns config for well known email service', async () => { + const licenseState = licenseStateMock.create(); + const router = httpServiceMock.createRouter(); + + getWellKnownEmailServiceRoute(router, licenseState); + + const [config, handler] = router.get.mock.calls[0]; + expect(config.path).toMatchInlineSnapshot( + `"/internal/actions/connector/_email_config/{service}"` + ); + + const [context, req, res] = mockHandlerArguments( + {}, + { + params: { service: 'gmail' }, + }, + ['ok'] + ); + + expect(await handler(context, req, res)).toMatchInlineSnapshot(` + Object { + "body": Object { + "host": "smtp.gmail.com", + "port": 465, + "secure": true, + }, + } + `); + + expect(res.ok).toHaveBeenCalledWith({ + body: { + host: 'smtp.gmail.com', + port: 465, + secure: true, + }, + }); + }); + + it('returns config for elastic cloud email service', async () => { + const licenseState = licenseStateMock.create(); + const router = httpServiceMock.createRouter(); + + getWellKnownEmailServiceRoute(router, licenseState); + + const [config, handler] = router.get.mock.calls[0]; + expect(config.path).toMatchInlineSnapshot( + `"/internal/actions/connector/_email_config/{service}"` + ); + + const [context, req, res] = mockHandlerArguments( + {}, + { + params: { service: 'elastic_cloud' }, + }, + ['ok'] + ); + + expect(await handler(context, req, res)).toMatchInlineSnapshot(` + Object { + "body": Object { + "host": "dockerhost", + "port": 10025, + "secure": false, + }, + } + `); + + expect(res.ok).toHaveBeenCalledWith({ + body: { + host: 'dockerhost', + port: 10025, + secure: false, + }, + }); + }); + + it('returns empty for unknown service', async () => { + const licenseState = licenseStateMock.create(); + const router = httpServiceMock.createRouter(); + + getWellKnownEmailServiceRoute(router, licenseState); + + const [config, handler] = router.get.mock.calls[0]; + expect(config.path).toMatchInlineSnapshot( + `"/internal/actions/connector/_email_config/{service}"` + ); + + const [context, req, res] = mockHandlerArguments( + {}, + { + params: { service: 'foo' }, + }, + ['ok'] + ); + + expect(await handler(context, req, res)).toMatchInlineSnapshot(` + Object { + "body": Object {}, + } + `); + + expect(res.ok).toHaveBeenCalledWith({ + body: {}, + }); + }); + + it('ensures the license allows getting well known email service config', async () => { + const licenseState = licenseStateMock.create(); + const router = httpServiceMock.createRouter(); + + getWellKnownEmailServiceRoute(router, licenseState); + + const [, handler] = router.get.mock.calls[0]; + + const [context, req, res] = mockHandlerArguments( + {}, + { + params: { service: 'gmail' }, + }, + ['ok'] + ); + + await handler(context, req, res); + + expect(verifyAccessAndContext).toHaveBeenCalledWith(licenseState, expect.any(Function)); + }); + + it('ensures the license check prevents getting well known email service config', async () => { + const licenseState = licenseStateMock.create(); + const router = httpServiceMock.createRouter(); + + (verifyAccessAndContext as jest.Mock).mockImplementation(() => async () => { + throw new Error('OMG'); + }); + + getWellKnownEmailServiceRoute(router, licenseState); + + const [, handler] = router.get.mock.calls[0]; + + const [context, req, res] = mockHandlerArguments( + {}, + { + params: { service: 'gmail' }, + }, + ['ok'] + ); + + expect(handler(context, req, res)).rejects.toMatchInlineSnapshot(`[Error: OMG]`); + + expect(verifyAccessAndContext).toHaveBeenCalledWith(licenseState, expect.any(Function)); + }); +}); diff --git a/x-pack/plugins/actions/server/routes/get_well_known_email_service.ts b/x-pack/plugins/actions/server/routes/get_well_known_email_service.ts new file mode 100644 index 0000000000000..837084f43b864 --- /dev/null +++ b/x-pack/plugins/actions/server/routes/get_well_known_email_service.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema } from '@kbn/config-schema'; +import { IRouter } from 'kibana/server'; +import nodemailerGetService from 'nodemailer/lib/well-known'; +import SMTPConnection from 'nodemailer/lib/smtp-connection'; +import { ILicenseState } from '../lib'; +import { INTERNAL_BASE_ACTION_API_PATH } from '../../common'; +import { ActionsRequestHandlerContext } from '../types'; +import { verifyAccessAndContext } from './verify_access_and_context'; +import { AdditionalEmailServices, ELASTIC_CLOUD_SERVICE } from '../builtin_action_types/email'; + +const paramSchema = schema.object({ + service: schema.string(), +}); + +export const getWellKnownEmailServiceRoute = ( + router: IRouter, + licenseState: ILicenseState +) => { + router.get( + { + path: `${INTERNAL_BASE_ACTION_API_PATH}/connector/_email_config/{service}`, + validate: { + params: paramSchema, + }, + }, + router.handleLegacyErrors( + verifyAccessAndContext(licenseState, async function (context, req, res) { + const { service } = req.params; + + let response: SMTPConnection.Options = {}; + if (service === AdditionalEmailServices.ELASTIC_CLOUD) { + response = ELASTIC_CLOUD_SERVICE; + } else { + const serviceEntry = nodemailerGetService(service); + if (serviceEntry) { + response = { + host: serviceEntry.host, + port: serviceEntry.port, + secure: serviceEntry.secure, + }; + } + } + + return res.ok({ + body: response, + }); + }) + ) + ); +}; diff --git a/x-pack/plugins/actions/server/routes/index.ts b/x-pack/plugins/actions/server/routes/index.ts index a236e514ef78d..0d39d87635d5e 100644 --- a/x-pack/plugins/actions/server/routes/index.ts +++ b/x-pack/plugins/actions/server/routes/index.ts @@ -15,6 +15,7 @@ import { getActionRoute } from './get'; import { getAllActionRoute } from './get_all'; import { connectorTypesRoute } from './connector_types'; import { updateActionRoute } from './update'; +import { getWellKnownEmailServiceRoute } from './get_well_known_email_service'; import { defineLegacyRoutes } from './legacy'; export function defineRoutes( @@ -30,4 +31,6 @@ export function defineRoutes( updateActionRoute(router, licenseState); connectorTypesRoute(router, licenseState); executeActionRoute(router, licenseState); + + getWellKnownEmailServiceRoute(router, licenseState); } diff --git a/x-pack/plugins/actions/server/saved_objects/actions_migrations.test.ts b/x-pack/plugins/actions/server/saved_objects/actions_migrations.test.ts index 7dc1426c13a4b..c094109a43d97 100644 --- a/x-pack/plugins/actions/server/saved_objects/actions_migrations.test.ts +++ b/x-pack/plugins/actions/server/saved_objects/actions_migrations.test.ts @@ -119,6 +119,54 @@ describe('successful migrations', () => { }); }); + describe('7.16.0', () => { + test('set service config property for .email connectors if service is undefined', () => { + const migration716 = getActionsMigrations(encryptedSavedObjectsSetup)['7.16.0']; + const action = getMockDataForEmail({ config: { service: undefined } }); + const migratedAction = migration716(action, context); + expect(migratedAction.attributes.config).toEqual({ + service: 'other', + }); + expect(migratedAction).toEqual({ + ...action, + attributes: { + ...action.attributes, + config: { + service: 'other', + }, + }, + }); + }); + + test('set service config property for .email connectors if service is null', () => { + const migration716 = getActionsMigrations(encryptedSavedObjectsSetup)['7.16.0']; + const action = getMockDataForEmail({ config: { service: null } }); + const migratedAction = migration716(action, context); + expect(migratedAction.attributes.config).toEqual({ + service: 'other', + }); + expect(migratedAction).toEqual({ + ...action, + attributes: { + ...action.attributes, + config: { + service: 'other', + }, + }, + }); + }); + + test('skips migrating .email connectors if service is defined, even if value is nonsense', () => { + const migration716 = getActionsMigrations(encryptedSavedObjectsSetup)['7.16.0']; + const action = getMockDataForEmail({ config: { service: 'gobbledygook' } }); + const migratedAction = migration716(action, context); + expect(migratedAction.attributes.config).toEqual({ + service: 'gobbledygook', + }); + expect(migratedAction).toEqual(action); + }); + }); + describe('8.0.0', () => { test('no op migration for rules SO', () => { const migration800 = getActionsMigrations(encryptedSavedObjectsSetup)['8.0.0']; diff --git a/x-pack/plugins/actions/server/saved_objects/actions_migrations.ts b/x-pack/plugins/actions/server/saved_objects/actions_migrations.ts index 7857a9e1f833f..e75f3eb41f2df 100644 --- a/x-pack/plugins/actions/server/saved_objects/actions_migrations.ts +++ b/x-pack/plugins/actions/server/saved_objects/actions_migrations.ts @@ -62,6 +62,12 @@ export function getActionsMigrations( pipeMigrations(addisMissingSecretsField) ); + const migrationEmailActionsSixteen = createEsoMigration( + encryptedSavedObjects, + (doc): doc is SavedObjectUnsanitizedDoc => doc.attributes.actionTypeId === '.email', + pipeMigrations(setServiceConfigIfNotSet) + ); + const migrationActions800 = createEsoMigration( encryptedSavedObjects, (doc: SavedObjectUnsanitizedDoc): doc is SavedObjectUnsanitizedDoc => @@ -73,6 +79,7 @@ export function getActionsMigrations( '7.10.0': executeMigrationWithErrorHandling(migrationActionsTen, '7.10.0'), '7.11.0': executeMigrationWithErrorHandling(migrationActionsEleven, '7.11.0'), '7.14.0': executeMigrationWithErrorHandling(migrationActionsFourteen, '7.14.0'), + '7.16.0': executeMigrationWithErrorHandling(migrationEmailActionsSixteen, '7.16.0'), '8.0.0': executeMigrationWithErrorHandling(migrationActions800, '8.0.0'), }; } @@ -157,6 +164,24 @@ const addHasAuthConfigurationObject = ( }; }; +const setServiceConfigIfNotSet = ( + doc: SavedObjectUnsanitizedDoc +): SavedObjectUnsanitizedDoc => { + if (doc.attributes.actionTypeId !== '.email' || null != doc.attributes.config.service) { + return doc; + } + return { + ...doc, + attributes: { + ...doc.attributes, + config: { + ...doc.attributes.config, + service: 'other', + }, + }, + }; +}; + const addisMissingSecretsField = ( doc: SavedObjectUnsanitizedDoc ): SavedObjectUnsanitizedDoc => { diff --git a/x-pack/plugins/triggers_actions_ui/kibana.json b/x-pack/plugins/triggers_actions_ui/kibana.json index 4033889d9811e..b72a7fe96817d 100644 --- a/x-pack/plugins/triggers_actions_ui/kibana.json +++ b/x-pack/plugins/triggers_actions_ui/kibana.json @@ -7,7 +7,7 @@ "version": "kibana", "server": true, "ui": true, - "optionalPlugins": ["alerting", "features", "home", "spaces"], + "optionalPlugins": ["alerting", "cloud", "features", "home", "spaces"], "requiredPlugins": ["management", "charts", "data", "kibanaReact", "kibanaUtils", "savedObjects"], "configPath": ["xpack", "trigger_actions_ui"], "extraPublicDirs": ["public/common", "public/common/constants"], diff --git a/x-pack/plugins/triggers_actions_ui/public/application/app.tsx b/x-pack/plugins/triggers_actions_ui/public/application/app.tsx index 9786f5dcb949d..ac0e6d95393b9 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/app.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/app.tsx @@ -37,6 +37,7 @@ export interface TriggersAndActionsUiServices extends CoreStart { alerting?: AlertingStart; spaces?: SpacesPluginStart; storage?: Storage; + isCloud: boolean; setBreadcrumbs: (crumbs: ChromeBreadcrumb[]) => void; actionTypeRegistry: ActionTypeRegistryContract; ruleTypeRegistry: RuleTypeRegistryContract; diff --git a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/email/api.ts b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/email/api.ts new file mode 100644 index 0000000000000..82c787426a38e --- /dev/null +++ b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/email/api.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { HttpSetup } from 'kibana/public'; +import { INTERNAL_BASE_ACTION_API_PATH } from '../../../constants'; +import { EmailConfig } from '../types'; + +export async function getServiceConfig({ + http, + service, +}: { + http: HttpSetup; + service: string; +}): Promise>> { + return await http.get(`${INTERNAL_BASE_ACTION_API_PATH}/connector/_email_config/${service}`); +} diff --git a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/email/email.test.tsx b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/email/email.test.tsx index 4d669ab4c76a1..0e1bf9ef53e15 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/email/email.test.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/email/email.test.tsx @@ -9,6 +9,7 @@ import { TypeRegistry } from '../../../type_registry'; import { registerBuiltInActionTypes } from '../index'; import { ActionTypeModel } from '../../../../types'; import { EmailActionConnector } from '../types'; +import { getEmailServices } from './email'; const ACTION_TYPE_ID = '.email'; let actionTypeModel: ActionTypeModel; @@ -29,6 +30,18 @@ describe('actionTypeRegistry.get() works', () => { }); }); +describe('getEmailServices', () => { + test('should return elastic cloud service if isCloudEnabled is true', () => { + const services = getEmailServices(true); + expect(services.find((service) => service.value === 'elastic_cloud')).toBeTruthy(); + }); + + test('should not return elastic cloud service if isCloudEnabled is false', () => { + const services = getEmailServices(false); + expect(services.find((service) => service.value === 'elastic_cloud')).toBeFalsy(); + }); +}); + describe('connector validation', () => { test('connector validation succeeds when connector config is valid', async () => { const actionConnector = { @@ -46,6 +59,7 @@ describe('connector validation', () => { host: 'localhost', test: 'test', hasAuth: true, + service: 'other', }, } as EmailActionConnector; @@ -55,6 +69,7 @@ describe('connector validation', () => { from: [], port: [], host: [], + service: [], }, }, secrets: { @@ -82,6 +97,7 @@ describe('connector validation', () => { host: 'localhost', test: 'test', hasAuth: false, + service: 'other', }, } as EmailActionConnector; @@ -91,6 +107,7 @@ describe('connector validation', () => { from: [], port: [], host: [], + service: [], }, }, secrets: { @@ -113,6 +130,7 @@ describe('connector validation', () => { config: { from: 'test@test.com', hasAuth: true, + service: 'other', }, } as EmailActionConnector; @@ -122,6 +140,7 @@ describe('connector validation', () => { from: [], port: ['Port is required.'], host: ['Host is required.'], + service: [], }, }, secrets: { @@ -148,6 +167,7 @@ describe('connector validation', () => { host: 'localhost', test: 'test', hasAuth: true, + service: 'other', }, } as EmailActionConnector; @@ -157,6 +177,7 @@ describe('connector validation', () => { from: [], port: [], host: [], + service: [], }, }, secrets: { @@ -183,6 +204,7 @@ describe('connector validation', () => { host: 'localhost', test: 'test', hasAuth: true, + service: 'other', }, } as EmailActionConnector; @@ -192,6 +214,7 @@ describe('connector validation', () => { from: [], port: [], host: [], + service: [], }, }, secrets: { @@ -202,6 +225,44 @@ describe('connector validation', () => { }, }); }); + test('connector validation fails when server type is not selected', async () => { + const actionConnector = { + secrets: { + user: 'user', + password: 'password', + }, + id: 'test', + actionTypeId: '.email', + isPreconfigured: false, + name: 'email', + config: { + from: 'test@test.com', + port: 2323, + host: 'localhost', + test: 'test', + hasAuth: true, + }, + }; + + expect( + await actionTypeModel.validateConnector((actionConnector as unknown) as EmailActionConnector) + ).toEqual({ + config: { + errors: { + from: [], + port: [], + host: [], + service: ['Service is required.'], + }, + }, + secrets: { + errors: { + user: [], + password: [], + }, + }, + }); + }); }); describe('action params validation', () => { diff --git a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/email/email.tsx b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/email/email.tsx index 5e23754621430..fe0b18b1b2e61 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/email/email.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/email/email.tsx @@ -7,6 +7,7 @@ import { lazy } from 'react'; import { i18n } from '@kbn/i18n'; +import { EuiSelectOption } from '@elastic/eui'; import { ActionTypeModel, ConnectorValidationResult, @@ -14,6 +15,69 @@ import { } from '../../../../types'; import { EmailActionParams, EmailConfig, EmailSecrets, EmailActionConnector } from '../types'; +const emailServices: EuiSelectOption[] = [ + { + text: i18n.translate( + 'xpack.triggersActionsUI.components.builtinActionTypes.emailAction.gmailServerTypeLabel', + { + defaultMessage: 'Gmail', + } + ), + value: 'gmail', + }, + { + text: i18n.translate( + 'xpack.triggersActionsUI.components.builtinActionTypes.emailAction.outlookServerTypeLabel', + { + defaultMessage: 'Outlook', + } + ), + value: 'outlook365', + }, + { + text: i18n.translate( + 'xpack.triggersActionsUI.components.builtinActionTypes.emailAction.amazonSesServerTypeLabel', + { + defaultMessage: 'Amazon SES', + } + ), + value: 'ses', + }, + { + text: i18n.translate( + 'xpack.triggersActionsUI.components.builtinActionTypes.emailAction.elasticCloudServerTypeLabel', + { + defaultMessage: 'Elastic Cloud', + } + ), + value: 'elastic_cloud', + }, + { + text: i18n.translate( + 'xpack.triggersActionsUI.components.builtinActionTypes.emailAction.exchangeServerTypeLabel', + { + defaultMessage: 'MS Exchange Server', + } + ), + value: 'exchange_server', + }, + { + text: i18n.translate( + 'xpack.triggersActionsUI.components.builtinActionTypes.emailAction.otherServerTypeLabel', + { + defaultMessage: 'Other', + } + ), + value: 'other', + }, +]; + +export function getEmailServices(isCloudEnabled: boolean) { + return isCloudEnabled + ? emailServices + : emailServices.filter((service) => service.value !== 'elastic_cloud'); +} + export function getActionType(): ActionTypeModel { const mailformat = /^[^@\s]+@[^@\s]+$/; return { @@ -41,6 +105,7 @@ export function getActionType(): ActionTypeModel(), port: new Array(), host: new Array(), + service: new Array(), }; const secretsErrors = { user: new Array(), @@ -69,6 +134,9 @@ export function getActionType(): ActionTypeModel { test('all connector fields is rendered', () => { const actionConnector = { @@ -29,7 +31,7 @@ describe('EmailActionConnectorFields renders', () => { const wrapper = mountWithIntl( {}} editActionSecrets={() => {}} readOnly={false} @@ -39,6 +41,7 @@ describe('EmailActionConnectorFields renders', () => { expect(wrapper.find('[data-test-subj="emailFromInput"]').first().prop('value')).toBe( 'test@test.com' ); + expect(wrapper.find('[data-test-subj="emailServiceSelectInput"]').length > 0).toBeTruthy(); expect(wrapper.find('[data-test-subj="emailHostInput"]').length > 0).toBeTruthy(); expect(wrapper.find('[data-test-subj="emailPortInput"]').length > 0).toBeTruthy(); expect(wrapper.find('[data-test-subj="emailUserInput"]').length > 0).toBeTruthy(); @@ -59,7 +62,7 @@ describe('EmailActionConnectorFields renders', () => { const wrapper = mountWithIntl( {}} editActionSecrets={() => {}} readOnly={false} @@ -75,6 +78,136 @@ describe('EmailActionConnectorFields renders', () => { expect(wrapper.find('[data-test-subj="emailPasswordInput"]').length > 0).toBeFalsy(); }); + test('service field defaults to empty when not defined', () => { + const actionConnector = { + secrets: { + user: 'user', + password: 'pass', + }, + id: 'test', + actionTypeId: '.email', + name: 'email', + config: { + from: 'test@test.com', + hasAuth: true, + }, + } as EmailActionConnector; + const wrapper = mountWithIntl( + {}} + editActionSecrets={() => {}} + readOnly={false} + /> + ); + expect(wrapper.find('[data-test-subj="emailFromInput"]').first().prop('value')).toBe( + 'test@test.com' + ); + expect(wrapper.find('[data-test-subj="emailServiceSelectInput"]').length > 0).toBeTruthy(); + expect(wrapper.find('select[data-test-subj="emailServiceSelectInput"]').prop('value')).toEqual( + '' + ); + }); + + test('service field is correctly selected when defined', () => { + const actionConnector = { + secrets: { + user: 'user', + password: 'pass', + }, + id: 'test', + actionTypeId: '.email', + name: 'email', + config: { + from: 'test@test.com', + hasAuth: true, + service: 'gmail', + }, + } as EmailActionConnector; + const wrapper = mountWithIntl( + {}} + editActionSecrets={() => {}} + readOnly={false} + /> + ); + expect(wrapper.find('[data-test-subj="emailServiceSelectInput"]').length > 0).toBeTruthy(); + expect(wrapper.find('select[data-test-subj="emailServiceSelectInput"]').prop('value')).toEqual( + 'gmail' + ); + }); + + test('host, port and secure fields should be disabled when service field is set to well known service', () => { + jest + .spyOn(hooks, 'useEmailConfig') + .mockImplementation(() => ({ emailServiceConfigurable: false, setEmailService: jest.fn() })); + const actionConnector = { + secrets: { + user: 'user', + password: 'pass', + }, + id: 'test', + actionTypeId: '.email', + name: 'email', + config: { + from: 'test@test.com', + hasAuth: true, + service: 'gmail', + }, + } as EmailActionConnector; + const wrapper = mountWithIntl( + {}} + editActionSecrets={() => {}} + readOnly={false} + /> + ); + expect(wrapper.find('[data-test-subj="emailHostInput"]').first().prop('disabled')).toBe(true); + expect(wrapper.find('[data-test-subj="emailPortInput"]').first().prop('disabled')).toBe(true); + expect(wrapper.find('[data-test-subj="emailSecureSwitch"]').first().prop('disabled')).toBe( + true + ); + }); + + test('host, port and secure fields should not be disabled when service field is set to other', () => { + jest + .spyOn(hooks, 'useEmailConfig') + .mockImplementation(() => ({ emailServiceConfigurable: true, setEmailService: jest.fn() })); + const actionConnector = { + secrets: { + user: 'user', + password: 'pass', + }, + id: 'test', + actionTypeId: '.email', + name: 'email', + config: { + from: 'test@test.com', + hasAuth: true, + service: 'other', + }, + } as EmailActionConnector; + const wrapper = mountWithIntl( + {}} + editActionSecrets={() => {}} + readOnly={false} + /> + ); + expect(wrapper.find('[data-test-subj="emailHostInput"]').first().prop('disabled')).toBe(false); + expect(wrapper.find('[data-test-subj="emailPortInput"]').first().prop('disabled')).toBe(false); + expect(wrapper.find('[data-test-subj="emailSecureSwitch"]').first().prop('disabled')).toBe( + false + ); + }); + test('should display a message to remember username and password when creating a connector with authentication', () => { const actionConnector = { actionTypeId: '.email', diff --git a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/email/email_connector.tsx b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/email/email_connector.tsx index e4d73ced1eb59..c37c3fc8355b1 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/email/email_connector.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/email/email_connector.tsx @@ -12,6 +12,7 @@ import { EuiFlexGroup, EuiFieldNumber, EuiFieldPassword, + EuiSelect, EuiSwitch, EuiFormRow, EuiTitle, @@ -24,13 +25,22 @@ import { ActionConnectorFieldsProps } from '../../../../types'; import { EmailActionConnector } from '../types'; import { useKibana } from '../../../../common/lib/kibana'; import { getEncryptedFieldNotifyLabel } from '../../get_encrypted_field_notify_label'; +import { getEmailServices } from './email'; +import { useEmailConfig } from './use_email_config'; export const EmailActionConnectorFields: React.FunctionComponent< ActionConnectorFieldsProps > = ({ action, editActionConfig, editActionSecrets, errors, readOnly }) => { - const { docLinks } = useKibana().services; - const { from, host, port, secure, hasAuth } = action.config; + const { docLinks, http, isCloud } = useKibana().services; + const { from, host, port, secure, hasAuth, service } = action.config; const { user, password } = action.secrets; + + const { emailServiceConfigurable, setEmailService } = useEmailConfig( + http, + service, + editActionConfig + ); + useEffect(() => { if (!action.id) { editActionConfig('hasAuth', true); @@ -42,6 +52,8 @@ export const EmailActionConnectorFields: React.FunctionComponent< from !== undefined && errors.from !== undefined && errors.from.length > 0; const isHostInvalid: boolean = host !== undefined && errors.host !== undefined && errors.host.length > 0; + const isServiceInvalid: boolean = + service !== undefined && errors.service !== undefined && errors.service.length > 0; const isPortInvalid: boolean = port !== undefined && errors.port !== undefined && errors.port.length > 0; @@ -93,6 +105,31 @@ export const EmailActionConnectorFields: React.FunctionComponent< + + + { + setEmailService(e.target.value); + }} + /> + + { editActionConfig('secure', e.target.checked); diff --git a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/email/translations.ts b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/email/translations.ts index 5da9145ecec0b..df68d0d1237ed 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/email/translations.ts +++ b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/email/translations.ts @@ -28,6 +28,13 @@ export const PORT_REQUIRED = i18n.translate( } ); +export const SERVICE_REQUIRED = i18n.translate( + 'xpack.triggersActionsUI.components.builtinActionTypes.error.requiredServiceText', + { + defaultMessage: 'Service is required.', + } +); + export const HOST_REQUIRED = i18n.translate( 'xpack.triggersActionsUI.components.builtinActionTypes.error.requiredHostText', { diff --git a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/email/use_email_config.test.ts b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/email/use_email_config.test.ts new file mode 100644 index 0000000000000..7d9cf15852748 --- /dev/null +++ b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/email/use_email_config.test.ts @@ -0,0 +1,118 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { renderHook, act } from '@testing-library/react-hooks'; +import { HttpSetup } from 'kibana/public'; +import { useEmailConfig } from './use_email_config'; + +const http = { + get: jest.fn(), +}; + +const editActionConfig = jest.fn(); + +const renderUseEmailConfigHook = (currentService?: string) => + renderHook(() => + useEmailConfig((http as unknown) as HttpSetup, currentService, editActionConfig) + ); + +describe('useEmailConfig', () => { + beforeEach(() => jest.clearAllMocks()); + + it('should call get email config API when service changes and handle result', async () => { + http.get.mockResolvedValueOnce({ + host: 'smtp.gmail.com', + port: 465, + secure: true, + }); + const { result, waitForNextUpdate } = renderUseEmailConfigHook(); + await act(async () => { + result.current.setEmailService('gmail'); + await waitForNextUpdate(); + }); + + expect(http.get).toHaveBeenCalledWith('/internal/actions/connector/_email_config/gmail'); + expect(editActionConfig).toHaveBeenCalledWith('service', 'gmail'); + + expect(editActionConfig).toHaveBeenCalledWith('host', 'smtp.gmail.com'); + expect(editActionConfig).toHaveBeenCalledWith('port', 465); + expect(editActionConfig).toHaveBeenCalledWith('secure', true); + + expect(result.current.emailServiceConfigurable).toEqual(false); + }); + + it('should call get email config API when service changes and handle partial result', async () => { + http.get.mockResolvedValueOnce({ + host: 'smtp.gmail.com', + port: 465, + }); + const { result, waitForNextUpdate } = renderUseEmailConfigHook(); + await act(async () => { + result.current.setEmailService('gmail'); + await waitForNextUpdate(); + }); + + expect(http.get).toHaveBeenCalledWith('/internal/actions/connector/_email_config/gmail'); + expect(editActionConfig).toHaveBeenCalledWith('service', 'gmail'); + + expect(editActionConfig).toHaveBeenCalledWith('host', 'smtp.gmail.com'); + expect(editActionConfig).toHaveBeenCalledWith('port', 465); + expect(editActionConfig).toHaveBeenCalledWith('secure', false); + + expect(result.current.emailServiceConfigurable).toEqual(false); + }); + + it('should call get email config API when service changes and handle empty result', async () => { + http.get.mockResolvedValueOnce({}); + const { result, waitForNextUpdate } = renderUseEmailConfigHook(); + await act(async () => { + result.current.setEmailService('foo'); + await waitForNextUpdate(); + }); + + expect(http.get).toHaveBeenCalledWith('/internal/actions/connector/_email_config/foo'); + expect(editActionConfig).toHaveBeenCalledWith('service', 'foo'); + + expect(editActionConfig).toHaveBeenCalledWith('host', ''); + expect(editActionConfig).toHaveBeenCalledWith('port', 0); + expect(editActionConfig).toHaveBeenCalledWith('secure', false); + + expect(result.current.emailServiceConfigurable).toEqual(true); + }); + + it('should call get email config API when service changes and handle errors', async () => { + http.get.mockImplementationOnce(() => { + throw new Error('no!'); + }); + const { result, waitForNextUpdate } = renderUseEmailConfigHook(); + await act(async () => { + result.current.setEmailService('foo'); + await waitForNextUpdate(); + }); + + expect(http.get).toHaveBeenCalledWith('/internal/actions/connector/_email_config/foo'); + expect(editActionConfig).toHaveBeenCalledWith('service', 'foo'); + + expect(editActionConfig).toHaveBeenCalledWith('host', ''); + expect(editActionConfig).toHaveBeenCalledWith('port', 0); + expect(editActionConfig).toHaveBeenCalledWith('secure', false); + + expect(result.current.emailServiceConfigurable).toEqual(true); + }); + + it('should call get email config API when initial service value is passed and determine if config is editable without overwriting config', async () => { + http.get.mockResolvedValueOnce({ + host: 'smtp.gmail.com', + port: 465, + secure: true, + }); + const { result } = renderUseEmailConfigHook('gmail'); + expect(http.get).toHaveBeenCalledWith('/internal/actions/connector/_email_config/gmail'); + expect(editActionConfig).not.toHaveBeenCalled(); + expect(result.current.emailServiceConfigurable).toEqual(false); + }); +}); diff --git a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/email/use_email_config.ts b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/email/use_email_config.ts new file mode 100644 index 0000000000000..fad71cf5d6385 --- /dev/null +++ b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/email/use_email_config.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useCallback, useEffect, useState } from 'react'; +import { HttpSetup } from 'kibana/public'; +import { isEmpty } from 'lodash'; +import { EmailConfig } from '../types'; +import { getServiceConfig } from './api'; + +export function useEmailConfig( + http: HttpSetup, + currentService: string | undefined, + editActionConfig: (property: string, value: unknown) => void +) { + const [emailServiceConfigurable, setEmailServiceConfigurable] = useState(false); + const [emailService, setEmailService] = useState(undefined); + + const getEmailServiceConfig = useCallback( + async (service: string) => { + let serviceConfig: Partial>; + try { + serviceConfig = await getServiceConfig({ http, service }); + setEmailServiceConfigurable(isEmpty(serviceConfig)); + } catch (err) { + serviceConfig = {}; + setEmailServiceConfigurable(true); + } + + return serviceConfig; + }, + // eslint-disable-next-line react-hooks/exhaustive-deps + [editActionConfig] + ); + + useEffect(() => { + (async () => { + if (emailService) { + const serviceConfig = await getEmailServiceConfig(emailService); + + editActionConfig('service', emailService); + editActionConfig('host', serviceConfig?.host ? serviceConfig.host : ''); + editActionConfig('port', serviceConfig?.port ? serviceConfig.port : 0); + editActionConfig('secure', null != serviceConfig?.secure ? serviceConfig.secure : false); + } + })(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [emailService]); + + useEffect(() => { + (async () => { + if (currentService) { + await getEmailServiceConfig(currentService); + } + })(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [currentService]); + + return { + emailServiceConfigurable, + setEmailService, + }; +} diff --git a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/types.ts b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/types.ts index 50410ba3c153d..60e0a0f14b897 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/types.ts +++ b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/types.ts @@ -78,6 +78,7 @@ export interface EmailConfig { port: number; secure?: boolean; hasAuth: boolean; + service: string; } export interface EmailSecrets { diff --git a/x-pack/plugins/triggers_actions_ui/public/application/constants/index.ts b/x-pack/plugins/triggers_actions_ui/public/application/constants/index.ts index cc04b8e7871cd..bed7b09110d87 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/constants/index.ts +++ b/x-pack/plugins/triggers_actions_ui/public/application/constants/index.ts @@ -11,7 +11,7 @@ export { BASE_ALERTING_API_PATH, INTERNAL_BASE_ALERTING_API_PATH, } from '../../../../alerting/common'; -export { BASE_ACTION_API_PATH } from '../../../../actions/common'; +export { BASE_ACTION_API_PATH, INTERNAL_BASE_ACTION_API_PATH } from '../../../../actions/common'; export type Section = 'connectors' | 'rules'; diff --git a/x-pack/plugins/triggers_actions_ui/public/common/lib/kibana/kibana_react.mock.ts b/x-pack/plugins/triggers_actions_ui/public/common/lib/kibana/kibana_react.mock.ts index 2985a5306ed51..de64906f75de3 100644 --- a/x-pack/plugins/triggers_actions_ui/public/common/lib/kibana/kibana_react.mock.ts +++ b/x-pack/plugins/triggers_actions_ui/public/common/lib/kibana/kibana_react.mock.ts @@ -40,6 +40,7 @@ export const createStartServicesMock = (): TriggersAndActionsUiServices => { list: jest.fn(), } as ActionTypeRegistryContract, charts: chartPluginMock.createStartContract(), + isCloud: false, kibanaFeatures: [], element: ({ style: { cursor: 'pointer' }, diff --git a/x-pack/plugins/triggers_actions_ui/public/plugin.ts b/x-pack/plugins/triggers_actions_ui/public/plugin.ts index 7661eefba7f65..17f0766a826e3 100644 --- a/x-pack/plugins/triggers_actions_ui/public/plugin.ts +++ b/x-pack/plugins/triggers_actions_ui/public/plugin.ts @@ -66,6 +66,7 @@ export interface TriggersAndActionsUIPublicPluginStart { interface PluginsSetup { management: ManagementSetup; home?: HomePublicPluginSetup; + cloud?: { isCloudEnabled: boolean }; } interface PluginsStart { @@ -148,6 +149,7 @@ export class Plugin charts: pluginsStart.charts, alerting: pluginsStart.alerting, spaces: pluginsStart.spaces, + isCloud: Boolean(plugins.cloud?.isCloudEnabled), element: params.element, storage: new Storage(window.localStorage), setBreadcrumbs: params.setBreadcrumbs, diff --git a/x-pack/test/alerting_api_integration/common/config.ts b/x-pack/test/alerting_api_integration/common/config.ts index 5a9d2a20fee16..dd43606cc79b7 100644 --- a/x-pack/test/alerting_api_integration/common/config.ts +++ b/x-pack/test/alerting_api_integration/common/config.ts @@ -149,7 +149,11 @@ export function createTestConfig(name: string, options: CreateTestConfigOptions) serverArgs: [ ...xPackApiIntegrationTestsConfig.get('kbnTestServer.serverArgs'), ...(options.publicBaseUrl ? ['--server.publicBaseUrl=https://localhost:5601'] : []), - `--xpack.actions.allowedHosts=${JSON.stringify(['localhost', 'some.non.existent.com'])}`, + `--xpack.actions.allowedHosts=${JSON.stringify([ + 'localhost', + 'some.non.existent.com', + 'smtp.live.com', + ])}`, '--xpack.encryptedSavedObjects.encryptionKey="wuGNaIhoMpk5sO4UBxgr3NyW1sFcLgIf"', '--xpack.alerting.invalidateApiKeysTask.interval="15s"', '--xpack.alerting.healthCheck.interval="1s"', diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/tests/actions/builtin_action_types/email.ts b/x-pack/test/alerting_api_integration/security_and_spaces/tests/actions/builtin_action_types/email.ts index 917246f09a99e..b3829824b7971 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/tests/actions/builtin_action_types/email.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/tests/actions/builtin_action_types/email.ts @@ -319,5 +319,122 @@ export default function emailTest({ getService }: FtrProviderContext) { }); }); }); + + it('should return 200 when creating an email action without defining service', async () => { + const { body: createdAction } = await supertest + .post('/api/actions/connector') + .set('kbn-xsrf', 'foo') + .send({ + name: 'An email action', + connector_type_id: '.email', + config: { + from: 'bob@example.com', + host: 'some.non.existent.com', + port: 25, + hasAuth: true, + }, + secrets: { + user: 'bob', + password: 'supersecret', + }, + }) + .expect(200); + + expect(createdAction).to.eql({ + id: createdAction.id, + is_preconfigured: false, + name: 'An email action', + connector_type_id: '.email', + is_missing_secrets: false, + config: { + service: 'other', + hasAuth: true, + host: 'some.non.existent.com', + port: 25, + secure: null, + from: 'bob@example.com', + }, + }); + + expect(typeof createdAction.id).to.be('string'); + + const { body: fetchedAction } = await supertest + .get(`/api/actions/connector/${createdAction.id}`) + .expect(200); + + expect(fetchedAction).to.eql({ + id: fetchedAction.id, + is_preconfigured: false, + name: 'An email action', + connector_type_id: '.email', + is_missing_secrets: false, + config: { + from: 'bob@example.com', + service: 'other', + hasAuth: true, + host: 'some.non.existent.com', + port: 25, + secure: null, + }, + }); + }); + + it('should return 200 when creating an email action with nodemailer well-defined service', async () => { + const { body: createdAction } = await supertest + .post('/api/actions/connector') + .set('kbn-xsrf', 'foo') + .send({ + name: 'An email action', + connector_type_id: '.email', + config: { + from: 'bob@example.com', + service: 'hotmail', + hasAuth: true, + }, + secrets: { + user: 'bob', + password: 'supersecret', + }, + }) + .expect(200); + + expect(createdAction).to.eql({ + id: createdAction.id, + is_preconfigured: false, + name: 'An email action', + connector_type_id: '.email', + is_missing_secrets: false, + config: { + service: 'hotmail', + hasAuth: true, + host: null, + port: null, + secure: null, + from: 'bob@example.com', + }, + }); + + expect(typeof createdAction.id).to.be('string'); + + const { body: fetchedAction } = await supertest + .get(`/api/actions/connector/${createdAction.id}`) + .expect(200); + + expect(fetchedAction).to.eql({ + id: fetchedAction.id, + is_preconfigured: false, + name: 'An email action', + connector_type_id: '.email', + is_missing_secrets: false, + config: { + from: 'bob@example.com', + service: 'hotmail', + hasAuth: true, + host: null, + port: null, + secure: null, + }, + }); + }); }); } diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/actions/migrations.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/actions/migrations.ts index 811a9470611eb..9b88dace13239 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/actions/migrations.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/actions/migrations.ts @@ -64,5 +64,23 @@ export default function createGetTests({ getService }: FtrProviderContext) { expect(responseWithisMissingSecrets.status).to.eql(200); expect(responseWithisMissingSecrets.body.isMissingSecrets).to.eql(false); }); + + it('7.16.0 migrates email connector configurations to set `service` property if not set', async () => { + const connectorWithService = await supertest.get( + `${getUrlPrefix(``)}/api/actions/action/0f8f2810-0a59-11ec-9a7c-fd0c2b83ff7c` + ); + + expect(connectorWithService.status).to.eql(200); + expect(connectorWithService.body.config).key('service'); + expect(connectorWithService.body.config.service).to.eql('someservice'); + + const connectorWithoutService = await supertest.get( + `${getUrlPrefix(``)}/api/actions/action/1e0824a0-0a59-11ec-9a7c-fd0c2b83ff7c` + ); + + expect(connectorWithoutService.status).to.eql(200); + expect(connectorWithoutService.body.config).key('service'); + expect(connectorWithoutService.body.config.service).to.eql('other'); + }); }); } diff --git a/x-pack/test/functional/es_archives/actions/data.json b/x-pack/test/functional/es_archives/actions/data.json index 18d67da1752bc..31d10005c0939 100644 --- a/x-pack/test/functional/es_archives/actions/data.json +++ b/x-pack/test/functional/es_archives/actions/data.json @@ -110,3 +110,67 @@ } } } + +{ + "type": "doc", + "value": { + "id": "action:0f8f2810-0a59-11ec-9a7c-fd0c2b83ff7c", + "index": ".kibana_1", + "source": { + "action": { + "actionTypeId" : ".email", + "name" : "test email connector with auth", + "isMissingSecrets" : false, + "config" : { + "hasAuth" : true, + "from" : "me@me.com", + "host" : "smtp.myhost.com", + "port" : 25, + "service" : "someservice", + "secure" : null + }, + "secrets" : "V2EJEtTv3yTFi1kdglhNahnKYWCS+J7aWCJQU+eEqGPZEz6n7G1NsBWoh7IY0FteLTilTteQXyY/Eg3k/7bb0G8Mz+WBZ1mRvUggGTFqgoOptyUsvHoBhv0R/1bCTCabN3Pe88AfnC+VDXqwuMifpmgKEEsKF3H8VONv7TYO02FW" + }, + "migrationVersion": { + "action": "7.14.0" + }, + "coreMigrationVersion" : "7.15.0", + "references": [ + ], + "type": "action", + "updated_at": "2021-08-31T12:43:37.117Z" + } + } +} + +{ + "type": "doc", + "value": { + "id": "action:1e0824a0-0a59-11ec-9a7c-fd0c2b83ff7c", + "index": ".kibana_1", + "source": { + "action": { + "actionTypeId" : ".email", + "name" : "test email connector no auth", + "isMissingSecrets" : false, + "config" : { + "hasAuth" : false, + "from" : "you@you.com", + "host" : "smtp.you.com", + "port" : 485, + "secure" : true, + "service" : null + }, + "secrets" : "iw/bRTXZQXOV0ODocb6FQnHR6AyeVyD91We03llNStyTNFwuHVWdFl6ZdiEEeDOadBMeJomvp/dAfQevGpbwWdclcu9F87x3CfeGqV9DtBy0dXRbx9PzKBwgJdK3ucHQDFAs8ZXQbefvCOFjCHGAsJDPhTKj5rTUyg==" + }, + "migrationVersion": { + "action": "7.14.0" + }, + "coreMigrationVersion" : "7.15.0", + "references": [ + ], + "type": "action", + "updated_at": "2021-08-31T12:44:01.396Z" + } + } +} diff --git a/x-pack/test/functional/es_archives/actions/mappings.json b/x-pack/test/functional/es_archives/actions/mappings.json index 737e0df57552e..8289174ffd57d 100644 --- a/x-pack/test/functional/es_archives/actions/mappings.json +++ b/x-pack/test/functional/es_archives/actions/mappings.json @@ -572,6 +572,9 @@ } } }, + "coreMigrationVersion": { + "type": "keyword" + }, "dashboard": { "properties": { "description": { From 21b4752dba84d8c2ebaa29ec4a65703ea416d60d Mon Sep 17 00:00:00 2001 From: Marco Liberati Date: Fri, 3 Sep 2021 15:57:57 +0200 Subject: [PATCH 11/20] [Lens] Fix transition to custom palette inconsistency when in number mode (#110852) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../public/services/palettes/helpers.ts | 5 +- .../coloring/palette_configuration.test.tsx | 182 ++++++++++++------ .../coloring/palette_configuration.tsx | 36 ++-- .../shared_components/coloring/utils.test.ts | 73 +++++++ .../shared_components/coloring/utils.ts | 5 +- x-pack/test/functional/apps/lens/table.ts | 21 ++ 6 files changed, 246 insertions(+), 76 deletions(-) diff --git a/src/plugins/charts/public/services/palettes/helpers.ts b/src/plugins/charts/public/services/palettes/helpers.ts index d4b1e98f94cc8..bd1f8350ba9f3 100644 --- a/src/plugins/charts/public/services/palettes/helpers.ts +++ b/src/plugins/charts/public/services/palettes/helpers.ts @@ -70,7 +70,10 @@ export function workoutColorForValue( const comparisonFn = (v: number, threshold: number) => v - threshold; // if steps are defined consider the specific rangeMax/Min as data boundaries - const maxRange = stops.length ? rangeMax : dataRangeArguments[1]; + // as of max reduce its value by 1/colors.length for correct continuity checks + const maxRange = stops.length + ? rangeMax + : dataRangeArguments[1] - (dataRangeArguments[1] - dataRangeArguments[0]) / colors.length; const minRange = stops.length ? rangeMin : dataRangeArguments[0]; // in case of shorter rangers, extends the steps on the sides to cover the whole set diff --git a/x-pack/plugins/lens/public/shared_components/coloring/palette_configuration.test.tsx b/x-pack/plugins/lens/public/shared_components/coloring/palette_configuration.test.tsx index ad1755bdbe85c..cda891871168e 100644 --- a/x-pack/plugins/lens/public/shared_components/coloring/palette_configuration.test.tsx +++ b/x-pack/plugins/lens/public/shared_components/coloring/palette_configuration.test.tsx @@ -6,7 +6,7 @@ */ import React from 'react'; -import { EuiColorPalettePickerPaletteProps } from '@elastic/eui'; +import { EuiButtonGroup, EuiColorPalettePickerPaletteProps } from '@elastic/eui'; import { mountWithIntl } from '@kbn/test/jest'; import { chartPluginMock } from 'src/plugins/charts/public/mocks'; import type { PaletteOutput, PaletteRegistry } from 'src/plugins/charts/public'; @@ -14,6 +14,7 @@ import { ReactWrapper } from 'enzyme'; import type { CustomPaletteParams } from '../../../common'; import { applyPaletteParams } from './utils'; import { CustomizablePalette } from './palette_configuration'; +import { act } from 'react-dom/test-utils'; // mocking random id generator function jest.mock('@elastic/eui', () => { @@ -128,71 +129,136 @@ describe('palette panel', () => { }); }); - describe('reverse option', () => { - beforeEach(() => { - props = { - activePalette: { type: 'palette', name: 'positive' }, - palettes: paletteRegistry, - setPalette: jest.fn(), - dataBounds: { min: 0, max: 100 }, - }; - }); + it('should rewrite the min/max range values on palette change', () => { + const instance = mountWithIntl(); + + changePaletteIn(instance, 'custom'); - function toggleReverse(instance: ReactWrapper, checked: boolean) { - return instance - .find('[data-test-subj="lnsPalettePanel_dynamicColoring_reverse"]') - .first() - .prop('onClick')!({} as React.MouseEvent); - } - - it('should reverse the colorStops on click', () => { - const instance = mountWithIntl(); - - toggleReverse(instance, true); - - expect(props.setPalette).toHaveBeenCalledWith( - expect.objectContaining({ - params: expect.objectContaining({ - reverse: true, - }), - }) - ); + expect(props.setPalette).toHaveBeenCalledWith({ + type: 'palette', + name: 'custom', + params: expect.objectContaining({ + rangeMin: 0, + rangeMax: 50, + }), }); }); + }); + + describe('reverse option', () => { + beforeEach(() => { + props = { + activePalette: { type: 'palette', name: 'positive' }, + palettes: paletteRegistry, + setPalette: jest.fn(), + dataBounds: { min: 0, max: 100 }, + }; + }); + + function toggleReverse(instance: ReactWrapper, checked: boolean) { + return instance + .find('[data-test-subj="lnsPalettePanel_dynamicColoring_reverse"]') + .first() + .prop('onClick')!({} as React.MouseEvent); + } + + it('should reverse the colorStops on click', () => { + const instance = mountWithIntl(); + + toggleReverse(instance, true); + + expect(props.setPalette).toHaveBeenCalledWith( + expect.objectContaining({ + params: expect.objectContaining({ + reverse: true, + }), + }) + ); + }); + }); + + describe('percentage / number modes', () => { + beforeEach(() => { + props = { + activePalette: { type: 'palette', name: 'positive' }, + palettes: paletteRegistry, + setPalette: jest.fn(), + dataBounds: { min: 5, max: 200 }, + }; + }); - describe('custom stops', () => { - beforeEach(() => { - props = { - activePalette: { type: 'palette', name: 'positive' }, - palettes: paletteRegistry, - setPalette: jest.fn(), - dataBounds: { min: 0, max: 100 }, - }; + it('should switch mode and range boundaries on click', () => { + const instance = mountWithIntl(); + act(() => { + instance + .find('[data-test-subj="lnsPalettePanel_dynamicColoring_custom_range_groups"]') + .find(EuiButtonGroup) + .prop('onChange')!('number'); }); - it('should be visible for predefined palettes', () => { - const instance = mountWithIntl(); - expect( - instance.find('[data-test-subj="lnsPalettePanel_dynamicColoring_custom_stops"]').exists() - ).toEqual(true); + + act(() => { + instance + .find('[data-test-subj="lnsPalettePanel_dynamicColoring_custom_range_groups"]') + .find(EuiButtonGroup) + .prop('onChange')!('percent'); }); - it('should be visible for custom palettes', () => { - const instance = mountWithIntl( - { + beforeEach(() => { + props = { + activePalette: { type: 'palette', name: 'positive' }, + palettes: paletteRegistry, + setPalette: jest.fn(), + dataBounds: { min: 0, max: 100 }, + }; + }); + it('should be visible for predefined palettes', () => { + const instance = mountWithIntl(); + expect( + instance.find('[data-test-subj="lnsPalettePanel_dynamicColoring_custom_stops"]').exists() + ).toEqual(true); + }); + + it('should be visible for custom palettes', () => { + const instance = mountWithIntl( + - ); - expect( - instance.find('[data-test-subj="lnsPalettePanel_dynamicColoring_custom_stops"]').exists() - ).toEqual(true); - }); + }, + }} + /> + ); + expect( + instance.find('[data-test-subj="lnsPalettePanel_dynamicColoring_custom_stops"]').exists() + ).toEqual(true); }); }); }); diff --git a/x-pack/plugins/lens/public/shared_components/coloring/palette_configuration.tsx b/x-pack/plugins/lens/public/shared_components/coloring/palette_configuration.tsx index bc6a590db0cb7..1d1e212b87c0c 100644 --- a/x-pack/plugins/lens/public/shared_components/coloring/palette_configuration.tsx +++ b/x-pack/plugins/lens/public/shared_components/coloring/palette_configuration.tsx @@ -108,16 +108,21 @@ export function CustomizablePalette({ colorStops: undefined, }; + const newColorStops = getColorStops(palettes, [], activePalette, dataBounds); if (isNewPaletteCustom) { - newParams.colorStops = getColorStops(palettes, [], activePalette, dataBounds); + newParams.colorStops = newColorStops; } newParams.stops = getPaletteStops(palettes, newParams, { prevPalette: isNewPaletteCustom || isCurrentPaletteCustom ? undefined : newPalette.name, dataBounds, + mapFromMinValue: true, }); + newParams.rangeMin = newColorStops[0].stop; + newParams.rangeMax = newColorStops[newColorStops.length - 1].stop; + setPalette({ ...newPalette, params: newParams, @@ -266,18 +271,18 @@ export function CustomizablePalette({ ) as RequiredPaletteParamTypes['rangeType']; const params: CustomPaletteParams = { rangeType: newRangeType }; + const { min: newMin, max: newMax } = getDataMinMax(newRangeType, dataBounds); + const { min: oldMin, max: oldMax } = getDataMinMax( + activePalette.params?.rangeType, + dataBounds + ); + const newColorStops = remapStopsByNewInterval(colorStopsToShow, { + oldInterval: oldMax - oldMin, + newInterval: newMax - newMin, + newMin, + oldMin, + }); if (isCurrentPaletteCustom) { - const { min: newMin, max: newMax } = getDataMinMax(newRangeType, dataBounds); - const { min: oldMin, max: oldMax } = getDataMinMax( - activePalette.params?.rangeType, - dataBounds - ); - const newColorStops = remapStopsByNewInterval(colorStopsToShow, { - oldInterval: oldMax - oldMin, - newInterval: newMax - newMin, - newMin, - oldMin, - }); const stops = getPaletteStops( palettes, { ...activePalette.params, colorStops: newColorStops, ...params }, @@ -285,8 +290,6 @@ export function CustomizablePalette({ ); params.colorStops = newColorStops; params.stops = stops; - params.rangeMin = newColorStops[0].stop; - params.rangeMax = newColorStops[newColorStops.length - 1].stop; } else { params.stops = getPaletteStops( palettes, @@ -294,6 +297,11 @@ export function CustomizablePalette({ { prevPalette: activePalette.name, dataBounds } ); } + // why not use newMin/newMax here? + // That's because there's the concept of continuity to accomodate, where in some scenarios it has to + // take into account the stop value rather than the data value + params.rangeMin = newColorStops[0].stop; + params.rangeMax = newColorStops[newColorStops.length - 1].stop; setPalette(mergePaletteParams(activePalette, params)); }} /> diff --git a/x-pack/plugins/lens/public/shared_components/coloring/utils.test.ts b/x-pack/plugins/lens/public/shared_components/coloring/utils.test.ts index 97dc2e45c96dc..07d93ca5c40c6 100644 --- a/x-pack/plugins/lens/public/shared_components/coloring/utils.test.ts +++ b/x-pack/plugins/lens/public/shared_components/coloring/utils.test.ts @@ -8,6 +8,7 @@ import { chartPluginMock } from 'src/plugins/charts/public/mocks'; import { applyPaletteParams, + getColorStops, getContrastColor, getDataMinMax, getPaletteStops, @@ -59,6 +60,78 @@ describe('applyPaletteParams', () => { }); }); +describe('getColorStops', () => { + const paletteRegistry = chartPluginMock.createPaletteRegistry(); + it('should return the same colorStops if a custom palette is passed, avoiding recomputation', () => { + const colorStops = [ + { stop: 0, color: 'red' }, + { stop: 100, color: 'blue' }, + ]; + expect( + getColorStops( + paletteRegistry, + colorStops, + { name: 'custom', type: 'palette' }, + { min: 0, max: 100 } + ) + ).toBe(colorStops); + }); + + it('should get a fresh list of colors', () => { + expect( + getColorStops( + paletteRegistry, + [ + { stop: 0, color: 'red' }, + { stop: 100, color: 'blue' }, + ], + { name: 'mocked', type: 'palette' }, + { min: 0, max: 100 } + ) + ).toEqual([ + { color: 'blue', stop: 0 }, + { color: 'yellow', stop: 50 }, + ]); + }); + + it('should get a fresh list of colors even if custom palette but empty colorStops', () => { + expect( + getColorStops(paletteRegistry, [], { name: 'mocked', type: 'palette' }, { min: 0, max: 100 }) + ).toEqual([ + { color: 'blue', stop: 0 }, + { color: 'yellow', stop: 50 }, + ]); + }); + + it('should correctly map the new colorStop to the current data bound and minValue', () => { + expect( + getColorStops( + paletteRegistry, + [], + { name: 'mocked', type: 'palette', params: { rangeType: 'number' } }, + { min: 100, max: 1000 } + ) + ).toEqual([ + { color: 'blue', stop: 100 }, + { color: 'yellow', stop: 550 }, + ]); + }); + + it('should reverse the colors', () => { + expect( + getColorStops( + paletteRegistry, + [], + { name: 'mocked', type: 'palette', params: { reverse: true } }, + { min: 100, max: 1000 } + ) + ).toEqual([ + { color: 'yellow', stop: 0 }, + { color: 'blue', stop: 50 }, + ]); + }); +}); + describe('remapStopsByNewInterval', () => { it('should correctly remap the current palette from 0..1 to 0...100', () => { expect( diff --git a/x-pack/plugins/lens/public/shared_components/coloring/utils.ts b/x-pack/plugins/lens/public/shared_components/coloring/utils.ts index b2969565f5390..413e3708e9c9b 100644 --- a/x-pack/plugins/lens/public/shared_components/coloring/utils.ts +++ b/x-pack/plugins/lens/public/shared_components/coloring/utils.ts @@ -269,11 +269,10 @@ export function getColorStops( palettes: PaletteRegistry, colorStops: Required['stops'], activePalette: PaletteOutput, - dataBounds: { min: number; max: number }, - defaultPalette?: string + dataBounds: { min: number; max: number } ) { // just forward the current stops if custom - if (activePalette?.name === CUSTOM_PALETTE) { + if (activePalette?.name === CUSTOM_PALETTE && colorStops?.length) { return colorStops; } // for predefined palettes create some stops, then drop the last one. diff --git a/x-pack/test/functional/apps/lens/table.ts b/x-pack/test/functional/apps/lens/table.ts index da079c0976db3..892534eec7033 100644 --- a/x-pack/test/functional/apps/lens/table.ts +++ b/x-pack/test/functional/apps/lens/table.ts @@ -122,7 +122,28 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(styleObj['background-color']).to.be('rgb(235, 239, 245)'); }); + it('should keep the coloring consistent when changing mode', async () => { + // Change mode from percent to number + await testSubjects.click('lnsPalettePanel_dynamicColoring_rangeType_groups_number'); + await PageObjects.header.waitUntilLoadingHasFinished(); + // check that all remained the same + const styleObj = await PageObjects.lens.getDatatableCellStyle(0, 2); + expect(styleObj['background-color']).to.be('rgb(235, 239, 245)'); + }); + + it('should keep the coloring consistent when moving to custom palette from default', async () => { + await PageObjects.lens.changePaletteTo('custom'); + await PageObjects.header.waitUntilLoadingHasFinished(); + // check that all remained the same + const styleObj = await PageObjects.lens.getDatatableCellStyle(0, 2); + expect(styleObj['background-color']).to.be('rgb(235, 239, 245)'); + }); + it('tweak the color stops numeric value', async () => { + // restore default palette and percent mode + await PageObjects.lens.changePaletteTo('temperature'); + await testSubjects.click('lnsPalettePanel_dynamicColoring_rangeType_groups_percent'); + // now tweak the value await testSubjects.setValue('lnsPalettePanel_dynamicColoring_stop_value_0', '30', { clearWithKeyboard: true, }); From ed18699e387cf661581e484a764f8e008de50d8f Mon Sep 17 00:00:00 2001 From: Josh Dover <1813008+joshdover@users.noreply.github.com> Date: Fri, 3 Sep 2021 16:01:28 +0200 Subject: [PATCH 12/20] Handle bulkGet errors on package retrieval from ES storage (#111114) --- .../server/services/epm/archive/storage.ts | 22 +++++++++++++++++++ .../fleet/server/services/epm/packages/get.ts | 5 ++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/services/epm/archive/storage.ts b/x-pack/plugins/fleet/server/services/epm/archive/storage.ts index dde6459addcbc..d3bc4afae6229 100644 --- a/x-pack/plugins/fleet/server/services/epm/archive/storage.ts +++ b/x-pack/plugins/fleet/server/services/epm/archive/storage.ts @@ -23,6 +23,8 @@ import type { } from '../../../../common'; import { pkgToPkgKey } from '../registry'; +import { appContextService } from '../../app_context'; + import { getArchiveEntry, setArchiveEntry, setArchiveFilelist, setPackageInfo } from './index'; import type { ArchiveEntry } from './index'; import { parseAndVerifyPolicyTemplates, parseAndVerifyStreams } from './validation'; @@ -165,6 +167,7 @@ export const getEsPackage = async ( references: PackageAssetReference[], savedObjectsClient: SavedObjectsClientContract ) => { + const logger = appContextService.getLogger(); const pkgKey = pkgToPkgKey({ name: pkgName, version: pkgVersion }); const bulkRes = await savedObjectsClient.bulkGet( references.map((reference) => ({ @@ -172,8 +175,27 @@ export const getEsPackage = async ( fields: ['asset_path', 'data_utf8', 'data_base64'], })) ); + const errors = bulkRes.saved_objects.filter((so) => so.error || !so.attributes); const assets = bulkRes.saved_objects.map((so) => so.attributes); + if (errors.length) { + const resolvedErrors = errors.map((so) => + so.error + ? { type: so.type, id: so.id, error: so.error } + : !so.attributes + ? { type: so.type, id: so.id, error: { error: `No attributes retrieved` } } + : { type: so.type, id: so.id, error: { error: `Unknown` } } + ); + + logger.warn( + `Failed to retrieve ${pkgName}-${pkgVersion} package from ES storage. bulkGet failed for assets: ${JSON.stringify( + resolvedErrors + )}` + ); + + return undefined; + } + const paths: string[] = []; const entries: ArchiveEntry[] = assets.map(packageAssetToArchiveEntry); entries.forEach(({ path, buffer }) => { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.ts index e493095bc4b36..0e23981b95fcd 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.ts @@ -217,7 +217,10 @@ export async function getPackageFromSource(options: { installedPkg.package_assets, savedObjectsClient ); - logger.debug(`retrieved installed package ${pkgName}-${pkgVersion} from ES`); + + if (res) { + logger.debug(`retrieved installed package ${pkgName}-${pkgVersion} from ES`); + } } // for packages not in cache or package storage and installed from registry, check registry if (!res && pkgInstallSource === 'registry') { From 66cb058fa79a3f53d25695e49f5ce04b93862c3a Mon Sep 17 00:00:00 2001 From: "Christiane (Tina) Heiligers" Date: Fri, 3 Sep 2021 07:22:14 -0700 Subject: [PATCH 13/20] Removes support for legacy exports (#110738) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../__snapshots__/flyout.test.tsx.snap | 276 +------------- .../components/flyout.test.mocks.ts | 5 - .../objects_table/components/flyout.test.tsx | 235 +----------- .../objects_table/components/flyout.tsx | 353 +----------------- .../components/import_mode_control.test.tsx | 9 +- .../components/import_mode_control.tsx | 29 +- test/functional/apps/dashboard/bwc_import.ts | 4 +- test/functional/apps/dashboard/time_zones.ts | 4 +- .../apps/management/_import_objects.ts | 281 -------------- .../management/_mgmt_import_saved_objects.js | 2 +- ...import_index_patterns_multiple_exists.json | 26 -- .../management/exports/_import_objects.json | 19 - ...ort_objects_connected_to_saved_search.json | 20 - .../exports/_import_objects_exists.json | 19 - ...rt_objects_missing_all_index_patterns.json | 121 ------ .../_import_objects_multiple_exists.json | 36 -- .../exports/_import_objects_saved_search.json | 25 -- .../_import_objects_with_index_patterns.json | 31 -- .../exports/mgmt_import_objects.json | 37 -- .../translations/translations/ja-JP.json | 15 - .../translations/translations/zh-CN.json | 15 - 21 files changed, 32 insertions(+), 1530 deletions(-) delete mode 100644 test/functional/apps/management/exports/_import_index_patterns_multiple_exists.json delete mode 100644 test/functional/apps/management/exports/_import_objects.json delete mode 100644 test/functional/apps/management/exports/_import_objects_connected_to_saved_search.json delete mode 100644 test/functional/apps/management/exports/_import_objects_exists.json delete mode 100644 test/functional/apps/management/exports/_import_objects_missing_all_index_patterns.json delete mode 100644 test/functional/apps/management/exports/_import_objects_multiple_exists.json delete mode 100644 test/functional/apps/management/exports/_import_objects_saved_search.json delete mode 100644 test/functional/apps/management/exports/_import_objects_with_index_patterns.json delete mode 100644 test/functional/apps/management/exports/mgmt_import_objects.json diff --git a/src/plugins/saved_objects_management/public/management_section/objects_table/components/__snapshots__/flyout.test.tsx.snap b/src/plugins/saved_objects_management/public/management_section/objects_table/components/__snapshots__/flyout.test.tsx.snap index bd97f2e6bffb1..015c7068d72b6 100644 --- a/src/plugins/saved_objects_management/public/management_section/objects_table/components/__snapshots__/flyout.test.tsx.snap +++ b/src/plugins/saved_objects_management/public/management_section/objects_table/components/__snapshots__/flyout.test.tsx.snap @@ -238,7 +238,6 @@ exports[`Flyout conflicts should allow conflict resolution 2`] = ` "title": undefined, }, ], - "isLegacyFile": false, "loadingMessage": undefined, "status": "loading", "successfulImports": Array [], @@ -276,278 +275,6 @@ exports[`Flyout conflicts should allow conflict resolution 2`] = ` } `; -exports[`Flyout legacy conflicts should allow conflict resolution 1`] = ` - - - -

- -

- - - - - - - } - > -

- -

- - - - - - - } - > -

- - - , - } - } - /> -

- - - - - - - - - - - - - - - - - - - -`; - -exports[`Flyout legacy conflicts should handle errors 2`] = ` -Array [ - - } - > -

- -

- , - - } - > -

- - - , - } - } - /> -

- , - - } - > -

- The file could not be processed due to error: "foobar" -

- , -] -`; - exports[`Flyout should render import step 1`] = ` diff --git a/src/plugins/saved_objects_management/public/management_section/objects_table/components/flyout.test.mocks.ts b/src/plugins/saved_objects_management/public/management_section/objects_table/components/flyout.test.mocks.ts index fc8558fa82c2f..7b716e1b813c9 100644 --- a/src/plugins/saved_objects_management/public/management_section/objects_table/components/flyout.test.mocks.ts +++ b/src/plugins/saved_objects_management/public/management_section/objects_table/components/flyout.test.mocks.ts @@ -16,11 +16,6 @@ jest.doMock('../../../lib/resolve_import_errors', () => ({ resolveImportErrors: resolveImportErrorsMock, })); -export const importLegacyFileMock = jest.fn(); -jest.doMock('../../../lib/import_legacy_file', () => ({ - importLegacyFile: importLegacyFileMock, -})); - export const resolveSavedObjectsMock = jest.fn(); export const resolveSavedSearchesMock = jest.fn(); export const resolveIndexPatternConflictsMock = jest.fn(); diff --git a/src/plugins/saved_objects_management/public/management_section/objects_table/components/flyout.test.tsx b/src/plugins/saved_objects_management/public/management_section/objects_table/components/flyout.test.tsx index a1eb94ab55cfa..28190e6bd872f 100644 --- a/src/plugins/saved_objects_management/public/management_section/objects_table/components/flyout.test.tsx +++ b/src/plugins/saved_objects_management/public/management_section/objects_table/components/flyout.test.tsx @@ -6,15 +6,7 @@ * Side Public License, v 1. */ -import { - importFileMock, - importLegacyFileMock, - resolveImportErrorsMock, - resolveIndexPatternConflictsMock, - resolveSavedObjectsMock, - resolveSavedSearchesMock, - saveObjectsMock, -} from './flyout.test.mocks'; +import { importFileMock, resolveImportErrorsMock } from './flyout.test.mocks'; import React from 'react'; import { shallowWithI18nProvider } from '@kbn/test/jest'; @@ -28,10 +20,6 @@ const mockFile = ({ name: 'foo.ndjson', path: '/home/foo.ndjson', } as unknown) as File; -const legacyMockFile = ({ - name: 'foo.json', - path: '/home/foo.json', -} as unknown) as File; describe('Flyout', () => { let defaultProps: FlyoutProps; @@ -107,31 +95,6 @@ describe('Flyout', () => { expect(component.state('file')).toBe(undefined); }); - it('should handle invalid files', async () => { - const component = shallowRender(defaultProps); - - // Ensure all promises resolve - await new Promise((resolve) => process.nextTick(resolve)); - // Ensure the state changes are reflected - component.update(); - - importLegacyFileMock.mockImplementation(() => { - throw new Error('foobar'); - }); - - await component.instance().legacyImport(); - expect(component.state('error')).toBe('The file could not be processed.'); - - importLegacyFileMock.mockImplementation(() => ({ - invalid: true, - })); - - await component.instance().legacyImport(); - expect(component.state('error')).toBe( - 'Saved objects file format is invalid and cannot be imported.' - ); - }); - describe('conflicts', () => { beforeEach(() => { importFileMock.mockImplementation(() => ({ @@ -169,7 +132,7 @@ describe('Flyout', () => { // Ensure the state changes are reflected component.update(); - component.setState({ file: mockFile, isLegacyFile: false }); + component.setState({ file: mockFile }); await component.instance().import(); expect(importFileMock).toHaveBeenCalledWith(defaultProps.http, mockFile, { @@ -207,7 +170,7 @@ describe('Flyout', () => { // Ensure the state changes are reflected component.update(); - component.setState({ file: mockFile, isLegacyFile: false }); + component.setState({ file: mockFile }); await component.instance().import(); // Ensure it looks right @@ -250,7 +213,7 @@ describe('Flyout', () => { successfulImports, })); - component.setState({ file: mockFile, isLegacyFile: false }); + component.setState({ file: mockFile }); // Go through the import flow await component.instance().import(); @@ -267,194 +230,4 @@ describe('Flyout', () => { expect(cancelButton.prop('disabled')).toBe(true); }); }); - - describe('legacy conflicts', () => { - const mockData = [ - { - _id: '1', - _type: 'search', - }, - { - _id: '2', - _type: 'index-pattern', - }, - { - _id: '3', - _type: 'invalid', - }, - ]; - - const mockConflictedIndexPatterns = [ - { - doc: { - _type: 'index-pattern', - _id: '1', - _source: { - title: 'MyIndexPattern*', - }, - }, - obj: { - searchSource: { - getOwnField: (field: string) => { - if (field === 'index') { - return 'MyIndexPattern*'; - } - if (field === 'filter') { - return [{ meta: { index: 'filterIndex' } }]; - } - }, - }, - _serialize: () => { - return { references: [{ id: 'MyIndexPattern*' }, { id: 'filterIndex' }] }; - }, - }, - }, - ]; - - const mockConflictedSavedObjectsLinkedToSavedSearches = [2]; - const mockConflictedSearchDocs = [3]; - - beforeEach(() => { - importLegacyFileMock.mockImplementation(() => mockData); - resolveSavedObjectsMock.mockImplementation(() => ({ - conflictedIndexPatterns: mockConflictedIndexPatterns, - conflictedSavedObjectsLinkedToSavedSearches: mockConflictedSavedObjectsLinkedToSavedSearches, - conflictedSearchDocs: mockConflictedSearchDocs, - importedObjectCount: 2, - confirmModalPromise: () => {}, - })); - }); - - it('should figure out unmatchedReferences', async () => { - const component = shallowRender(defaultProps); - - // Ensure all promises resolve - await new Promise((resolve) => process.nextTick(resolve)); - // Ensure the state changes are reflected - component.update(); - - component.setState({ file: legacyMockFile, isLegacyFile: true }); - await component.instance().legacyImport(); - - expect(importLegacyFileMock).toHaveBeenCalledWith(legacyMockFile); - // Remove the last element from data since it should be filtered out - expect(resolveSavedObjectsMock).toHaveBeenCalledWith( - mockData.slice(0, 2).map((doc) => ({ ...doc, _migrationVersion: {} })), - true, - defaultProps.serviceRegistry.all().map((s) => s.service), - defaultProps.indexPatterns, - defaultProps.overlays.openConfirm - ); - - expect(component.state()).toMatchObject({ - conflictedIndexPatterns: mockConflictedIndexPatterns, - conflictedSavedObjectsLinkedToSavedSearches: mockConflictedSavedObjectsLinkedToSavedSearches, - conflictedSearchDocs: mockConflictedSearchDocs, - importCount: 2, - status: 'idle', - error: undefined, - unmatchedReferences: [ - { - existingIndexPatternId: 'MyIndexPattern*', - newIndexPatternId: undefined, - list: [ - { - id: 'MyIndexPattern*', - title: 'MyIndexPattern*', - type: 'index-pattern', - }, - ], - }, - { - existingIndexPatternId: 'filterIndex', - list: [ - { - id: 'filterIndex', - title: 'MyIndexPattern*', - type: 'index-pattern', - }, - ], - newIndexPatternId: undefined, - }, - ], - }); - }); - - it('should allow conflict resolution', async () => { - const component = shallowRender(defaultProps); - - // Ensure all promises resolve - await new Promise((resolve) => process.nextTick(resolve)); - // Ensure the state changes are reflected - component.update(); - - component.setState({ file: legacyMockFile, isLegacyFile: true }); - await component.instance().legacyImport(); - - // Ensure it looks right - component.update(); - expect(component).toMatchSnapshot(); - - // Ensure we can change the resolution - component.instance().onIndexChanged('MyIndexPattern*', { target: { value: '2' } }); - expect(component.state('unmatchedReferences')![0].newIndexPatternId).toBe('2'); - - // Let's resolve now - await component - .find('EuiButton[data-test-subj="importSavedObjectsConfirmBtn"]') - .simulate('click'); - // Ensure all promises resolve - await new Promise((resolve) => process.nextTick(resolve)); - expect(resolveIndexPatternConflictsMock).toHaveBeenCalledWith( - component.instance().resolutions, - mockConflictedIndexPatterns, - true, - { - search: defaultProps.search, - indexPatterns: defaultProps.indexPatterns, - } - ); - expect(saveObjectsMock).toHaveBeenCalledWith( - mockConflictedSavedObjectsLinkedToSavedSearches, - true - ); - expect(resolveSavedSearchesMock).toHaveBeenCalledWith( - mockConflictedSearchDocs, - defaultProps.serviceRegistry.all().map((s) => s.service), - defaultProps.indexPatterns, - true - ); - }); - - it('should handle errors', async () => { - const component = shallowRender(defaultProps); - - // Ensure all promises resolve - await new Promise((resolve) => process.nextTick(resolve)); - // Ensure the state changes are reflected - component.update(); - - resolveIndexPatternConflictsMock.mockImplementation(() => { - throw new Error('foobar'); - }); - - component.setState({ file: legacyMockFile, isLegacyFile: true }); - - // Go through the import flow - await component.instance().legacyImport(); - component.update(); - // Set a resolution - component.instance().onIndexChanged('MyIndexPattern*', { target: { value: '2' } }); - await component - .find('EuiButton[data-test-subj="importSavedObjectsConfirmBtn"]') - .simulate('click'); - // Ensure all promises resolve - await new Promise((resolve) => process.nextTick(resolve)); - - expect(component.state('error')).toMatchInlineSnapshot( - `"The file could not be processed due to error: \\"foobar\\""` - ); - expect(component.find('EuiFlyoutBody EuiCallOut')).toMatchSnapshot(); - }); - }); }); diff --git a/src/plugins/saved_objects_management/public/management_section/objects_table/components/flyout.tsx b/src/plugins/saved_objects_management/public/management_section/objects_table/components/flyout.tsx index 8f4940ffb05c9..aca229b9a70ed 100644 --- a/src/plugins/saved_objects_management/public/management_section/objects_table/components/flyout.tsx +++ b/src/plugins/saved_objects_management/public/management_section/objects_table/components/flyout.tsx @@ -7,7 +7,7 @@ */ import React, { Component, Fragment, ReactNode } from 'react'; -import { take, get as getField } from 'lodash'; +import { take } from 'lodash'; import { EuiFlyout, EuiFlyoutBody, @@ -39,18 +39,10 @@ import { } from '../../../../../data/public'; import { importFile, - importLegacyFile, resolveImportErrors, - logLegacyImport, processImportResponse, ProcessedImportResponse, } from '../../../lib'; -import { - resolveSavedObjects, - resolveSavedSearches, - resolveIndexPatternConflicts, - saveObjects, -} from '../../../lib/resolve_saved_objects'; import { ISavedObjectsManagementServiceRegistry } from '../../../services'; import { FailedImportConflict, RetryDecision } from '../../../lib/resolve_import_errors'; import { OverwriteModal } from './overwrite_modal'; @@ -89,7 +81,6 @@ export interface FlyoutState { indexPatterns?: IndexPattern[]; importMode: ImportMode; loadingMessage?: string; - isLegacyFile: boolean; status: string; } @@ -129,7 +120,6 @@ export class Flyout extends Component { indexPatterns: undefined, importMode: { createNewCopies: CREATE_NEW_COPIES_DEFAULT, overwrite: OVERWRITE_ALL_DEFAULT }, loadingMessage: undefined, - isLegacyFile: false, status: 'idle', }; } @@ -152,14 +142,11 @@ export class Flyout extends Component { setImportFile = (files: FileList | null) => { if (!files || !files[0]) { - this.setState({ file: undefined, isLegacyFile: false }); + this.setState({ file: undefined }); return; } const file = files[0]; - this.setState({ - file, - isLegacyFile: /\.json$/i.test(file.name) || file.type === 'application/json', - }); + this.setState({ file }); }; /** @@ -246,103 +233,6 @@ export class Flyout extends Component { } }; - legacyImport = async () => { - const { serviceRegistry, indexPatterns, overlays, http, allowedTypes } = this.props; - const { file, importMode } = this.state; - - this.setState({ status: 'loading', error: undefined }); - - // Log warning on server, don't wait for response - logLegacyImport(http); - - let contents; - try { - contents = await importLegacyFile(file!); - } catch (e) { - this.setState({ - status: 'error', - error: i18n.translate( - 'savedObjectsManagement.objectsTable.flyout.importLegacyFileErrorMessage', - { defaultMessage: 'The file could not be processed.' } - ), - }); - return; - } - - if (!Array.isArray(contents)) { - this.setState({ - status: 'error', - error: i18n.translate( - 'savedObjectsManagement.objectsTable.flyout.invalidFormatOfImportedFileErrorMessage', - { defaultMessage: 'Saved objects file format is invalid and cannot be imported.' } - ), - }); - return; - } - - contents = contents - .filter((content) => allowedTypes.includes(content._type)) - .map((doc) => ({ - ...doc, - // The server assumes that documents with no migrationVersion are up to date. - // That assumption enables Kibana and other API consumers to not have to build - // up migrationVersion prior to creating new objects. But it means that imports - // need to set migrationVersion to something other than undefined, so that imported - // docs are not seen as automatically up-to-date. - _migrationVersion: doc._migrationVersion || {}, - })); - - const { - conflictedIndexPatterns, - conflictedSavedObjectsLinkedToSavedSearches, - conflictedSearchDocs, - importedObjectCount, - failedImports, - } = await resolveSavedObjects( - contents, - importMode.overwrite, - serviceRegistry.all().map((e) => e.service), - indexPatterns, - overlays.openConfirm - ); - - const byId: Record = {}; - conflictedIndexPatterns - .map(({ doc, obj }) => { - return { doc, obj: obj._serialize() }; - }) - .forEach(({ doc, obj }) => - obj.references.forEach((ref: Record) => { - byId[ref.id] = byId[ref.id] != null ? byId[ref.id].concat({ doc, obj }) : [{ doc, obj }]; - }) - ); - const unmatchedReferences = Object.entries(byId).reduce( - (accum, [existingIndexPatternId, list]) => { - accum.push({ - existingIndexPatternId, - newIndexPatternId: undefined, - list: list.map(({ doc }) => ({ - id: existingIndexPatternId, - type: doc._type, - title: doc._source.title, - })), - }); - return accum; - }, - [] as any[] - ); - - this.setState({ - conflictedIndexPatterns, - conflictedSavedObjectsLinkedToSavedSearches, - conflictedSearchDocs, - failedImports, - unmatchedReferences, - importCount: importedObjectCount, - status: unmatchedReferences.length === 0 ? 'success' : 'idle', - }); - }; - public get hasUnmatchedReferences() { return this.state.unmatchedReferences && this.state.unmatchedReferences.length > 0; } @@ -362,89 +252,6 @@ export class Flyout extends Component { ); } - confirmLegacyImport = async () => { - const { - conflictedIndexPatterns, - importMode, - conflictedSavedObjectsLinkedToSavedSearches, - conflictedSearchDocs, - failedImports, - } = this.state; - - const { serviceRegistry, indexPatterns, search } = this.props; - - this.setState({ - error: undefined, - status: 'loading', - loadingMessage: undefined, - }); - - let importCount = this.state.importCount; - - if (this.hasUnmatchedReferences) { - try { - const resolutions = this.resolutions; - - // Do not Promise.all these calls as the order matters - this.setState({ - loadingMessage: i18n.translate( - 'savedObjectsManagement.objectsTable.flyout.confirmLegacyImport.resolvingConflictsLoadingMessage', - { defaultMessage: 'Resolving conflicts…' } - ), - }); - if (resolutions.length) { - importCount += await resolveIndexPatternConflicts( - resolutions, - conflictedIndexPatterns!, - importMode.overwrite, - { indexPatterns, search } - ); - } - this.setState({ - loadingMessage: i18n.translate( - 'savedObjectsManagement.objectsTable.flyout.confirmLegacyImport.savingConflictsLoadingMessage', - { defaultMessage: 'Saving conflicts…' } - ), - }); - importCount += await saveObjects( - conflictedSavedObjectsLinkedToSavedSearches!, - importMode.overwrite - ); - this.setState({ - loadingMessage: i18n.translate( - 'savedObjectsManagement.objectsTable.flyout.confirmLegacyImport.savedSearchAreLinkedProperlyLoadingMessage', - { defaultMessage: 'Ensure saved searches are linked properly…' } - ), - }); - importCount += await resolveSavedSearches( - conflictedSearchDocs!, - serviceRegistry.all().map((e) => e.service), - indexPatterns, - importMode.overwrite - ); - this.setState({ - loadingMessage: i18n.translate( - 'savedObjectsManagement.objectsTable.flyout.confirmLegacyImport.retryingFailedObjectsLoadingMessage', - { defaultMessage: 'Retrying failed objects…' } - ), - }); - importCount += await saveObjects( - failedImports!.map(({ obj }) => obj) as any[], - importMode.overwrite - ); - } catch (e) { - this.setState({ - status: 'error', - error: getErrorMessage(e), - loadingMessage: undefined, - }); - return; - } - } - - this.setState({ status: 'success', importCount }); - }; - onIndexChanged = (id: string, e: any) => { const value = e.target.value; this.setState((state) => { @@ -613,10 +420,8 @@ export class Flyout extends Component { const { status, loadingMessage, - importCount, failedImports = [], successfulImports = [], - isLegacyFile, importMode, importWarnings, } = this.state; @@ -635,7 +440,8 @@ export class Flyout extends Component { ); } - if (!isLegacyFile && status === 'success') { + // Import summary for completed import + if (status === 'success') { return ( { ); } - // Import summary for failed legacy import - if (failedImports.length && !this.hasUnmatchedReferences) { - return ( - - } - color="warning" - iconType="help" - > -

- -

-

- {failedImports - .map(({ error, obj }) => { - if (error.type === 'missing_references') { - return error.references.map((reference) => { - return i18n.translate( - 'savedObjectsManagement.objectsTable.flyout.importFailedMissingReference', - { - defaultMessage: '{type} [id={id}] could not locate {refType} [id={refId}]', - values: { - id: obj.id, - type: obj.type, - refId: reference.id, - refType: reference.type, - }, - } - ); - }); - } else if (error.type === 'unsupported_type') { - return i18n.translate( - 'savedObjectsManagement.objectsTable.flyout.importFailedUnsupportedType', - { - defaultMessage: '{type} [id={id}] unsupported type', - values: { - id: obj.id, - type: obj.type, - }, - } - ); - } - return getField(error, 'body.message', (error as any).message ?? ''); - }) - .join(' ')} -

- - ); - } - - // Import summary for completed legacy import - if (status === 'success') { - if (importCount === 0) { - return ( - - } - color="primary" - /> - ); - } - - return ( - - } - color="success" - iconType="check" - > -

- -

- - ); - } - + // Failed imports if (this.hasUnmatchedReferences) { return this.renderUnmatchedReferences(); } @@ -768,7 +473,7 @@ export class Flyout extends Component { } > { this.changeImportMode(newValues)} /> @@ -791,7 +495,7 @@ export class Flyout extends Component { } renderFooter() { - const { isLegacyFile, status } = this.state; + const { status } = this.state; const { done, close } = this.props; let confirmButton; @@ -808,7 +512,7 @@ export class Flyout extends Component { } else if (this.hasUnmatchedReferences) { confirmButton = ( { } else { confirmButton = ( { { return null; } - let legacyFileWarning; - if (this.state.isLegacyFile) { - legacyFileWarning = ( - <> - - } - color="warning" - iconType="help" - > -

- -

- - - - ); - } - let indexPatternConflictsWarning; if (this.hasUnmatchedReferences) { indexPatternConflictsWarning = ( @@ -925,18 +602,12 @@ export class Flyout extends Component { ); } - if (!legacyFileWarning && !indexPatternConflictsWarning) { + if (!indexPatternConflictsWarning) { return null; } return ( - {legacyFileWarning && ( - - - {legacyFileWarning} - - )} {indexPatternConflictsWarning && ( diff --git a/src/plugins/saved_objects_management/public/management_section/objects_table/components/import_mode_control.test.tsx b/src/plugins/saved_objects_management/public/management_section/objects_table/components/import_mode_control.test.tsx index 2ece421238635..fbf50e0ee0c86 100644 --- a/src/plugins/saved_objects_management/public/management_section/objects_table/components/import_mode_control.test.tsx +++ b/src/plugins/saved_objects_management/public/management_section/objects_table/components/import_mode_control.test.tsx @@ -32,14 +32,9 @@ describe('ImportModeControl', () => { jest.resetAllMocks(); }); - const props: ImportModeControlProps = { initialValues, updateSelection, isLegacyFile: false }; + const props: ImportModeControlProps = { initialValues, updateSelection }; - it('returns partial import mode control when used with a legacy file', async () => { - const wrapper = shallowWithI18nProvider(); - expect(wrapper.find('EuiFormFieldset')).toHaveLength(0); - }); - - it('returns full import mode control when used without a legacy file', async () => { + it('returns full import mode control', async () => { const wrapper = shallowWithI18nProvider(); expect(wrapper.find('EuiFormFieldset')).toHaveLength(1); }); diff --git a/src/plugins/saved_objects_management/public/management_section/objects_table/components/import_mode_control.tsx b/src/plugins/saved_objects_management/public/management_section/objects_table/components/import_mode_control.tsx index ee36ef67ee96a..6641e53be8f57 100644 --- a/src/plugins/saved_objects_management/public/management_section/objects_table/components/import_mode_control.tsx +++ b/src/plugins/saved_objects_management/public/management_section/objects_table/components/import_mode_control.tsx @@ -22,7 +22,6 @@ import { i18n } from '@kbn/i18n'; export interface ImportModeControlProps { initialValues: ImportMode; - isLegacyFile: boolean; updateSelection: (result: ImportMode) => void; } @@ -87,11 +86,7 @@ const createLabel = ({ text, tooltip }: { text: string; tooltip: string }) => ( ); -export const ImportModeControl = ({ - initialValues, - isLegacyFile, - updateSelection, -}: ImportModeControlProps) => { +export const ImportModeControl = ({ initialValues, updateSelection }: ImportModeControlProps) => { const [createNewCopies, setCreateNewCopies] = useState(initialValues.createNewCopies); const [overwrite, setOverwrite] = useState(initialValues.overwrite); @@ -104,20 +99,6 @@ export const ImportModeControl = ({ updateSelection({ createNewCopies, overwrite, ...partial }); }; - const overwriteRadio = ( - onChange({ overwrite: id === overwriteEnabled.id })} - disabled={createNewCopies && !isLegacyFile} - data-test-subj={'savedObjectsManagement-importModeControl-overwriteRadioGroup'} - /> - ); - - if (isLegacyFile) { - return overwriteRadio; - } - return ( onChange({ createNewCopies: false })} > - {overwriteRadio} + onChange({ overwrite: id === overwriteEnabled.id })} + disabled={createNewCopies} + data-test-subj={'savedObjectsManagement-importModeControl-overwriteRadioGroup'} + /> diff --git a/test/functional/apps/dashboard/bwc_import.ts b/test/functional/apps/dashboard/bwc_import.ts index 03f1f126338fa..ebb9d2b99ffa7 100644 --- a/test/functional/apps/dashboard/bwc_import.ts +++ b/test/functional/apps/dashboard/bwc_import.ts @@ -12,8 +12,8 @@ import { FtrProviderContext } from '../../ftr_provider_context'; export default function ({ getService, getPageObjects }: FtrProviderContext) { const PageObjects = getPageObjects(['dashboard', 'header', 'settings', 'savedObjects', 'common']); const dashboardExpect = getService('dashboardExpect'); - - describe('bwc import', function describeIndexTests() { + // Legacy imports are no longer supported https://github.com/elastic/kibana/issues/103921 + describe.skip('bwc import', function describeIndexTests() { before(async function () { await PageObjects.dashboard.initTests(); await PageObjects.settings.navigateTo(); diff --git a/test/functional/apps/dashboard/time_zones.ts b/test/functional/apps/dashboard/time_zones.ts index e5c532537b6f0..f60792b3f292a 100644 --- a/test/functional/apps/dashboard/time_zones.ts +++ b/test/functional/apps/dashboard/time_zones.ts @@ -22,8 +22,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { 'common', 'savedObjects', ]); - - describe('dashboard time zones', function () { + // Legacy imports are no longer supported https://github.com/elastic/kibana/issues/103921 + describe.skip('dashboard time zones', function () { this.tags('includeFirefox'); before(async () => { diff --git a/test/functional/apps/management/_import_objects.ts b/test/functional/apps/management/_import_objects.ts index 6ef0bfd5a09e8..81350b3542c44 100644 --- a/test/functional/apps/management/_import_objects.ts +++ b/test/functional/apps/management/_import_objects.ts @@ -11,8 +11,6 @@ import path from 'path'; import { keyBy } from 'lodash'; import { FtrProviderContext } from '../../ftr_provider_context'; -const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); - function uniq(input: T[]): T[] { return [...new Set(input)]; } @@ -210,284 +208,5 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(isSavedObjectImported).to.be(true); }); }); - - describe('.json file', () => { - beforeEach(async function () { - await esArchiver.load('test/functional/fixtures/es_archiver/saved_objects_imports'); - await kibanaServer.uiSettings.replace({}); - await PageObjects.settings.navigateTo(); - await PageObjects.settings.clickKibanaSavedObjects(); - }); - - afterEach(async function () { - await esArchiver.unload('test/functional/fixtures/es_archiver/saved_objects_imports'); - }); - - it('should import saved objects', async function () { - await PageObjects.savedObjects.importFile( - path.join(__dirname, 'exports', '_import_objects.json') - ); - await PageObjects.savedObjects.checkImportSucceeded(); - await PageObjects.savedObjects.clickImportDone(); - const objects = await PageObjects.savedObjects.getRowTitles(); - const isSavedObjectImported = objects.includes('Log Agents'); - expect(isSavedObjectImported).to.be(true); - }); - - it('should provide dialog to allow the importing of saved objects with index pattern conflicts', async function () { - await PageObjects.savedObjects.importFile( - path.join(__dirname, 'exports', '_import_objects-conflicts.json') - ); - await PageObjects.savedObjects.checkImportLegacyWarning(); - await PageObjects.savedObjects.checkImportConflictsWarning(); - await PageObjects.settings.associateIndexPattern( - 'd1e4c910-a2e6-11e7-bb30-233be9be6a15', - 'logstash-*' - ); - await PageObjects.savedObjects.clickConfirmChanges(); - await PageObjects.header.waitUntilLoadingHasFinished(); - await PageObjects.savedObjects.clickImportDone(); - const objects = await PageObjects.savedObjects.getRowTitles(); - const isSavedObjectImported = objects.includes('saved object with index pattern conflict'); - expect(isSavedObjectImported).to.be(true); - }); - - it('should allow the user to override duplicate saved objects', async function () { - // This data has already been loaded by the "visualize" esArchive. We'll load it again - // so that we can override the existing visualization. - await PageObjects.savedObjects.importFile( - path.join(__dirname, 'exports', '_import_objects_exists.json'), - false - ); - - await PageObjects.savedObjects.checkImportLegacyWarning(); - await PageObjects.savedObjects.checkImportConflictsWarning(); - await PageObjects.settings.associateIndexPattern('logstash-*', 'logstash-*'); - await PageObjects.savedObjects.clickConfirmChanges(); - - // Override the visualization. - await PageObjects.common.clickConfirmOnModal(); - - const isSuccessful = await testSubjects.exists('importSavedObjectsSuccess'); - expect(isSuccessful).to.be(true); - }); - - it('should allow the user to cancel overriding duplicate saved objects', async function () { - // This data has already been loaded by the "visualize" esArchive. We'll load it again - // so that we can be prompted to override the existing visualization. - await PageObjects.savedObjects.importFile( - path.join(__dirname, 'exports', '_import_objects_exists.json'), - false - ); - - await PageObjects.savedObjects.checkImportLegacyWarning(); - await PageObjects.savedObjects.checkImportConflictsWarning(); - await PageObjects.settings.associateIndexPattern('logstash-*', 'logstash-*'); - await PageObjects.savedObjects.clickConfirmChanges(); - - // *Don't* override the visualization. - await PageObjects.common.clickCancelOnModal(); - - const isSuccessful = await testSubjects.exists('importSavedObjectsSuccessNoneImported'); - expect(isSuccessful).to.be(true); - }); - - it('should allow the user to confirm overriding multiple duplicate saved objects', async function () { - // This data has already been loaded by the "visualize" esArchive. We'll load it again - // so that we can override the existing visualization. - await PageObjects.savedObjects.importFile( - path.join(__dirname, 'exports', '_import_objects_multiple_exists.json'), - false - ); - - await PageObjects.savedObjects.checkImportLegacyWarning(); - await PageObjects.savedObjects.checkImportConflictsWarning(); - - await PageObjects.settings.associateIndexPattern('logstash-*', 'logstash-*'); - await PageObjects.savedObjects.clickConfirmChanges(); - - // Override the visualizations. - await PageObjects.common.clickConfirmOnModal(false); - // as the second confirm can pop instantly, we can't wait for it to be hidden - // with is why we call clickConfirmOnModal with ensureHidden: false in previous statement - // but as the initial popin can take a few ms before fading, we need to wait a little - // to avoid clicking twice on the same modal. - await delay(1000); - await PageObjects.common.clickConfirmOnModal(true); - - const isSuccessful = await testSubjects.exists('importSavedObjectsSuccess'); - expect(isSuccessful).to.be(true); - }); - - it('should allow the user to confirm overriding multiple duplicate index patterns', async function () { - // This data has already been loaded by the "visualize" esArchive. We'll load it again - // so that we can override the existing visualization. - await PageObjects.savedObjects.importFile( - path.join(__dirname, 'exports', '_import_index_patterns_multiple_exists.json'), - false - ); - - // Override the index patterns. - await PageObjects.common.clickConfirmOnModal(false); - // as the second confirm can pop instantly, we can't wait for it to be hidden - // with is why we call clickConfirmOnModal with ensureHidden: false in previous statement - // but as the initial popin can take a few ms before fading, we need to wait a little - // to avoid clicking twice on the same modal. - await delay(1000); - await PageObjects.common.clickConfirmOnModal(true); - - const isSuccessful = await testSubjects.exists('importSavedObjectsSuccess'); - expect(isSuccessful).to.be(true); - }); - - it('should import saved objects linked to saved searches', async function () { - await PageObjects.savedObjects.importFile( - path.join(__dirname, 'exports', '_import_objects_saved_search.json') - ); - await PageObjects.savedObjects.checkImportSucceeded(); - await PageObjects.savedObjects.clickImportDone(); - - await PageObjects.savedObjects.importFile( - path.join(__dirname, 'exports', '_import_objects_connected_to_saved_search.json') - ); - await PageObjects.savedObjects.checkImportSucceeded(); - await PageObjects.savedObjects.clickImportDone(); - - const objects = await PageObjects.savedObjects.getRowTitles(); - const isSavedObjectImported = objects.includes('saved object connected to saved search'); - expect(isSavedObjectImported).to.be(true); - }); - - it('should not import saved objects linked to saved searches when saved search does not exist', async function () { - await PageObjects.savedObjects.importFile( - path.join(__dirname, 'exports', '_import_objects_connected_to_saved_search.json') - ); - await PageObjects.savedObjects.checkImportFailedWarning(); - await PageObjects.savedObjects.clickImportDone(); - - const objects = await PageObjects.savedObjects.getRowTitles(); - const isSavedObjectImported = objects.includes('saved object connected to saved search'); - expect(isSavedObjectImported).to.be(false); - }); - - it('should not import saved objects linked to saved searches when saved search index pattern does not exist', async function () { - // First, import the saved search - await PageObjects.savedObjects.importFile( - path.join(__dirname, 'exports', '_import_objects_saved_search.json') - ); - // Wait for all the saves to happen - await PageObjects.savedObjects.checkImportSucceeded(); - await PageObjects.savedObjects.clickImportDone(); - - // Second, we need to delete the index pattern - await PageObjects.savedObjects.clickCheckboxByTitle('logstash-*'); - await PageObjects.savedObjects.clickDelete(); - - // Last, import a saved object connected to the saved search - // This should NOT show the conflicts - await PageObjects.savedObjects.importFile( - path.join(__dirname, 'exports', '_import_objects_connected_to_saved_search.json') - ); - // Wait for all the saves to happen - await PageObjects.savedObjects.checkNoneImported(); - await PageObjects.savedObjects.clickImportDone(); - - const objects = await PageObjects.savedObjects.getRowTitles(); - const isSavedObjectImported = objects.includes('saved object connected to saved search'); - expect(isSavedObjectImported).to.be(false); - }); - - it('should import saved objects with index patterns when index patterns already exists', async () => { - // First, import the objects - await PageObjects.savedObjects.importFile( - path.join(__dirname, 'exports', '_import_objects_with_index_patterns.json') - ); - await PageObjects.savedObjects.clickImportDone(); - - const objects = await PageObjects.savedObjects.getRowTitles(); - const isSavedObjectImported = objects.includes('saved object imported with index pattern'); - expect(isSavedObjectImported).to.be(true); - }); - - it('should preserve index patterns selection when switching between pages', async () => { - await PageObjects.savedObjects.importFile( - path.join(__dirname, 'exports', '_import_objects_missing_all_index_patterns.json') - ); - - await PageObjects.savedObjects.setOverriddenIndexPatternValue( - 'missing-index-pattern-1', - 'index-pattern-test-1' - ); - - const flyout = await testSubjects.find('importSavedObjectsFlyout'); - - await (await flyout.findByTestSubject('pagination-button-next')).click(); - - await PageObjects.savedObjects.setOverriddenIndexPatternValue( - 'missing-index-pattern-7', - 'index-pattern-test-2' - ); - - await (await flyout.findByTestSubject('pagination-button-previous')).click(); - - const selectedIdForMissingIndexPattern1 = await testSubjects.getAttribute( - 'managementChangeIndexSelection-missing-index-pattern-1', - 'value' - ); - - expect(selectedIdForMissingIndexPattern1).to.eql('f1e4c910-a2e6-11e7-bb30-233be9be6a20'); - - await (await flyout.findByTestSubject('pagination-button-next')).click(); - - const selectedIdForMissingIndexPattern7 = await testSubjects.getAttribute( - 'managementChangeIndexSelection-missing-index-pattern-7', - 'value' - ); - - expect(selectedIdForMissingIndexPattern7).to.eql('f1e4c910-a2e6-11e7-bb30-233be9be6a87'); - }); - - it('should display an explicit error message when importing object from a higher Kibana version', async () => { - await PageObjects.savedObjects.importFile( - path.join(__dirname, 'exports', '_import_higher_version.ndjson') - ); - - await PageObjects.savedObjects.checkImportError(); - - const errorText = await PageObjects.savedObjects.getImportErrorText(); - - expect(errorText).to.contain( - `has property "visualization" which belongs to a more recent version of Kibana [9.15.82]` - ); - }); - - describe('when bigger than savedObjects.maxImportPayloadBytes (not Cloud)', function () { - // see --savedObjects.maxImportPayloadBytes in config file - this.tags(['skipCloud']); - it('should display an explicit error message when importing a file bigger than allowed', async () => { - await PageObjects.savedObjects.importFile( - path.join(__dirname, 'exports', '_import_too_big.ndjson') - ); - - await PageObjects.savedObjects.checkImportError(); - - const errorText = await PageObjects.savedObjects.getImportErrorText(); - - expect(errorText).to.contain(`Payload content length greater than maximum allowed`); - }); - }); - - it('should display an explicit error message when importing an invalid file', async () => { - await PageObjects.savedObjects.importFile( - path.join(__dirname, 'exports', '_import_invalid_format.ndjson') - ); - - await PageObjects.savedObjects.checkImportError(); - - const errorText = await PageObjects.savedObjects.getImportErrorText(); - - expect(errorText).to.contain(`Unexpected token T in JSON at position 0`); - }); - }); }); } diff --git a/test/functional/apps/management/_mgmt_import_saved_objects.js b/test/functional/apps/management/_mgmt_import_saved_objects.js index 84e57a798c006..b7bca79a25940 100644 --- a/test/functional/apps/management/_mgmt_import_saved_objects.js +++ b/test/functional/apps/management/_mgmt_import_saved_objects.js @@ -30,7 +30,7 @@ export default function ({ getService, getPageObjects }) { it('should import saved objects mgmt', async function () { await PageObjects.settings.clickKibanaSavedObjects(); await PageObjects.savedObjects.importFile( - path.join(__dirname, 'exports', 'mgmt_import_objects.json') + path.join(__dirname, 'exports', 'mgmt_import_objects.ndjson') ); await PageObjects.settings.associateIndexPattern( '4c3f3c30-ac94-11e8-a651-614b2788174a', diff --git a/test/functional/apps/management/exports/_import_index_patterns_multiple_exists.json b/test/functional/apps/management/exports/_import_index_patterns_multiple_exists.json deleted file mode 100644 index 2eb64b1c7ca9f..0000000000000 --- a/test/functional/apps/management/exports/_import_index_patterns_multiple_exists.json +++ /dev/null @@ -1,26 +0,0 @@ -[ - { - "_id": "f1e4c910-a2e6-11e7-bb30-233be9be6a20", - "_type": "index-pattern", - "_source": { - "title": "index-pattern-test-1", - "timeFieldName": "@timestamp", - "fields": "[{\"name\":\"referer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"agent\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:image:width\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"xss.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"headings.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:description\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"meta.user.lastname\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.article:tag.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"geo.dest\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.twitter:image\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.article:section.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"utc_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.twitter:card\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"meta.char\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"clientip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:image:height\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"host\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"machine.ram\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"links\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"id\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"@tags.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"phpmemory\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.twitter:card.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:image\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.article:modified_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.url\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:site_name.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"request.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.article:tag\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"agent.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"spaces\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:site.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"headings\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"relatedContent.og:image.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"request\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"index.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"extension\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"memory\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"relatedContent.twitter:site\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:description\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:url\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"geo.coordinates\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.url.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"meta.related\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:title.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:title.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"response.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"@message.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"machine.os\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.article:section\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:url.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"xss\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"links.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:title\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"geo.srcdest\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"url.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"extension.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"machine.os.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"@tags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"host.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:type.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"geo.src\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"spaces.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:image:height.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"url\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:description.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:site_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:title\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"@message\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:image.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"@timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"response\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"meta.user.firstname\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:image:width.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:description.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.article:published_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"expression script\",\"type\":\"number\",\"count\":0,\"scripted\":true,\"script\":\"doc['bytes'].value\",\"lang\":\"expression\",\"indexed\":true,\"analyzed\":false,\"doc_values\":false}]" - }, - "_meta": { - "savedObjectVersion": 1 - } - }, - { - "_id": "f1e4c910-a2e6-11e7-bb30-233be9be6a87", - "_type": "index-pattern", - "_source": { - "title": "index-pattern-test-2", - "timeFieldName": "@timestamp", - "fields": "[{\"name\":\"referer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"agent\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:image:width\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"xss.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"headings.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:description\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"meta.user.lastname\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.article:tag.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"geo.dest\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.twitter:image\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.article:section.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"utc_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.twitter:card\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"meta.char\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"clientip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:image:height\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"host\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"machine.ram\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"links\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"id\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"@tags.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"phpmemory\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.twitter:card.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:image\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.article:modified_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.url\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:site_name.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"request.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.article:tag\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"agent.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"spaces\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:site.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"headings\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"relatedContent.og:image.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"request\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"index.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"extension\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"memory\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"relatedContent.twitter:site\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:description\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:url\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"geo.coordinates\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.url.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"meta.related\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:title.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:title.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"response.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"@message.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"machine.os\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.article:section\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:url.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"xss\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"links.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:title\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"geo.srcdest\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"url.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"extension.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"machine.os.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"@tags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"host.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:type.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"geo.src\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"spaces.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:image:height.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"url\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:description.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:site_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:title\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"@message\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:image.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"@timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"response\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"meta.user.firstname\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:image:width.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:description.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.article:published_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"expression script\",\"type\":\"number\",\"count\":0,\"scripted\":true,\"script\":\"doc['bytes'].value\",\"lang\":\"expression\",\"indexed\":true,\"analyzed\":false,\"doc_values\":false}]" - }, - "_meta": { - "savedObjectVersion": 1 - } - } -] diff --git a/test/functional/apps/management/exports/_import_objects.json b/test/functional/apps/management/exports/_import_objects.json deleted file mode 100644 index 48015d64133fb..0000000000000 --- a/test/functional/apps/management/exports/_import_objects.json +++ /dev/null @@ -1,19 +0,0 @@ -[ - { - "_id": "082f1d60-a2e7-11e7-bb30-233be9be6a15", - "_type": "visualization", - "_source": { - "title": "Log Agents", - "visState": "{\"title\":\"Log Agents\",\"type\":\"area\",\"params\":{\"type\":\"area\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{\"text\":\"agent.raw: Descending\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"area\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"interpolate\":\"linear\",\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"agent.raw\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}]}", - "uiStateJSON": "{}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"f1e4c910-a2e6-11e7-bb30-233be9be6a15\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}" - } - }, - "_meta": { - "savedObjectVersion": 2 - } - } -] diff --git a/test/functional/apps/management/exports/_import_objects_connected_to_saved_search.json b/test/functional/apps/management/exports/_import_objects_connected_to_saved_search.json deleted file mode 100644 index 7088e1ab34b64..0000000000000 --- a/test/functional/apps/management/exports/_import_objects_connected_to_saved_search.json +++ /dev/null @@ -1,20 +0,0 @@ -[ - { - "_id": "saved_object_connected_to_saved_search", - "_type": "visualization", - "_source": { - "title": "saved object connected to saved search", - "visState": "{\"title\":\"PHP Viz\",\"type\":\"horizontal_bar\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":200},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":75,\"filter\":true,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"normal\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}]}", - "uiStateJSON": "{}", - "description": "", - "savedSearchId": "c45e6c50-ba72-11e7-a8f9-ad70f02e633d", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}" - } - }, - "_meta": { - "savedObjectVersion": 2 - } - } -] diff --git a/test/functional/apps/management/exports/_import_objects_exists.json b/test/functional/apps/management/exports/_import_objects_exists.json deleted file mode 100644 index 5356d1fdf6477..0000000000000 --- a/test/functional/apps/management/exports/_import_objects_exists.json +++ /dev/null @@ -1,19 +0,0 @@ -[ - { - "_id": "Shared-Item-Visualization-AreaChart", - "_type": "visualization", - "_source": { - "title": "Shared-Item Visualization AreaChart", - "visState": "{\"title\":\"New Visualization\",\"type\":\"area\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"smoothLines\":false,\"scale\":\"linear\",\"interpolate\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}}],\"listeners\":{}}", - "uiStateJSON": "{}", - "description": "AreaChart", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" - } - }, - "_meta": { - "savedObjectVersion": 2 - } - } -] diff --git a/test/functional/apps/management/exports/_import_objects_missing_all_index_patterns.json b/test/functional/apps/management/exports/_import_objects_missing_all_index_patterns.json deleted file mode 100644 index 45572b0bf34fe..0000000000000 --- a/test/functional/apps/management/exports/_import_objects_missing_all_index_patterns.json +++ /dev/null @@ -1,121 +0,0 @@ -[ - { - "_id": "test-vis-1", - "_type": "visualization", - "_source": { - "title": "Test VIS 1", - "visState": "{\"title\":\"test vis 1\",\"type\":\"histogram\"}", - "uiStateJSON": "{}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"missing-index-pattern-1\",\"query\":{}}" - } - }, - "_meta": { - "savedObjectVersion": 2 - } - }, - { - "_id": "test-vis-2", - "_type": "visualization", - "_source": { - "title": "Test VIS 2", - "visState": "{\"title\":\"test vis 2\",\"type\":\"histogram\"}", - "uiStateJSON": "{}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"missing-index-pattern-2\",\"query\":{}}" - } - }, - "_meta": { - "savedObjectVersion": 2 - } - }, - { - "_id": "test-vis-3", - "_type": "visualization", - "_source": { - "title": "Test VIS 3", - "visState": "{\"title\":\"test vis 3\",\"type\":\"histogram\"}", - "uiStateJSON": "{}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"missing-index-pattern-3\",\"query\":{}}" - } - }, - "_meta": { - "savedObjectVersion": 2 - } - }, - { - "_id": "test-vis-4", - "_type": "visualization", - "_source": { - "title": "Test VIS 4", - "visState": "{\"title\":\"test vis 4\",\"type\":\"histogram\"}", - "uiStateJSON": "{}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"missing-index-pattern-4\",\"query\":{}}" - } - }, - "_meta": { - "savedObjectVersion": 2 - } - }, - { - "_id": "test-vis-5", - "_type": "visualization", - "_source": { - "title": "Test VIS 5", - "visState": "{\"title\":\"test vis 5\",\"type\":\"histogram\"}", - "uiStateJSON": "{}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"missing-index-pattern-5\",\"query\":{}}" - } - }, - "_meta": { - "savedObjectVersion": 2 - } - }, - { - "_id": "test-vis-6", - "_type": "visualization", - "_source": { - "title": "Test VIS 6", - "visState": "{\"title\":\"test vis 6\",\"type\":\"histogram\"}", - "uiStateJSON": "{}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"missing-index-pattern-6\",\"query\":{}}" - } - }, - "_meta": { - "savedObjectVersion": 2 - } - }, - { - "_id": "test-vis-7", - "_type": "visualization", - "_source": { - "title": "Test VIS 7", - "visState": "{\"title\":\"test vis 7\",\"type\":\"histogram\"}", - "uiStateJSON": "{}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"missing-index-pattern-7\",\"query\":{}}" - } - }, - "_meta": { - "savedObjectVersion": 2 - } - } -] diff --git a/test/functional/apps/management/exports/_import_objects_multiple_exists.json b/test/functional/apps/management/exports/_import_objects_multiple_exists.json deleted file mode 100644 index 9e554aecd9f7a..0000000000000 --- a/test/functional/apps/management/exports/_import_objects_multiple_exists.json +++ /dev/null @@ -1,36 +0,0 @@ -[ - { - "_id": "test-1", - "_type": "visualization", - "_source": { - "title": "Visualization test 1", - "visState": "{\"title\":\"New Visualization\",\"type\":\"area\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"smoothLines\":false,\"scale\":\"linear\",\"interpolate\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}}],\"listeners\":{}}", - "uiStateJSON": "{}", - "description": "AreaChart", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" - } - }, - "_meta": { - "savedObjectVersion": 2 - } - }, - { - "_id": "test-2", - "_type": "visualization", - "_source": { - "title": "Visualization test 2", - "visState": "{\"title\":\"New Visualization\",\"type\":\"area\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"smoothLines\":false,\"scale\":\"linear\",\"interpolate\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}}],\"listeners\":{}}", - "uiStateJSON": "{}", - "description": "AreaChart", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" - } - }, - "_meta": { - "savedObjectVersion": 2 - } - } -] diff --git a/test/functional/apps/management/exports/_import_objects_saved_search.json b/test/functional/apps/management/exports/_import_objects_saved_search.json deleted file mode 100644 index bfd034a7086d2..0000000000000 --- a/test/functional/apps/management/exports/_import_objects_saved_search.json +++ /dev/null @@ -1,25 +0,0 @@ -[ - { - "_id": "c45e6c50-ba72-11e7-a8f9-ad70f02e633d", - "_type": "search", - "_source": { - "title": "PHP saved search", - "description": "", - "hits": 0, - "columns": [ - "_source" - ], - "sort": [ - "@timestamp", - "desc" - ], - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"f1e4c910-a2e6-11e7-bb30-233be9be6a15\",\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":\"php\"},\"filter\":[]}" - } - }, - "_meta": { - "savedObjectVersion": 2 - } - } -] diff --git a/test/functional/apps/management/exports/_import_objects_with_index_patterns.json b/test/functional/apps/management/exports/_import_objects_with_index_patterns.json deleted file mode 100644 index a0288652dddac..0000000000000 --- a/test/functional/apps/management/exports/_import_objects_with_index_patterns.json +++ /dev/null @@ -1,31 +0,0 @@ -[ - { - "_id": "f1e4c910-a2e6-11e7-bb30-233be9be6a15", - "_type": "index-pattern", - "_source": { - "title": "logstash-*", - "timeFieldName": "@timestamp", - "fields": "[{\"name\":\"referer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"agent\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:image:width\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"xss.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"headings.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:description\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"meta.user.lastname\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.article:tag.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"geo.dest\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.twitter:image\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.article:section.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"utc_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.twitter:card\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"meta.char\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"clientip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:image:height\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"host\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"machine.ram\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"links\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"id\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"@tags.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"phpmemory\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.twitter:card.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:image\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.article:modified_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.url\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:site_name.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"request.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.article:tag\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"agent.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"spaces\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:site.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"headings\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"relatedContent.og:image.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"request\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"index.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"extension\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"memory\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"relatedContent.twitter:site\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:description\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:url\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"geo.coordinates\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.url.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"meta.related\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:title.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:title.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"response.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"@message.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"machine.os\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.article:section\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:url.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"xss\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"links.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:title\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"geo.srcdest\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"url.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"extension.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"machine.os.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"@tags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"host.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:type.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"geo.src\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"spaces.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:image:height.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"url\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:description.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:site_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:title\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"@message\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:image.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"@timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"response\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"meta.user.firstname\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:image:width.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:description.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.article:published_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"expression script\",\"type\":\"number\",\"count\":0,\"scripted\":true,\"script\":\"doc['bytes'].value\",\"lang\":\"expression\",\"indexed\":true,\"analyzed\":false,\"doc_values\":false}]" - }, - "_meta": { - "savedObjectVersion": 2 - } - }, - { - "_id": "saved_object_imported_with_index_pattern", - "_type": "visualization", - "_source": { - "title": "saved object imported with index pattern", - "visState": "{\"title\":\"New Visualization\",\"type\":\"area\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"smoothLines\":false,\"scale\":\"linear\",\"interpolate\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}}],\"listeners\":{}}", - "uiStateJSON": "{}", - "description": "AreaChart", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"f1e4c910-a2e6-11e7-bb30-233be9be6a15\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" - } - }, - "_meta": { - "savedObjectVersion": 2 - } - } -] diff --git a/test/functional/apps/management/exports/mgmt_import_objects.json b/test/functional/apps/management/exports/mgmt_import_objects.json deleted file mode 100644 index 88e03585bf1ee..0000000000000 --- a/test/functional/apps/management/exports/mgmt_import_objects.json +++ /dev/null @@ -1,37 +0,0 @@ -[ - { - "_id": "6aea5700-ac94-11e8-a651-614b2788174a", - "_type": "search", - "_source": { - "title": "mysavedsearch", - "description": "", - "hits": 0, - "columns": [ - "_source" - ], - "sort": [ - "@timestamp", - "desc" - ], - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"4c3f3c30-ac94-11e8-a651-614b2788174a\",\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" - } - } - }, - { - "_id": "8411daa0-ac94-11e8-a651-614b2788174a", - "_type": "visualization", - "_source": { - "title": "mysavedviz", - "visState": "{\"title\":\"mysavedviz\",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}]}", - "uiStateJSON": "{}", - "description": "", - "savedSearchId": "6aea5700-ac94-11e8-a651-614b2788174a", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" - } - } - } -] diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index fd445f34317e0..ffd3adce378f5 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -3302,32 +3302,17 @@ "savedObjectsManagement.objectsTable.exportObjectsConfirmModal.exportOptionsLabel": "オプション", "savedObjectsManagement.objectsTable.exportObjectsConfirmModal.includeReferencesDeepLabel": "関連オブジェクトを含める", "savedObjectsManagement.objectsTable.exportObjectsConfirmModalDescription": "エクスポートするタイプを選択してください", - "savedObjectsManagement.objectsTable.flyout.confirmLegacyImport.resolvingConflictsLoadingMessage": "矛盾を解決中…", - "savedObjectsManagement.objectsTable.flyout.confirmLegacyImport.retryingFailedObjectsLoadingMessage": "失敗したオブジェクトを再試行中…", - "savedObjectsManagement.objectsTable.flyout.confirmLegacyImport.savedSearchAreLinkedProperlyLoadingMessage": "保存された検索が正しくリンクされていることを確認してください…", - "savedObjectsManagement.objectsTable.flyout.confirmLegacyImport.savingConflictsLoadingMessage": "矛盾を保存中…", "savedObjectsManagement.objectsTable.flyout.errorCalloutTitle": "申し訳ございません、エラーが発生しました", "savedObjectsManagement.objectsTable.flyout.import.cancelButtonLabel": "キャンセル", "savedObjectsManagement.objectsTable.flyout.import.confirmButtonLabel": "インポート", - "savedObjectsManagement.objectsTable.flyout.importFailedDescription": "{totalImportCount}個中{failedImportCount}個のオブジェクトのインポートに失敗しました。インポート失敗", - "savedObjectsManagement.objectsTable.flyout.importFailedMissingReference": "{type} [id={id}]は{refType} [id={refId}]を見つけられませんでした", - "savedObjectsManagement.objectsTable.flyout.importFailedTitle": "インポート失敗", - "savedObjectsManagement.objectsTable.flyout.importFailedUnsupportedType": "{type} [id={id}]サポートされていないタイプ", "savedObjectsManagement.objectsTable.flyout.importFileErrorMessage": "エラーのためファイルを処理できませんでした：「{error}」", - "savedObjectsManagement.objectsTable.flyout.importLegacyFileErrorMessage": "ファイルを処理できませんでした。", "savedObjectsManagement.objectsTable.flyout.importPromptText": "インポート", "savedObjectsManagement.objectsTable.flyout.importSavedObjectTitle": "保存されたオブジェクトのインポート", "savedObjectsManagement.objectsTable.flyout.importSuccessful.confirmAllChangesButtonLabel": "すべての変更を確定", "savedObjectsManagement.objectsTable.flyout.importSuccessful.confirmButtonLabel": "完了", - "savedObjectsManagement.objectsTable.flyout.importSuccessfulCallout.noObjectsImportedTitle": "オブジェクトがインポートされませんでした", - "savedObjectsManagement.objectsTable.flyout.importSuccessfulDescription": "{importCount}個のオブジェクトがインポートされました。", - "savedObjectsManagement.objectsTable.flyout.importSuccessfulTitle": "インポート成功", "savedObjectsManagement.objectsTable.flyout.indexPatternConflictsCalloutLinkText": "新規インデックスパターンを作成", "savedObjectsManagement.objectsTable.flyout.indexPatternConflictsDescription": "次の保存されたオブジェクトは、存在しないインデックスパターンを使用しています。関連付け直す別のインデックスパターンを選択してください。必要に応じて、{indexPatternLink}できます。", "savedObjectsManagement.objectsTable.flyout.indexPatternConflictsTitle": "インデックスパターンの矛盾", - "savedObjectsManagement.objectsTable.flyout.invalidFormatOfImportedFileErrorMessage": "保存されたオブジェクトのファイル形式が無効なため、インポートできません。", - "savedObjectsManagement.objectsTable.flyout.legacyFileUsedBody": "最新のレポートでNDJSONファイルを作成すれば完了です。", - "savedObjectsManagement.objectsTable.flyout.legacyFileUsedTitle": "JSONファイルのサポートが終了します", "savedObjectsManagement.objectsTable.flyout.renderConflicts.columnCountDescription": "影響されるオブジェクトの数です", "savedObjectsManagement.objectsTable.flyout.renderConflicts.columnCountName": "カウント", "savedObjectsManagement.objectsTable.flyout.renderConflicts.columnIdDescription": "インデックスパターンのIDです", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index beb4f3d0c2860..0fc6f0ec119eb 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -3320,32 +3320,17 @@ "savedObjectsManagement.objectsTable.exportObjectsConfirmModal.includeReferencesDeepLabel": "包括相关对象", "savedObjectsManagement.objectsTable.exportObjectsConfirmModalDescription": "选择要导出的类型", "savedObjectsManagement.objectsTable.exportObjectsConfirmModalTitle": "导出 {filteredItemCount, plural, other {# 个对象}}", - "savedObjectsManagement.objectsTable.flyout.confirmLegacyImport.resolvingConflictsLoadingMessage": "正在解决冲突……", - "savedObjectsManagement.objectsTable.flyout.confirmLegacyImport.retryingFailedObjectsLoadingMessage": "正在重试失败的对象……", - "savedObjectsManagement.objectsTable.flyout.confirmLegacyImport.savedSearchAreLinkedProperlyLoadingMessage": "确保已保存搜索已正确链接……", - "savedObjectsManagement.objectsTable.flyout.confirmLegacyImport.savingConflictsLoadingMessage": "正在保存冲突……", "savedObjectsManagement.objectsTable.flyout.errorCalloutTitle": "抱歉，有错误", "savedObjectsManagement.objectsTable.flyout.import.cancelButtonLabel": "取消", "savedObjectsManagement.objectsTable.flyout.import.confirmButtonLabel": "导入", - "savedObjectsManagement.objectsTable.flyout.importFailedDescription": "{totalImportCount} 个对象中有 {failedImportCount} 个无法导入。导入失败", - "savedObjectsManagement.objectsTable.flyout.importFailedMissingReference": "{type} [id={id}] 无法找到 {refType} [id={refId}]", - "savedObjectsManagement.objectsTable.flyout.importFailedTitle": "导入失败", - "savedObjectsManagement.objectsTable.flyout.importFailedUnsupportedType": "{type} [id={id}] 不受支持的类型", "savedObjectsManagement.objectsTable.flyout.importFileErrorMessage": "由于以下错误，无法处理文件：“{error}”", - "savedObjectsManagement.objectsTable.flyout.importLegacyFileErrorMessage": "无法处理该文件。", "savedObjectsManagement.objectsTable.flyout.importPromptText": "导入", "savedObjectsManagement.objectsTable.flyout.importSavedObjectTitle": "导入已保存对象", "savedObjectsManagement.objectsTable.flyout.importSuccessful.confirmAllChangesButtonLabel": "确认所有更改", "savedObjectsManagement.objectsTable.flyout.importSuccessful.confirmButtonLabel": "完成", - "savedObjectsManagement.objectsTable.flyout.importSuccessfulCallout.noObjectsImportedTitle": "未导入任何对象", - "savedObjectsManagement.objectsTable.flyout.importSuccessfulDescription": "已成功导入 {importCount} 个对象。", - "savedObjectsManagement.objectsTable.flyout.importSuccessfulTitle": "导入成功", "savedObjectsManagement.objectsTable.flyout.indexPatternConflictsCalloutLinkText": "创建新的索引模式", "savedObjectsManagement.objectsTable.flyout.indexPatternConflictsDescription": "以下已保存对象使用不存在的索引模式。请选择要重新关联的索引模式。必要时可以{indexPatternLink}。", "savedObjectsManagement.objectsTable.flyout.indexPatternConflictsTitle": "索引模式冲突", - "savedObjectsManagement.objectsTable.flyout.invalidFormatOfImportedFileErrorMessage": "已保存对象文件格式无效，无法导入。", - "savedObjectsManagement.objectsTable.flyout.legacyFileUsedBody": "只需使用更新的导出功能生成 NDJSON 文件，便万事俱备。", - "savedObjectsManagement.objectsTable.flyout.legacyFileUsedTitle": "将不再支持 JSON 文件", "savedObjectsManagement.objectsTable.flyout.renderConflicts.columnCountDescription": "受影响对象数目", "savedObjectsManagement.objectsTable.flyout.renderConflicts.columnCountName": "计数", "savedObjectsManagement.objectsTable.flyout.renderConflicts.columnIdDescription": "索引模式的 ID", From a18cc31924c58c9a3f2d80a6e0b6c2a3ae0ab535 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Fri, 3 Sep 2021 15:36:03 +0100 Subject: [PATCH 14/20] [ML] Datafeed preview based job validation check (#109080) * [ML] Datafeed preview based job validation check * updating warning text * fix tests * adding jest test * updating tests * fixing translation ids * fixing more tests * changes based on review * disabled validation step next button when validation fails * disabling nano job test * adding test skip comment Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../ml/common/constants/messages.test.mock.ts | 3 + .../ml/common/constants/messages.test.ts | 6 + .../plugins/ml/common/constants/messages.ts | 24 +++ .../components/validation_step/validation.tsx | 6 +- .../job_validation/job_validation.test.ts | 150 ++++++++++++++---- .../models/job_validation/job_validation.ts | 9 +- .../validate_datafeed_preview.ts | 38 +++++ .../ml/server/routes/job_validation.ts | 6 +- .../server/routes/schemas/datafeeds_schema.ts | 2 +- .../apis/ml/job_validation/validate.ts | 2 +- .../ml/anomaly_detection/date_nanos_job.ts | 3 +- 11 files changed, 209 insertions(+), 40 deletions(-) create mode 100644 x-pack/plugins/ml/server/models/job_validation/validate_datafeed_preview.ts diff --git a/x-pack/plugins/ml/common/constants/messages.test.mock.ts b/x-pack/plugins/ml/common/constants/messages.test.mock.ts index 6e539617604c1..fbfff20adc5c2 100644 --- a/x-pack/plugins/ml/common/constants/messages.test.mock.ts +++ b/x-pack/plugins/ml/common/constants/messages.test.mock.ts @@ -78,4 +78,7 @@ export const nonBasicIssuesMessages = [ { id: 'missing_summary_count_field_name', }, + { + id: 'datafeed_preview_failed', + }, ]; diff --git a/x-pack/plugins/ml/common/constants/messages.test.ts b/x-pack/plugins/ml/common/constants/messages.test.ts index 59fc50757b674..c46eba458d1d2 100644 --- a/x-pack/plugins/ml/common/constants/messages.test.ts +++ b/x-pack/plugins/ml/common/constants/messages.test.ts @@ -173,6 +173,12 @@ describe('Constants: Messages parseMessages()', () => { text: 'A job configured with a datafeed with aggregations must set summary_count_field_name; use doc_count or suitable alternative.', }, + { + id: 'datafeed_preview_failed', + status: 'error', + text: + 'The datafeed preview failed. This may be due to an error in the job or datafeed configurations.', + }, ]); }); }); diff --git a/x-pack/plugins/ml/common/constants/messages.ts b/x-pack/plugins/ml/common/constants/messages.ts index 0327e8746c7d8..fd3b9aa9d19b9 100644 --- a/x-pack/plugins/ml/common/constants/messages.ts +++ b/x-pack/plugins/ml/common/constants/messages.ts @@ -626,6 +626,30 @@ export const getMessages = once((docLinks?: DocLinksStart) => { 'the UNIX epoch beginning. Timestamps before 01/01/1970 00:00:00 (UTC) are not supported for machine learning jobs.', }), }, + datafeed_preview_no_documents: { + status: VALIDATION_STATUS.WARNING, + heading: i18n.translate( + 'xpack.ml.models.jobValidation.messages.datafeedPreviewNoDocumentsHeading', + { + defaultMessage: 'Datafeed preview', + } + ), + text: i18n.translate( + 'xpack.ml.models.jobValidation.messages.datafeedPreviewNoDocumentsMessage', + { + defaultMessage: + 'Running the datafeed preview over the current job configuration produces no results. ' + + 'If the index contains no documents this warning can be ignored, otherwise the job may be misconfigured.', + } + ), + }, + datafeed_preview_failed: { + status: VALIDATION_STATUS.ERROR, + text: i18n.translate('xpack.ml.models.jobValidation.messages.datafeedPreviewFailedMessage', { + defaultMessage: + 'The datafeed preview failed. This may be due to an error in the job or datafeed configurations.', + }), + }, }; }); diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/validation_step/validation.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/validation_step/validation.tsx index a1c7eab6b746f..d2a0e83200d92 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/validation_step/validation.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/validation_step/validation.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { Fragment, FC, useContext, useEffect } from 'react'; +import React, { Fragment, FC, useContext, useEffect, useState } from 'react'; import { WizardNav } from '../wizard_nav'; import { WIZARD_STEPS, StepProps } from '../step_types'; import { JobCreatorContext } from '../job_creator_context'; @@ -22,6 +22,7 @@ const idFilterList = [ export const ValidationStep: FC = ({ setCurrentStep, isCurrentStep }) => { const { jobCreator, jobCreatorUpdate, jobValidator } = useContext(JobCreatorContext); + const [nextActive, setNextActive] = useState(false); if (jobCreator.type === JOB_TYPE.ADVANCED) { // for advanced jobs, ignore time range warning as the @@ -52,6 +53,7 @@ export const ValidationStep: FC = ({ setCurrentStep, isCurrentStep }) // keep a record of the advanced validation in the jobValidator function setIsValid(valid: boolean) { jobValidator.advancedValid = valid; + setNextActive(valid); } return ( @@ -69,7 +71,7 @@ export const ValidationStep: FC = ({ setCurrentStep, isCurrentStep }) setCurrentStep(WIZARD_STEPS.JOB_DETAILS)} next={() => setCurrentStep(WIZARD_STEPS.SUMMARY)} - nextActive={true} + nextActive={nextActive} /> )} diff --git a/x-pack/plugins/ml/server/models/job_validation/job_validation.test.ts b/x-pack/plugins/ml/server/models/job_validation/job_validation.test.ts index a5483491f1357..e890020eb726d 100644 --- a/x-pack/plugins/ml/server/models/job_validation/job_validation.test.ts +++ b/x-pack/plugins/ml/server/models/job_validation/job_validation.test.ts @@ -10,6 +10,7 @@ import { IScopedClusterClient } from 'kibana/server'; import { validateJob, ValidateJobPayload } from './job_validation'; import { ES_CLIENT_TOTAL_HITS_RELATION } from '../../../common/types/es_client'; import type { MlClient } from '../../lib/ml_client'; +import type { AuthorizationHeader } from '../../lib/request_authorization'; const callAs = { fieldCaps: () => Promise.resolve({ body: { fields: [] } }), @@ -19,6 +20,8 @@ const callAs = { }), }; +const authHeader: AuthorizationHeader = {}; + const mlClusterClient = ({ asCurrentUser: callAs, asInternalUser: callAs, @@ -34,18 +37,19 @@ const mlClient = ({ }, }, }), + previewDatafeed: () => Promise.resolve({ body: [{}] }), } as unknown) as MlClient; // Note: The tests cast `payload` as any // so we can simulate possible runtime payloads // that don't satisfy the TypeScript specs. describe('ML - validateJob', () => { - it('basic validation messages', () => { + it('basic validation messages', async () => { const payload = ({ job: { analysis_config: { detectors: [] } }, } as unknown) as ValidateJobPayload; - return validateJob(mlClusterClient, mlClient, payload).then((messages) => { + return validateJob(mlClusterClient, mlClient, payload, authHeader).then((messages) => { const ids = messages.map((m) => m.id); expect(ids).toStrictEqual([ @@ -58,14 +62,14 @@ describe('ML - validateJob', () => { }); const jobIdTests = (testIds: string[], messageId: string) => { - const promises = testIds.map((id) => { + const promises = testIds.map(async (id) => { const payload = ({ job: { analysis_config: { detectors: [] }, job_id: id, }, } as unknown) as ValidateJobPayload; - return validateJob(mlClusterClient, mlClient, payload).catch(() => { + return validateJob(mlClusterClient, mlClient, payload, authHeader).catch(() => { new Error('Promise should not fail for jobIdTests.'); }); }); @@ -86,7 +90,7 @@ describe('ML - validateJob', () => { job: { analysis_config: { detectors: [] }, groups: testIds }, } as unknown) as ValidateJobPayload; - return validateJob(mlClusterClient, mlClient, payload).then((messages) => { + return validateJob(mlClusterClient, mlClient, payload, authHeader).then((messages) => { const ids = messages.map((m) => m.id); expect(ids.includes(messageId)).toBe(true); }); @@ -126,7 +130,7 @@ describe('ML - validateJob', () => { const payload = ({ job: { analysis_config: { bucket_span: format, detectors: [] } }, } as unknown) as ValidateJobPayload; - return validateJob(mlClusterClient, mlClient, payload).catch(() => { + return validateJob(mlClusterClient, mlClient, payload, authHeader).catch(() => { new Error('Promise should not fail for bucketSpanFormatTests.'); }); }); @@ -150,7 +154,7 @@ describe('ML - validateJob', () => { return bucketSpanFormatTests(validBucketSpanFormats, 'bucket_span_valid'); }); - it('at least one detector function is empty', () => { + it('at least one detector function is empty', async () => { const payload = ({ job: { analysis_config: { detectors: [] as Array<{ function?: string }> } }, } as unknown) as ValidateJobPayload; @@ -165,13 +169,13 @@ describe('ML - validateJob', () => { function: undefined, }); - return validateJob(mlClusterClient, mlClient, payload).then((messages) => { + return validateJob(mlClusterClient, mlClient, payload, authHeader).then((messages) => { const ids = messages.map((m) => m.id); expect(ids.includes('detectors_function_empty')).toBe(true); }); }); - it('detector function is not empty', () => { + it('detector function is not empty', async () => { const payload = ({ job: { analysis_config: { detectors: [] as Array<{ function?: string }> } }, } as unknown) as ValidateJobPayload; @@ -179,37 +183,37 @@ describe('ML - validateJob', () => { function: 'count', }); - return validateJob(mlClusterClient, mlClient, payload).then((messages) => { + return validateJob(mlClusterClient, mlClient, payload, authHeader).then((messages) => { const ids = messages.map((m) => m.id); expect(ids.includes('detectors_function_not_empty')).toBe(true); }); }); - it('invalid index fields', () => { + it('invalid index fields', async () => { const payload = ({ job: { analysis_config: { detectors: [] } }, fields: {}, } as unknown) as ValidateJobPayload; - return validateJob(mlClusterClient, mlClient, payload).then((messages) => { + return validateJob(mlClusterClient, mlClient, payload, authHeader).then((messages) => { const ids = messages.map((m) => m.id); expect(ids.includes('index_fields_invalid')).toBe(true); }); }); - it('valid index fields', () => { + it('valid index fields', async () => { const payload = ({ job: { analysis_config: { detectors: [] } }, fields: { testField: {} }, } as unknown) as ValidateJobPayload; - return validateJob(mlClusterClient, mlClient, payload).then((messages) => { + return validateJob(mlClusterClient, mlClient, payload, authHeader).then((messages) => { const ids = messages.map((m) => m.id); expect(ids.includes('index_fields_valid')).toBe(true); }); }); - const getBasicPayload = (): any => ({ + const getBasicPayload = (): ValidateJobPayload => ({ job: { job_id: 'test', analysis_config: { @@ -231,7 +235,7 @@ describe('ML - validateJob', () => { const payload = getBasicPayload() as any; delete payload.job.analysis_config.influencers; - validateJob(mlClusterClient, mlClient, payload).then( + validateJob(mlClusterClient, mlClient, payload, authHeader).then( () => done( new Error('Promise should not resolve for this test when influencers is not an Array.') @@ -240,10 +244,10 @@ describe('ML - validateJob', () => { ); }); - it('detect duplicate detectors', () => { + it('detect duplicate detectors', async () => { const payload = getBasicPayload() as any; payload.job.analysis_config.detectors.push({ function: 'count' }); - return validateJob(mlClusterClient, mlClient, payload).then((messages) => { + return validateJob(mlClusterClient, mlClient, payload, authHeader).then((messages) => { const ids = messages.map((m) => m.id); expect(ids).toStrictEqual([ 'job_id_valid', @@ -256,7 +260,7 @@ describe('ML - validateJob', () => { }); }); - it('dedupe duplicate messages', () => { + it('dedupe duplicate messages', async () => { const payload = getBasicPayload() as any; // in this test setup, the following configuration passes // the duplicate detectors check, but would return the same @@ -266,7 +270,7 @@ describe('ML - validateJob', () => { { function: 'count', by_field_name: 'airline' }, { function: 'count', partition_field_name: 'airline' }, ]; - return validateJob(mlClusterClient, mlClient, payload).then((messages) => { + return validateJob(mlClusterClient, mlClient, payload, authHeader).then((messages) => { const ids = messages.map((m) => m.id); expect(ids).toStrictEqual([ 'job_id_valid', @@ -278,9 +282,9 @@ describe('ML - validateJob', () => { }); }); - it('basic validation passes, extended checks return some messages', () => { + it('basic validation passes, extended checks return some messages', async () => { const payload = getBasicPayload(); - return validateJob(mlClusterClient, mlClient, payload).then((messages) => { + return validateJob(mlClusterClient, mlClient, payload, authHeader).then((messages) => { const ids = messages.map((m) => m.id); expect(ids).toStrictEqual([ 'job_id_valid', @@ -291,8 +295,8 @@ describe('ML - validateJob', () => { }); }); - it('categorization job using mlcategory passes aggregatable field check', () => { - const payload: any = { + it('categorization job using mlcategory passes aggregatable field check', async () => { + const payload: ValidateJobPayload = { job: { job_id: 'categorization_test', analysis_config: { @@ -312,7 +316,7 @@ describe('ML - validateJob', () => { fields: { testField: {} }, }; - return validateJob(mlClusterClient, mlClient, payload).then((messages) => { + return validateJob(mlClusterClient, mlClient, payload, authHeader).then((messages) => { const ids = messages.map((m) => m.id); expect(ids).toStrictEqual([ 'job_id_valid', @@ -325,8 +329,8 @@ describe('ML - validateJob', () => { }); }); - it('non-existent field reported as non aggregatable', () => { - const payload: any = { + it('non-existent field reported as non aggregatable', async () => { + const payload: ValidateJobPayload = { job: { job_id: 'categorization_test', analysis_config: { @@ -345,7 +349,7 @@ describe('ML - validateJob', () => { fields: { testField: {} }, }; - return validateJob(mlClusterClient, mlClient, payload).then((messages) => { + return validateJob(mlClusterClient, mlClient, payload, authHeader).then((messages) => { const ids = messages.map((m) => m.id); expect(ids).toStrictEqual([ 'job_id_valid', @@ -357,8 +361,8 @@ describe('ML - validateJob', () => { }); }); - it('script field not reported as non aggregatable', () => { - const payload: any = { + it('script field not reported as non aggregatable', async () => { + const payload: ValidateJobPayload = { job: { job_id: 'categorization_test', analysis_config: { @@ -387,7 +391,7 @@ describe('ML - validateJob', () => { fields: { testField: {} }, }; - return validateJob(mlClusterClient, mlClient, payload).then((messages) => { + return validateJob(mlClusterClient, mlClient, payload, authHeader).then((messages) => { const ids = messages.map((m) => m.id); expect(ids).toStrictEqual([ 'job_id_valid', @@ -399,4 +403,88 @@ describe('ML - validateJob', () => { ]); }); }); + + it('datafeed preview contains no docs', async () => { + const payload: ValidateJobPayload = { + job: { + job_id: 'categorization_test', + analysis_config: { + bucket_span: '15m', + detectors: [ + { + function: 'count', + partition_field_name: 'custom_script_field', + }, + ], + influencers: [''], + }, + data_description: { time_field: '@timestamp' }, + datafeed_config: { + indices: [], + }, + }, + fields: { testField: {} }, + }; + + const mlClientEmptyDatafeedPreview = ({ + ...mlClient, + previewDatafeed: () => Promise.resolve({ body: [] }), + } as unknown) as MlClient; + + return validateJob(mlClusterClient, mlClientEmptyDatafeedPreview, payload, authHeader).then( + (messages) => { + const ids = messages.map((m) => m.id); + expect(ids).toStrictEqual([ + 'job_id_valid', + 'detectors_function_not_empty', + 'index_fields_valid', + 'field_not_aggregatable', + 'time_field_invalid', + 'datafeed_preview_no_documents', + ]); + } + ); + }); + + it('datafeed preview failed', async () => { + const payload: ValidateJobPayload = { + job: { + job_id: 'categorization_test', + analysis_config: { + bucket_span: '15m', + detectors: [ + { + function: 'count', + partition_field_name: 'custom_script_field', + }, + ], + influencers: [''], + }, + data_description: { time_field: '@timestamp' }, + datafeed_config: { + indices: [], + }, + }, + fields: { testField: {} }, + }; + + const mlClientEmptyDatafeedPreview = ({ + ...mlClient, + previewDatafeed: () => Promise.reject({}), + } as unknown) as MlClient; + + return validateJob(mlClusterClient, mlClientEmptyDatafeedPreview, payload, authHeader).then( + (messages) => { + const ids = messages.map((m) => m.id); + expect(ids).toStrictEqual([ + 'job_id_valid', + 'detectors_function_not_empty', + 'index_fields_valid', + 'field_not_aggregatable', + 'time_field_invalid', + 'datafeed_preview_failed', + ]); + } + ); + }); }); diff --git a/x-pack/plugins/ml/server/models/job_validation/job_validation.ts b/x-pack/plugins/ml/server/models/job_validation/job_validation.ts index 80eba7b864051..838f188455d44 100644 --- a/x-pack/plugins/ml/server/models/job_validation/job_validation.ts +++ b/x-pack/plugins/ml/server/models/job_validation/job_validation.ts @@ -6,7 +6,7 @@ */ import Boom from '@hapi/boom'; -import { IScopedClusterClient } from 'kibana/server'; +import type { IScopedClusterClient } from 'kibana/server'; import { TypeOf } from '@kbn/config-schema'; import { fieldsServiceProvider } from '../fields_service'; import { getMessages, MessageId, JobValidationMessage } from '../../../common/constants/messages'; @@ -17,12 +17,14 @@ import { basicJobValidation, uniqWithIsEqual } from '../../../common/util/job_ut import { validateBucketSpan } from './validate_bucket_span'; import { validateCardinality } from './validate_cardinality'; import { validateInfluencers } from './validate_influencers'; +import { validateDatafeedPreview } from './validate_datafeed_preview'; import { validateModelMemoryLimit } from './validate_model_memory_limit'; import { validateTimeRange, isValidTimeField } from './validate_time_range'; import { validateJobSchema } from '../../routes/schemas/job_validation_schema'; -import { CombinedJob } from '../../../common/types/anomaly_detection_jobs'; +import type { CombinedJob } from '../../../common/types/anomaly_detection_jobs'; import type { MlClient } from '../../lib/ml_client'; import { getDatafeedAggregations } from '../../../common/util/datafeed_utils'; +import type { AuthorizationHeader } from '../../lib/request_authorization'; export type ValidateJobPayload = TypeOf; @@ -34,6 +36,7 @@ export async function validateJob( client: IScopedClusterClient, mlClient: MlClient, payload: ValidateJobPayload, + authHeader: AuthorizationHeader, isSecurityDisabled?: boolean ) { const messages = getMessages(); @@ -107,6 +110,8 @@ export async function validateJob( if (datafeedAggregations !== undefined && !job.analysis_config?.summary_count_field_name) { validationMessages.push({ id: 'missing_summary_count_field_name' }); } + + validationMessages.push(...(await validateDatafeedPreview(mlClient, authHeader, job))); } else { validationMessages = basicValidation.messages; validationMessages.push({ id: 'skipped_extended_tests' }); diff --git a/x-pack/plugins/ml/server/models/job_validation/validate_datafeed_preview.ts b/x-pack/plugins/ml/server/models/job_validation/validate_datafeed_preview.ts new file mode 100644 index 0000000000000..e009dcf49fdab --- /dev/null +++ b/x-pack/plugins/ml/server/models/job_validation/validate_datafeed_preview.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { MlClient } from '../../lib/ml_client'; +import type { AuthorizationHeader } from '../../lib/request_authorization'; +import type { CombinedJob } from '../../../common/types/anomaly_detection_jobs'; +import type { JobValidationMessage } from '../../../common/constants/messages'; + +export async function validateDatafeedPreview( + mlClient: MlClient, + authHeader: AuthorizationHeader, + job: CombinedJob +): Promise { + const { datafeed_config: datafeed, ...tempJob } = job; + try { + const { body } = ((await mlClient.previewDatafeed( + { + body: { + job_config: tempJob, + datafeed_config: datafeed, + }, + }, + authHeader + // previewDatafeed response type is incorrect + )) as unknown) as { body: unknown[] }; + + if (Array.isArray(body) === false || body.length === 0) { + return [{ id: 'datafeed_preview_no_documents' }]; + } + return []; + } catch (error) { + return [{ id: 'datafeed_preview_failed' }]; + } +} diff --git a/x-pack/plugins/ml/server/routes/job_validation.ts b/x-pack/plugins/ml/server/routes/job_validation.ts index 9309592dfc474..b75eab20e7bc0 100644 --- a/x-pack/plugins/ml/server/routes/job_validation.ts +++ b/x-pack/plugins/ml/server/routes/job_validation.ts @@ -8,9 +8,9 @@ import Boom from '@hapi/boom'; import { IScopedClusterClient } from 'kibana/server'; import { TypeOf } from '@kbn/config-schema'; -import { AnalysisConfig, Datafeed } from '../../common/types/anomaly_detection_jobs'; +import type { AnalysisConfig, Datafeed } from '../../common/types/anomaly_detection_jobs'; import { wrapError } from '../client/error_wrapper'; -import { RouteInitialization } from '../types'; +import type { RouteInitialization } from '../types'; import { estimateBucketSpanSchema, modelMemoryLimitSchema, @@ -20,6 +20,7 @@ import { import { estimateBucketSpanFactory } from '../models/bucket_span_estimator'; import { calculateModelMemoryLimitProvider } from '../models/calculate_model_memory_limit'; import { validateJob, validateCardinality } from '../models/job_validation'; +import { getAuthorizationHeader } from '../lib/request_authorization'; import type { MlClient } from '../lib/ml_client'; type CalculateModelMemoryLimitPayload = TypeOf; @@ -192,6 +193,7 @@ export function jobValidationRoutes({ router, mlLicense, routeGuard }: RouteInit client, mlClient, request.body, + getAuthorizationHeader(request), mlLicense.isSecurityEnabled() === false ); diff --git a/x-pack/plugins/ml/server/routes/schemas/datafeeds_schema.ts b/x-pack/plugins/ml/server/routes/schemas/datafeeds_schema.ts index 118d2e4140ced..27e1b6afe3364 100644 --- a/x-pack/plugins/ml/server/routes/schemas/datafeeds_schema.ts +++ b/x-pack/plugins/ml/server/routes/schemas/datafeeds_schema.ts @@ -52,7 +52,7 @@ export const datafeedConfigSchema = schema.object({ runtime_mappings: schema.maybe(schema.any()), scroll_size: schema.maybe(schema.number()), delayed_data_check_config: schema.maybe(schema.any()), - indices_options: indicesOptionsSchema, + indices_options: schema.maybe(indicesOptionsSchema), }); export const datafeedIdSchema = schema.object({ datafeedId: schema.string() }); diff --git a/x-pack/test/api_integration/apis/ml/job_validation/validate.ts b/x-pack/test/api_integration/apis/ml/job_validation/validate.ts index 06d966851abfd..293b0e94351d0 100644 --- a/x-pack/test/api_integration/apis/ml/job_validation/validate.ts +++ b/x-pack/test/api_integration/apis/ml/job_validation/validate.ts @@ -184,7 +184,7 @@ export default ({ getService }: FtrProviderContext) => { expect(body.length).to.eql( expectedResponse.length, - `Response body should have ${expectedResponse.length} entries (got ${body})` + `Response body should have ${expectedResponse.length} entries (got ${JSON.stringify(body)})` ); for (const entry of expectedResponse) { const responseEntry = body.find((obj: any) => obj.id === entry.id); diff --git a/x-pack/test/functional/apps/ml/anomaly_detection/date_nanos_job.ts b/x-pack/test/functional/apps/ml/anomaly_detection/date_nanos_job.ts index d351e8f7057e4..5f8d346ee4473 100644 --- a/x-pack/test/functional/apps/ml/anomaly_detection/date_nanos_job.ts +++ b/x-pack/test/functional/apps/ml/anomaly_detection/date_nanos_job.ts @@ -114,7 +114,8 @@ export default function ({ getService }: FtrProviderContext) { }, ]; - describe('job on data set with date_nanos time field', function () { + // test skipped until https://github.com/elastic/elasticsearch/pull/77109 is fixed + describe.skip('job on data set with date_nanos time field', function () { this.tags(['mlqa']); before(async () => { await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/event_rate_nanos'); From 98014d0cb9dcdb767f9cdd4d6efefe64706aaca2 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Fri, 3 Sep 2021 15:36:45 +0100 Subject: [PATCH 15/20] [ML] Datafeed preview based job validation check (#109080) * [ML] Datafeed preview based job validation check * updating warning text * fix tests * adding jest test * updating tests * fixing translation ids * fixing more tests * changes based on review * disabled validation step next button when validation fails * disabling nano job test * adding test skip comment Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> From 19260d5f02361c28276306b37cbd39dc2732020b Mon Sep 17 00:00:00 2001 From: Dmitry Shevchenko Date: Fri, 3 Sep 2021 16:38:58 +0200 Subject: [PATCH 16/20] Fix "Expression produces a union type that is too complex to represent" TS error (#111111) --- .../rule_registry_log_client.ts | 126 +++++++++--------- 1 file changed, 64 insertions(+), 62 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_adapter/rule_registry_log_client/rule_registry_log_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_adapter/rule_registry_log_client/rule_registry_log_client.ts index f0da8dad16ab0..a5515f8db8552 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_adapter/rule_registry_log_client/rule_registry_log_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_adapter/rule_registry_log_client/rule_registry_log_client.ts @@ -142,76 +142,78 @@ export class RuleRegistryLogClient implements IRuleRegistryLogClient { invariant(result.aggregations, 'Search response should contain aggregations'); return Object.fromEntries( - result.aggregations.rules.buckets.map((bucket) => [ - bucket.key, - bucket.most_recent_logs.hits.hits.map((event) => { - const logEntry = parseRuleExecutionLog(event._source); - invariant( - logEntry[ALERT_RULE_UUID] ?? '', - 'Malformed execution log entry: rule.id field not found' - ); + result.aggregations.rules.buckets.map<[ruleId: string, logs: IRuleStatusSOAttributes[]]>( + (bucket) => [ + bucket.key as string, + bucket.most_recent_logs.hits.hits.map((event) => { + const logEntry = parseRuleExecutionLog(event._source); + invariant( + logEntry[ALERT_RULE_UUID] ?? '', + 'Malformed execution log entry: rule.id field not found' + ); - const lastFailure = bucket.last_failure.event.hits.hits[0] - ? parseRuleExecutionLog(bucket.last_failure.event.hits.hits[0]._source) - : undefined; + const lastFailure = bucket.last_failure.event.hits.hits[0] + ? parseRuleExecutionLog(bucket.last_failure.event.hits.hits[0]._source) + : undefined; - const lastSuccess = bucket.last_success.event.hits.hits[0] - ? parseRuleExecutionLog(bucket.last_success.event.hits.hits[0]._source) - : undefined; + const lastSuccess = bucket.last_success.event.hits.hits[0] + ? parseRuleExecutionLog(bucket.last_success.event.hits.hits[0]._source) + : undefined; - const lookBack = bucket.indexing_lookback.event.hits.hits[0] - ? parseRuleExecutionLog(bucket.indexing_lookback.event.hits.hits[0]._source) - : undefined; + const lookBack = bucket.indexing_lookback.event.hits.hits[0] + ? parseRuleExecutionLog(bucket.indexing_lookback.event.hits.hits[0]._source) + : undefined; - const executionGap = bucket.execution_gap.event.hits.hits[0] - ? parseRuleExecutionLog(bucket.execution_gap.event.hits.hits[0]._source)[ - getMetricField(ExecutionMetric.executionGap) - ] - : undefined; + const executionGap = bucket.execution_gap.event.hits.hits[0] + ? parseRuleExecutionLog(bucket.execution_gap.event.hits.hits[0]._source)[ + getMetricField(ExecutionMetric.executionGap) + ] + : undefined; - const searchDuration = bucket.search_duration_max.event.hits.hits[0] - ? parseRuleExecutionLog(bucket.search_duration_max.event.hits.hits[0]._source)[ - getMetricField(ExecutionMetric.searchDurationMax) - ] - : undefined; + const searchDuration = bucket.search_duration_max.event.hits.hits[0] + ? parseRuleExecutionLog(bucket.search_duration_max.event.hits.hits[0]._source)[ + getMetricField(ExecutionMetric.searchDurationMax) + ] + : undefined; - const indexingDuration = bucket.indexing_duration_max.event.hits.hits[0] - ? parseRuleExecutionLog(bucket.indexing_duration_max.event.hits.hits[0]._source)[ - getMetricField(ExecutionMetric.indexingDurationMax) - ] - : undefined; + const indexingDuration = bucket.indexing_duration_max.event.hits.hits[0] + ? parseRuleExecutionLog(bucket.indexing_duration_max.event.hits.hits[0]._source)[ + getMetricField(ExecutionMetric.indexingDurationMax) + ] + : undefined; - const alertId = logEntry[ALERT_RULE_UUID] ?? ''; - const statusDate = logEntry[TIMESTAMP]; - const lastFailureAt = lastFailure?.[TIMESTAMP]; - const lastFailureMessage = lastFailure?.[MESSAGE]; - const lastSuccessAt = lastSuccess?.[TIMESTAMP]; - const lastSuccessMessage = lastSuccess?.[MESSAGE]; - const status = (logEntry[RULE_STATUS] as RuleExecutionStatus) || null; - const lastLookBackDate = lookBack?.[getMetricField(ExecutionMetric.indexingLookback)]; - const gap = executionGap ? moment.duration(executionGap).humanize() : null; - const bulkCreateTimeDurations = indexingDuration - ? [makeFloatString(indexingDuration)] - : null; - const searchAfterTimeDurations = searchDuration - ? [makeFloatString(searchDuration)] - : null; + const alertId = logEntry[ALERT_RULE_UUID] ?? ''; + const statusDate = logEntry[TIMESTAMP]; + const lastFailureAt = lastFailure?.[TIMESTAMP]; + const lastFailureMessage = lastFailure?.[MESSAGE]; + const lastSuccessAt = lastSuccess?.[TIMESTAMP]; + const lastSuccessMessage = lastSuccess?.[MESSAGE]; + const status = (logEntry[RULE_STATUS] as RuleExecutionStatus) || null; + const lastLookBackDate = lookBack?.[getMetricField(ExecutionMetric.indexingLookback)]; + const gap = executionGap ? moment.duration(executionGap).humanize() : null; + const bulkCreateTimeDurations = indexingDuration + ? [makeFloatString(indexingDuration)] + : null; + const searchAfterTimeDurations = searchDuration + ? [makeFloatString(searchDuration)] + : null; - return { - alertId, - statusDate, - lastFailureAt, - lastFailureMessage, - lastSuccessAt, - lastSuccessMessage, - status, - lastLookBackDate, - gap, - bulkCreateTimeDurations, - searchAfterTimeDurations, - }; - }), - ]) + return { + alertId, + statusDate, + lastFailureAt, + lastFailureMessage, + lastSuccessAt, + lastSuccessMessage, + status, + lastLookBackDate, + gap, + bulkCreateTimeDurations, + searchAfterTimeDurations, + }; + }), + ] + ) ); } From 9a459806ad12526caf7b099db2664b0ef65b647b Mon Sep 17 00:00:00 2001 From: Frank Hassanabad Date: Fri, 3 Sep 2021 08:41:28 -0600 Subject: [PATCH 17/20] [Security Solutions][Detection Engine] Adds ability to ignore fields during alert indexing and a workaround for an EQL bug (#110927) ## Summary Adds a workaround for EQL bug: https://github.com/elastic/elasticsearch/issues/77152 Adds the safety feature mentioned here: https://github.com/elastic/kibana/issues/110802 Adds the ability to ignore particular [fields](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-fields.html#search-fields-param) when the field is merged with [_source](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-fields.html#source-filtering). Also fixes an EQL bug where EQL is introducing the meta field of `_ignored` within the fields and causing documents to not be indexable when we merge with the fields from EQL. Alerting document creation uses the fields API to get [runtime field](https://www.elastic.co/guide/en/elasticsearch/reference/current/runtime.html), [constant keyword](https://www.elastic.co/guide/en/elasticsearch/reference/master/keyword.html#constant-keyword-field-type), etc... that are only available within the [fields API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-fields.html#search-fields-param) and then merges the field values not found within the `_source` document with the `_source` document and then finally indexes this merged document as an alert document. This fix/ability is a "safety feature" in that if a problematic [runtime field](https://www.elastic.co/guide/en/elasticsearch/reference/current/runtime.html), [constant keyword](https://www.elastic.co/guide/en/elasticsearch/reference/master/keyword.html#constant-keyword-field-type) is discovered or another bug along the stack we can set a `kibana.yml` key/value pair to ignore the problematic field. This _WILL NOT_ remove problematic fields from the `_source` document. This will only ignore problematic constant keyword, runtime fields, aliases, or anything else found in the fields API that is causing merge issues. This PR: * Adds a `alertIgnoreFields` `kibana.yml` array key with a default of an empty array if not specified. * Plumbs the `alertIgnoreFields` through the stack and into the fields/_source merge strategies of `missingFields` and `allFields` * Adds a temporary `isEqlBug77152` where it hard codes an ignore of `_ignored` until the EQL problem is fixed and then we will remove the workaround * Adds unit tests * Adds e2e tests which covers the described use cases above. The `alertIgnoreFields` key/value within `kibana.yml` if set should be an array of strings of each field you want to ignore. This can also contain regular expressions as long as they are of the form, `"/regex/"` in the array. Example if you want to ignore fields that are problematic called "host.name" and then one in which you want to ignore all fields that start with "user." using a regular expression: ```yml xpack.securitySolution.alertIgnoreFields: ['host.name', '/user\..*/'] ``` Although there are e2e tests which exercise the use cases... If you want to manual test the EQL bug fix you would add these documents in dev tools: ```json # Delete and add a mapping with a small ignore_above. DELETE eql-issue-ignore-fields-delme PUT eql-issue-ignore-fields-delme { "mappings" : { "dynamic": "strict", "properties" : { "@timestamp": { "type": "date" }, "some_keyword" : { "ignore_above": 5, "type" : "keyword" }, "other_keyword" : { "ignore_above": 10, "type" : "keyword" } } } } # Add a single document with one field that will be truncated and a second that will not. PUT eql-issue-ignore-fields-delme/_doc/1 { "@timestamp": "2021-09-02T04:13:05.626Z", "some_keyword": "longer than normal", "other_keyword": "normal" } ``` Then create an alert which queries everything from it: Screen Shot 2021-09-01 at 10 15 06 PM and ensure signals are created: Screen Shot 2021-09-01 at 10 30 18 PM To test the manual exclusions of any other problematic fields, create any index which has runtime fields or `constant keywords` but does not have anything within the `_source` document using dev tools. For example you can use `constant keyword` like so ```json PUT constant-keywords-deleme { "mappings": { "dynamic": "strict", "properties": { "@timestamp": { "type": "date" }, "testing_ignored": { "properties": { "constant": { "type": "constant_keyword", "value": "constant_value" } } }, "testing_regex": { "type": "constant_keyword", "value": "constant_value" }, "normal_constant": { "type": "constant_keyword", "value": "constant_value" }, "small_field": { "type": "keyword", "ignore_above": 10 } } } } PUT constant-keywords-deleme/_doc/1 { "@timestamp": "2021-09-02T04:20:01.760Z" } ``` Set in your `kibana.yml` the key/value of: ```yml xpack.securitySolution.alertIgnoreFields: ['testing_ignored.constant', '/.*_regex/'] ``` Setup a rule to run: Screen Shot 2021-09-01 at 10 23 23 PM Once it runs you should notice that the constant values for testing are not on the signals table since it only typically exists in the fields API: Screen Shot 2021-09-01 at 10 26 16 PM But the normal one still exists: Screen Shot 2021-09-01 at 10 26 31 PM If you change the `xpack.securitySolution.alertIgnoreFields` by removing it and re-generate the signals you will see these values added back. ### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [x] If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/master/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker) --- .../resources/base/bin/kibana-docker | 1 + .../security_solution/server/config.test.ts | 80 +++++ .../security_solution/server/config.ts | 49 +++ .../routes/__mocks__/index.ts | 1 + .../create_security_rule_type_factory.ts | 2 + .../factories/utils/build_bulk_body.ts | 3 +- .../rule_types/factories/wrap_hits_factory.ts | 7 +- .../create_indicator_match_alert_type.test.ts | 3 + .../create_indicator_match_alert_type.ts | 2 + .../ml/create_ml_alert_type.test.ts | 1 + .../rule_types/ml/create_ml_alert_type.ts | 11 +- .../query/create_query_alert_type.test.ts | 2 + .../query/create_query_alert_type.ts | 2 + .../lib/detection_engine/rule_types/types.ts | 2 + .../signals/build_bulk_body.test.ts | 8 + .../signals/build_bulk_body.ts | 16 +- .../signals/search_after_bulk_create.test.ts | 1 + .../signals/signal_rule_alert_type.test.ts | 1 + .../signals/signal_rule_alert_type.ts | 4 + .../merge_all_fields_with_source.test.ts | 326 ++++++++++++------ .../merge_all_fields_with_source.ts | 8 +- .../merge_missing_fields_with_source.test.ts | 278 +++++++++------ .../merge_missing_fields_with_source.ts | 8 +- .../strategies/merge_no_fields.ts | 3 +- .../signals/source_fields_merging/types.ts | 10 +- .../utils/filter_field_entries.test.ts | 37 +- .../utils/filter_field_entries.ts | 9 +- .../source_fields_merging/utils/index.ts | 1 + .../utils/is_eql_bug_77152.test.ts | 29 ++ .../utils/is_eql_bug_77152.ts | 18 + .../utils/is_ignored.test.ts | 84 +++++ .../source_fields_merging/utils/is_ignored.ts | 23 ++ .../signals/wrap_hits_factory.ts | 11 +- .../signals/wrap_sequences_factory.ts | 3 + .../security_solution/server/plugin.ts | 2 + .../common/config.ts | 4 + .../tests/ignore_fields.ts | 133 +++++++ .../security_and_spaces/tests/index.ts | 1 + .../security_solution/ignore_fields/data.json | 51 +++ .../ignore_fields/mappings.json | 41 +++ 40 files changed, 1044 insertions(+), 232 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/config.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_eql_bug_77152.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_eql_bug_77152.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_ignored.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_ignored.ts create mode 100644 x-pack/test/detection_engine_api_integration/security_and_spaces/tests/ignore_fields.ts create mode 100644 x-pack/test/functional/es_archives/security_solution/ignore_fields/data.json create mode 100644 x-pack/test/functional/es_archives/security_solution/ignore_fields/mappings.json diff --git a/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker b/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker index 0af087f1427d7..4bb89e1b7e606 100755 --- a/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker +++ b/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker @@ -383,6 +383,7 @@ kibana_vars=( xpack.security.session.lifespan xpack.security.sessionTimeout xpack.securitySolution.alertMergeStrategy + xpack.securitySolution.alertIgnoreFields xpack.securitySolution.endpointResultListDefaultFirstPageIndex xpack.securitySolution.endpointResultListDefaultPageSize xpack.securitySolution.maxRuleImportExportSize diff --git a/x-pack/plugins/security_solution/server/config.test.ts b/x-pack/plugins/security_solution/server/config.test.ts new file mode 100644 index 0000000000000..67956acd6656f --- /dev/null +++ b/x-pack/plugins/security_solution/server/config.test.ts @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { configSchema } from './config'; + +describe('config', () => { + describe('alertIgnoreFields', () => { + test('should default to an empty array', () => { + expect(configSchema.validate({}).alertIgnoreFields).toEqual([]); + }); + + test('should accept an array of strings', () => { + expect( + configSchema.validate({ alertIgnoreFields: ['foo.bar', 'mars.bar'] }).alertIgnoreFields + ).toEqual(['foo.bar', 'mars.bar']); + }); + + test('should throw if a non string is being sent in', () => { + expect( + () => + configSchema.validate({ + alertIgnoreFields: 5, + }).alertIgnoreFields + ).toThrow('[alertIgnoreFields]: expected value of type [array] but got [number]'); + }); + + test('should throw if we send in an invalid regular expression as a string', () => { + expect( + () => + configSchema.validate({ + alertIgnoreFields: ['/(/'], + }).alertIgnoreFields + ).toThrow( + '[alertIgnoreFields]: "Invalid regular expression: /(/: Unterminated group" at array position 0' + ); + }); + + test('should throw with two errors if we send two invalid regular expressions', () => { + expect( + () => + configSchema.validate({ + alertIgnoreFields: ['/(/', '/(invalid/'], + }).alertIgnoreFields + ).toThrow( + '[alertIgnoreFields]: "Invalid regular expression: /(/: Unterminated group" at array position 0. "Invalid regular expression: /(invalid/: Unterminated group" at array position 1' + ); + }); + + test('should throw with two errors with a valid string mixed in if we send two invalid regular expressions', () => { + expect( + () => + configSchema.validate({ + alertIgnoreFields: ['/(/', 'valid.string', '/(invalid/'], + }).alertIgnoreFields + ).toThrow( + '[alertIgnoreFields]: "Invalid regular expression: /(/: Unterminated group" at array position 0. "Invalid regular expression: /(invalid/: Unterminated group" at array position 2' + ); + }); + + test('should accept a valid regular expression within the string', () => { + expect( + configSchema.validate({ + alertIgnoreFields: ['/(.*)/'], + }).alertIgnoreFields + ).toEqual(['/(.*)/']); + }); + + test('should accept two valid regular expressions', () => { + expect( + configSchema.validate({ + alertIgnoreFields: ['/(.*)/', '/(.valid*)/'], + }).alertIgnoreFields + ).toEqual(['/(.*)/', '/(.valid*)/']); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/config.ts b/x-pack/plugins/security_solution/server/config.ts index a1c6601520a54..0850e43b21eda 100644 --- a/x-pack/plugins/security_solution/server/config.ts +++ b/x-pack/plugins/security_solution/server/config.ts @@ -21,12 +21,61 @@ export const configSchema = schema.object({ maxRuleImportPayloadBytes: schema.number({ defaultValue: 10485760 }), maxTimelineImportExportSize: schema.number({ defaultValue: 10000 }), maxTimelineImportPayloadBytes: schema.number({ defaultValue: 10485760 }), + + /** + * This is used within the merge strategies: + * server/lib/detection_engine/signals/source_fields_merging + * + * For determining which strategy for merging "fields" and "_source" together to get + * runtime fields, constant keywords, etc... + * + * "missingFields" (default) This will only merge fields that are missing from the _source and exist in the fields. + * "noFields" This will turn off all merging of runtime fields, constant keywords from fields. + * "allFields" This will merge and overwrite anything found within "fields" into "_source" before indexing the data. + */ alertMergeStrategy: schema.oneOf( [schema.literal('allFields'), schema.literal('missingFields'), schema.literal('noFields')], { defaultValue: 'missingFields', } ), + + /** + * This is used within the merge strategies: + * server/lib/detection_engine/signals/source_fields_merging + * + * For determining if we need to ignore particular "fields" and not merge them with "_source" such as + * runtime fields, constant keywords, etc... + * + * This feature and functionality is mostly as "safety feature" meaning that we have had bugs in the past + * where something down the stack unexpectedly ends up in the fields API which causes documents to not + * be indexable. Rather than changing alertMergeStrategy to be "noFields", you can use this array to add + * any problematic values. + * + * You can use plain dotted notation strings such as "host.name" or a regular expression such as "/host\..+/" + */ + alertIgnoreFields: schema.arrayOf(schema.string(), { + defaultValue: [], + validate(ignoreFields) { + const errors = ignoreFields.flatMap((ignoreField, index) => { + if (ignoreField.startsWith('/') && ignoreField.endsWith('/')) { + try { + new RegExp(ignoreField.slice(1, -1)); + return []; + } catch (error) { + return [`"${error.message}" at array position ${index}`]; + } + } else { + return []; + } + }); + if (errors.length !== 0) { + return errors.join('. '); + } else { + return undefined; + } + }, + }), [SIGNALS_INDEX_KEY]: schema.string({ defaultValue: DEFAULT_SIGNALS_INDEX }), /** diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts index a768273c9d147..1ac85f9a27969 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts @@ -26,6 +26,7 @@ export const createMockConfig = (): ConfigType => ({ endpointResultListDefaultPageSize: 10, packagerTaskInterval: '60s', alertMergeStrategy: 'missingFields', + alertIgnoreFields: [], prebuiltRulesFromFileSystem: true, prebuiltRulesFromSavedObjects: false, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_factory.ts index 879d776f83df2..3992c3afaa302 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_factory.ts @@ -40,6 +40,7 @@ export const createSecurityRuleTypeFactory: CreateSecurityRuleTypeFactory = ({ lists, logger, mergeStrategy, + ignoreFields, ruleDataClient, ruleDataService, }) => (type) => { @@ -208,6 +209,7 @@ export const createSecurityRuleTypeFactory: CreateSecurityRuleTypeFactory = ({ const wrapHits = wrapHitsFactory({ logger, + ignoreFields, mergeStrategy, ruleSO, spaceId, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts index ae2ebc787451b..c09d707fe484e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts @@ -36,10 +36,11 @@ export const buildBulkBody = ( ruleSO: SavedObject, doc: SignalSourceHit, mergeStrategy: ConfigType['alertMergeStrategy'], + ignoreFields: ConfigType['alertIgnoreFields'], applyOverrides: boolean, buildReasonMessage: BuildReasonMessage ): RACAlert => { - const mergedDoc = getMergeStrategy(mergeStrategy)({ doc }); + const mergedDoc = getMergeStrategy(mergeStrategy)({ doc, ignoreFields }); const rule = applyOverrides ? buildRuleWithOverrides(ruleSO, mergedDoc._source ?? {}) : buildRuleWithoutOverrides(ruleSO); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts index 62946c52b7f40..95c7b4e90b29c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts @@ -7,7 +7,7 @@ import { Logger } from 'kibana/server'; -import { SearchAfterAndBulkCreateParams, SignalSourceHit, WrapHits } from '../../signals/types'; +import { SearchAfterAndBulkCreateParams, WrapHits } from '../../signals/types'; import { buildBulkBody } from './utils/build_bulk_body'; import { generateId } from '../../signals/utils'; import { filterDuplicateSignals } from '../../signals/filter_duplicate_signals'; @@ -16,6 +16,7 @@ import { WrappedRACAlert } from '../types'; export const wrapHitsFactory = ({ logger, + ignoreFields, mergeStrategy, ruleSO, spaceId, @@ -23,6 +24,7 @@ export const wrapHitsFactory = ({ logger: Logger; ruleSO: SearchAfterAndBulkCreateParams['ruleSO']; mergeStrategy: ConfigType['alertMergeStrategy']; + ignoreFields: ConfigType['alertIgnoreFields']; spaceId: string | null | undefined; }): WrapHits => (events, buildReasonMessage) => { try { @@ -38,8 +40,9 @@ export const wrapHitsFactory = ({ _source: buildBulkBody( spaceId, ruleSO, - doc as SignalSourceHit, + doc, mergeStrategy, + ignoreFields, true, buildReasonMessage ), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.test.ts index f13a5a5e0e715..fe836c872dcad 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.test.ts @@ -56,6 +56,7 @@ describe('Indicator Match Alerts', () => { experimentalFeatures: allowedExperimentalValues, lists: dependencies.lists, logger: dependencies.logger, + ignoreFields: [], mergeStrategy: 'allFields', ruleDataClient: dependencies.ruleDataClient, ruleDataService: dependencies.ruleDataService, @@ -97,6 +98,7 @@ describe('Indicator Match Alerts', () => { lists: dependencies.lists, logger: dependencies.logger, mergeStrategy: 'allFields', + ignoreFields: [], ruleDataClient: dependencies.ruleDataClient, ruleDataService: dependencies.ruleDataService, version: '1.0.0', @@ -135,6 +137,7 @@ describe('Indicator Match Alerts', () => { lists: dependencies.lists, logger: dependencies.logger, mergeStrategy: 'allFields', + ignoreFields: [], ruleDataClient: dependencies.ruleDataClient, ruleDataService: dependencies.ruleDataService, version: '1.0.0', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.ts index 71acc2e1cee85..f2dfe69debed0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.ts @@ -19,6 +19,7 @@ export const createIndicatorMatchAlertType = (createOptions: CreateRuleOptions) lists, logger, mergeStrategy, + ignoreFields, ruleDataClient, version, ruleDataService, @@ -27,6 +28,7 @@ export const createIndicatorMatchAlertType = (createOptions: CreateRuleOptions) lists, logger, mergeStrategy, + ignoreFields, ruleDataClient, ruleDataService, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.test.ts index 40566ffa04e6a..23cd2e94aedf8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.test.ts @@ -98,6 +98,7 @@ describe('Machine Learning Alerts', () => { lists: dependencies.lists, logger: dependencies.logger, mergeStrategy: 'allFields', + ignoreFields: [], ml: mlMock, ruleDataClient: dependencies.ruleDataClient, ruleDataService: dependencies.ruleDataService, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.ts index 1d872df35de3a..cdaeb4be76d02 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.ts @@ -14,11 +14,20 @@ import { createSecurityRuleTypeFactory } from '../create_security_rule_type_fact import { CreateRuleOptions } from '../types'; export const createMlAlertType = (createOptions: CreateRuleOptions) => { - const { lists, logger, mergeStrategy, ml, ruleDataClient, ruleDataService } = createOptions; + const { + lists, + logger, + mergeStrategy, + ignoreFields, + ml, + ruleDataClient, + ruleDataService, + } = createOptions; const createSecurityRuleType = createSecurityRuleTypeFactory({ lists, logger, mergeStrategy, + ignoreFields, ruleDataClient, ruleDataService, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts index 903cf6adadd43..ed791af08890c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts @@ -32,6 +32,7 @@ describe('Custom query alerts', () => { lists: dependencies.lists, logger: dependencies.logger, mergeStrategy: 'allFields', + ignoreFields: [], ruleDataClient: dependencies.ruleDataClient, ruleDataService: dependencies.ruleDataService, version: '1.0.0', @@ -79,6 +80,7 @@ describe('Custom query alerts', () => { lists: dependencies.lists, logger: dependencies.logger, mergeStrategy: 'allFields', + ignoreFields: [], ruleDataClient: dependencies.ruleDataClient, ruleDataService: dependencies.ruleDataService, version: '1.0.0', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.ts index e59037f38ce56..2f185853754b3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.ts @@ -19,6 +19,7 @@ export const createQueryAlertType = (createOptions: CreateRuleOptions) => { lists, logger, mergeStrategy, + ignoreFields, ruleDataClient, version, ruleDataService, @@ -27,6 +28,7 @@ export const createQueryAlertType = (createOptions: CreateRuleOptions) => { lists, logger, mergeStrategy, + ignoreFields, ruleDataClient, ruleDataService, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts index f061240c4a6e5..d50ab566c75cb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts @@ -96,6 +96,7 @@ export type CreateSecurityRuleTypeFactory = (options: { lists: SetupPlugins['lists']; logger: Logger; mergeStrategy: ConfigType['alertMergeStrategy']; + ignoreFields: ConfigType['alertIgnoreFields']; ruleDataClient: IRuleDataClient; ruleDataService: IRuleDataPluginService; }) => < @@ -124,6 +125,7 @@ export interface CreateRuleOptions { lists: SetupPlugins['lists']; logger: Logger; mergeStrategy: ConfigType['alertMergeStrategy']; + ignoreFields: ConfigType['alertIgnoreFields']; ml?: SetupPlugins['ml']; ruleDataClient: IRuleDataClient; version: string; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts index 206f3ae59d246..5f392bed75f76 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts @@ -43,6 +43,7 @@ describe('buildBulkBody', () => { ruleSO, doc, 'missingFields', + [], buildReasonMessage ); // Timestamp will potentially always be different so remove it for the test @@ -114,6 +115,7 @@ describe('buildBulkBody', () => { ruleSO, doc, 'missingFields', + [], buildReasonMessage ); // Timestamp will potentially always be different so remove it for the test @@ -199,6 +201,7 @@ describe('buildBulkBody', () => { ruleSO, doc, 'missingFields', + [], buildReasonMessage ); // Timestamp will potentially always be different so remove it for the test @@ -270,6 +273,7 @@ describe('buildBulkBody', () => { ruleSO, doc, 'missingFields', + [], buildReasonMessage ); // Timestamp will potentially always be different so remove it for the test @@ -338,6 +342,7 @@ describe('buildBulkBody', () => { ruleSO, doc, 'missingFields', + [], buildReasonMessage ); // Timestamp will potentially always be different so remove it for the test @@ -405,6 +410,7 @@ describe('buildBulkBody', () => { ruleSO, doc, 'missingFields', + [], buildReasonMessage ); const expected: Omit & { someKey: string } = { @@ -468,6 +474,7 @@ describe('buildBulkBody', () => { ruleSO, doc, 'missingFields', + [], buildReasonMessage ); const expected: Omit & { someKey: string } = { @@ -712,6 +719,7 @@ describe('buildSignalFromEvent', () => { ruleSO, true, 'missingFields', + [], buildReasonMessage ); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts index a4e812e8f111a..f8e39964523d0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts @@ -37,9 +37,10 @@ export const buildBulkBody = ( ruleSO: SavedObject, doc: SignalSourceHit, mergeStrategy: ConfigType['alertMergeStrategy'], + ignoreFields: ConfigType['alertIgnoreFields'], buildReasonMessage: BuildReasonMessage ): SignalHit => { - const mergedDoc = getMergeStrategy(mergeStrategy)({ doc }); + const mergedDoc = getMergeStrategy(mergeStrategy)({ doc, ignoreFields }); const rule = buildRuleWithOverrides(ruleSO, mergedDoc._source ?? {}); const timestamp = new Date().toISOString(); const reason = buildReasonMessage({ mergedDoc, rule }); @@ -76,11 +77,19 @@ export const buildSignalGroupFromSequence = ( ruleSO: SavedObject, outputIndex: string, mergeStrategy: ConfigType['alertMergeStrategy'], + ignoreFields: ConfigType['alertIgnoreFields'], buildReasonMessage: BuildReasonMessage ): WrappedSignalHit[] => { const wrappedBuildingBlocks = wrapBuildingBlocks( sequence.events.map((event) => { - const signal = buildSignalFromEvent(event, ruleSO, false, mergeStrategy, buildReasonMessage); + const signal = buildSignalFromEvent( + event, + ruleSO, + false, + mergeStrategy, + ignoreFields, + buildReasonMessage + ); signal.signal.rule.building_block_type = 'default'; return signal; }), @@ -146,9 +155,10 @@ export const buildSignalFromEvent = ( ruleSO: SavedObject, applyOverrides: boolean, mergeStrategy: ConfigType['alertMergeStrategy'], + ignoreFields: ConfigType['alertIgnoreFields'], buildReasonMessage: BuildReasonMessage ): SignalHit => { - const mergedEvent = getMergeStrategy(mergeStrategy)({ doc: event }); + const mergedEvent = getMergeStrategy(mergeStrategy)({ doc: event, ignoreFields }); const rule = applyOverrides ? buildRuleWithOverrides(ruleSO, mergedEvent._source ?? {}) : buildRuleWithoutOverrides(ruleSO); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts index 8bf0c986b9c25..55a184a1c0bcc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts @@ -74,6 +74,7 @@ describe('searchAfterAndBulkCreate', () => { ruleSO, signalsIndex: DEFAULT_SIGNALS_INDEX, mergeStrategy: 'missingFields', + ignoreFields: [], }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts index 39728235db39c..9af8680ec726a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts @@ -195,6 +195,7 @@ describe('signal_rule_alert_type', () => { ml: mlMock, lists: listMock.createSetup(), mergeStrategy: 'missingFields', + ignoreFields: [], ruleDataService, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 1c4efea0a1d59..68d60f7757e4a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -82,6 +82,7 @@ export const signalRulesAlertType = ({ ml, lists, mergeStrategy, + ignoreFields, ruleDataService, }: { logger: Logger; @@ -91,6 +92,7 @@ export const signalRulesAlertType = ({ ml: SetupPlugins['ml']; lists: SetupPlugins['lists'] | undefined; mergeStrategy: ConfigType['alertMergeStrategy']; + ignoreFields: ConfigType['alertIgnoreFields']; ruleDataService: IRuleDataPluginService; }): SignalRuleAlertTypeDefinition => { return { @@ -275,12 +277,14 @@ export const signalRulesAlertType = ({ ruleSO: savedObject, signalsIndex: params.outputIndex, mergeStrategy, + ignoreFields, }); const wrapSequences = wrapSequencesFactory({ ruleSO: savedObject, signalsIndex: params.outputIndex, mergeStrategy, + ignoreFields, }); if (isMlRule(type)) { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.test.ts index b900ea268fd6e..6af82d3a71028 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.test.ts @@ -44,7 +44,7 @@ describe('merge_all_fields_with_source', () => { test('when source is "undefined", merged doc is "undefined"', () => { const _source: SignalSourceHit['_source'] = {}; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -53,7 +53,7 @@ describe('merge_all_fields_with_source', () => { foo: [], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -62,7 +62,7 @@ describe('merge_all_fields_with_source', () => { foo: 'value', }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -71,7 +71,7 @@ describe('merge_all_fields_with_source', () => { foo: ['value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -80,7 +80,7 @@ describe('merge_all_fields_with_source', () => { foo: ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -89,7 +89,7 @@ describe('merge_all_fields_with_source', () => { foo: { bar: 'some value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -98,7 +98,7 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'some value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -107,7 +107,7 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'some value' }, { foo: 'some other value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -133,7 +133,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -142,7 +142,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': 'value', }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -151,7 +151,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -160,7 +160,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -169,7 +169,7 @@ describe('merge_all_fields_with_source', () => { foo: { bar: 'some value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -178,7 +178,7 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'some value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -187,7 +187,7 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'some value' }, { foo: 'some other value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -217,7 +217,7 @@ describe('merge_all_fields_with_source', () => { foo: { bar: [] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -226,7 +226,7 @@ describe('merge_all_fields_with_source', () => { foo: { bar: 'value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -235,7 +235,7 @@ describe('merge_all_fields_with_source', () => { foo: { bar: ['value'] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -244,7 +244,7 @@ describe('merge_all_fields_with_source', () => { foo: { bar: ['value_1', 'value_2'] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -253,7 +253,7 @@ describe('merge_all_fields_with_source', () => { foo: { bar: { mars: 'some value' } }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -262,7 +262,7 @@ describe('merge_all_fields_with_source', () => { foo: { bar: [{ mars: 'some value' }] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -271,7 +271,7 @@ describe('merge_all_fields_with_source', () => { foo: { bar: [{ mars: 'some value' }, { mars: 'some other value' }] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -299,7 +299,7 @@ describe('merge_all_fields_with_source', () => { 'bar.foo': [], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -308,7 +308,7 @@ describe('merge_all_fields_with_source', () => { 'bar.foo': 'value', }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -317,7 +317,7 @@ describe('merge_all_fields_with_source', () => { 'bar.foo': ['value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -326,7 +326,7 @@ describe('merge_all_fields_with_source', () => { 'bar.foo': ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -335,7 +335,7 @@ describe('merge_all_fields_with_source', () => { foo: { bar: 'some value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -344,7 +344,7 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'some value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -353,7 +353,7 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'some value' }, { foo: 'some other value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -376,7 +376,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: 'other_value_1', @@ -389,7 +389,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1', 'other_value_2'], @@ -402,7 +402,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: { zed: 'other_value_1' } }, }); @@ -413,7 +413,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }] }, }); @@ -440,7 +440,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: 'other_value_1', @@ -453,7 +453,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1', 'other_value_2'] }, }); @@ -464,7 +464,7 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: 'other_value_1', @@ -477,7 +477,7 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'other_value_1' }, { bar: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: [{ bar: 'other_value_1' }, { bar: 'other_value_2' }], }); @@ -503,7 +503,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ 'foo.bar': 'other_value_1', }); @@ -514,7 +514,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); @@ -523,7 +523,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ 'foo.bar': { zed: 'other_value_1' }, }); @@ -534,7 +534,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); }); @@ -560,7 +560,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1'] }, }); @@ -571,7 +571,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1', 'other_value_2'] }, }); @@ -582,7 +582,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }] }, }); @@ -593,7 +593,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }] }, }); @@ -619,7 +619,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); @@ -628,7 +628,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); @@ -637,7 +637,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); @@ -646,7 +646,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); }); @@ -670,7 +670,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1'] }, }); @@ -681,7 +681,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1', 'other_value_2'] }, }); @@ -692,7 +692,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }] }, }); @@ -703,7 +703,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }] }, }); @@ -729,7 +729,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); @@ -738,7 +738,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); @@ -747,7 +747,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); @@ -756,7 +756,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); }); @@ -782,7 +782,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1'], @@ -795,7 +795,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1', 'other_value_2'], @@ -808,7 +808,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }], @@ -821,7 +821,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], @@ -849,7 +849,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); @@ -858,7 +858,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); @@ -867,7 +867,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); @@ -876,7 +876,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); }); @@ -902,7 +902,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -911,7 +911,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -920,7 +920,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: { zed: 'other_value_1' } }, }); @@ -931,7 +931,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }] }, }); @@ -957,7 +957,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -966,7 +966,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -975,7 +975,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ 'foo.bar': { mars: 'other_value_1' }, }); @@ -986,7 +986,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], }); @@ -1014,7 +1014,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1023,7 +1023,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1032,7 +1032,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }] }, }); @@ -1043,7 +1043,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }] }, }); @@ -1069,7 +1069,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1078,7 +1078,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1087,7 +1087,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }] }, }); @@ -1098,7 +1098,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }] }, }); @@ -1124,7 +1124,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1133,7 +1133,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1142,7 +1142,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); @@ -1151,7 +1151,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); }); @@ -1175,7 +1175,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1184,7 +1184,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1193,7 +1193,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); @@ -1202,7 +1202,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); }); @@ -1228,7 +1228,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: 'value_1' }, 'foo.bar': 'other_value_1', @@ -1243,7 +1243,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: 'value_1' }, // <--- We have duplicated value_1 twice which is a bug 'foo.bar': ['value_1', 'value_2'], // <-- We have merged the array value because we do not understand if we should or not @@ -1270,7 +1270,7 @@ describe('merge_all_fields_with_source', () => { 'bar.keyword': ['bar_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: 'foo_other_value_1', bar: 'bar_other_value_1', @@ -1291,7 +1291,7 @@ describe('merge_all_fields_with_source', () => { 'host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ host: { hostname: 'hostname_other_value_1', @@ -1316,7 +1316,7 @@ describe('merge_all_fields_with_source', () => { 'foo.host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { host: { @@ -1334,7 +1334,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: 'other_value_1', }); @@ -1354,7 +1354,7 @@ describe('merge_all_fields_with_source', () => { 'host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ 'host.name': 'host_name_other_value_1', 'host.hostname': 'hostname_other_value_1', @@ -1373,7 +1373,7 @@ describe('merge_all_fields_with_source', () => { 'foo.host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ 'foo.host.name': 'host_name_other_value_1', 'foo.host.hostname': 'hostname_other_value_1', @@ -1388,7 +1388,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar.zed': ['zed_other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: 'other_value_1', }); @@ -1415,7 +1415,7 @@ describe('merge_all_fields_with_source', () => { 'foo.mars': ['other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1435,7 +1435,7 @@ describe('merge_all_fields_with_source', () => { 'foo.zed.mars': ['other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -1450,7 +1450,7 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: ['single_value'], zed: ['single_value'] }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: 'single_value', zed: 'single_value' }, }); @@ -1469,10 +1469,132 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: ['single_value'], zed: ['single_value'] }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: [{ bar: ['single_value'], zed: ['single_value'] }], }); }); }); + + /** + * Small set of tests to ensure that ignore fields are wired up at the strategy level + */ + describe('ignore fields', () => { + test('Does not merge an ignored field if it does not exist already in the _source', () => { + const _source: SignalSourceHit['_source'] = {}; + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + 'value.should.ignore': ['other_value_2'], // string value should ignore this + '_odd.value': ['other_value_2'], // Regex should ignore this value of: /[_]+/ + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: ['value.should.ignore', '/[_]+/'], + })._source; + expect(merged).toEqual({ + foo: { + bar: 'other_value_1', + }, + }); + }); + + test('Does merge fields when no matching happens', () => { + const _source: SignalSourceHit['_source'] = {}; + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + 'value.should.work': ['other_value_2'], + '_odd.value': ['other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: ['other.string', '/[z]+/'], // Neither of these two should match anything + })._source; + expect(merged).toEqual({ + foo: { + bar: 'other_value_1', + }, + _odd: { + value: 'other_value_2', + }, + value: { + should: { + work: 'other_value_2', + }, + }, + }); + }); + + test('Does not update an ignored field and keeps the original value if it matches in the ignoreFields', () => { + const _source: SignalSourceHit['_source'] = { + 'value.should.ignore': ['value_1'], + '_odd.value': ['value_2'], + }; + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + 'value.should.ignore': ['other_value_2'], // string value should ignore this + '_odd.value': ['other_value_2'], // Regex should ignore this value of: /[_]+/ + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: ['value.should.ignore', '/[_]+/'], + })._source; + expect(merged).toEqual({ + foo: { + bar: 'other_value_1', + }, + 'value.should.ignore': ['value_1'], + '_odd.value': ['value_2'], + }); + }); + + test('Does not ignore anything when no matching happens and overwrites the expected fields', () => { + const _source: SignalSourceHit['_source'] = { + 'value.should.ignore': ['value_1'], + '_odd.value': ['value_2'], + }; + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + 'value.should.ignore': ['other_value_2'], + '_odd.value': ['other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: ['nothing.to.match', '/[z]+/'], // these match nothing + })._source; + expect(merged).toEqual({ + foo: { + bar: 'other_value_1', + }, + 'value.should.ignore': ['other_value_2'], + '_odd.value': ['other_value_2'], + }); + }); + }); + + /** + * Test that the EQL bug workaround is wired up. Remove this once the bug is fixed. + */ + describe('Works around EQL bug 77152 (https://github.com/elastic/elasticsearch/issues/77152)', () => { + test('Does not merge field that contains _ignored', () => { + const _source: SignalSourceHit['_source'] = {}; + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + _ignored: ['other_value_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: [], + })._source; + expect(merged).toEqual({ + foo: { + bar: 'other_value_1', + }, + }); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.ts index da2eea9d2c61e..ade83b88d526b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.ts @@ -23,14 +23,16 @@ import { isTypeObject } from '../utils/is_type_object'; * on this function and the general strategies. * * @param doc The document with "_source" and "fields" - * @param throwOnFailSafe Defaults to false, but if set to true it will cause a throw if the fail safe is triggered to indicate we need to add a new explicit test condition + * @param ignoreFields Any fields that we should ignore and never merge from "fields". If the value exists + * within doc._source it will be untouched and used. If the value does not exist within the doc._source, + * it will not be added from fields. * @returns The two merged together in one object where we can */ -export const mergeAllFieldsWithSource: MergeStrategyFunction = ({ doc }) => { +export const mergeAllFieldsWithSource: MergeStrategyFunction = ({ doc, ignoreFields }) => { const source = doc._source ?? {}; const fields = doc.fields ?? {}; const fieldEntries = Object.entries(fields); - const filteredEntries = filterFieldEntries(fieldEntries); + const filteredEntries = filterFieldEntries(fieldEntries, ignoreFields); const transformedSource = filteredEntries.reduce( (merged, [fieldsKey, fieldsValue]: [string, FieldsType]) => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.test.ts index 70d1e79580e84..612bff75792da 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.test.ts @@ -44,7 +44,7 @@ describe('merge_missing_fields_with_source', () => { test('when source is "undefined", merged doc is "undefined"', () => { const _source: SignalSourceHit['_source'] = {}; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -53,7 +53,7 @@ describe('merge_missing_fields_with_source', () => { foo: [], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -62,7 +62,7 @@ describe('merge_missing_fields_with_source', () => { foo: 'value', }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -71,7 +71,7 @@ describe('merge_missing_fields_with_source', () => { foo: ['value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -80,7 +80,7 @@ describe('merge_missing_fields_with_source', () => { foo: ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -89,7 +89,7 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: 'some value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -98,7 +98,7 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'some value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -107,7 +107,7 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'some value' }, { foo: 'some other value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -133,7 +133,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -142,7 +142,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': 'value', }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -151,7 +151,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -160,7 +160,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -169,7 +169,7 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: 'some value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -178,7 +178,7 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'some value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -187,7 +187,7 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'some value' }, { foo: 'some other value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -217,7 +217,7 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: [] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -226,7 +226,7 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: 'value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -235,7 +235,7 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: ['value'] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -244,7 +244,7 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: ['value_1', 'value_2'] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -253,7 +253,7 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: { mars: 'some value' } }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -262,7 +262,7 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: [{ mars: 'some value' }] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -271,7 +271,7 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: [{ mars: 'some value' }, { mars: 'some other value' }] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -299,7 +299,7 @@ describe('merge_missing_fields_with_source', () => { 'bar.foo': [], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -308,7 +308,7 @@ describe('merge_missing_fields_with_source', () => { 'bar.foo': 'value', }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -317,7 +317,7 @@ describe('merge_missing_fields_with_source', () => { 'bar.foo': ['value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -326,7 +326,7 @@ describe('merge_missing_fields_with_source', () => { 'bar.foo': ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -335,7 +335,7 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: 'some value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -344,7 +344,7 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'some value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -353,7 +353,7 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'some value' }, { foo: 'some other value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -376,7 +376,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: 'other_value_1', @@ -389,7 +389,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1', 'other_value_2'], @@ -402,7 +402,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({}); }); @@ -411,7 +411,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({}); }); }); @@ -436,7 +436,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -445,7 +445,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -454,7 +454,7 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -463,7 +463,7 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'other_value_1' }, { bar: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -487,7 +487,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -496,7 +496,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -505,7 +505,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -514,7 +514,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -540,7 +540,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -549,7 +549,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -558,7 +558,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -567,7 +567,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -591,7 +591,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -600,7 +600,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -609,7 +609,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -618,7 +618,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -642,7 +642,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -651,7 +651,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -660,7 +660,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -669,7 +669,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -693,7 +693,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -702,7 +702,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -711,7 +711,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -720,7 +720,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -746,7 +746,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -755,7 +755,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -764,7 +764,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -773,7 +773,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -797,7 +797,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -806,7 +806,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -815,7 +815,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -824,7 +824,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -850,7 +850,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -859,7 +859,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -868,7 +868,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -877,7 +877,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -901,7 +901,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -910,7 +910,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -919,7 +919,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -928,7 +928,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -954,7 +954,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -963,7 +963,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -972,7 +972,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -981,7 +981,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -1005,7 +1005,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1014,7 +1014,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1023,7 +1023,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1032,7 +1032,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -1056,7 +1056,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1065,7 +1065,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1074,7 +1074,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1083,7 +1083,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -1107,7 +1107,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1116,7 +1116,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1125,7 +1125,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1134,7 +1134,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -1160,7 +1160,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1172,7 +1172,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -1196,7 +1196,7 @@ describe('merge_missing_fields_with_source', () => { 'bar.keyword': ['bar_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1214,7 +1214,7 @@ describe('merge_missing_fields_with_source', () => { 'host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1234,7 +1234,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1245,7 +1245,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: 'other_value_1', }); @@ -1265,7 +1265,7 @@ describe('merge_missing_fields_with_source', () => { 'host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1281,7 +1281,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1293,7 +1293,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar.zed': ['zed_other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: 'other_value_1', }); @@ -1320,7 +1320,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.mars': ['other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1340,7 +1340,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.zed.mars': ['other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -1355,7 +1355,7 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: ['single_value'], zed: ['single_value'] }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({}); }); @@ -1372,8 +1372,82 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: ['single_value'], zed: ['single_value'] }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); + + /** + * Small set of tests to ensure that ignore fields are wired up at the strategy level + */ + describe('ignore fields', () => { + test('Does not merge an ignored field if it does not exist already in the _source', () => { + const _source: SignalSourceHit['_source'] = {}; + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + 'value.should.ignore': ['other_value_2'], // string value should ignore this + '_odd.value': ['other_value_2'], // Regex should ignore this value of: /[_]+/ + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: ['value.should.ignore', '/[_]+/'], + })._source; + expect(merged).toEqual({ + foo: { + bar: 'other_value_1', + }, + }); + }); + + test('Does merge fields when no matching happens', () => { + const _source: SignalSourceHit['_source'] = {}; + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + 'value.should.work': ['other_value_2'], + '_odd.value': ['other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: ['other.string', '/[z]+/'], // Neither of these two should match anything + })._source; + expect(merged).toEqual({ + foo: { + bar: 'other_value_1', + }, + _odd: { + value: 'other_value_2', + }, + value: { + should: { + work: 'other_value_2', + }, + }, + }); + }); + }); + + /** + * Test that the EQL bug workaround is wired up. Remove this once the bug is fixed. + */ + describe('Works around EQL bug 77152 (https://github.com/elastic/elasticsearch/issues/77152)', () => { + test('Does not merge field that contains _ignored', () => { + const _source: SignalSourceHit['_source'] = {}; + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + _ignored: ['other_value_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: [], + })._source; + expect(merged).toEqual({ + foo: { + bar: 'other_value_1', + }, + }); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.ts index b66c46ccbf0ca..611a3ad879705 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.ts @@ -19,14 +19,16 @@ import { isNestedObject } from '../utils/is_nested_object'; * Merges only missing sections of "doc._source" with its "doc.fields" on a "best effort" basis. See ../README.md for more information * on this function and the general strategies. * @param doc The document with "_source" and "fields" - * @param throwOnFailSafe Defaults to false, but if set to true it will cause a throw if the fail safe is triggered to indicate we need to add a new explicit test condition + * @param ignoreFields Any fields that we should ignore and never merge from "fields". If the value exists + * within doc._source it will be untouched and used. If the value does not exist within the doc._source, + * it will not be added from fields. * @returns The two merged together in one object where we can */ -export const mergeMissingFieldsWithSource: MergeStrategyFunction = ({ doc }) => { +export const mergeMissingFieldsWithSource: MergeStrategyFunction = ({ doc, ignoreFields }) => { const source = doc._source ?? {}; const fields = doc.fields ?? {}; const fieldEntries = Object.entries(fields); - const filteredEntries = filterFieldEntries(fieldEntries); + const filteredEntries = filterFieldEntries(fieldEntries, ignoreFields); const transformedSource = filteredEntries.reduce( (merged, [fieldsKey, fieldsValue]: [string, FieldsType]) => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_no_fields.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_no_fields.ts index 6c2daf2526715..5e26b619fbdfa 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_no_fields.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_no_fields.ts @@ -10,6 +10,7 @@ import { MergeStrategyFunction } from '../types'; /** * Does nothing and does not merge source with fields * @param doc The doc to return and do nothing + * @param ignoreFields We do nothing with this value and ignore it if set * @returns The doc as a no operation and do nothing */ -export const mergeNoFields: MergeStrategyFunction = ({ doc }) => doc; +export const mergeNoFields: MergeStrategyFunction = ({ doc, ignoreFields }) => doc; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/types.ts index 1438d2844949c..0b847064d5d62 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/types.ts @@ -14,5 +14,13 @@ export type FieldsType = string[] | number[] | boolean[] | object[]; /** * The type of the merge strategy functions which must implement to be part of the strategy group + * @param doc The document to send in to merge + * @param ignoreFields Fields you want to ignore and not merge. */ -export type MergeStrategyFunction = ({ doc }: { doc: SignalSourceHit }) => SignalSourceHit; +export type MergeStrategyFunction = ({ + doc, + ignoreFields, +}: { + doc: SignalSourceHit; + ignoreFields: string[]; +}) => SignalSourceHit; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/filter_field_entries.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/filter_field_entries.test.ts index 9cc2478290885..031a2013b462e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/filter_field_entries.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/filter_field_entries.test.ts @@ -27,7 +27,9 @@ describe('filter_field_entries', () => { test('returns a single valid fieldEntries as expected', () => { const fieldEntries: Array<[string, FieldsType]> = [['foo.bar', dummyValue]]; - expect(filterFieldEntries(fieldEntries)).toEqual(fieldEntries); + expect(filterFieldEntries(fieldEntries, [])).toEqual( + fieldEntries + ); }); test('removes invalid dotted entries', () => { @@ -37,7 +39,7 @@ describe('filter_field_entries', () => { ['..', dummyValue], ['foo..bar', dummyValue], ]; - expect(filterFieldEntries(fieldEntries)).toEqual([ + expect(filterFieldEntries(fieldEntries, [])).toEqual([ ['foo.bar', dummyValue], ]); }); @@ -49,7 +51,7 @@ describe('filter_field_entries', () => { ['bar.keyword', dummyValue], // <-- "bar.keyword" multi-field should be removed ['bar', dummyValue], ]; - expect(filterFieldEntries(fieldEntries)).toEqual([ + expect(filterFieldEntries(fieldEntries, [])).toEqual([ ['foo', dummyValue], ['bar', dummyValue], ]); @@ -62,7 +64,7 @@ describe('filter_field_entries', () => { ['host.hostname', dummyValue], ['host.hostname.keyword', dummyValue], // <-- multi-field should be removed ]; - expect(filterFieldEntries(fieldEntries)).toEqual([ + expect(filterFieldEntries(fieldEntries, [])).toEqual([ ['host.name', dummyValue], ['host.hostname', dummyValue], ]); @@ -75,9 +77,34 @@ describe('filter_field_entries', () => { ['foo.host.hostname', dummyValue], ['foo.host.hostname.keyword', dummyValue], // <-- multi-field should be removed ]; - expect(filterFieldEntries(fieldEntries)).toEqual([ + expect(filterFieldEntries(fieldEntries, [])).toEqual([ ['foo.host.name', dummyValue], ['foo.host.hostname', dummyValue], ]); }); + + test('ignores fields of "_ignore", for eql bug https://github.com/elastic/elasticsearch/issues/77152', () => { + const fieldEntries: Array<[string, FieldsType]> = [ + ['_ignored', dummyValue], + ['foo.host.hostname', dummyValue], + ]; + expect(filterFieldEntries(fieldEntries, [])).toEqual([ + ['foo.host.hostname', dummyValue], + ]); + }); + + test('ignores fields given strings and regular expressions in the ignoreFields list', () => { + const fieldEntries: Array<[string, FieldsType]> = [ + ['host.name', dummyValue], + ['user.name', dummyValue], // <-- string from ignoreFields should ignore this + ['host.hostname', dummyValue], + ['_odd.value', dummyValue], // <-- regular expression from ignoreFields should ignore this + ]; + expect( + filterFieldEntries(fieldEntries, ['user.name', '/[_]+/']) + ).toEqual([ + ['host.name', dummyValue], + ['host.hostname', dummyValue], + ]); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/filter_field_entries.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/filter_field_entries.ts index 221cdabc62847..4ee5fa1db52f5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/filter_field_entries.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/filter_field_entries.ts @@ -9,6 +9,8 @@ import { isMultiField } from './is_multifield'; import { isInvalidKey } from './is_invalid_key'; import { isTypeObject } from './is_type_object'; import { FieldsType } from '../types'; +import { isIgnored } from './is_ignored'; +import { isEqlBug77152 } from './is_eql_bug_77152'; /** * Filters field entries by removing invalid field entries such as any invalid characters @@ -17,13 +19,18 @@ import { FieldsType } from '../types'; * those and don't try to merge those. * * @param fieldEntries The field entries to filter + * @param ignoreFields Array of fields to ignore. If a value starts and ends with "/", such as: "/[_]+/" then the field will be treated as a regular expression. + * If you have an object structure to ignore such as "{ a: { b: c: {} } } ", then you need to ignore it as the string "a.b.c" * @returns The field entries filtered */ export const filterFieldEntries = ( - fieldEntries: Array<[string, FieldsType]> + fieldEntries: Array<[string, FieldsType]>, + ignoreFields: string[] ): Array<[string, FieldsType]> => { return fieldEntries.filter(([fieldsKey, fieldsValue]: [string, FieldsType]) => { return ( + !isEqlBug77152(fieldsKey) && + !isIgnored(fieldsKey, ignoreFields) && !isInvalidKey(fieldsKey) && !isMultiField(fieldsKey, fieldEntries) && !isTypeObject(fieldsValue) // TODO: Look at not filtering this and instead transform it so it can be inserted correctly in the strategies which does an overwrite of everything from fields diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/index.ts index baf9efca511e2..87b1097dd9bca 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/index.ts @@ -7,6 +7,7 @@ export * from './array_in_path_exists'; export * from './filter_field_entries'; export * from './is_array_of_primitives'; +export * from './is_ignored'; export * from './is_invalid_key'; export * from './is_multifield'; export * from './is_nested_object'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_eql_bug_77152.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_eql_bug_77152.test.ts new file mode 100644 index 0000000000000..47a56e096649b --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_eql_bug_77152.test.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isEqlBug77152 } from './is_eql_bug_77152'; + +/** + * @deprecated Remove this test once https://github.com/elastic/elasticsearch/issues/77152 is fixed. + */ +describe('is_eql_bug_77152', () => { + beforeAll(() => { + jest.resetAllMocks(); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + it('returns true if it encounters the bug which is _ignored is returned in the fields', () => { + expect(isEqlBug77152('_ignored')).toEqual(true); + }); + + it('returns false if it encounters a normal field', () => { + expect(isEqlBug77152('some.field')).toEqual(false); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_eql_bug_77152.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_eql_bug_77152.ts new file mode 100644 index 0000000000000..e9a642cd5c382 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_eql_bug_77152.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Ignores any field that is "_ignored". Customers are not allowed to have this field and more importantly this shows up as a bug + * from EQL as seen here: https://github.com/elastic/elasticsearch/issues/77152 + * Once this ticket is fixed, please remove this function. + * @param fieldsKey The fields key to match against "_ignored" + * @returns true if it is a "_ignored", otherwise false + * @deprecated Remove this once https://github.com/elastic/elasticsearch/issues/77152 is fixed. + */ +export const isEqlBug77152 = (fieldsKey: string): boolean => { + return fieldsKey === '_ignored'; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_ignored.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_ignored.test.ts new file mode 100644 index 0000000000000..e4a7093ef127c --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_ignored.test.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isIgnored } from './is_ignored'; + +describe('is_ignored', () => { + beforeAll(() => { + jest.resetAllMocks(); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + describe('string matching', () => { + test('it returns false if given an empty array', () => { + expect(isIgnored('simple.value', [])).toEqual(false); + }); + + test('it returns true if a simple string value matches', () => { + expect(isIgnored('simple.value', ['simple.value'])).toEqual(true); + }); + + test('it returns false if a simple string value does not match', () => { + expect(isIgnored('simple', ['simple.value'])).toEqual(false); + }); + + test('it returns true if a simple string value matches with two strings', () => { + expect(isIgnored('simple.value', ['simple.value', 'simple.second.value'])).toEqual(true); + }); + + test('it returns true if a simple string value matches the second string', () => { + expect(isIgnored('simple.second.value', ['simple.value', 'simple.second.value'])).toEqual( + true + ); + }); + + test('it returns false if a simple string value does not match two strings', () => { + expect(isIgnored('simple', ['simple.value', 'simple.second.value'])).toEqual(false); + }); + + test('it returns true if mixed with a regular expression in the list', () => { + expect(isIgnored('simple', ['simple', '/[_]+/'])).toEqual(true); + }); + }); + + describe('regular expression matching', () => { + test('it returns true if a simple regular expression matches', () => { + expect(isIgnored('_ignored', ['/[_]+/'])).toEqual(true); + }); + + test('it returns false if a simple regular expression does not match', () => { + expect(isIgnored('simple', ['/[_]+/'])).toEqual(false); + }); + + test('it returns true if a simple regular expression matches a longer string', () => { + expect(isIgnored('___ignored', ['/[_]+/'])).toEqual(true); + }); + + test('it returns true if mixed with regular stings', () => { + expect(isIgnored('___ignored', ['simple', '/[_]+/'])).toEqual(true); + }); + + test('it returns true with start anchor', () => { + expect(isIgnored('_ignored', ['simple', '/^[_]+/'])).toEqual(true); + }); + + test('it returns false with start anchor', () => { + expect(isIgnored('simple.something_', ['simple', '/^[_]+/'])).toEqual(false); + }); + + test('it returns true with end anchor', () => { + expect(isIgnored('something_', ['simple', '/[_]+$/'])).toEqual(true); + }); + + test('it returns false with end anchor', () => { + expect(isIgnored('_something', ['simple', '/[_]+$/'])).toEqual(false); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_ignored.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_ignored.ts new file mode 100644 index 0000000000000..a418ce735626d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_ignored.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Matches against anything you want to ignore and if it matches that field is ignored. + * @param fieldsKey The fields key to match against + * @param ignoreFields Array of fields to ignore. If a value starts and ends with "/", such as: "/[_]+/" then the field will be treated as a regular expression. + * If you have an object structure to ignore such as "{ a: { b: c: {} } } ", then you need to ignore it as the string "a.b.c" + * @returns true if it is a field to ignore, otherwise false + */ +export const isIgnored = (fieldsKey: string, ignoreFields: string[]): boolean => { + return ignoreFields.some((ignoreField) => { + if (ignoreField.startsWith('/') && ignoreField.endsWith('/')) { + return new RegExp(ignoreField.slice(1, -1)).test(fieldsKey); + } else { + return fieldsKey === ignoreField; + } + }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_hits_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_hits_factory.ts index 19bdd58140a33..27220d80ebd6e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_hits_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_hits_factory.ts @@ -5,12 +5,7 @@ * 2.0. */ -import { - SearchAfterAndBulkCreateParams, - SignalSourceHit, - WrapHits, - WrappedSignalHit, -} from './types'; +import { SearchAfterAndBulkCreateParams, WrapHits, WrappedSignalHit } from './types'; import { generateId } from './utils'; import { buildBulkBody } from './build_bulk_body'; import { filterDuplicateSignals } from './filter_duplicate_signals'; @@ -20,10 +15,12 @@ export const wrapHitsFactory = ({ ruleSO, signalsIndex, mergeStrategy, + ignoreFields, }: { ruleSO: SearchAfterAndBulkCreateParams['ruleSO']; signalsIndex: string; mergeStrategy: ConfigType['alertMergeStrategy']; + ignoreFields: ConfigType['alertIgnoreFields']; }): WrapHits => (events, buildReasonMessage) => { const wrappedDocs: WrappedSignalHit[] = events.flatMap((doc) => [ { @@ -34,7 +31,7 @@ export const wrapHitsFactory = ({ String(doc._version), ruleSO.attributes.params.ruleId ?? '' ), - _source: buildBulkBody(ruleSO, doc as SignalSourceHit, mergeStrategy, buildReasonMessage), + _source: buildBulkBody(ruleSO, doc, mergeStrategy, ignoreFields, buildReasonMessage), }, ]); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_sequences_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_sequences_factory.ts index 0ca4b9688f971..d4a4c55db1d23 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_sequences_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_sequences_factory.ts @@ -13,10 +13,12 @@ export const wrapSequencesFactory = ({ ruleSO, signalsIndex, mergeStrategy, + ignoreFields, }: { ruleSO: SearchAfterAndBulkCreateParams['ruleSO']; signalsIndex: string; mergeStrategy: ConfigType['alertMergeStrategy']; + ignoreFields: ConfigType['alertIgnoreFields']; }): WrapSequences => (sequences, buildReasonMessage) => sequences.reduce( (acc: WrappedSignalHit[], sequence) => [ @@ -26,6 +28,7 @@ export const wrapSequencesFactory = ({ ruleSO, signalsIndex, mergeStrategy, + ignoreFields, buildReasonMessage ), ], diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index d657d7e06b1a6..4e4d0be5a7411 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -252,6 +252,7 @@ export class Plugin implements IPlugin `--xpack.${key}.enabled=false`), ...(ssl ? [ diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/ignore_fields.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/ignore_fields.ts new file mode 100644 index 0000000000000..409128523ea40 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/ignore_fields.ts @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; + +import { FtrProviderContext } from '../../../common/ftr_provider_context'; +import { + createRule, + createSignalsIndex, + deleteAllAlerts, + deleteSignalsIndex, + getEqlRuleForSignalTesting, + getSignalsById, + waitForRuleSuccessOrStatus, + waitForSignalsToBePresent, +} from '../../utils'; + +interface Ignore { + normal_constant?: string; + small_field?: string; + testing_ignored?: string; + testing_regex?: string; +} + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext): void => { + /** + * See the config file (detection_engine_api_integration/common/config.ts) for which field values were added to be ignored + * for testing. The values should be in the config around the area of: + * --xpack.securitySolution.alertIgnoreFields=[testing.ignore_1,/[testingRegex] + * meaning that the ignore fields values should be the array: ["testing.ignore_1", "/[testingRegex]/"] + * + * This test exercises the ability to be able to ignore particular values within the fields API and merge strategies. + * These values can be defined in your kibana.yml file as "xpack.securitySolution.alertIgnoreFields". This is useful + * for users that find bugs or regressions within query languages or bugs within the merge strategies + * where one or more fields are causing problems and they need to turn disable that particular field. + * + * Ref: + * https://github.com/elastic/kibana/issues/110802 + * https://github.com/elastic/elasticsearch/issues/77152 + * + * Files ref: + * server/lib/detection_engine/signals/source_fields_merging/utils/is_ignored.ts + * server/lib/detection_engine/signals/source_fields_merging/utils/is_eql_bug_77152.ts + */ + describe('ignore_fields', () => { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/security_solution/ignore_fields'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/security_solution/ignore_fields'); + }); + + beforeEach(async () => { + await createSignalsIndex(supertest); + }); + + afterEach(async () => { + await deleteSignalsIndex(supertest); + await deleteAllAlerts(supertest); + }); + + it('should ignore the field of "testing_ignored"', async () => { + const rule = getEqlRuleForSignalTesting(['ignore_fields']); + + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 4, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits + .map((hit) => (hit._source as Ignore).testing_ignored) + .sort(); + + // Value should be "undefined for all records" + expect(hits).to.eql([undefined, undefined, undefined, undefined]); + }); + + it('should ignore the field of "testing_regex"', async () => { + const rule = getEqlRuleForSignalTesting(['ignore_fields']); + + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 4, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => (hit._source as Ignore).testing_regex).sort(); + + // Value should be "undefined for all records" + expect(hits).to.eql([undefined, undefined, undefined, undefined]); + }); + + it('should have the field of "normal_constant"', async () => { + const rule = getEqlRuleForSignalTesting(['ignore_fields']); + + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 4, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits + .map((hit) => (hit._source as Ignore).normal_constant) + .sort(); + + // Value should be "constant_value for all records" + expect(hits).to.eql(['constant_value', 'constant_value', 'constant_value', 'constant_value']); + }); + + // TODO: Remove this test once https://github.com/elastic/elasticsearch/issues/77152 is fixed + it('should ignore the field of "_ignored" when using EQL and index the data', async () => { + const rule = getEqlRuleForSignalTesting(['ignore_fields']); + + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 4, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => (hit._source as Ignore).small_field).sort(); + + // We just test a constant value to ensure this did not blow up on us and did index data. + expect(hits).to.eql([ + '1 indexed', + '2 large not indexed', + '3 large not indexed', + '4 large not indexed', + ]); + }); + }); +}; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts index 27474fe563a36..41a3d084e084e 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts @@ -48,6 +48,7 @@ export default ({ loadTestFile }: FtrProviderContext): void => { loadTestFile(require.resolve('./timestamps')); loadTestFile(require.resolve('./runtime')); loadTestFile(require.resolve('./throttle')); + loadTestFile(require.resolve('./ignore_fields')); }); // That split here enable us on using a different ciGroup to run the tests diff --git a/x-pack/test/functional/es_archives/security_solution/ignore_fields/data.json b/x-pack/test/functional/es_archives/security_solution/ignore_fields/data.json new file mode 100644 index 0000000000000..7a33785c0bc41 --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/ignore_fields/data.json @@ -0,0 +1,51 @@ +{ + "type": "doc", + "value": { + "id": "1", + "index": "ignore_fields", + "source": { + "@timestamp": "2020-10-28T05:00:53.000Z", + "small_field": "1 indexed" + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "2", + "index": "ignore_fields", + "source": { + "@timestamp": "2020-10-28T05:01:53.000Z", + "small_field": "2 large not indexed" + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "3", + "index": "ignore_fields", + "source": { + "@timestamp": "2020-10-28T05:02:53.000Z", + "small_field": "3 large not indexed" + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "4", + "index": "ignore_fields", + "source": { + "@timestamp": "2020-10-28T05:03:53.000Z", + "small_field": "4 large not indexed" + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/security_solution/ignore_fields/mappings.json b/x-pack/test/functional/es_archives/security_solution/ignore_fields/mappings.json new file mode 100644 index 0000000000000..e2c8ca3c2bc89 --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/ignore_fields/mappings.json @@ -0,0 +1,41 @@ +{ + "type": "index", + "value": { + "index": "ignore_fields", + "mappings": { + "dynamic": "strict", + "properties": { + "@timestamp": { + "type": "date" + }, + "testing_ignored": { + "properties": { + "constant": { + "type": "constant_keyword", + "value": "constant_value" + } + } + }, + "testing_regex": { + "type": "constant_keyword", + "value": "constant_value" + }, + "normal_constant": { + "type": "constant_keyword", + "value": "constant_value" + }, + "small_field": { + "type": "keyword", + "ignore_above": 10 + } + } + }, + "settings": { + "index": { + "refresh_interval": "1s", + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +} From d4c03eb9b4776a90d5b2c48aec55ef27084a3e79 Mon Sep 17 00:00:00 2001 From: Marta Bondyra Date: Fri, 3 Sep 2021 16:44:12 +0200 Subject: [PATCH 18/20] [Lens] Switch to SavedObjectClient.resolve (#110059) * Step 2: Update client code to use resolve() method instead of get() Following sharing Saved Objects developer guide: Step 2 This step demonstrates the changes to update client code to use the new SavedObjectsClient `resolve()` method instead of `get()`. * Step 3 Lens --- x-pack/plugins/lens/kibana.json | 3 +- .../lens/public/app_plugin/app.test.tsx | 31 ++++ x-pack/plugins/lens/public/app_plugin/app.tsx | 28 ++++ .../lens/public/app_plugin/mounter.tsx | 6 +- .../plugins/lens/public/app_plugin/types.ts | 3 + .../workspace_panel/workspace_panel.tsx | 10 +- .../lens/public/editor_frame_service/types.ts | 2 +- .../public/embeddable/embeddable.test.tsx | 56 ++++++- .../lens/public/embeddable/embeddable.tsx | 27 +++- .../public/embeddable/expression_wrapper.tsx | 64 +++++++- .../lens/public/lens_attribute_service.ts | 51 +++++-- x-pack/plugins/lens/public/mocks.tsx | 13 +- .../persistence/saved_object_store.test.ts | 21 +-- .../public/persistence/saved_object_store.ts | 22 +-- x-pack/plugins/lens/public/plugin.ts | 2 + .../state_management/init_middleware/index.ts | 8 +- .../init_middleware/load_initial.test.tsx | 143 ++++++++++++++---- .../init_middleware/load_initial.ts | 63 ++++++-- .../public/state_management/lens_slice.ts | 2 + .../lens/public/state_management/types.ts | 4 +- x-pack/plugins/lens/public/types.ts | 10 +- .../public/xy_visualization/visualization.tsx | 4 +- x-pack/plugins/lens/tsconfig.json | 1 + 23 files changed, 466 insertions(+), 108 deletions(-) diff --git a/x-pack/plugins/lens/kibana.json b/x-pack/plugins/lens/kibana.json index 2ec7a1962da82..f82f3366448da 100644 --- a/x-pack/plugins/lens/kibana.json +++ b/x-pack/plugins/lens/kibana.json @@ -24,7 +24,8 @@ "usageCollection", "taskManager", "globalSearch", - "savedObjectsTagging" + "savedObjectsTagging", + "spaces" ], "configPath": [ "xpack", diff --git a/x-pack/plugins/lens/public/app_plugin/app.test.tsx b/x-pack/plugins/lens/public/app_plugin/app.test.tsx index 8cb4a7c4c8433..5617b5b0edeea 100644 --- a/x-pack/plugins/lens/public/app_plugin/app.test.tsx +++ b/x-pack/plugins/lens/public/app_plugin/app.test.tsx @@ -383,6 +383,9 @@ describe('Lens App', () => { savedObjectId: savedObjectId || 'aaa', })); services.attributeService.unwrapAttributes = jest.fn().mockResolvedValue({ + sharingSavedObjectProps: { + outcome: 'exactMatch', + }, savedObjectId: initialSavedObjectId ?? 'aaa', references: [], state: { @@ -1256,4 +1259,32 @@ describe('Lens App', () => { expect(defaultLeave).not.toHaveBeenCalled(); }); }); + it('should display a conflict callout if saved object conflicts', async () => { + const history = createMemoryHistory(); + const { services } = await mountWith({ + props: { + ...makeDefaultProps(), + history: { + ...history, + location: { + ...history.location, + search: '?_g=test', + }, + }, + }, + preloadedState: { + persistedDoc: defaultDoc, + sharingSavedObjectProps: { + outcome: 'conflict', + aliasTargetId: '2', + }, + }, + }); + expect(services.spaces.ui.components.getLegacyUrlConflict).toHaveBeenCalledWith({ + currentObjectId: '1234', + objectNoun: 'Lens visualization', + otherObjectId: '2', + otherObjectPath: '#/edit/2?_g=test', + }); + }); }); diff --git a/x-pack/plugins/lens/public/app_plugin/app.tsx b/x-pack/plugins/lens/public/app_plugin/app.tsx index 63cb7d3002542..ae2edaa1b98d3 100644 --- a/x-pack/plugins/lens/public/app_plugin/app.tsx +++ b/x-pack/plugins/lens/public/app_plugin/app.tsx @@ -38,6 +38,7 @@ import { runSaveLensVisualization, } from './save_modal_container'; import { getLensInspectorService, LensInspector } from '../lens_inspector_service'; +import { getEditPath } from '../../common'; export type SaveProps = Omit & { returnToOrigin: boolean; @@ -70,6 +71,8 @@ export function App({ notifications, savedObjectsTagging, getOriginatingAppName, + spaces, + http, // Temporarily required until the 'by value' paradigm is default. dashboardFeatureFlag, } = lensAppServices; @@ -82,6 +85,7 @@ export function App({ const { persistedDoc, + sharingSavedObjectProps, isLinkedToOriginatingApp, searchSessionId, isLoading, @@ -166,6 +170,28 @@ export function App({ }); }, [onAppLeave, lastKnownDoc, isSaveable, persistedDoc, application.capabilities.visualize.save]); + const getLegacyUrlConflictCallout = useCallback(() => { + // This function returns a callout component *if* we have encountered a "legacy URL conflict" scenario + if (spaces && sharingSavedObjectProps?.outcome === 'conflict' && persistedDoc?.savedObjectId) { + // We have resolved to one object, but another object has a legacy URL alias associated with this ID/page. We should display a + // callout with a warning for the user, and provide a way for them to navigate to the other object. + const currentObjectId = persistedDoc.savedObjectId; + const otherObjectId = sharingSavedObjectProps?.aliasTargetId!; // This is always defined if outcome === 'conflict' + const otherObjectPath = http.basePath.prepend( + `${getEditPath(otherObjectId)}${history.location.search}` + ); + return spaces.ui.components.getLegacyUrlConflict({ + objectNoun: i18n.translate('xpack.lens.appName', { + defaultMessage: 'Lens visualization', + }), + currentObjectId, + otherObjectId, + otherObjectPath, + }); + } + return null; + }, [persistedDoc, sharingSavedObjectProps, spaces, http, history]); + // Sync Kibana breadcrumbs any time the saved document's title changes useEffect(() => { const isByValueMode = getIsByValueMode(); @@ -273,6 +299,8 @@ export function App({ title={persistedDoc?.title} lensInspector={lensInspector} /> + + {getLegacyUrlConflictCallout()} {(!isLoading || persistedDoc) && ( diff --git a/x-pack/plugins/lens/public/app_plugin/types.ts b/x-pack/plugins/lens/public/app_plugin/types.ts index 4ccf441799b1c..8a3a848ffa204 100644 --- a/x-pack/plugins/lens/public/app_plugin/types.ts +++ b/x-pack/plugins/lens/public/app_plugin/types.ts @@ -7,6 +7,7 @@ import type { History } from 'history'; import type { OnSaveProps } from 'src/plugins/saved_objects/public'; +import { SpacesApi } from '../../../spaces/public'; import type { ApplicationStart, AppMountParameters, @@ -116,6 +117,8 @@ export interface LensAppServices { savedObjectsTagging?: SavedObjectTaggingPluginStart; getOriginatingAppName: () => string | undefined; presentationUtil: PresentationUtilPluginStart; + spaces: SpacesApi; + // Temporarily required until the 'by value' paradigm is default. dashboardFeatureFlag: DashboardFeatureFlagConfig; } diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx index 31705d6b92933..c34e3c4137368 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx @@ -79,7 +79,7 @@ export interface WorkspacePanelProps { interface WorkspaceState { expressionBuildError?: Array<{ shortMessage: string; - longMessage: string; + longMessage: React.ReactNode; fixAction?: DatasourceFixAction; }>; expandError: boolean; @@ -416,10 +416,10 @@ export const VisualizationWrapper = ({ localState: WorkspaceState & { configurationValidationError?: Array<{ shortMessage: string; - longMessage: string; + longMessage: React.ReactNode; fixAction?: DatasourceFixAction; }>; - missingRefsErrors?: Array<{ shortMessage: string; longMessage: string }>; + missingRefsErrors?: Array<{ shortMessage: string; longMessage: React.ReactNode }>; }; ExpressionRendererComponent: ReactExpressionRendererType; application: ApplicationStart; @@ -454,7 +454,7 @@ export const VisualizationWrapper = ({ validationError: | { shortMessage: string; - longMessage: string; + longMessage: React.ReactNode; fixAction?: DatasourceFixAction; } | undefined @@ -499,7 +499,7 @@ export const VisualizationWrapper = ({ .map((validationError) => ( <>

diff --git a/x-pack/plugins/lens/public/editor_frame_service/types.ts b/x-pack/plugins/lens/public/editor_frame_service/types.ts index ebfd098b5fb19..9435faf374420 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/types.ts +++ b/x-pack/plugins/lens/public/editor_frame_service/types.ts @@ -11,6 +11,6 @@ export type TableInspectorAdapter = Record; export interface ErrorMessage { shortMessage: string; - longMessage: string; + longMessage: React.ReactNode; type?: 'fixable' | 'critical'; } diff --git a/x-pack/plugins/lens/public/embeddable/embeddable.test.tsx b/x-pack/plugins/lens/public/embeddable/embeddable.test.tsx index 74aac932a6861..a0831e8a73b57 100644 --- a/x-pack/plugins/lens/public/embeddable/embeddable.test.tsx +++ b/x-pack/plugins/lens/public/embeddable/embeddable.test.tsx @@ -11,6 +11,7 @@ import { LensByReferenceInput, LensSavedObjectAttributes, LensEmbeddableInput, + ResolvedLensSavedObjectAttributes, } from './embeddable'; import { ReactExpressionRendererProps } from 'src/plugins/expressions/public'; import { Query, TimeRange, Filter, IndexPatternsContract } from 'src/plugins/data/public'; @@ -68,12 +69,17 @@ const options = { const attributeServiceMockFromSavedVis = (document: Document): LensAttributeService => { const core = coreMock.createStart(); const service = new AttributeService< - LensSavedObjectAttributes, + ResolvedLensSavedObjectAttributes, LensByValueInput, LensByReferenceInput >('lens', jest.fn(), core.i18n.Context, core.notifications.toasts, options); service.unwrapAttributes = jest.fn((input: LensByValueInput | LensByReferenceInput) => { - return Promise.resolve({ ...document } as LensSavedObjectAttributes); + return Promise.resolve({ + ...document, + sharingSavedObjectProps: { + outcome: 'exactMatch', + }, + } as ResolvedLensSavedObjectAttributes); }); service.wrapAttributes = jest.fn(); return service; @@ -86,7 +92,7 @@ describe('embeddable', () => { let trigger: { exec: jest.Mock }; let basePath: IBasePath; let attributeService: AttributeService< - LensSavedObjectAttributes, + ResolvedLensSavedObjectAttributes, LensByValueInput, LensByReferenceInput >; @@ -223,6 +229,50 @@ describe('embeddable', () => { expect(expressionRenderer).toHaveBeenCalledTimes(0); }); + it('should not render the vis if loaded saved object conflicts', async () => { + attributeService.unwrapAttributes = jest.fn( + (input: LensByValueInput | LensByReferenceInput) => { + return Promise.resolve({ + ...savedVis, + sharingSavedObjectProps: { + outcome: 'conflict', + errorJSON: '{targetType: "lens", sourceId: "1", targetSpace: "space"}', + aliasTargetId: '2', + }, + } as ResolvedLensSavedObjectAttributes); + } + ); + const embeddable = new Embeddable( + { + timefilter: dataPluginMock.createSetupContract().query.timefilter.timefilter, + attributeService, + inspector: inspectorPluginMock.createStartContract(), + expressionRenderer, + basePath, + indexPatternService: {} as IndexPatternsContract, + capabilities: { + canSaveDashboards: true, + canSaveVisualizations: true, + }, + getTrigger, + documentToExpression: () => + Promise.resolve({ + ast: { + type: 'expression', + chain: [ + { type: 'function', function: 'my', arguments: {} }, + { type: 'function', function: 'expression', arguments: {} }, + ], + }, + errors: undefined, + }), + }, + {} as LensEmbeddableInput + ); + await embeddable.initializeSavedVis({} as LensEmbeddableInput); + expect(expressionRenderer).toHaveBeenCalledTimes(0); + }); + it('should initialize output with deduped list of index patterns', async () => { attributeService = attributeServiceMockFromSavedVis({ ...savedVis, diff --git a/x-pack/plugins/lens/public/embeddable/embeddable.tsx b/x-pack/plugins/lens/public/embeddable/embeddable.tsx index 172274b1f90bc..7e87dd3076faa 100644 --- a/x-pack/plugins/lens/public/embeddable/embeddable.tsx +++ b/x-pack/plugins/lens/public/embeddable/embeddable.tsx @@ -41,7 +41,11 @@ import { ReferenceOrValueEmbeddable, } from '../../../../../src/plugins/embeddable/public'; import { Document, injectFilterReferences } from '../persistence'; -import { ExpressionWrapper, ExpressionWrapperProps } from './expression_wrapper'; +import { + ExpressionWrapper, + ExpressionWrapperProps, + savedObjectConflictError, +} from './expression_wrapper'; import { UiActionsStart } from '../../../../../src/plugins/ui_actions/public'; import { isLensBrushEvent, @@ -58,8 +62,12 @@ import { IBasePath } from '../../../../../src/core/public'; import { LensAttributeService } from '../lens_attribute_service'; import type { ErrorMessage } from '../editor_frame_service/types'; import { getLensInspectorService, LensInspector } from '../lens_inspector_service'; +import { SharingSavedObjectProps } from '../types'; export type LensSavedObjectAttributes = Omit; +export interface ResolvedLensSavedObjectAttributes extends LensSavedObjectAttributes { + sharingSavedObjectProps?: SharingSavedObjectProps; +} interface LensBaseEmbeddableInput extends EmbeddableInput { filters?: Filter[]; @@ -76,7 +84,7 @@ interface LensBaseEmbeddableInput extends EmbeddableInput { } export type LensByValueInput = { - attributes: LensSavedObjectAttributes; + attributes: ResolvedLensSavedObjectAttributes; } & LensBaseEmbeddableInput; export type LensByReferenceInput = SavedObjectEmbeddableInput & LensBaseEmbeddableInput; @@ -253,15 +261,18 @@ export class Embeddable } async initializeSavedVis(input: LensEmbeddableInput) { - const attributes: - | LensSavedObjectAttributes + const attrs: + | ResolvedLensSavedObjectAttributes | false = await this.deps.attributeService.unwrapAttributes(input).catch((e: Error) => { this.onFatalError(e); return false; }); - if (!attributes || this.isDestroyed) { + if (!attrs || this.isDestroyed) { return; } + + const { sharingSavedObjectProps, ...attributes } = attrs; + this.savedVis = { ...attributes, type: this.type, @@ -269,8 +280,12 @@ export class Embeddable }; const { ast, errors } = await this.deps.documentToExpression(this.savedVis); this.errors = errors; + if (sharingSavedObjectProps?.outcome === 'conflict') { + const conflictError = savedObjectConflictError(sharingSavedObjectProps.errorJSON!); + this.errors = this.errors ? [...this.errors, conflictError] : [conflictError]; + } this.expression = ast ? toExpression(ast) : null; - if (errors) { + if (this.errors) { this.logError('validation'); } await this.initializeOutput(); diff --git a/x-pack/plugins/lens/public/embeddable/expression_wrapper.tsx b/x-pack/plugins/lens/public/embeddable/expression_wrapper.tsx index d57e1c450fea2..1116b4a0d3963 100644 --- a/x-pack/plugins/lens/public/embeddable/expression_wrapper.tsx +++ b/x-pack/plugins/lens/public/embeddable/expression_wrapper.tsx @@ -5,10 +5,20 @@ * 2.0. */ -import React from 'react'; +import React, { useState } from 'react'; import { I18nProvider } from '@kbn/i18n/react'; import { FormattedMessage } from '@kbn/i18n/react'; -import { EuiFlexGroup, EuiFlexItem, EuiText, EuiIcon, EuiEmptyPrompt } from '@elastic/eui'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiText, + EuiIcon, + EuiEmptyPrompt, + EuiButtonEmpty, + EuiCallOut, + EuiSpacer, + EuiLink, +} from '@elastic/eui'; import { ExpressionRendererEvent, ReactExpressionRendererType, @@ -18,6 +28,7 @@ import type { KibanaExecutionContext } from 'src/core/public'; import { ExecutionContextSearch } from 'src/plugins/data/public'; import { DefaultInspectorAdapters, RenderMode } from 'src/plugins/expressions'; import classNames from 'classnames'; +import { i18n } from '@kbn/i18n'; import { getOriginalRequestErrorMessages } from '../editor_frame_service/error_helper'; import { ErrorMessage } from '../editor_frame_service/types'; import { LensInspector } from '../lens_inspector_service'; @@ -158,3 +169,52 @@ export function ExpressionWrapper({ ); } + +const SavedObjectConflictMessage = ({ json }: { json: string }) => { + const [expandError, setExpandError] = useState(false); + return ( + <> + + {i18n.translate('xpack.lens.embeddable.legacyURLConflict.documentationLinkText', { + defaultMessage: 'legacy URL alias', + })} + + ), + }} + /> + + {expandError ? ( + + ) : ( + setExpandError(true)}> + {i18n.translate('xpack.lens.embeddable.legacyURLConflict.expandError', { + defaultMessage: `Show more`, + })} + + )} + + ); +}; + +export const savedObjectConflictError = (json: string): ErrorMessage => ({ + shortMessage: i18n.translate('xpack.lens.embeddable.legacyURLConflict.shortMessage', { + defaultMessage: `You've encountered a URL conflict`, + }), + longMessage: , +}); diff --git a/x-pack/plugins/lens/public/lens_attribute_service.ts b/x-pack/plugins/lens/public/lens_attribute_service.ts index 39a1903c6d0c4..09c98b3dcba72 100644 --- a/x-pack/plugins/lens/public/lens_attribute_service.ts +++ b/x-pack/plugins/lens/public/lens_attribute_service.ts @@ -9,47 +9,68 @@ import { CoreStart } from '../../../../src/core/public'; import { LensPluginStartDependencies } from './plugin'; import { AttributeService } from '../../../../src/plugins/embeddable/public'; import { - LensSavedObjectAttributes, + ResolvedLensSavedObjectAttributes, LensByValueInput, LensByReferenceInput, } from './embeddable/embeddable'; -import { SavedObjectIndexStore, Document } from './persistence'; +import { SavedObjectIndexStore } from './persistence'; import { checkForDuplicateTitle, OnSaveProps } from '../../../../src/plugins/saved_objects/public'; import { DOC_TYPE } from '../common'; export type LensAttributeService = AttributeService< - LensSavedObjectAttributes, + ResolvedLensSavedObjectAttributes, LensByValueInput, LensByReferenceInput >; -function documentToAttributes(doc: Document): LensSavedObjectAttributes { - delete doc.savedObjectId; - delete doc.type; - return { ...doc }; -} - export function getLensAttributeService( core: CoreStart, startDependencies: LensPluginStartDependencies ): LensAttributeService { const savedObjectStore = new SavedObjectIndexStore(core.savedObjects.client); return startDependencies.embeddable.getAttributeService< - LensSavedObjectAttributes, + ResolvedLensSavedObjectAttributes, LensByValueInput, LensByReferenceInput >(DOC_TYPE, { - saveMethod: async (attributes: LensSavedObjectAttributes, savedObjectId?: string) => { + saveMethod: async (attributes: ResolvedLensSavedObjectAttributes, savedObjectId?: string) => { + const { sharingSavedObjectProps, ...attributesToSave } = attributes; const savedDoc = await savedObjectStore.save({ - ...attributes, + ...attributesToSave, savedObjectId, type: DOC_TYPE, }); return { id: savedDoc.savedObjectId }; }, - unwrapMethod: async (savedObjectId: string): Promise => { - const attributes = documentToAttributes(await savedObjectStore.load(savedObjectId)); - return attributes; + unwrapMethod: async (savedObjectId: string): Promise => { + const { + saved_object: savedObject, + outcome, + alias_target_id: aliasTargetId, + } = await savedObjectStore.load(savedObjectId); + const { attributes, references, type, id } = savedObject; + const document = { + ...attributes, + references, + }; + + const sharingSavedObjectProps = { + aliasTargetId, + outcome, + errorJSON: + outcome === 'conflict' + ? JSON.stringify({ + targetType: type, + sourceId: id, + targetSpace: (await startDependencies.spaces.getActiveSpace()).id, + }) + : undefined, + }; + + return { + sharingSavedObjectProps, + ...document, + }; }, checkForDuplicateTitle: (props: OnSaveProps) => { const savedObjectsClient = core.savedObjects.client; diff --git a/x-pack/plugins/lens/public/mocks.tsx b/x-pack/plugins/lens/public/mocks.tsx index b2c8d3948b285..8fbd263fe909e 100644 --- a/x-pack/plugins/lens/public/mocks.tsx +++ b/x-pack/plugins/lens/public/mocks.tsx @@ -24,11 +24,12 @@ import { LensAppServices } from './app_plugin/types'; import { DOC_TYPE, layerTypes } from '../common'; import { DataPublicPluginStart, esFilters, UI_SETTINGS } from '../../../../src/plugins/data/public'; import { inspectorPluginMock } from '../../../../src/plugins/inspector/public/mocks'; +import { spacesPluginMock } from '../../spaces/public/mocks'; import { dashboardPluginMock } from '../../../../src/plugins/dashboard/public/mocks'; import type { LensByValueInput, - LensSavedObjectAttributes, LensByReferenceInput, + ResolvedLensSavedObjectAttributes, } from './embeddable/embeddable'; import { mockAttributeService, @@ -352,7 +353,7 @@ export function makeDefaultServices( function makeAttributeService(): LensAttributeService { const attributeServiceMock = mockAttributeService< - LensSavedObjectAttributes, + ResolvedLensSavedObjectAttributes, LensByValueInput, LensByReferenceInput >( @@ -365,7 +366,12 @@ export function makeDefaultServices( core ); - attributeServiceMock.unwrapAttributes = jest.fn().mockResolvedValue(doc); + attributeServiceMock.unwrapAttributes = jest.fn().mockResolvedValue({ + ...doc, + sharingSavedObjectProps: { + outcome: 'exactMatch', + }, + }); attributeServiceMock.wrapAttributes = jest.fn().mockResolvedValue({ savedObjectId: ((doc as unknown) as LensByReferenceInput).savedObjectId, }); @@ -404,6 +410,7 @@ export function makeDefaultServices( remove: jest.fn(), clear: jest.fn(), }, + spaces: spacesPluginMock.createStartContract(), }; } diff --git a/x-pack/plugins/lens/public/persistence/saved_object_store.test.ts b/x-pack/plugins/lens/public/persistence/saved_object_store.test.ts index ab0708d99f082..5a42ea054b4d9 100644 --- a/x-pack/plugins/lens/public/persistence/saved_object_store.test.ts +++ b/x-pack/plugins/lens/public/persistence/saved_object_store.test.ts @@ -15,7 +15,7 @@ describe('LensStore', () => { bulkUpdate: jest.fn(([{ id }]: SavedObjectsBulkUpdateObject[]) => Promise.resolve({ savedObjects: [{ id }, { id }] }) ), - get: jest.fn(), + resolve: jest.fn(), }; return { @@ -142,15 +142,18 @@ describe('LensStore', () => { describe('load', () => { test('throws if an error is returned', async () => { const { client, store } = testStore(); - client.get = jest.fn(async () => ({ - id: 'Paul', - type: 'lens', - attributes: { - title: 'Hope clouds observation.', - visualizationType: 'dune', - state: '{ "datasource": { "giantWorms": true } }', + client.resolve = jest.fn(async () => ({ + outcome: 'exactMatch', + saved_object: { + id: 'Paul', + type: 'lens', + attributes: { + title: 'Hope clouds observation.', + visualizationType: 'dune', + state: '{ "datasource": { "giantWorms": true } }', + }, + error: new Error('shoot dang!'), }, - error: new Error('shoot dang!'), })); await expect(store.load('Paul')).rejects.toThrow('shoot dang!'); diff --git a/x-pack/plugins/lens/public/persistence/saved_object_store.ts b/x-pack/plugins/lens/public/persistence/saved_object_store.ts index c87548daf53dc..79d7b78f768ae 100644 --- a/x-pack/plugins/lens/public/persistence/saved_object_store.ts +++ b/x-pack/plugins/lens/public/persistence/saved_object_store.ts @@ -9,9 +9,11 @@ import { SavedObjectAttributes, SavedObjectsClientContract, SavedObjectReference, + ResolvedSimpleSavedObject, } from 'kibana/public'; import { Query } from '../../../../../src/plugins/data/public'; import { DOC_TYPE, PersistableFilter } from '../../common'; +import { LensSavedObjectAttributes } from '../async_services'; export interface Document { savedObjectId?: string; @@ -37,7 +39,7 @@ export interface DocumentSaver { } export interface DocumentLoader { - load: (savedObjectId: string) => Promise; + load: (savedObjectId: string) => Promise; } export type SavedObjectStore = DocumentLoader & DocumentSaver; @@ -87,18 +89,16 @@ export class SavedObjectIndexStore implements SavedObjectStore { ).savedObjects[1]; } - async load(savedObjectId: string): Promise { - const { type, attributes, references, error } = await this.client.get(DOC_TYPE, savedObjectId); + async load(savedObjectId: string): Promise> { + const resolveResult = await this.client.resolve( + DOC_TYPE, + savedObjectId + ); - if (error) { - throw error; + if (resolveResult.saved_object.error) { + throw resolveResult.saved_object.error; } - return { - ...(attributes as SavedObjectAttributes), - references, - savedObjectId, - type, - } as Document; + return resolveResult; } } diff --git a/x-pack/plugins/lens/public/plugin.ts b/x-pack/plugins/lens/public/plugin.ts index 95f2e13cbc464..26278f446c558 100644 --- a/x-pack/plugins/lens/public/plugin.ts +++ b/x-pack/plugins/lens/public/plugin.ts @@ -9,6 +9,7 @@ import { AppMountParameters, CoreSetup, CoreStart } from 'kibana/public'; import type { Start as InspectorStartContract } from 'src/plugins/inspector/public'; import type { FieldFormatsSetup, FieldFormatsStart } from 'src/plugins/field_formats/public'; import { UsageCollectionSetup, UsageCollectionStart } from 'src/plugins/usage_collection/public'; +import { SpacesPluginStart } from '../../spaces/public'; import { DataPublicPluginSetup, DataPublicPluginStart } from '../../../../src/plugins/data/public'; import { EmbeddableSetup, EmbeddableStart } from '../../../../src/plugins/embeddable/public'; import { DashboardStart } from '../../../../src/plugins/dashboard/public'; @@ -100,6 +101,7 @@ export interface LensPluginStartDependencies { presentationUtil: PresentationUtilPluginStart; indexPatternFieldEditor: IndexPatternFieldEditorStart; inspector: InspectorStartContract; + spaces: SpacesPluginStart; usageCollection?: UsageCollectionStart; } diff --git a/x-pack/plugins/lens/public/state_management/init_middleware/index.ts b/x-pack/plugins/lens/public/state_management/init_middleware/index.ts index bf13ca69e82c0..256684c5dbc25 100644 --- a/x-pack/plugins/lens/public/state_management/init_middleware/index.ts +++ b/x-pack/plugins/lens/public/state_management/init_middleware/index.ts @@ -19,13 +19,7 @@ export const initMiddleware = (storeDeps: LensStoreDeps) => (store: MiddlewareAP ); return (next: Dispatch) => (action: PayloadAction) => { if (lensSlice.actions.loadInitial.match(action)) { - return loadInitial( - store, - storeDeps, - action.payload.redirectCallback, - action.payload.initialInput, - action.payload.emptyState - ); + return loadInitial(store, storeDeps, action.payload); } else if (lensSlice.actions.navigateAway.match(action)) { return unsubscribeFromExternalContext(); } diff --git a/x-pack/plugins/lens/public/state_management/init_middleware/load_initial.test.tsx b/x-pack/plugins/lens/public/state_management/init_middleware/load_initial.test.tsx index 79402b698af98..6d3b77c6476e5 100644 --- a/x-pack/plugins/lens/public/state_management/init_middleware/load_initial.test.tsx +++ b/x-pack/plugins/lens/public/state_management/init_middleware/load_initial.test.tsx @@ -12,6 +12,7 @@ import { createMockDatasource, DatasourceMock, } from '../../mocks'; +import { Location, History } from 'history'; import { act } from 'react-dom/test-utils'; import { loadInitial } from './load_initial'; import { LensEmbeddableInput } from '../../embeddable'; @@ -65,7 +66,12 @@ describe('Mounter', () => { it('should initialize initial datasource', async () => { const services = makeDefaultServices(); - services.attributeService.unwrapAttributes = jest.fn().mockResolvedValue(defaultDoc); + services.attributeService.unwrapAttributes = jest.fn().mockResolvedValue({ + ...defaultDoc, + sharingSavedObjectProps: { + outcome: 'exactMatch', + }, + }); const lensStore = await makeLensStore({ data: services.data, @@ -79,8 +85,10 @@ describe('Mounter', () => { datasourceMap, visualizationMap, }, - jest.fn(), - ({ savedObjectId: defaultSavedObjectId } as unknown) as LensEmbeddableInput + { + redirectCallback: jest.fn(), + initialInput: ({ savedObjectId: defaultSavedObjectId } as unknown) as LensEmbeddableInput, + } ); }); expect(mockDatasource.initialize).toHaveBeenCalled(); @@ -88,7 +96,12 @@ describe('Mounter', () => { it('should have initialized only the initial datasource and visualization', async () => { const services = makeDefaultServices(); - services.attributeService.unwrapAttributes = jest.fn().mockResolvedValue(defaultDoc); + services.attributeService.unwrapAttributes = jest.fn().mockResolvedValue({ + ...defaultDoc, + sharingSavedObjectProps: { + outcome: 'exactMatch', + }, + }); const lensStore = await makeLensStore({ data: services.data, preloadedState }); await act(async () => { @@ -99,7 +112,7 @@ describe('Mounter', () => { datasourceMap, visualizationMap, }, - jest.fn() + { redirectCallback: jest.fn() } ); }); expect(mockDatasource.initialize).toHaveBeenCalled(); @@ -129,7 +142,7 @@ describe('Mounter', () => { datasourceMap, visualizationMap, }, - jest.fn() + { redirectCallback: jest.fn() } ); expect(services.attributeService.unwrapAttributes).not.toHaveBeenCalled(); }); @@ -170,7 +183,11 @@ describe('Mounter', () => { const emptyState = getPreloadedState(storeDeps) as LensAppState; services.attributeService.unwrapAttributes = jest.fn(); await act(async () => { - await loadInitial(lensStore, storeDeps, jest.fn(), undefined, emptyState); + await loadInitial(lensStore, storeDeps, { + redirectCallback: jest.fn(), + initialInput: undefined, + emptyState, + }); }); expect(lensStore.getState()).toEqual({ @@ -189,20 +206,28 @@ describe('Mounter', () => { it('loads a document and uses query and filters if initial input is provided', async () => { const services = makeDefaultServices(); - services.attributeService.unwrapAttributes = jest.fn().mockResolvedValue(defaultDoc); + services.attributeService.unwrapAttributes = jest.fn().mockResolvedValue({ + ...defaultDoc, + sharingSavedObjectProps: { + outcome: 'exactMatch', + }, + }); + const storeDeps = { + lensServices: services, + datasourceMap, + visualizationMap, + }; + const emptyState = getPreloadedState(storeDeps) as LensAppState; const lensStore = await makeLensStore({ data: services.data, preloadedState }); await act(async () => { - await loadInitial( - lensStore, - { - lensServices: services, - datasourceMap, - visualizationMap, - }, - jest.fn(), - ({ savedObjectId: defaultSavedObjectId } as unknown) as LensEmbeddableInput - ); + await loadInitial(lensStore, storeDeps, { + redirectCallback: jest.fn(), + initialInput: ({ + savedObjectId: defaultSavedObjectId, + } as unknown) as LensEmbeddableInput, + emptyState, + }); }); expect(services.attributeService.unwrapAttributes).toHaveBeenCalledWith({ @@ -235,8 +260,12 @@ describe('Mounter', () => { datasourceMap, visualizationMap, }, - jest.fn(), - ({ savedObjectId: defaultSavedObjectId } as unknown) as LensEmbeddableInput + { + redirectCallback: jest.fn(), + initialInput: ({ + savedObjectId: defaultSavedObjectId, + } as unknown) as LensEmbeddableInput, + } ); }); @@ -248,8 +277,12 @@ describe('Mounter', () => { datasourceMap, visualizationMap, }, - jest.fn(), - ({ savedObjectId: defaultSavedObjectId } as unknown) as LensEmbeddableInput + { + redirectCallback: jest.fn(), + initialInput: ({ + savedObjectId: defaultSavedObjectId, + } as unknown) as LensEmbeddableInput, + } ); }); @@ -263,8 +296,10 @@ describe('Mounter', () => { datasourceMap, visualizationMap, }, - jest.fn(), - ({ savedObjectId: '5678' } as unknown) as LensEmbeddableInput + { + redirectCallback: jest.fn(), + initialInput: ({ savedObjectId: '5678' } as unknown) as LensEmbeddableInput, + } ); }); @@ -287,8 +322,12 @@ describe('Mounter', () => { datasourceMap, visualizationMap, }, - redirectCallback, - ({ savedObjectId: defaultSavedObjectId } as unknown) as LensEmbeddableInput + { + redirectCallback, + initialInput: ({ + savedObjectId: defaultSavedObjectId, + } as unknown) as LensEmbeddableInput, + } ); }); expect(services.attributeService.unwrapAttributes).toHaveBeenCalledWith({ @@ -298,6 +337,50 @@ describe('Mounter', () => { expect(redirectCallback).toHaveBeenCalled(); }); + it('redirects if saved object is an aliasMatch', async () => { + const services = makeDefaultServices(); + + const lensStore = makeLensStore({ data: services.data, preloadedState }); + + services.attributeService.unwrapAttributes = jest.fn().mockResolvedValue({ + ...defaultDoc, + sharingSavedObjectProps: { + outcome: 'aliasMatch', + aliasTargetId: 'id2', + }, + }); + + await act(async () => { + await loadInitial( + lensStore, + { + lensServices: services, + datasourceMap, + visualizationMap, + }, + { + redirectCallback: jest.fn(), + initialInput: ({ + savedObjectId: defaultSavedObjectId, + } as unknown) as LensEmbeddableInput, + history: { + location: { + search: '?search', + } as Location, + } as History, + } + ); + }); + expect(services.attributeService.unwrapAttributes).toHaveBeenCalledWith({ + savedObjectId: defaultSavedObjectId, + }); + + expect(services.spaces.ui.redirectLegacyUrl).toHaveBeenCalledWith( + '#/edit/id2?search', + 'Lens visualization' + ); + }); + it('adds to the recently accessed list on load', async () => { const services = makeDefaultServices(); const lensStore = makeLensStore({ data: services.data, preloadedState }); @@ -309,8 +392,12 @@ describe('Mounter', () => { datasourceMap, visualizationMap, }, - jest.fn(), - ({ savedObjectId: defaultSavedObjectId } as unknown) as LensEmbeddableInput + { + redirectCallback: jest.fn(), + initialInput: ({ + savedObjectId: defaultSavedObjectId, + } as unknown) as LensEmbeddableInput, + } ); }); diff --git a/x-pack/plugins/lens/public/state_management/init_middleware/load_initial.ts b/x-pack/plugins/lens/public/state_management/init_middleware/load_initial.ts index 0be2bc9cfc00e..8ae6e58019c91 100644 --- a/x-pack/plugins/lens/public/state_management/init_middleware/load_initial.ts +++ b/x-pack/plugins/lens/public/state_management/init_middleware/load_initial.ts @@ -8,8 +8,10 @@ import { MiddlewareAPI } from '@reduxjs/toolkit'; import { isEqual } from 'lodash'; import { i18n } from '@kbn/i18n'; +import { History } from 'history'; import { LensAppState, setState } from '..'; import { updateLayer, updateVisualizationState, LensStoreDeps } from '..'; +import { SharingSavedObjectProps } from '../../types'; import { LensEmbeddableInput, LensByReferenceInput } from '../../embeddable/embeddable'; import { getInitialDatasourceId } from '../../utils'; import { initializeDatasources } from '../../editor_frame_service/editor_frame'; @@ -19,22 +21,50 @@ import { switchToSuggestion, } from '../../editor_frame_service/editor_frame/suggestion_helpers'; import { LensAppServices } from '../../app_plugin/types'; -import { getFullPath, LENS_EMBEDDABLE_TYPE } from '../../../common/constants'; +import { getEditPath, getFullPath, LENS_EMBEDDABLE_TYPE } from '../../../common/constants'; import { Document, injectFilterReferences } from '../../persistence'; export const getPersisted = async ({ initialInput, lensServices, + history, }: { initialInput: LensEmbeddableInput; lensServices: LensAppServices; -}): Promise<{ doc: Document } | undefined> => { - const { notifications, attributeService } = lensServices; + history?: History; +}): Promise< + { doc: Document; sharingSavedObjectProps: Omit } | undefined +> => { + const { notifications, spaces, attributeService } = lensServices; let doc: Document; try { - const attributes = await attributeService.unwrapAttributes(initialInput); - + const result = await attributeService.unwrapAttributes(initialInput); + if (!result) { + return { + doc: ({ + ...initialInput, + type: LENS_EMBEDDABLE_TYPE, + } as unknown) as Document, + sharingSavedObjectProps: { + outcome: 'exactMatch', + }, + }; + } + const { sharingSavedObjectProps, ...attributes } = result; + if (spaces && sharingSavedObjectProps?.outcome === 'aliasMatch' && history) { + // We found this object by a legacy URL alias from its old ID; redirect the user to the page with its new ID, preserving any URL hash + const newObjectId = sharingSavedObjectProps?.aliasTargetId; // This is always defined if outcome === 'aliasMatch' + const newPath = lensServices.http.basePath.prepend( + `${getEditPath(newObjectId)}${history.location.search}` + ); + await spaces.ui.redirectLegacyUrl( + newPath, + i18n.translate('xpack.lens.legacyUrlConflict.objectNoun', { + defaultMessage: 'Lens visualization', + }) + ); + } doc = { ...initialInput, ...attributes, @@ -43,6 +73,10 @@ export const getPersisted = async ({ return { doc, + sharingSavedObjectProps: { + aliasTargetId: sharingSavedObjectProps?.aliasTargetId, + outcome: sharingSavedObjectProps?.outcome, + }, }; } catch (e) { notifications.toasts.addDanger( @@ -62,9 +96,17 @@ export function loadInitial( embeddableEditorIncomingState, initialContext, }: LensStoreDeps, - redirectCallback: (savedObjectId?: string) => void, - initialInput?: LensEmbeddableInput, - emptyState?: LensAppState + { + redirectCallback, + initialInput, + emptyState, + history, + }: { + redirectCallback: (savedObjectId?: string) => void; + initialInput?: LensEmbeddableInput; + emptyState?: LensAppState; + history?: History; + } ) { const { getState, dispatch } = store; const { attributeService, notifications, data, dashboardFeatureFlag } = lensServices; @@ -146,11 +188,11 @@ export function loadInitial( redirectCallback(); }); } - getPersisted({ initialInput, lensServices }) + getPersisted({ initialInput, lensServices, history }) .then( (persisted) => { if (persisted) { - const { doc } = persisted; + const { doc, sharingSavedObjectProps } = persisted; if (attributeService.inputIsRefType(initialInput)) { lensServices.chrome.recentlyAccessed.add( getFullPath(initialInput.savedObjectId), @@ -190,6 +232,7 @@ export function loadInitial( dispatch( setState({ + sharingSavedObjectProps, query: doc.state.query, searchSessionId: dashboardFeatureFlag.allowByValueEmbeddables && diff --git a/x-pack/plugins/lens/public/state_management/lens_slice.ts b/x-pack/plugins/lens/public/state_management/lens_slice.ts index 85cb79f6ea5da..6cf0529b34575 100644 --- a/x-pack/plugins/lens/public/state_management/lens_slice.ts +++ b/x-pack/plugins/lens/public/state_management/lens_slice.ts @@ -6,6 +6,7 @@ */ import { createSlice, current, PayloadAction } from '@reduxjs/toolkit'; +import { History } from 'history'; import { LensEmbeddableInput } from '..'; import { TableInspectorAdapter } from '../editor_frame_service/types'; import { getInitialDatasourceId, getResolvedDateRange } from '../utils'; @@ -301,6 +302,7 @@ export const lensSlice = createSlice({ initialInput?: LensEmbeddableInput; redirectCallback: (savedObjectId?: string) => void; emptyState: LensAppState; + history: History; }> ) => state, }, diff --git a/x-pack/plugins/lens/public/state_management/types.ts b/x-pack/plugins/lens/public/state_management/types.ts index 7321f72386b42..33f311a982f05 100644 --- a/x-pack/plugins/lens/public/state_management/types.ts +++ b/x-pack/plugins/lens/public/state_management/types.ts @@ -13,8 +13,7 @@ import { Document } from '../persistence'; import { TableInspectorAdapter } from '../editor_frame_service/types'; import { DateRange } from '../../common'; import { LensAppServices } from '../app_plugin/types'; -import { DatasourceMap, VisualizationMap } from '../types'; - +import { DatasourceMap, VisualizationMap, SharingSavedObjectProps } from '../types'; export interface VisualizationState { activeId: string | null; state: unknown; @@ -44,6 +43,7 @@ export interface LensAppState extends EditorFrameState { savedQuery?: SavedQuery; searchSessionId: string; resolvedDateRange: DateRange; + sharingSavedObjectProps?: Omit; } export type DispatchSetState = ( diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index 399e226a711db..844541cd2ad3e 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -256,7 +256,7 @@ export interface Datasource { ) => | Array<{ shortMessage: string; - longMessage: string; + longMessage: React.ReactNode; fixAction?: { label: string; newState: () => Promise }; }> | undefined; @@ -729,7 +729,7 @@ export interface Visualization { ) => | Array<{ shortMessage: string; - longMessage: string; + longMessage: React.ReactNode; }> | undefined; @@ -813,3 +813,9 @@ export interface ILensInterpreterRenderHandlers extends IInterpreterRenderHandle | LensTableRowContextMenuEvent ) => void; } + +export interface SharingSavedObjectProps { + outcome?: 'aliasMatch' | 'exactMatch' | 'conflict'; + aliasTargetId?: string; + errorJSON?: string; +} diff --git a/x-pack/plugins/lens/public/xy_visualization/visualization.tsx b/x-pack/plugins/lens/public/xy_visualization/visualization.tsx index 0a4b18f554f31..026c2827cedbd 100644 --- a/x-pack/plugins/lens/public/xy_visualization/visualization.tsx +++ b/x-pack/plugins/lens/public/xy_visualization/visualization.tsx @@ -383,7 +383,7 @@ export const getXyVisualization = ({ const errors: Array<{ shortMessage: string; - longMessage: string; + longMessage: React.ReactNode; }> = []; // check if the layers in the state are compatible with this type of chart @@ -488,7 +488,7 @@ function validateLayersForDimension( | { valid: true } | { valid: false; - payload: { shortMessage: string; longMessage: string }; + payload: { shortMessage: string; longMessage: React.ReactNode }; } { // Multiple layers must be consistent: // * either a dimension is missing in ALL of them diff --git a/x-pack/plugins/lens/tsconfig.json b/x-pack/plugins/lens/tsconfig.json index 04d3838df2063..16287ae596df3 100644 --- a/x-pack/plugins/lens/tsconfig.json +++ b/x-pack/plugins/lens/tsconfig.json @@ -15,6 +15,7 @@ "../../../typings/**/*" ], "references": [ + { "path": "../spaces/tsconfig.json" }, { "path": "../../../src/core/tsconfig.json" }, { "path": "../task_manager/tsconfig.json" }, { "path": "../global_search/tsconfig.json"}, From 1f06cafa19e7a1479adf37eba2dc634360d75ebd Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Fri, 3 Sep 2021 17:08:56 +0200 Subject: [PATCH 19/20] [Reporting/Visualization] Migrate Visualize to V2 reporting (#110206) * added initial version of locator * removed unused params and added jest test * updated functional test to expect PDF reports to be available when vis is new * fix TS: remove unkown field * added some docs and removed unused code * AggsConfigOption -> AggsConfigSerialized * moved locator to common * fixed building of "create" path and updated test snapshots * updated import * update encoding behaviour * added time range from timefilter to locator params request * add index pattern and search id to URL params * reading index pattern from search source if it is there for the locator * remove "type" from locator params, update comments and test * removed duplicate identifier * remove unused type Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../usage_collector/get_usage_collector.ts | 2 +- src/plugins/visualizations/common/types.ts | 10 +- .../saved_visualization_references.test.ts | 8 +- src/plugins/visualizations/public/types.ts | 4 +- src/plugins/visualizations/public/vis.ts | 6 +- src/plugins/visualize/common/constants.ts | 13 ++ src/plugins/visualize/common/locator.test.ts | 133 ++++++++++++++++++ src/plugins/visualize/common/locator.ts | 133 ++++++++++++++++++ src/plugins/visualize/common/types.ts | 10 ++ .../visualize/public/application/types.ts | 10 +- .../utils/create_visualize_app_state.ts | 1 - .../application/utils/get_top_nav_config.tsx | 32 ++++- .../public/application/utils/stubs.ts | 1 - .../utils/use/use_visualize_app_state.test.ts | 2 - .../public/application/visualize_constants.ts | 13 +- src/plugins/visualize/public/plugin.ts | 8 +- src/plugins/visualize/tsconfig.json | 8 +- .../functional/apps/visualize/reporting.ts | 4 +- 18 files changed, 355 insertions(+), 43 deletions(-) create mode 100644 src/plugins/visualize/common/locator.test.ts create mode 100644 src/plugins/visualize/common/locator.ts create mode 100644 src/plugins/visualize/common/types.ts diff --git a/src/plugins/vis_types/vega/server/usage_collector/get_usage_collector.ts b/src/plugins/vis_types/vega/server/usage_collector/get_usage_collector.ts index 67fcb742c4240..0e67af1c2e890 100644 --- a/src/plugins/vis_types/vega/server/usage_collector/get_usage_collector.ts +++ b/src/plugins/vis_types/vega/server/usage_collector/get_usage_collector.ts @@ -78,7 +78,7 @@ export const getStats = async ( const doTelemetry = ({ params }: SavedVisState) => { try { - const spec = parse(params.spec, { legacyRoot: false }); + const spec = parse(params.spec as string, { legacyRoot: false }); if (spec) { shouldPublishTelemetry = true; diff --git a/src/plugins/visualizations/common/types.ts b/src/plugins/visualizations/common/types.ts index 0bdae26f01f93..187e55566c9d9 100644 --- a/src/plugins/visualizations/common/types.ts +++ b/src/plugins/visualizations/common/types.ts @@ -7,18 +7,20 @@ */ import { SavedObjectAttributes } from 'kibana/server'; -import { AggConfigOptions } from 'src/plugins/data/common'; +import type { SerializableRecord } from '@kbn/utility-types'; +import { AggConfigSerialized } from 'src/plugins/data/common'; export interface VisParams { [key: string]: any; } -export interface SavedVisState { +// eslint-disable-next-line @typescript-eslint/consistent-type-definitions +export type SavedVisState = { title: string; type: string; params: TVisParams; - aggs: AggConfigOptions[]; -} + aggs: AggConfigSerialized[]; +}; export interface VisualizationSavedObjectAttributes extends SavedObjectAttributes { description: string; diff --git a/src/plugins/visualizations/public/saved_visualizations/saved_visualization_references/saved_visualization_references.test.ts b/src/plugins/visualizations/public/saved_visualizations/saved_visualization_references/saved_visualization_references.test.ts index 867febd2544b0..9c832414e7f00 100644 --- a/src/plugins/visualizations/public/saved_visualizations/saved_visualization_references/saved_visualization_references.test.ts +++ b/src/plugins/visualizations/public/saved_visualizations/saved_visualization_references/saved_visualization_references.test.ts @@ -115,7 +115,7 @@ describe('injectReferences', () => { }); test('injects references into context', () => { - const context = { + const context = ({ id: '1', title: 'test', savedSearchRefName: 'search_0', @@ -133,7 +133,7 @@ describe('injectReferences', () => { ], }, } as unknown) as SavedVisState, - } as VisSavedObject; + } as unknown) as VisSavedObject; const references = [ { name: 'search_0', @@ -182,7 +182,7 @@ describe('injectReferences', () => { }); test(`fails when it can't find the index pattern reference in the array`, () => { - const context = { + const context = ({ id: '1', title: 'test', visState: ({ @@ -196,7 +196,7 @@ describe('injectReferences', () => { ], }, } as unknown) as SavedVisState, - } as VisSavedObject; + } as unknown) as VisSavedObject; expect(() => injectReferences(context, [])).toThrowErrorMatchingInlineSnapshot( `"Could not find index pattern reference \\"control_0_index_pattern\\""` ); diff --git a/src/plugins/visualizations/public/types.ts b/src/plugins/visualizations/public/types.ts index c6eceb86b3450..d68599c0724f6 100644 --- a/src/plugins/visualizations/public/types.ts +++ b/src/plugins/visualizations/public/types.ts @@ -8,10 +8,10 @@ import { SavedObject } from '../../../plugins/saved_objects/public'; import { - AggConfigOptions, IAggConfigs, SearchSourceFields, TimefilterContract, + AggConfigSerialized, } from '../../../plugins/data/public'; import { ExpressionAstExpression } from '../../expressions/public'; @@ -24,7 +24,7 @@ export interface SavedVisState { title: string; type: string; params: VisParams; - aggs: AggConfigOptions[]; + aggs: AggConfigSerialized[]; } export interface ISavedVis { diff --git a/src/plugins/visualizations/public/vis.ts b/src/plugins/visualizations/public/vis.ts index ff4e8a3794e0d..dfab4ecfc3cd8 100644 --- a/src/plugins/visualizations/public/vis.ts +++ b/src/plugins/visualizations/public/vis.ts @@ -26,7 +26,7 @@ import { IAggConfigs, IndexPattern, ISearchSource, - AggConfigOptions, + AggConfigSerialized, SearchSourceFields, } from '../../../plugins/data/public'; import { BaseVisType } from './vis_types'; @@ -34,7 +34,7 @@ import { VisParams } from '../common/types'; export interface SerializedVisData { expression?: string; - aggs: AggConfigOptions[]; + aggs: AggConfigSerialized[]; searchSource: SearchSourceFields; savedSearchId?: string; } @@ -194,7 +194,7 @@ export class Vis { } } - private initializeDefaultsFromSchemas(configStates: AggConfigOptions[], schemas: any) { + private initializeDefaultsFromSchemas(configStates: AggConfigSerialized[], schemas: any) { // Set the defaults for any schema which has them. If the defaults // for some reason has more then the max only set the max number // of defaults (not sure why a someone define more... diff --git a/src/plugins/visualize/common/constants.ts b/src/plugins/visualize/common/constants.ts index 5fe8ed7e095a2..10a4498193e3d 100644 --- a/src/plugins/visualize/common/constants.ts +++ b/src/plugins/visualize/common/constants.ts @@ -8,3 +8,16 @@ export const STATE_STORAGE_KEY = '_a'; export const GLOBAL_STATE_STORAGE_KEY = '_g'; + +export const APP_NAME = 'visualize'; + +export const VisualizeConstants = { + VISUALIZE_BASE_PATH: '/app/visualize', + LANDING_PAGE_PATH: '/', + WIZARD_STEP_1_PAGE_PATH: '/new', + WIZARD_STEP_2_PAGE_PATH: '/new/configure', + CREATE_PATH: '/create', + EDIT_PATH: '/edit', + EDIT_BY_VALUE_PATH: '/edit_by_value', + APP_ID: 'visualize', +}; diff --git a/src/plugins/visualize/common/locator.test.ts b/src/plugins/visualize/common/locator.test.ts new file mode 100644 index 0000000000000..c08c6a910327e --- /dev/null +++ b/src/plugins/visualize/common/locator.test.ts @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { VisualizeLocatorDefinition } from './locator'; +import { FilterStateStore } from '../../data/common'; + +describe('visualize locator', () => { + let definition: VisualizeLocatorDefinition; + + beforeEach(() => { + definition = new VisualizeLocatorDefinition(); + }); + + it('returns a location for "create" path', async () => { + const location = await definition.getLocation({}); + + expect(location.app).toMatchInlineSnapshot(`"visualize"`); + expect(location.path).toMatchInlineSnapshot(`"#/create?_g=()&_a=()"`); + expect(location.state).toMatchInlineSnapshot(`Object {}`); + }); + + it('returns a location for "edit" path', async () => { + const location = await definition.getLocation({ + visId: 'test', + vis: { + title: 'test', + type: 'test', + aggs: [], + params: {}, + }, + }); + + expect(location.app).toMatchInlineSnapshot(`"visualize"`); + expect(location.path).toMatchInlineSnapshot( + `"#/edit/test?_g=()&_a=(vis:(aggs:!(),params:(),title:test,type:test))&type=test"` + ); + expect(location.state).toMatchInlineSnapshot(`Object {}`); + }); + + it('creates a location with query, filters (global and app), refresh interval and time range', async () => { + const location = await definition.getLocation({ + visId: '123', + vis: { + title: 'test', + type: 'test', + aggs: [], + params: {}, + }, + timeRange: { to: 'now', from: 'now-15m', mode: 'relative' }, + refreshInterval: { pause: false, value: 300 }, + filters: [ + { + meta: { + alias: null, + disabled: false, + negate: false, + }, + query: { query: 'hi' }, + }, + { + meta: { + alias: null, + disabled: false, + negate: false, + }, + query: { query: 'hi' }, + $state: { + store: FilterStateStore.GLOBAL_STATE, + }, + }, + ], + query: { query: 'bye', language: 'kuery' }, + }); + + expect(location.app).toMatchInlineSnapshot(`"visualize"`); + + expect(location.path.match(/filters:/g)?.length).toBe(2); + expect(location.path.match(/refreshInterval:/g)?.length).toBe(1); + expect(location.path.match(/time:/g)?.length).toBe(1); + expect(location.path).toMatchInlineSnapshot( + `"#/edit/123?_g=(filters:!(('$state':(store:globalState),meta:(alias:!n,disabled:!f,negate:!f),query:(query:hi))),refreshInterval:(pause:!f,value:300),time:(from:now-15m,mode:relative,to:now))&_a=(filters:!((meta:(alias:!n,disabled:!f,negate:!f),query:(query:hi))),query:(language:kuery,query:bye),vis:(aggs:!(),params:(),title:test,type:test))&type=test"` + ); + + expect(location.state).toMatchInlineSnapshot(`Object {}`); + }); + + it('creates a location with all values provided', async () => { + const indexPattern = 'indexPatternTest'; + const savedSearchId = 'savedSearchIdTest'; + const location = await definition.getLocation({ + visId: '123', + vis: { + title: 'test', + type: 'test', + aggs: [], + params: {}, + }, + timeRange: { to: 'now', from: 'now-15m', mode: 'relative' }, + refreshInterval: { pause: false, value: 300 }, + filters: [ + { + meta: { + alias: null, + disabled: false, + negate: false, + }, + query: { query: 'hi' }, + }, + ], + query: { query: 'bye', language: 'kuery' }, + linked: true, + uiState: { + fakeUIState: 'fakeUIState', + this: 'value contains a spaces that should be encoded', + }, + indexPattern, + savedSearchId, + }); + + expect(location.app).toMatchInlineSnapshot(`"visualize"`); + expect(location.path).toContain(indexPattern); + expect(location.path).toContain(savedSearchId); + expect(location.path).toMatchInlineSnapshot( + `"#/edit/123?_g=(filters:!(),refreshInterval:(pause:!f,value:300),time:(from:now-15m,mode:relative,to:now))&_a=(filters:!((meta:(alias:!n,disabled:!f,negate:!f),query:(query:hi))),linked:!t,query:(language:kuery,query:bye),uiState:(fakeUIState:fakeUIState,this:'value%20contains%20a%20spaces%20that%20should%20be%20encoded'),vis:(aggs:!(),params:(),title:test,type:test))&indexPattern=indexPatternTest&savedSearchId=savedSearchIdTest&type=test"` + ); + expect(location.state).toMatchInlineSnapshot(`Object {}`); + }); +}); diff --git a/src/plugins/visualize/common/locator.ts b/src/plugins/visualize/common/locator.ts new file mode 100644 index 0000000000000..23fde918780f2 --- /dev/null +++ b/src/plugins/visualize/common/locator.ts @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { SerializableRecord, Serializable } from '@kbn/utility-types'; +import { omitBy } from 'lodash'; +import type { ParsedQuery } from 'query-string'; +import { stringify } from 'query-string'; +import rison from 'rison-node'; +import type { Filter, Query, RefreshInterval, TimeRange } from 'src/plugins/data/common'; +import type { LocatorDefinition, LocatorPublic } from 'src/plugins/share/common'; +import { isFilterPinned } from '../../data/common'; +import { url } from '../../kibana_utils/common'; +import { GLOBAL_STATE_STORAGE_KEY, STATE_STORAGE_KEY, VisualizeConstants } from './constants'; +import { PureVisState } from './types'; + +const removeEmptyKeys = (o: Record): Record => + omitBy(o, (v) => v == null); + +// eslint-disable-next-line @typescript-eslint/consistent-type-definitions +export type VisualizeLocatorParams = { + /** + * The ID of the saved visualization to load. + */ + visId?: string; + + /** + * Global- and app-level filters to apply to data loaded by visualize. + */ + filters?: Filter[]; + + /** + * Time range to apply to data loaded by visualize. + */ + timeRange?: TimeRange; + + /** + * How frequently to poll for data. + */ + refreshInterval?: RefreshInterval; + + /** + * The query to use in to load data in visualize. + */ + query?: Query; + + /** + * UI state to be passed on to the current visualization. This value is opaque from the perspective of visualize. + */ + uiState?: SerializableRecord; + + /** + * Serialized visualization. + * + * @note This is required to navigate to "create" page (i.e., when no `visId` has been provided). + */ + vis?: PureVisState; + + /** + * Whether this visualization is linked a saved search. + */ + linked?: boolean; + + /** + * The saved search used as the source of the visualization. + */ + savedSearchId?: string; + + /** + * The saved search used as the source of the visualization. + */ + indexPattern?: string; +}; + +export type VisualizeAppLocator = LocatorPublic; + +export const VISUALIZE_APP_LOCATOR = 'VISUALIZE_APP_LOCATOR'; + +export class VisualizeLocatorDefinition implements LocatorDefinition { + id = VISUALIZE_APP_LOCATOR; + + public async getLocation({ + visId, + timeRange, + filters, + refreshInterval, + linked, + uiState, + query, + vis, + savedSearchId, + indexPattern, + }: VisualizeLocatorParams) { + let path = visId + ? `#${VisualizeConstants.EDIT_PATH}/${visId}` + : `#${VisualizeConstants.CREATE_PATH}`; + + const urlState: ParsedQuery = { + [GLOBAL_STATE_STORAGE_KEY]: rison.encode( + removeEmptyKeys({ + time: timeRange, + filters: filters?.filter((f) => isFilterPinned(f)), + refreshInterval, + }) + ), + [STATE_STORAGE_KEY]: rison.encode( + removeEmptyKeys({ + linked, + filters: filters?.filter((f) => !isFilterPinned(f)), + uiState, + query, + vis, + }) + ), + }; + + path += `?${stringify(url.encodeQuery(urlState), { encode: false, sort: false })}`; + + const otherParams = stringify({ type: vis?.type, savedSearchId, indexPattern }); + + if (otherParams) path += `&${otherParams}`; + + return { + app: VisualizeConstants.APP_ID, + path, + state: {}, + }; + } +} diff --git a/src/plugins/visualize/common/types.ts b/src/plugins/visualize/common/types.ts new file mode 100644 index 0000000000000..189c44ba15cc8 --- /dev/null +++ b/src/plugins/visualize/common/types.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import type { SavedVisState } from 'src/plugins/visualizations/common/types'; + +export type PureVisState = SavedVisState; diff --git a/src/plugins/visualize/public/application/types.ts b/src/plugins/visualize/public/application/types.ts index f850aedc33366..7e9f69163f5a6 100644 --- a/src/plugins/visualize/public/application/types.ts +++ b/src/plugins/visualize/public/application/types.ts @@ -9,6 +9,8 @@ import type { EventEmitter } from 'events'; import type { History } from 'history'; +import type { SerializableRecord } from '@kbn/utility-types'; + import type { CoreStart, PluginInitializerContext, @@ -19,7 +21,6 @@ import type { } from 'kibana/public'; import type { - SavedVisState, VisualizationsStart, Vis, VisualizeEmbeddableContract, @@ -45,11 +46,11 @@ import type { DashboardStart } from '../../../dashboard/public'; import type { SavedObjectsTaggingApi } from '../../../saved_objects_tagging_oss/public'; import type { UsageCollectionStart } from '../../../usage_collection/public'; -export type PureVisState = SavedVisState; +import { PureVisState } from '../../common/types'; export interface VisualizeAppState { filters: Filter[]; - uiState: Record; + uiState: SerializableRecord; vis: PureVisState; query: Query; savedQuery?: string; @@ -103,6 +104,7 @@ export interface VisualizeServices extends CoreStart { savedObjectsTagging?: SavedObjectsTaggingApi; presentationUtil: PresentationUtilPluginStart; usageCollection?: UsageCollectionStart; + getKibanaVersion: () => string; } export interface SavedVisInstance { @@ -146,3 +148,5 @@ export interface EditorRenderProps { */ linked: boolean; } + +export { PureVisState }; diff --git a/src/plugins/visualize/public/application/utils/create_visualize_app_state.ts b/src/plugins/visualize/public/application/utils/create_visualize_app_state.ts index e288996fa6f3d..10c573090da34 100644 --- a/src/plugins/visualize/public/application/utils/create_visualize_app_state.ts +++ b/src/plugins/visualize/public/application/utils/create_visualize_app_state.ts @@ -63,7 +63,6 @@ const pureTransitions = { function createVisualizeByValueAppState(stateDefaults: VisualizeAppState) { const initialState = migrateAppState({ ...stateDefaults, - ...stateDefaults, }); const stateContainer = createStateContainer( initialState, diff --git a/src/plugins/visualize/public/application/utils/get_top_nav_config.tsx b/src/plugins/visualize/public/application/utils/get_top_nav_config.tsx index ed361bbdb104d..a4421d9535c71 100644 --- a/src/plugins/visualize/public/application/utils/get_top_nav_config.tsx +++ b/src/plugins/visualize/public/application/utils/get_top_nav_config.tsx @@ -7,8 +7,10 @@ */ import React from 'react'; +import moment from 'moment'; import { i18n } from '@kbn/i18n'; import { METRIC_TYPE } from '@kbn/analytics'; +import { parse } from 'query-string'; import { Capabilities } from 'src/core/public'; import { TopNavMenuData } from 'src/plugins/navigation/public'; @@ -33,6 +35,7 @@ import { import { APP_NAME, VisualizeConstants } from '../visualize_constants'; import { getEditBreadcrumbs } from './breadcrumbs'; import { EmbeddableStateTransfer } from '../../../../embeddable/public'; +import { VISUALIZE_APP_LOCATOR, VisualizeLocatorParams } from '../../../common/locator'; interface VisualizeCapabilities { createShortUrl: boolean; @@ -95,6 +98,7 @@ export const getTopNavConfig = ( savedObjectsTagging, presentationUtil, usageCollection, + getKibanaVersion, }: VisualizeServices ) => { const { vis, embeddableHandler } = visInstance; @@ -279,6 +283,22 @@ export const getTopNavConfig = ( testId: 'shareTopNavButton', run: (anchorElement) => { if (share && !embeddableId) { + const currentState = stateContainer.getState(); + const searchParams = parse(history.location.search); + const params: VisualizeLocatorParams = { + visId: savedVis?.id, + filters: currentState.filters, + refreshInterval: undefined, + timeRange: data.query.timefilter.timefilter.getTime(), + uiState: currentState.uiState, + query: currentState.query, + vis: currentState.vis, + linked: currentState.linked, + indexPattern: + visInstance.savedSearch?.searchSource?.getField('index')?.id ?? + (searchParams.indexPattern as string), + savedSearchId: visInstance.savedSearch?.id ?? (searchParams.savedSearchId as string), + }; // TODO: support sharing in by-value mode share.toggleShareContextMenu({ anchorElement, @@ -288,7 +308,17 @@ export const getTopNavConfig = ( objectId: savedVis?.id, objectType: 'visualization', sharingData: { - title: savedVis?.title, + title: + savedVis?.title || + i18n.translate('visualize.reporting.defaultReportTitle', { + defaultMessage: 'Visualization [{date}]', + values: { date: moment().toISOString(true) }, + }), + locatorParams: { + id: VISUALIZE_APP_LOCATOR, + version: getKibanaVersion(), + params, + }, }, isDirty: hasUnappliedChanges || hasUnsavedChanges, showPublicUrlSwitch, diff --git a/src/plugins/visualize/public/application/utils/stubs.ts b/src/plugins/visualize/public/application/utils/stubs.ts index 41a017306dc0a..086811df02baa 100644 --- a/src/plugins/visualize/public/application/utils/stubs.ts +++ b/src/plugins/visualize/public/application/utils/stubs.ts @@ -27,7 +27,6 @@ export const visualizeAppStateStub: VisualizeAppState = { { id: '1', enabled: true, - // @ts-expect-error type: 'avg', schema: 'metric', params: { field: 'total_quantity', customLabel: 'average items' }, diff --git a/src/plugins/visualize/public/application/utils/use/use_visualize_app_state.test.ts b/src/plugins/visualize/public/application/utils/use/use_visualize_app_state.test.ts index b101a3c2feae9..26f866d22ce4e 100644 --- a/src/plugins/visualize/public/application/utils/use/use_visualize_app_state.test.ts +++ b/src/plugins/visualize/public/application/utils/use/use_visualize_app_state.test.ts @@ -157,7 +157,6 @@ describe('useVisualizeAppState', () => { }; it('should successfully update vis state and set up app state container', async () => { - // @ts-expect-error stateContainerGetStateMock.mockImplementation(() => state); const { result, waitForNextUpdate } = renderHook(() => useVisualizeAppState(mockServices, eventEmitter, savedVisInstance) @@ -204,7 +203,6 @@ describe('useVisualizeAppState', () => { it(`should add warning toast and redirect to the landing page if setting new vis state was not successful, e.x. invalid query params`, async () => { - // @ts-expect-error stateContainerGetStateMock.mockImplementation(() => state); // @ts-expect-error savedVisInstance.vis.setState.mockRejectedValue({ diff --git a/src/plugins/visualize/public/application/visualize_constants.ts b/src/plugins/visualize/public/application/visualize_constants.ts index 6e901882a9365..19327ac940e9d 100644 --- a/src/plugins/visualize/public/application/visualize_constants.ts +++ b/src/plugins/visualize/public/application/visualize_constants.ts @@ -6,15 +6,4 @@ * Side Public License, v 1. */ -export const APP_NAME = 'visualize'; - -export const VisualizeConstants = { - VISUALIZE_BASE_PATH: '/app/visualize', - LANDING_PAGE_PATH: '/', - WIZARD_STEP_1_PAGE_PATH: '/new', - WIZARD_STEP_2_PAGE_PATH: '/new/configure', - CREATE_PATH: '/create', - EDIT_PATH: '/edit', - EDIT_BY_VALUE_PATH: '/edit_by_value', - APP_ID: 'visualize', -}; +export { VisualizeConstants, APP_NAME } from '../../common/constants'; diff --git a/src/plugins/visualize/public/plugin.ts b/src/plugins/visualize/public/plugin.ts index 00c3545034b32..d71e7fd81f1d9 100644 --- a/src/plugins/visualize/public/plugin.ts +++ b/src/plugins/visualize/public/plugin.ts @@ -47,6 +47,7 @@ import type { UsageCollectionStart } from '../../usage_collection/public'; import { setVisEditorsRegistry, setUISettings, setUsageCollector } from './services'; import { createVisEditorsRegistry, VisEditorsRegistry } from './vis_editors_registry'; +import { VisualizeLocatorDefinition } from '../common/locator'; export interface VisualizePluginStartDependencies { data: DataPublicPluginStart; @@ -92,7 +93,7 @@ export class VisualizePlugin public setup( core: CoreSetup, - { home, urlForwarding, data }: VisualizePluginSetupDependencies + { home, urlForwarding, data, share }: VisualizePluginSetupDependencies ) { const { appMounted, @@ -209,6 +210,7 @@ export class VisualizePlugin savedObjectsTagging: pluginsStart.savedObjectsTaggingOss?.getTaggingApi(), presentationUtil: pluginsStart.presentationUtil, usageCollection: pluginsStart.usageCollection, + getKibanaVersion: () => this.initializerContext.env.packageInfo.version, }; params.element.classList.add('visAppWrapper'); @@ -241,6 +243,10 @@ export class VisualizePlugin }); } + if (share) { + share.url.locators.create(new VisualizeLocatorDefinition()); + } + return { visEditorsRegistry: this.visEditorsRegistry, } as VisualizePluginSetup; diff --git a/src/plugins/visualize/tsconfig.json b/src/plugins/visualize/tsconfig.json index 4dcf43dadf8ba..3f1f7487085bf 100644 --- a/src/plugins/visualize/tsconfig.json +++ b/src/plugins/visualize/tsconfig.json @@ -6,11 +6,7 @@ "declaration": true, "declarationMap": true }, - "include": [ - "common/**/*", - "public/**/*", - "server/**/*" - ], + "include": ["common/**/*", "public/**/*", "server/**/*", "../../../typings/**/*"], "references": [ { "path": "../../core/tsconfig.json" }, { "path": "../data/tsconfig.json" }, @@ -28,6 +24,6 @@ { "path": "../kibana_react/tsconfig.json" }, { "path": "../home/tsconfig.json" }, { "path": "../presentation_util/tsconfig.json" }, - { "path": "../discover/tsconfig.json" }, + { "path": "../discover/tsconfig.json" } ] } diff --git a/x-pack/test/functional/apps/visualize/reporting.ts b/x-pack/test/functional/apps/visualize/reporting.ts index c43747c346ca7..1e629927ffb4d 100644 --- a/x-pack/test/functional/apps/visualize/reporting.ts +++ b/x-pack/test/functional/apps/visualize/reporting.ts @@ -42,13 +42,13 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); describe('Print PDF button', () => { - it('is not available if new', async () => { + it('is available if new', async () => { await PageObjects.common.navigateToUrl('visualize', 'new', { useActualUrl: true }); await PageObjects.visualize.clickAggBasedVisualizations(); await PageObjects.visualize.clickAreaChart(); await PageObjects.visualize.clickNewSearch('ecommerce'); await PageObjects.reporting.openPdfReportingPanel(); - expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true'); + expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null); }); it('becomes available when saved', async () => { From 7f6c6e44eaec4e881c8d107394a6c8e96ca993de Mon Sep 17 00:00:00 2001 From: Pete Harverson Date: Fri, 3 Sep 2021 16:32:10 +0100 Subject: [PATCH 20/20] [ML] Add API integration tests for start and stop datafeeds (#110961) * [ML] Add API integration tests for start and stop datafeeds * [ML] Edits to setup and clean-up steps following review --- .../apis/ml/jobs/force_start_datafeeds.ts | 249 ++++++++++++++++++ .../ml/jobs/force_start_datafeeds_spaces.ts | 126 +++++++++ .../api_integration/apis/ml/jobs/index.ts | 4 + .../apis/ml/jobs/stop_datafeeds.ts | 246 +++++++++++++++++ .../apis/ml/jobs/stop_datafeeds_spaces.ts | 123 +++++++++ 5 files changed, 748 insertions(+) create mode 100644 x-pack/test/api_integration/apis/ml/jobs/force_start_datafeeds.ts create mode 100644 x-pack/test/api_integration/apis/ml/jobs/force_start_datafeeds_spaces.ts create mode 100644 x-pack/test/api_integration/apis/ml/jobs/stop_datafeeds.ts create mode 100644 x-pack/test/api_integration/apis/ml/jobs/stop_datafeeds_spaces.ts diff --git a/x-pack/test/api_integration/apis/ml/jobs/force_start_datafeeds.ts b/x-pack/test/api_integration/apis/ml/jobs/force_start_datafeeds.ts new file mode 100644 index 0000000000000..04ab308a0d7b2 --- /dev/null +++ b/x-pack/test/api_integration/apis/ml/jobs/force_start_datafeeds.ts @@ -0,0 +1,249 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { USER } from '../../../../functional/services/ml/security_common'; +import { JOB_STATE, DATAFEED_STATE } from '../../../../../plugins/ml/common/constants/states'; +import { MULTI_METRIC_JOB_CONFIG, SINGLE_METRIC_JOB_CONFIG, DATAFEED_CONFIG } from './common_jobs'; +import { COMMON_REQUEST_HEADERS } from '../../../../functional/services/ml/common_api'; + +export default ({ getService }: FtrProviderContext) => { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertestWithoutAuth'); + const ml = getService('ml'); + + const testSetupJobConfigs = [SINGLE_METRIC_JOB_CONFIG, MULTI_METRIC_JOB_CONFIG]; + + async function runStartDatafeedsRequest( + user: USER, + requestBody: object, + expectedResponsecode: number + ): Promise> { + const { body } = await supertest + .post('/api/ml/jobs/force_start_datafeeds') + .auth(user, ml.securityCommon.getPasswordForUser(user)) + .set(COMMON_REQUEST_HEADERS) + .send(requestBody) + .expect(expectedResponsecode); + + return body; + } + + const testDataList = [ + { + testTitle: 'as ML Poweruser', + jobIds: [SINGLE_METRIC_JOB_CONFIG.job_id, MULTI_METRIC_JOB_CONFIG.job_id], + user: USER.ML_POWERUSER, + requestBody: { + datafeedIds: [ + `datafeed-${SINGLE_METRIC_JOB_CONFIG.job_id}`, + `datafeed-${MULTI_METRIC_JOB_CONFIG.job_id}`, + ], + start: 1454803200000, // Starts in real-time from Feb 7 2016 00:00 + }, + expected: { + responseCode: 200, + responseBody: { + [`datafeed-${SINGLE_METRIC_JOB_CONFIG.job_id}`]: { started: true }, + [`datafeed-${MULTI_METRIC_JOB_CONFIG.job_id}`]: { started: true }, + }, + }, + }, + ]; + + const invalidTestDataList = [ + { + testTitle: 'as ML Poweruser with datafeed ID that does not exist', + jobIds: [SINGLE_METRIC_JOB_CONFIG.job_id], + user: USER.ML_POWERUSER, + requestBody: { + datafeedIds: [`invalid-datafeed-${SINGLE_METRIC_JOB_CONFIG.job_id}`], + start: 1454803200000, // Feb 7 2016 00:00 + }, + expected: { + responseCode: 200, + responseBody: { + [`invalid-datafeed-${SINGLE_METRIC_JOB_CONFIG.job_id}`]: { started: false }, + }, + }, + }, + ]; + + const testDataListUnauthorized = [ + { + testTitle: 'as ML Unauthorized user', + user: USER.ML_UNAUTHORIZED, + jobIds: [SINGLE_METRIC_JOB_CONFIG.job_id, MULTI_METRIC_JOB_CONFIG.job_id], + requestBody: { + datafeedIds: [ + `datafeed-${SINGLE_METRIC_JOB_CONFIG.job_id}`, + `datafeed-${MULTI_METRIC_JOB_CONFIG.job_id}`, + ], + start: 1454803200000, // Feb 7 2016 00:00 + end: 1455235200000, // Feb 12 2016 00:00 + }, + expected: { + responseCode: 403, + error: 'Forbidden', + }, + }, + { + testTitle: 'as ML Viewer', + user: USER.ML_VIEWER, + jobIds: [SINGLE_METRIC_JOB_CONFIG.job_id, MULTI_METRIC_JOB_CONFIG.job_id], + requestBody: { + datafeedIds: [ + `datafeed-${SINGLE_METRIC_JOB_CONFIG.job_id}`, + `datafeed-${MULTI_METRIC_JOB_CONFIG.job_id}`, + ], + start: 1454803200000, // Feb 7 2016 00:00 + end: 1455235200000, // Feb 12 2016 00:00 + }, + expected: { + responseCode: 403, + error: 'Forbidden', + }, + }, + ]; + + describe('force_start_datafeeds', function () { + before(async () => { + await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote'); + await ml.testResources.setKibanaTimeZoneToUTC(); + + for (const job of testSetupJobConfigs) { + const datafeedId = `datafeed-${job.job_id}`; + await ml.api.createAnomalyDetectionJob(job); + await ml.api.createDatafeed({ + ...DATAFEED_CONFIG, + datafeed_id: datafeedId, + job_id: job.job_id, + }); + } + }); + + after(async () => { + for (const job of testSetupJobConfigs) { + await ml.api.deleteAnomalyDetectionJobES(job.job_id); + } + await ml.api.cleanMlIndices(); + }); + + describe('rejects requests for unauthorized users', function () { + for (const testData of testDataListUnauthorized) { + describe('fails to force start supplied datafeed IDs', function () { + it(`${testData.testTitle}`, async () => { + const body = await runStartDatafeedsRequest( + testData.user, + testData.requestBody, + testData.expected.responseCode + ); + + expect(body).to.have.property('error').eql(testData.expected.error); + + // check jobs are still closed + for (const id of testData.jobIds) { + await ml.api.waitForJobState(id, JOB_STATE.CLOSED); + } + + // check datafeeds are still stopped + for (const id of testData.requestBody.datafeedIds) { + await ml.api.waitForDatafeedState(id, DATAFEED_STATE.STOPPED); + } + }); + }); + } + }); + + describe('starts datafeeds with supplied IDs', function () { + for (const testData of testDataList) { + it(`${testData.testTitle}`, async () => { + const body = await runStartDatafeedsRequest( + testData.user, + testData.requestBody, + testData.expected.responseCode + ); + + const expectedResponse = testData.expected.responseBody; + const expectedRspDatafeedIds = Object.keys(expectedResponse).sort((a, b) => + a.localeCompare(b) + ); + const actualRspDatafeedIds = Object.keys(body).sort((a, b) => a.localeCompare(b)); + + expect(actualRspDatafeedIds).to.have.length(expectedRspDatafeedIds.length); + expect(actualRspDatafeedIds).to.eql(expectedRspDatafeedIds); + + // check jobs are open + for (const id of testData.jobIds) { + await ml.api.waitForJobState(id, JOB_STATE.OPENED); + } + + // check datafeeds have started + for (const id of testData.requestBody.datafeedIds) { + await ml.api.waitForDatafeedState(id, DATAFEED_STATE.STARTED); + } + }); + } + }); + + describe('succeeds with datafeed already started', function () { + for (const testData of testDataList) { + it(`${testData.testTitle}`, async () => { + const body = await runStartDatafeedsRequest( + testData.user, + testData.requestBody, + testData.expected.responseCode + ); + + const expectedResponse = testData.expected.responseBody; + const expectedRspDatafeedIds = Object.keys(expectedResponse).sort((a, b) => + a.localeCompare(b) + ); + const actualRspDatafeedIds = Object.keys(body).sort((a, b) => a.localeCompare(b)); + + expect(actualRspDatafeedIds).to.have.length(expectedRspDatafeedIds.length); + expect(actualRspDatafeedIds).to.eql(expectedRspDatafeedIds); + + // check jobs are still open + for (const id of testData.jobIds) { + await ml.api.waitForJobState(id, JOB_STATE.OPENED); + } + + // check datafeeds are still started + for (const id of testData.requestBody.datafeedIds) { + await ml.api.waitForDatafeedState(id, DATAFEED_STATE.STARTED); + } + }); + } + }); + + describe('returns expected response for invalid request', function () { + for (const testData of invalidTestDataList) { + it(`${testData.testTitle}`, async () => { + const body = await runStartDatafeedsRequest( + testData.user, + testData.requestBody, + testData.expected.responseCode + ); + const expectedResponse = testData.expected.responseBody; + const expectedRspDatafeedIds = Object.keys(expectedResponse).sort((a, b) => + a.localeCompare(b) + ); + const actualRspDatafeedIds = Object.keys(body).sort((a, b) => a.localeCompare(b)); + + expect(actualRspDatafeedIds).to.have.length(expectedRspDatafeedIds.length); + expect(actualRspDatafeedIds).to.eql(expectedRspDatafeedIds); + + expectedRspDatafeedIds.forEach((id) => { + expect(body[id].started).to.eql(expectedResponse[id].started); + }); + }); + } + }); + }); +}; diff --git a/x-pack/test/api_integration/apis/ml/jobs/force_start_datafeeds_spaces.ts b/x-pack/test/api_integration/apis/ml/jobs/force_start_datafeeds_spaces.ts new file mode 100644 index 0000000000000..1ebc6c5b78424 --- /dev/null +++ b/x-pack/test/api_integration/apis/ml/jobs/force_start_datafeeds_spaces.ts @@ -0,0 +1,126 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; + +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { COMMON_REQUEST_HEADERS } from '../../../../functional/services/ml/common_api'; +import { USER } from '../../../../functional/services/ml/security_common'; +import { DATAFEED_STATE } from '../../../../../plugins/ml/common/constants/states'; + +export default ({ getService }: FtrProviderContext) => { + const esArchiver = getService('esArchiver'); + const ml = getService('ml'); + const spacesService = getService('spaces'); + const supertest = getService('supertestWithoutAuth'); + + const idSpace1 = 'space1'; + const idSpace2 = 'space2'; + const jobIdSpace1 = `fq_single_${idSpace1}`; + const jobIdSpace2 = `fq_single_${idSpace2}`; + const datafeedIdSpace1 = `datafeed-${jobIdSpace1}`; + const datafeedIdSpace2 = `datafeed-${jobIdSpace2}`; + const startMs = 1454803200000; // Feb 7 2016 00:00 + const endMs = 1455235200000; // Feb 12 2016 00:00 + + async function runRequest( + space: string, + expectedStatusCode: number, + datafeedIds: string[], + start: number, + end: number + ): Promise> { + const { body } = await supertest + .post(`/s/${space}/api/ml/jobs/force_start_datafeeds`) + .auth( + USER.ML_POWERUSER_ALL_SPACES, + ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER_ALL_SPACES) + ) + .set(COMMON_REQUEST_HEADERS) + .send({ datafeedIds, start, end }) + .expect(expectedStatusCode); + + return body; + } + + describe('force_start_datafeeds with spaces', function () { + before(async () => { + await spacesService.create({ id: idSpace1, name: 'space_one', disabledFeatures: [] }); + await spacesService.create({ id: idSpace2, name: 'space_two', disabledFeatures: [] }); + + await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote'); + await ml.testResources.setKibanaTimeZoneToUTC(); + }); + + beforeEach(async () => { + const jobConfigSpace1 = ml.commonConfig.getADFqSingleMetricJobConfig(jobIdSpace1); + const datafeedConfigSpace1 = ml.commonConfig.getADFqDatafeedConfig(jobIdSpace1); + await ml.api.createAnomalyDetectionJob(jobConfigSpace1, idSpace1); + await ml.api.createDatafeed( + { + ...datafeedConfigSpace1, + datafeed_id: datafeedIdSpace1, + job_id: jobIdSpace1, + }, + idSpace1 + ); + const jobConfigSpace2 = ml.commonConfig.getADFqSingleMetricJobConfig(jobIdSpace2); + const datafeedConfigSpace2 = ml.commonConfig.getADFqDatafeedConfig(jobIdSpace2); + await ml.api.createAnomalyDetectionJob(jobConfigSpace2, idSpace2); + await ml.api.createDatafeed( + { + ...datafeedConfigSpace2, + datafeed_id: datafeedIdSpace2, + job_id: jobIdSpace2, + }, + idSpace2 + ); + }); + + afterEach(async () => { + await ml.api.closeAnomalyDetectionJob(jobIdSpace1); + await ml.api.closeAnomalyDetectionJob(jobIdSpace2); + await ml.api.deleteAnomalyDetectionJobES(jobIdSpace1); + await ml.api.deleteAnomalyDetectionJobES(jobIdSpace2); + await ml.api.cleanMlIndices(); + await ml.testResources.cleanMLSavedObjects(); + }); + + after(async () => { + await spacesService.delete(idSpace1); + await spacesService.delete(idSpace2); + }); + + it('should start single datafeed from same space', async () => { + const body = await runRequest(idSpace1, 200, [datafeedIdSpace1], startMs, endMs); + expect(body).to.eql({ [datafeedIdSpace1]: { started: true } }); + await ml.api.waitForDatafeedState(datafeedIdSpace1, DATAFEED_STATE.STARTED); + }); + + it('should not start single datafeed from different space', async () => { + const body = await runRequest(idSpace2, 200, [datafeedIdSpace1], startMs, endMs); + expect(body).to.eql({ [datafeedIdSpace1]: { error: 'Job has no datafeed', started: false } }); + await ml.api.waitForDatafeedState(datafeedIdSpace1, DATAFEED_STATE.STOPPED); + }); + + it('should only start datafeed from same space when called with a list of datafeeds', async () => { + const body = await runRequest( + idSpace1, + 200, + [datafeedIdSpace1, datafeedIdSpace2], + startMs, + endMs + ); + expect(body).to.eql({ + [datafeedIdSpace1]: { started: true }, + [datafeedIdSpace2]: { error: 'Job has no datafeed', started: false }, + }); + await ml.api.waitForDatafeedState(datafeedIdSpace1, DATAFEED_STATE.STARTED); + await ml.api.waitForDatafeedState(datafeedIdSpace2, DATAFEED_STATE.STOPPED); + }); + }); +}; diff --git a/x-pack/test/api_integration/apis/ml/jobs/index.ts b/x-pack/test/api_integration/apis/ml/jobs/index.ts index 4c52f2ef862c3..91368251ff2d7 100644 --- a/x-pack/test/api_integration/apis/ml/jobs/index.ts +++ b/x-pack/test/api_integration/apis/ml/jobs/index.ts @@ -18,5 +18,9 @@ export default function ({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./close_jobs_spaces')); loadTestFile(require.resolve('./delete_jobs_spaces')); loadTestFile(require.resolve('./datafeed_preview')); + loadTestFile(require.resolve('./force_start_datafeeds')); + loadTestFile(require.resolve('./force_start_datafeeds_spaces')); + loadTestFile(require.resolve('./stop_datafeeds')); + loadTestFile(require.resolve('./stop_datafeeds_spaces')); }); } diff --git a/x-pack/test/api_integration/apis/ml/jobs/stop_datafeeds.ts b/x-pack/test/api_integration/apis/ml/jobs/stop_datafeeds.ts new file mode 100644 index 0000000000000..593dfdd2fdfe7 --- /dev/null +++ b/x-pack/test/api_integration/apis/ml/jobs/stop_datafeeds.ts @@ -0,0 +1,246 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { USER } from '../../../../functional/services/ml/security_common'; +import { JOB_STATE, DATAFEED_STATE } from '../../../../../plugins/ml/common/constants/states'; +import { MULTI_METRIC_JOB_CONFIG, SINGLE_METRIC_JOB_CONFIG, DATAFEED_CONFIG } from './common_jobs'; +import { COMMON_REQUEST_HEADERS } from '../../../../functional/services/ml/common_api'; + +export default ({ getService }: FtrProviderContext) => { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertestWithoutAuth'); + const ml = getService('ml'); + + const testSetupJobConfigs = [SINGLE_METRIC_JOB_CONFIG, MULTI_METRIC_JOB_CONFIG]; + + async function runStopDatafeedsRequest( + user: USER, + requestBody: object, + expectedResponsecode: number + ): Promise> { + const { body } = await supertest + .post('/api/ml/jobs/stop_datafeeds') + .auth(user, ml.securityCommon.getPasswordForUser(user)) + .set(COMMON_REQUEST_HEADERS) + .send(requestBody) + .expect(expectedResponsecode); + + return body; + } + + const testDataList = [ + { + testTitle: 'as ML Poweruser', + jobIds: [SINGLE_METRIC_JOB_CONFIG.job_id, MULTI_METRIC_JOB_CONFIG.job_id], + user: USER.ML_POWERUSER, + requestBody: { + datafeedIds: [ + `datafeed-${SINGLE_METRIC_JOB_CONFIG.job_id}`, + `datafeed-${MULTI_METRIC_JOB_CONFIG.job_id}`, + ], + }, + expected: { + responseCode: 200, + responseBody: { + [`datafeed-${SINGLE_METRIC_JOB_CONFIG.job_id}`]: { stopped: true }, + [`datafeed-${MULTI_METRIC_JOB_CONFIG.job_id}`]: { stopped: true }, + }, + }, + }, + ]; + + const invalidTestDataList = [ + { + testTitle: 'as ML Poweruser with datafeed ID that does not exist', + jobIds: [SINGLE_METRIC_JOB_CONFIG.job_id], + user: USER.ML_POWERUSER, + requestBody: { + datafeedIds: [`invalid-datafeed-${SINGLE_METRIC_JOB_CONFIG.job_id}`], + }, + expected: { + responseCode: 200, + responseBody: { + [`invalid-datafeed-${SINGLE_METRIC_JOB_CONFIG.job_id}`]: { stopped: false }, + }, + }, + }, + ]; + + const testDataListUnauthorized = [ + { + testTitle: 'as ML Unauthorized user', + user: USER.ML_UNAUTHORIZED, + jobIds: [SINGLE_METRIC_JOB_CONFIG.job_id, MULTI_METRIC_JOB_CONFIG.job_id], + requestBody: { + datafeedIds: [ + `datafeed-${SINGLE_METRIC_JOB_CONFIG.job_id}`, + `datafeed-${MULTI_METRIC_JOB_CONFIG.job_id}`, + ], + }, + expected: { + responseCode: 403, + error: 'Forbidden', + }, + }, + { + testTitle: 'as ML Viewer', + user: USER.ML_VIEWER, + jobIds: [SINGLE_METRIC_JOB_CONFIG.job_id, MULTI_METRIC_JOB_CONFIG.job_id], + requestBody: { + datafeedIds: [ + `datafeed-${SINGLE_METRIC_JOB_CONFIG.job_id}`, + `datafeed-${MULTI_METRIC_JOB_CONFIG.job_id}`, + ], + }, + expected: { + responseCode: 403, + error: 'Forbidden', + }, + }, + ]; + + describe('stop_datafeeds', function () { + before(async () => { + await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote'); + await ml.testResources.setKibanaTimeZoneToUTC(); + + for (const job of testSetupJobConfigs) { + const datafeedId = `datafeed-${job.job_id}`; + await ml.api.createAnomalyDetectionJob(job); + await ml.api.openAnomalyDetectionJob(job.job_id); + await ml.api.createDatafeed({ + ...DATAFEED_CONFIG, + datafeed_id: datafeedId, + job_id: job.job_id, + }); + await ml.api.startDatafeed(datafeedId, { start: '0' }); + await ml.api.waitForDatafeedState(datafeedId, DATAFEED_STATE.STARTED); + } + }); + + after(async () => { + for (const job of testSetupJobConfigs) { + await ml.api.deleteAnomalyDetectionJobES(job.job_id); + } + await ml.api.cleanMlIndices(); + }); + + describe('rejects requests for unauthorized users', function () { + for (const testData of testDataListUnauthorized) { + describe('fails to stop supplied datafeed IDs', function () { + it(`${testData.testTitle}`, async () => { + const body = await runStopDatafeedsRequest( + testData.user, + testData.requestBody, + testData.expected.responseCode + ); + + expect(body).to.have.property('error').eql(testData.expected.error); + + // check jobs are still opened + for (const id of testData.jobIds) { + await ml.api.waitForJobState(id, JOB_STATE.OPENED); + } + + // check datafeeds are still started + for (const id of testData.requestBody.datafeedIds) { + await ml.api.waitForDatafeedState(id, DATAFEED_STATE.STARTED); + } + }); + }); + } + }); + + describe('succeeds for ML Poweruser with datafeed started', function () { + for (const testData of testDataList) { + it(`${testData.testTitle}`, async () => { + const body = await runStopDatafeedsRequest( + testData.user, + testData.requestBody, + testData.expected.responseCode + ); + + const expectedResponse = testData.expected.responseBody; + const expectedRspDatafeedIds = Object.keys(expectedResponse).sort((a, b) => + a.localeCompare(b) + ); + const actualRspDatafeedIds = Object.keys(body).sort((a, b) => a.localeCompare(b)); + + expect(actualRspDatafeedIds).to.have.length(expectedRspDatafeedIds.length); + expect(actualRspDatafeedIds).to.eql(expectedRspDatafeedIds); + + // check datafeeds have stopped + for (const id of testData.requestBody.datafeedIds) { + await ml.api.waitForDatafeedState(id, DATAFEED_STATE.STOPPED, 4 * 60 * 1000); + } + + // check jobs are still open + for (const id of testData.jobIds) { + await ml.api.waitForJobState(id, JOB_STATE.OPENED); + } + }); + } + }); + + describe('succeeds for ML Poweruser with datafeed already stopped', function () { + for (const testData of testDataList) { + it(`${testData.testTitle}`, async () => { + const body = await runStopDatafeedsRequest( + testData.user, + testData.requestBody, + testData.expected.responseCode + ); + + const expectedResponse = testData.expected.responseBody; + const expectedRspDatafeedIds = Object.keys(expectedResponse).sort((a, b) => + a.localeCompare(b) + ); + const actualRspDatafeedIds = Object.keys(body).sort((a, b) => a.localeCompare(b)); + + expect(actualRspDatafeedIds).to.have.length(expectedRspDatafeedIds.length); + expect(actualRspDatafeedIds).to.eql(expectedRspDatafeedIds); + + // check datafeeds have stopped + for (const id of testData.requestBody.datafeedIds) { + await ml.api.waitForDatafeedState(id, DATAFEED_STATE.STOPPED, 4 * 60 * 1000); + } + + // check jobs are still open + for (const id of testData.jobIds) { + await ml.api.waitForJobState(id, JOB_STATE.OPENED); + } + }); + } + }); + + describe('returns expected response for invalid request', function () { + for (const testData of invalidTestDataList) { + it(`${testData.testTitle}`, async () => { + const body = await runStopDatafeedsRequest( + testData.user, + testData.requestBody, + testData.expected.responseCode + ); + const expectedResponse = testData.expected.responseBody; + const expectedRspDatafeedIds = Object.keys(expectedResponse).sort((a, b) => + a.localeCompare(b) + ); + const actualRspDatafeedIds = Object.keys(body).sort((a, b) => a.localeCompare(b)); + + expect(actualRspDatafeedIds).to.have.length(expectedRspDatafeedIds.length); + expect(actualRspDatafeedIds).to.eql(expectedRspDatafeedIds); + + expectedRspDatafeedIds.forEach((id) => { + expect(body[id].stopped).to.eql(expectedResponse[id].stopped); + }); + }); + } + }); + }); +}; diff --git a/x-pack/test/api_integration/apis/ml/jobs/stop_datafeeds_spaces.ts b/x-pack/test/api_integration/apis/ml/jobs/stop_datafeeds_spaces.ts new file mode 100644 index 0000000000000..0e1ac038dc962 --- /dev/null +++ b/x-pack/test/api_integration/apis/ml/jobs/stop_datafeeds_spaces.ts @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; + +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { COMMON_REQUEST_HEADERS } from '../../../../functional/services/ml/common_api'; +import { USER } from '../../../../functional/services/ml/security_common'; +import { DATAFEED_STATE } from '../../../../../plugins/ml/common/constants/states'; + +export default ({ getService }: FtrProviderContext) => { + const esArchiver = getService('esArchiver'); + const ml = getService('ml'); + const spacesService = getService('spaces'); + const supertest = getService('supertestWithoutAuth'); + + const idSpace1 = 'space1'; + const idSpace2 = 'space2'; + const jobIdSpace1 = `fq_single_${idSpace1}`; + const jobIdSpace2 = `fq_single_${idSpace2}`; + const datafeedIdSpace1 = `datafeed-${jobIdSpace1}`; + const datafeedIdSpace2 = `datafeed-${jobIdSpace2}`; + + async function runRequest( + space: string, + expectedStatusCode: number, + datafeedIds: string[] + ): Promise> { + const { body } = await supertest + .post(`/s/${space}/api/ml/jobs/stop_datafeeds`) + .auth( + USER.ML_POWERUSER_ALL_SPACES, + ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER_ALL_SPACES) + ) + .set(COMMON_REQUEST_HEADERS) + .send({ datafeedIds }) + .expect(expectedStatusCode); + + return body; + } + + describe('stop_datafeeds with spaces', function () { + before(async () => { + await spacesService.create({ id: idSpace1, name: 'space_one', disabledFeatures: [] }); + await spacesService.create({ id: idSpace2, name: 'space_two', disabledFeatures: [] }); + + await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote'); + await ml.testResources.setKibanaTimeZoneToUTC(); + }); + + beforeEach(async () => { + const jobConfigSpace1 = ml.commonConfig.getADFqSingleMetricJobConfig(jobIdSpace1); + const datafeedConfigSpace1 = ml.commonConfig.getADFqDatafeedConfig(jobIdSpace1); + await ml.api.createAnomalyDetectionJob(jobConfigSpace1, idSpace1); + await ml.api.openAnomalyDetectionJob(jobIdSpace1); + await ml.api.createDatafeed( + { + ...datafeedConfigSpace1, + datafeed_id: datafeedIdSpace1, + job_id: jobIdSpace1, + }, + idSpace1 + ); + await ml.api.startDatafeed(datafeedIdSpace1, { start: '0' }); + await ml.api.waitForDatafeedState(datafeedIdSpace1, DATAFEED_STATE.STARTED); + + const jobConfigSpace2 = ml.commonConfig.getADFqSingleMetricJobConfig(jobIdSpace2); + const datafeedConfigSpace2 = ml.commonConfig.getADFqDatafeedConfig(jobIdSpace2); + await ml.api.createAnomalyDetectionJob(jobConfigSpace2, idSpace2); + await ml.api.openAnomalyDetectionJob(jobIdSpace2); + await ml.api.createDatafeed( + { + ...datafeedConfigSpace2, + datafeed_id: datafeedIdSpace2, + job_id: jobIdSpace2, + }, + idSpace2 + ); + await ml.api.startDatafeed(datafeedIdSpace2, { start: '0' }); + await ml.api.waitForDatafeedState(datafeedIdSpace2, DATAFEED_STATE.STARTED); + }); + + afterEach(async () => { + await ml.api.closeAnomalyDetectionJob(jobIdSpace1); + await ml.api.closeAnomalyDetectionJob(jobIdSpace2); + await ml.api.deleteAnomalyDetectionJobES(jobIdSpace1); + await ml.api.deleteAnomalyDetectionJobES(jobIdSpace2); + await ml.api.cleanMlIndices(); + await ml.testResources.cleanMLSavedObjects(); + }); + + after(async () => { + await spacesService.delete(idSpace1); + await spacesService.delete(idSpace2); + }); + + it('should stop single datafeed from same space', async () => { + const body = await runRequest(idSpace1, 200, [datafeedIdSpace1]); + expect(body).to.eql({ [datafeedIdSpace1]: { stopped: true } }); + await ml.api.waitForDatafeedState(datafeedIdSpace1, DATAFEED_STATE.STOPPED); + }); + + it('should not stop single datafeed from different space', async () => { + const body = await runRequest(idSpace2, 200, [datafeedIdSpace1]); + expect(body).to.eql({ [datafeedIdSpace1]: { stopped: false } }); + await ml.api.waitForDatafeedState(datafeedIdSpace1, DATAFEED_STATE.STARTED); + }); + + it('should only stop datafeed from same space when called with a list of datafeeds', async () => { + const body = await runRequest(idSpace1, 200, [datafeedIdSpace1, datafeedIdSpace2]); + expect(body).to.eql({ + [datafeedIdSpace1]: { stopped: true }, + [datafeedIdSpace2]: { stopped: false }, + }); + await ml.api.waitForDatafeedState(datafeedIdSpace1, DATAFEED_STATE.STOPPED); + await ml.api.waitForDatafeedState(datafeedIdSpace2, DATAFEED_STATE.STARTED); + }); + }); +};