From d0900f844df7020ffeeb4aa4acf26ffa40f15eab Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Wed, 10 Feb 2021 21:18:41 +0100 Subject: [PATCH 01/13] Limit cardinality of transaction.name (#90955) --- src/optimize/bundles_route/bundles_route.ts | 3 +++ .../server/task_running/task_runner.ts | 17 +++++++++-------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/optimize/bundles_route/bundles_route.ts b/src/optimize/bundles_route/bundles_route.ts index 6debf4b476590..b88ca7e5c22b1 100644 --- a/src/optimize/bundles_route/bundles_route.ts +++ b/src/optimize/bundles_route/bundles_route.ts @@ -10,6 +10,7 @@ import { extname, join } from 'path'; import Hapi from '@hapi/hapi'; import * as UiSharedDeps from '@kbn/ui-shared-deps'; +import agent from 'elastic-apm-node'; import { createDynamicAssetResponse } from './dynamic_asset_response'; import { FileHashCache } from './file_hash_cache'; @@ -101,6 +102,8 @@ function buildRouteForBundles({ method(request: Hapi.Request, h: Hapi.ResponseToolkit) { const ext = extname(request.params.path); + agent.setTransactionName('GET ?/bundles/?'); + if (ext !== '.js' && ext !== '.css') { return h.continue; } diff --git a/x-pack/plugins/task_manager/server/task_running/task_runner.ts b/x-pack/plugins/task_manager/server/task_running/task_runner.ts index 40213b3743d62..ad5a2e11409ec 100644 --- a/x-pack/plugins/task_manager/server/task_running/task_runner.ts +++ b/x-pack/plugins/task_manager/server/task_running/task_runner.ts @@ -201,10 +201,10 @@ export class TaskManagerRunner implements TaskRunner { }); const stopTaskTimer = startTaskTimer(); - const apmTrans = apm.startTransaction( - `taskManager run ${this.instance.taskType}`, - 'taskManager' - ); + const apmTrans = apm.startTransaction(`taskManager run`, 'taskManager'); + apmTrans?.addLabels({ + taskType: this.taskType, + }); try { this.task = this.definition.createTaskRunner(modifiedContext); const result = await this.task.run(); @@ -232,10 +232,11 @@ export class TaskManagerRunner implements TaskRunner { public async markTaskAsRunning(): Promise { performance.mark('markTaskAsRunning_start'); - const apmTrans = apm.startTransaction( - `taskManager markTaskAsRunning ${this.instance.taskType}`, - 'taskManager' - ); + const apmTrans = apm.startTransaction(`taskManager markTaskAsRunning`, 'taskManager'); + + apmTrans?.addLabels({ + taskType: this.taskType, + }); const now = new Date(); try { From 65a3f16c0fa13308c107e28137ef2e9b075ef6c2 Mon Sep 17 00:00:00 2001 From: Zacqary Adam Xeper Date: Wed, 10 Feb 2021 14:20:09 -0600 Subject: [PATCH 02/13] [Metrics UI] Fix saving/loading saved views from URL (#90216) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../saved_views/toolbar_control.tsx | 6 +- .../containers/saved_view/saved_view.tsx | 64 +++++++++++++++---- .../infra/public/pages/metrics/index.tsx | 6 +- .../pages/metrics/inventory_view/index.tsx | 6 +- 4 files changed, 60 insertions(+), 22 deletions(-) diff --git a/x-pack/plugins/infra/public/components/saved_views/toolbar_control.tsx b/x-pack/plugins/infra/public/components/saved_views/toolbar_control.tsx index 09e319b9935d3..ade43638deb68 100644 --- a/x-pack/plugins/infra/public/components/saved_views/toolbar_control.tsx +++ b/x-pack/plugins/infra/public/components/saved_views/toolbar_control.tsx @@ -6,7 +6,7 @@ */ import { EuiFlexGroup } from '@elastic/eui'; -import React, { useCallback, useState, useEffect, useContext } from 'react'; +import React, { useCallback, useState, useEffect } from 'react'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; import { @@ -21,7 +21,7 @@ import { SavedViewCreateModal } from './create_modal'; import { SavedViewUpdateModal } from './update_modal'; import { SavedViewManageViewsFlyout } from './manage_views_flyout'; import { useKibana } from '../../../../../../src/plugins/kibana_react/public'; -import { SavedView } from '../../containers/saved_view/saved_view'; +import { useSavedViewContext } from '../../containers/saved_view/saved_view'; import { SavedViewListModal } from './view_list_modal'; interface Props { @@ -47,7 +47,7 @@ export function SavedViewsToolbarControls(props: Props) { updatedView, currentView, setCurrentView, - } = useContext(SavedView.Context); + } = useSavedViewContext(); const [modalOpen, setModalOpen] = useState(false); const [viewListModalOpen, setViewListModalOpen] = useState(false); const [isInvalid, setIsInvalid] = useState(false); diff --git a/x-pack/plugins/infra/public/containers/saved_view/saved_view.tsx b/x-pack/plugins/infra/public/containers/saved_view/saved_view.tsx index e867cf800f4b4..4c4835cbe4cdb 100644 --- a/x-pack/plugins/infra/public/containers/saved_view/saved_view.tsx +++ b/x-pack/plugins/infra/public/containers/saved_view/saved_view.tsx @@ -6,9 +6,14 @@ */ import createContainer from 'constate'; +import * as rt from 'io-ts'; +import { pipe } from 'fp-ts/lib/pipeable'; +import { fold } from 'fp-ts/lib/Either'; +import { constant, identity } from 'fp-ts/lib/function'; import { useCallback, useMemo, useState, useEffect, useContext } from 'react'; import { i18n } from '@kbn/i18n'; import { SimpleSavedObject, SavedObjectAttributes } from 'kibana/public'; +import { useUrlState } from '../../utils/use_url_state'; import { useFindSavedObject } from '../../hooks/use_find_saved_object'; import { useCreateSavedObject } from '../../hooks/use_create_saved_object'; import { useDeleteSavedObject } from '../../hooks/use_delete_saved_object'; @@ -39,6 +44,14 @@ interface Props { shouldLoadDefault: boolean; } +const savedViewUrlStateRT = rt.type({ + viewId: rt.string, +}); +type SavedViewUrlState = rt.TypeOf; +const DEFAULT_SAVED_VIEW_STATE: SavedViewUrlState = { + viewId: '0', +}; + export const useSavedView = (props: Props) => { const { source, @@ -52,6 +65,13 @@ export const useSavedView = (props: Props) => { const { data, loading, find, error: errorOnFind, hasView } = useFindSavedObject< SavedViewSavedObject >(viewType); + const [urlState, setUrlState] = useUrlState({ + defaultState: DEFAULT_SAVED_VIEW_STATE, + decodeUrlState, + encodeUrlState, + urlStateKey: 'savedView', + }); + const [shouldLoadDefault] = useState(props.shouldLoadDefault); const [currentView, setCurrentView] = useState | null>(null); const [loadingDefaultView, setLoadingDefaultView] = useState(null); @@ -212,25 +232,35 @@ export const useSavedView = (props: Props) => { }); }, [setCurrentView, defaultViewId, defaultViewState]); - useEffect(() => { - if (loadingDefaultView || currentView || !shouldLoadDefault) { - return; - } - + const loadDefaultViewIfSet = useCallback(() => { if (defaultViewId !== '0') { loadDefaultView(); } else { setDefault(); setLoadingDefaultView(false); } - }, [ - loadDefaultView, - shouldLoadDefault, - setDefault, - loadingDefaultView, - currentView, - defaultViewId, - ]); + }, [defaultViewId, loadDefaultView, setDefault, setLoadingDefaultView]); + + useEffect(() => { + if (loadingDefaultView || currentView || !shouldLoadDefault) { + return; + } + + loadDefaultViewIfSet(); + }, [loadDefaultViewIfSet, loadingDefaultView, currentView, shouldLoadDefault]); + + useEffect(() => { + if (currentView && urlState.viewId !== currentView.id && data) + setUrlState({ viewId: currentView.id }); + }, [urlState, setUrlState, currentView, defaultViewId, data]); + + useEffect(() => { + if (!currentView && !loading && data) { + const viewToSet = views.find((v) => v.id === urlState.viewId); + if (viewToSet) setCurrentView(viewToSet); + else loadDefaultViewIfSet(); + } + }, [loading, currentView, data, views, setCurrentView, loadDefaultViewIfSet, urlState.viewId]); return { views, @@ -260,3 +290,11 @@ export const useSavedView = (props: Props) => { export const SavedView = createContainer(useSavedView); export const [SavedViewProvider, useSavedViewContext] = SavedView; + +const encodeUrlState = (state: SavedViewUrlState) => { + return savedViewUrlStateRT.encode(state); +}; +const decodeUrlState = (value: unknown) => { + const state = pipe(savedViewUrlStateRT.decode(value), fold(constant(undefined), identity)); + return state; +}; diff --git a/x-pack/plugins/infra/public/pages/metrics/index.tsx b/x-pack/plugins/infra/public/pages/metrics/index.tsx index 8fd32bda7fbc8..240cb778275b1 100644 --- a/x-pack/plugins/infra/public/pages/metrics/index.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/index.tsx @@ -36,7 +36,7 @@ import { WaffleTimeProvider } from './inventory_view/hooks/use_waffle_time'; import { WaffleFiltersProvider } from './inventory_view/hooks/use_waffle_filters'; import { MetricsAlertDropdown } from '../../alerting/common/components/metrics_alert_dropdown'; -import { SavedView } from '../../containers/saved_view/saved_view'; +import { SavedViewProvider } from '../../containers/saved_view/saved_view'; import { AlertPrefillProvider } from '../../alerting/use_alert_prefill'; import { InfraMLCapabilitiesProvider } from '../../containers/ml/infra_ml_capabilities'; import { AnomalyDetectionFlyout } from './inventory_view/components/ml/anomaly_detection/anomaly_detection_flyout'; @@ -195,7 +195,7 @@ const PageContent = (props: { const { options } = useContext(MetricsExplorerOptionsContainer.Context); return ( - - + ); }; diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/index.tsx b/x-pack/plugins/infra/public/pages/metrics/inventory_view/index.tsx index 7123c022538e9..6b980d33c2559 100644 --- a/x-pack/plugins/infra/public/pages/metrics/inventory_view/index.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/index.tsx @@ -23,7 +23,7 @@ import { useTrackPageview } from '../../../../../observability/public'; import { useKibana } from '../../../../../../../src/plugins/kibana_react/public'; import { Layout } from './components/layout'; import { useLinkProps } from '../../../hooks/use_link_props'; -import { SavedView } from '../../../containers/saved_view/saved_view'; +import { SavedViewProvider } from '../../../containers/saved_view/saved_view'; import { DEFAULT_WAFFLE_VIEW_STATE } from './hooks/use_waffle_view_state'; import { useWaffleOptionsContext } from './hooks/use_waffle_options'; @@ -64,13 +64,13 @@ export const SnapshotPage = () => { ) : metricIndicesExist ? ( <> - - + ) : hasFailedLoadingSource ? ( From a9e6cff88d775c32dce93d76d7295c977382ccd9 Mon Sep 17 00:00:00 2001 From: Jason Stoltzfus Date: Wed, 10 Feb 2021 15:35:56 -0500 Subject: [PATCH 03/13] [App Search] Relevance Tuning logic listeners (#89461) --- .../components/relevance_tuning/constants.ts | 26 + .../relevance_tuning_logic.test.ts | 913 +++++++++++++++++- .../relevance_tuning_logic.ts | 377 +++++++- .../components/relevance_tuning/types.ts | 22 +- .../components/relevance_tuning/utils.test.ts | 147 +++ .../components/relevance_tuning/utils.ts | 63 ++ 6 files changed, 1512 insertions(+), 36 deletions(-) create mode 100644 x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/utils.test.ts create mode 100644 x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/utils.ts diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/constants.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/constants.ts index 3655c60bde3bf..211995b2a7d18 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/constants.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/constants.ts @@ -11,3 +11,29 @@ export const RELEVANCE_TUNING_TITLE = i18n.translate( 'xpack.enterpriseSearch.appSearch.engine.relevanceTuning.title', { defaultMessage: 'Relevance Tuning' } ); + +export const UPDATE_SUCCESS_MESSAGE = i18n.translate( + 'xpack.enterpriseSearch.appSearch.relevanceTuning.messages.updateSuccess', + { + defaultMessage: 'Relevance successfully tuned. The changes will impact your results shortly.', + } +); +export const DELETE_SUCCESS_MESSAGE = i18n.translate( + 'xpack.enterpriseSearch.appSearch.relevanceTuning.messages.deleteSuccess', + { + defaultMessage: + 'Relevance has been reset to default values. The change will impact your results shortly.', + } +); +export const RESET_CONFIRMATION_MESSAGE = i18n.translate( + 'xpack.enterpriseSearch.appSearch.relevanceTuning.messages.resetConfirmation', + { + defaultMessage: 'Are you sure you want to restore relevance defaults?', + } +); +export const DELETE_CONFIRMATION_MESSAGE = i18n.translate( + 'xpack.enterpriseSearch.appSearch.relevanceTuning.messages.deleteConfirmation', + { + defaultMessage: 'Are you sure you want to delete this boost?', + } +); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_logic.test.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_logic.test.ts index 7f7bce1b7ba95..194848bcfc86c 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_logic.test.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_logic.test.ts @@ -5,12 +5,18 @@ * 2.0. */ -import { LogicMounter } from '../../../__mocks__'; +import { LogicMounter, mockFlashMessageHelpers, mockHttpValues } from '../../../__mocks__'; -import { BoostType } from './types'; +import { nextTick } from '@kbn/test/jest'; + +import { Boost, BoostType } from './types'; import { RelevanceTuningLogic } from './'; +jest.mock('../engine', () => ({ + EngineLogic: { values: { engineName: 'test-engine' } }, +})); + describe('RelevanceTuningLogic', () => { const { mount } = new LogicMounter(RelevanceTuningLogic); @@ -32,13 +38,27 @@ describe('RelevanceTuningLogic', () => { schema, schemaConflicts, }; - const searchResults = [{}, {}]; + const searchResults = [ + { + id: { + raw: '1', + }, + _meta: { + id: '1', + score: 100, + engine: 'my-engine', + }, + }, + ]; const DEFAULT_VALUES = { dataLoading: true, schema: {}, schemaConflicts: {}, - searchSettings: {}, + searchSettings: { + boosts: {}, + search_fields: {}, + }, unsavedChanges: false, filterInputValue: '', query: '', @@ -188,6 +208,873 @@ describe('RelevanceTuningLogic', () => { }); }); }); + + describe('setSearchSettingsResponse', () => { + it('should set searchSettings state and unsavedChanges to false', () => { + mount({ + unsavedChanges: true, + }); + RelevanceTuningLogic.actions.setSearchSettingsResponse(searchSettings); + + expect(RelevanceTuningLogic.values).toEqual({ + ...DEFAULT_VALUES, + searchSettings, + unsavedChanges: false, + }); + }); + }); + }); + + describe('listeners', () => { + const { http } = mockHttpValues; + const { flashAPIErrors, setSuccessMessage } = mockFlashMessageHelpers; + let scrollToSpy: jest.SpyInstance; + let confirmSpy: jest.SpyInstance; + + const searchSettingsWithBoost = (boost: Boost) => ({ + ...searchSettings, + boosts: { + foo: [ + { + factor: 1, + type: 'functional', + }, + boost, + ], + }, + }); + + beforeAll(() => { + scrollToSpy = jest.spyOn(window, 'scrollTo').mockImplementation(() => true); + confirmSpy = jest.spyOn(window, 'confirm'); + }); + + afterAll(() => { + scrollToSpy.mockRestore(); + confirmSpy.mockRestore(); + }); + + describe('initializeRelevanceTuning', () => { + it('should make an API call and set state based on the normalized response', async () => { + mount(); + http.get.mockReturnValueOnce( + Promise.resolve({ + ...relevanceTuningProps, + searchSettings: { + ...relevanceTuningProps.searchSettings, + boosts: { + foo: [ + { + type: 'value' as BoostType, + factor: 5, + value: 5, + }, + ], + }, + }, + }) + ); + jest.spyOn(RelevanceTuningLogic.actions, 'onInitializeRelevanceTuning'); + + RelevanceTuningLogic.actions.initializeRelevanceTuning(); + await nextTick(); + + expect(http.get).toHaveBeenCalledWith( + '/api/app_search/engines/test-engine/search_settings/details' + ); + expect(RelevanceTuningLogic.actions.onInitializeRelevanceTuning).toHaveBeenCalledWith({ + ...relevanceTuningProps, + searchSettings: { + ...relevanceTuningProps.searchSettings, + boosts: { + foo: [ + { + type: 'value' as BoostType, + factor: 5, + value: ['5'], + }, + ], + }, + }, + }); + }); + + it('handles errors', async () => { + mount(); + http.get.mockReturnValueOnce(Promise.reject('error')); + + RelevanceTuningLogic.actions.initializeRelevanceTuning(); + await nextTick(); + + expect(flashAPIErrors).toHaveBeenCalledWith('error'); + }); + }); + + describe('getSearchResults', () => { + beforeAll(() => { + jest.useFakeTimers(); + }); + + afterAll(() => { + jest.useRealTimers(); + }); + + it('should make an API call and set state based on the response', async () => { + const searchSettingsWithNewBoostProp = { + boosts: { + foo: [ + { + type: 'value' as BoostType, + factor: 5, + newBoost: true, // This should be deleted before sent to the server + }, + ], + }, + }; + + const searchSettingsWithoutNewBoostProp = { + boosts: { + foo: [ + { + type: 'value' as BoostType, + factor: 5, + }, + ], + }, + }; + + mount({ + query: 'foo', + searchSettings: searchSettingsWithNewBoostProp, + }); + jest.spyOn(RelevanceTuningLogic.actions, 'setSearchResults'); + jest.spyOn(RelevanceTuningLogic.actions, 'setResultsLoading'); + http.post.mockReturnValueOnce( + Promise.resolve({ + results: searchResults, + }) + ); + + RelevanceTuningLogic.actions.getSearchResults(); + jest.runAllTimers(); + await nextTick(); + + expect(RelevanceTuningLogic.actions.setResultsLoading).toHaveBeenCalledWith(true); + expect(http.post).toHaveBeenCalledWith( + '/api/app_search/engines/test-engine/search_settings_search', + { + body: JSON.stringify(searchSettingsWithoutNewBoostProp), + query: { + query: 'foo', + }, + } + ); + expect(RelevanceTuningLogic.actions.setSearchResults).toHaveBeenCalledWith(searchResults); + }); + + it("won't send boosts or search_fields on the API call if there are none", async () => { + mount({ + query: 'foo', + searchSettings: { + searchField: {}, + boosts: {}, + }, + }); + jest.spyOn(RelevanceTuningLogic.actions, 'setSearchResults'); + http.post.mockReturnValueOnce( + Promise.resolve({ + results: searchResults, + }) + ); + + RelevanceTuningLogic.actions.getSearchResults(); + + jest.runAllTimers(); + await nextTick(); + + expect(http.post).toHaveBeenCalledWith( + '/api/app_search/engines/test-engine/search_settings_search', + { + body: '{}', + query: { + query: 'foo', + }, + } + ); + }); + + it('will call clearSearchResults if there is no query', async () => { + mount({ + query: '', + }); + jest.spyOn(RelevanceTuningLogic.actions, 'setSearchResults'); + jest.spyOn(RelevanceTuningLogic.actions, 'setResultsLoading'); + jest.spyOn(RelevanceTuningLogic.actions, 'clearSearchResults'); + + RelevanceTuningLogic.actions.getSearchResults(); + jest.runAllTimers(); + await nextTick(); + + expect(RelevanceTuningLogic.actions.clearSearchResults).toHaveBeenCalled(); + expect(RelevanceTuningLogic.actions.setSearchResults).not.toHaveBeenCalled(); + expect(RelevanceTuningLogic.actions.setResultsLoading).not.toHaveBeenCalled(); + }); + + it('handles errors', async () => { + mount({ + query: 'foo', + }); + http.post.mockReturnValueOnce(Promise.reject('error')); + RelevanceTuningLogic.actions.getSearchResults(); + + jest.runAllTimers(); + await nextTick(); + + expect(flashAPIErrors).toHaveBeenCalledWith('error'); + }); + }); + + describe('setSearchSettings', () => { + it('updates search results whenever search settings are changed', () => { + mount(); + jest.spyOn(RelevanceTuningLogic.actions, 'getSearchResults'); + + RelevanceTuningLogic.actions.setSearchSettings(searchSettings); + + expect(RelevanceTuningLogic.actions.getSearchResults).toHaveBeenCalled(); + }); + }); + + describe('onSearchSettingsSuccess', () => { + it('should save the response, trigger a new search, and then scroll to the top', () => { + mount(); + jest.spyOn(RelevanceTuningLogic.actions, 'setSearchSettingsResponse'); + jest.spyOn(RelevanceTuningLogic.actions, 'getSearchResults'); + + RelevanceTuningLogic.actions.onSearchSettingsSuccess(searchSettings); + + expect(RelevanceTuningLogic.actions.setSearchSettingsResponse).toHaveBeenCalledWith( + searchSettings + ); + expect(RelevanceTuningLogic.actions.getSearchResults).toHaveBeenCalled(); + expect(scrollToSpy).toHaveBeenCalledWith(0, 0); + }); + }); + + describe('onSearchSettingsError', () => { + it('scrolls to the top', () => { + mount(); + RelevanceTuningLogic.actions.onSearchSettingsError(); + expect(scrollToSpy).toHaveBeenCalledWith(0, 0); + }); + }); + + describe('updateSearchSettings', () => { + it('calls an API endpoint and handles success response', async () => { + const searchSettingsWithNewBoostProp = { + boosts: { + foo: [ + { + type: 'value' as BoostType, + factor: 5, + newBoost: true, // This should be deleted before sent to the server + }, + ], + }, + }; + + const searchSettingsWithoutNewBoostProp = { + boosts: { + foo: [ + { + type: 'value' as BoostType, + factor: 5, + }, + ], + }, + }; + mount({ + searchSettings: searchSettingsWithNewBoostProp, + }); + jest.spyOn(RelevanceTuningLogic.actions, 'onSearchSettingsSuccess'); + http.put.mockReturnValueOnce(Promise.resolve(searchSettingsWithoutNewBoostProp)); + + RelevanceTuningLogic.actions.updateSearchSettings(); + await nextTick(); + + expect(http.put).toHaveBeenCalledWith( + '/api/app_search/engines/test-engine/search_settings', + { + body: JSON.stringify(searchSettingsWithoutNewBoostProp), + } + ); + expect(setSuccessMessage).toHaveBeenCalledWith( + 'Relevance successfully tuned. The changes will impact your results shortly.' + ); + expect(RelevanceTuningLogic.actions.onSearchSettingsSuccess).toHaveBeenCalledWith( + searchSettingsWithoutNewBoostProp + ); + }); + + it('handles errors', async () => { + mount(); + jest.spyOn(RelevanceTuningLogic.actions, 'onSearchSettingsError'); + http.put.mockReturnValueOnce(Promise.reject('error')); + + RelevanceTuningLogic.actions.updateSearchSettings(); + await nextTick(); + + expect(flashAPIErrors).toHaveBeenCalledWith('error'); + expect(RelevanceTuningLogic.actions.onSearchSettingsError).toHaveBeenCalled(); + }); + }); + + describe('resetSearchSettings', () => { + it('calls and API endpoint, shows a success message, and saves the response', async () => { + mount(); + jest.spyOn(RelevanceTuningLogic.actions, 'onSearchSettingsSuccess'); + confirmSpy.mockImplementation(() => true); + http.post.mockReturnValueOnce(Promise.resolve(searchSettings)); + + RelevanceTuningLogic.actions.resetSearchSettings(); + await nextTick(); + + expect(http.post).toHaveBeenCalledWith( + '/api/app_search/engines/test-engine/search_settings/reset' + ); + expect(setSuccessMessage).toHaveBeenCalledWith( + 'Relevance has been reset to default values. The change will impact your results shortly.' + ); + expect(RelevanceTuningLogic.actions.onSearchSettingsSuccess).toHaveBeenCalledWith( + searchSettings + ); + }); + + it('does nothing if the user does not confirm', async () => { + mount(); + confirmSpy.mockImplementation(() => false); + + RelevanceTuningLogic.actions.resetSearchSettings(); + + expect(http.post).not.toHaveBeenCalledWith( + '/api/app_search/engines/test-engine/search_settings/reset' + ); + }); + + it('handles errors', async () => { + mount(); + jest.spyOn(RelevanceTuningLogic.actions, 'onSearchSettingsError'); + confirmSpy.mockImplementation(() => true); + http.post.mockReturnValueOnce(Promise.reject('error')); + + RelevanceTuningLogic.actions.resetSearchSettings(); + await nextTick(); + + expect(flashAPIErrors).toHaveBeenCalledWith('error'); + expect(RelevanceTuningLogic.actions.onSearchSettingsError).toHaveBeenCalled(); + }); + }); + + describe('toggleSearchField', () => { + it('updates search weight to 1 in search fields when enabling', () => { + mount({ + searchSettings: { + ...searchSettings, + search_fields: { + bar: { + weight: 1, + }, + }, + }, + }); + jest.spyOn(RelevanceTuningLogic.actions, 'setSearchSettings'); + + RelevanceTuningLogic.actions.toggleSearchField('foo', false); + + expect(RelevanceTuningLogic.actions.setSearchSettings).toHaveBeenCalledWith({ + ...searchSettings, + search_fields: { + bar: { + weight: 1, + }, + foo: { + weight: 1, + }, + }, + }); + }); + + it('removes fields from search fields when disabling', () => { + mount({ + searchSettings: { + ...searchSettings, + search_fields: { + bar: { + weight: 1, + }, + }, + }, + }); + jest.spyOn(RelevanceTuningLogic.actions, 'setSearchSettings'); + + RelevanceTuningLogic.actions.toggleSearchField('bar', true); + + expect(RelevanceTuningLogic.actions.setSearchSettings).toHaveBeenCalledWith({ + ...searchSettings, + }); + }); + }); + + describe('updateFieldWeight', () => { + it('updates the search weight in search fields', () => { + mount({ + searchSettings, + }); + jest.spyOn(RelevanceTuningLogic.actions, 'setSearchSettings'); + + RelevanceTuningLogic.actions.updateFieldWeight('foo', 3); + + expect(RelevanceTuningLogic.actions.setSearchSettings).toHaveBeenCalledWith({ + ...searchSettings, + search_fields: { + foo: { + weight: 3, + }, + }, + }); + }); + + it('will round decimal numbers', () => { + mount({ + searchSettings, + }); + jest.spyOn(RelevanceTuningLogic.actions, 'setSearchSettings'); + + RelevanceTuningLogic.actions.updateFieldWeight('foo', 3.9393939); + + expect(RelevanceTuningLogic.actions.setSearchSettings).toHaveBeenCalledWith({ + ...searchSettings, + search_fields: { + foo: { + weight: 3.9, + }, + }, + }); + }); + }); + + describe('addBoost', () => { + it('adds a boost of given type for the given field', () => { + mount({ + searchSettings: { + ...searchSettings, + boosts: { + foo: [ + { + factor: 2, + type: 'value', + }, + ], + }, + }, + }); + jest.spyOn(RelevanceTuningLogic.actions, 'setSearchSettings'); + + RelevanceTuningLogic.actions.addBoost('foo', 'functional'); + + expect(RelevanceTuningLogic.actions.setSearchSettings).toHaveBeenCalledWith({ + ...searchSettings, + boosts: { + foo: [ + { + factor: 2, + type: 'value', + }, + { + factor: 1, + newBoost: true, + type: 'functional', + }, + ], + }, + }); + }); + + it('works even if there are no boosts yet', () => { + mount({ + searchSettings: { + ...searchSettings, + boosts: {}, + }, + }); + jest.spyOn(RelevanceTuningLogic.actions, 'setSearchSettings'); + + RelevanceTuningLogic.actions.addBoost('foo', 'functional'); + + expect(RelevanceTuningLogic.actions.setSearchSettings).toHaveBeenCalledWith({ + ...searchSettings, + boosts: { + foo: [ + { + factor: 1, + newBoost: true, + type: 'functional', + }, + ], + }, + }); + }); + }); + + describe('deleteBoost', () => { + it('deletes the boost with the given name and index', () => { + mount({ + searchSettings: { + ...searchSettings, + boosts: { + foo: [ + { + factor: 1, + type: 'functional', + }, + { + factor: 2, + type: 'value', + }, + ], + }, + }, + }); + confirmSpy.mockImplementation(() => true); + jest.spyOn(RelevanceTuningLogic.actions, 'setSearchSettings'); + + RelevanceTuningLogic.actions.deleteBoost('foo', 1); + + expect(RelevanceTuningLogic.actions.setSearchSettings).toHaveBeenCalledWith({ + ...searchSettings, + boosts: { + foo: [ + { + factor: 1, + type: 'functional', + }, + ], + }, + }); + }); + + it('will delete they field key in boosts if this is the last boost or that field', () => { + mount({ + searchSettings: { + ...searchSettings, + boosts: { + foo: [ + { + factor: 1, + type: 'functional', + }, + ], + }, + }, + }); + confirmSpy.mockImplementation(() => true); + jest.spyOn(RelevanceTuningLogic.actions, 'setSearchSettings'); + + RelevanceTuningLogic.actions.deleteBoost('foo', 0); + + expect(RelevanceTuningLogic.actions.setSearchSettings).toHaveBeenCalledWith({ + ...searchSettings, + boosts: {}, + }); + }); + + it('will do nothing if the user does not confirm', () => { + mount({ + searchSettings: { + ...searchSettings, + boosts: { + foo: [ + { + factor: 1, + type: 'functional', + }, + ], + }, + }, + }); + confirmSpy.mockImplementation(() => false); + jest.spyOn(RelevanceTuningLogic.actions, 'setSearchSettings'); + + RelevanceTuningLogic.actions.deleteBoost('foo', 0); + + expect(RelevanceTuningLogic.actions.setSearchSettings).not.toHaveBeenCalled(); + }); + }); + + describe('updateBoostFactor', () => { + it('updates the boost factor of the target boost', () => { + mount({ + searchSettings: searchSettingsWithBoost({ + factor: 1, + type: 'functional', + }), + }); + jest.spyOn(RelevanceTuningLogic.actions, 'setSearchSettings'); + + RelevanceTuningLogic.actions.updateBoostFactor('foo', 1, 5); + + expect(RelevanceTuningLogic.actions.setSearchSettings).toHaveBeenCalledWith( + searchSettingsWithBoost({ + factor: 5, + type: 'functional', + }) + ); + }); + + it('will round decimal numbers', () => { + mount({ + searchSettings: searchSettingsWithBoost({ + factor: 1, + type: 'functional', + }), + }); + jest.spyOn(RelevanceTuningLogic.actions, 'setSearchSettings'); + + RelevanceTuningLogic.actions.updateBoostFactor('foo', 1, 5.293191); + + expect(RelevanceTuningLogic.actions.setSearchSettings).toHaveBeenCalledWith( + searchSettingsWithBoost({ + factor: 5.3, + type: 'functional', + }) + ); + }); + }); + + describe('updateBoostValue', () => { + it('will update the boost value and update search reuslts', () => { + mount({ + searchSettings: searchSettingsWithBoost({ + factor: 1, + type: 'functional', + value: ['a', 'b', 'c'], + }), + }); + jest.spyOn(RelevanceTuningLogic.actions, 'setSearchSettings'); + + RelevanceTuningLogic.actions.updateBoostValue('foo', 1, 1, 'a'); + + expect(RelevanceTuningLogic.actions.setSearchSettings).toHaveBeenCalledWith( + searchSettingsWithBoost({ + factor: 1, + type: 'functional', + value: ['a', 'a', 'c'], + }) + ); + }); + + it('will create a new array if no array exists yet for value', () => { + mount({ + searchSettings: searchSettingsWithBoost({ + factor: 1, + type: 'functional', + }), + }); + jest.spyOn(RelevanceTuningLogic.actions, 'setSearchSettings'); + + RelevanceTuningLogic.actions.updateBoostValue('foo', 1, 0, 'a'); + + expect(RelevanceTuningLogic.actions.setSearchSettings).toHaveBeenCalledWith( + searchSettingsWithBoost({ + factor: 1, + type: 'functional', + value: ['a'], + }) + ); + }); + }); + + describe('updateBoostCenter', () => { + it('will parse the provided provided value and set the center to that parsed value', () => { + mount({ + schema: { + foo: 'number', + }, + searchSettings: searchSettingsWithBoost({ + factor: 1, + type: 'proximity', + center: 1, + }), + }); + jest.spyOn(RelevanceTuningLogic.actions, 'setSearchSettings'); + + RelevanceTuningLogic.actions.updateBoostCenter('foo', 1, '4'); + + expect(RelevanceTuningLogic.actions.setSearchSettings).toHaveBeenCalledWith( + searchSettingsWithBoost({ + factor: 1, + type: 'proximity', + center: 4, + }) + ); + }); + }); + + describe('addBoostValue', () => { + it('will add an empty boost value', () => { + mount({ + searchSettings: searchSettingsWithBoost({ + factor: 1, + type: 'functional', + value: ['a'], + }), + }); + jest.spyOn(RelevanceTuningLogic.actions, 'setSearchSettings'); + + RelevanceTuningLogic.actions.addBoostValue('foo', 1); + + expect(RelevanceTuningLogic.actions.setSearchSettings).toHaveBeenCalledWith( + searchSettingsWithBoost({ + factor: 1, + type: 'functional', + value: ['a', ''], + }) + ); + }); + + it('will add two empty boost values if none exist yet', () => { + mount({ + searchSettings: searchSettingsWithBoost({ + factor: 1, + type: 'functional', + }), + }); + jest.spyOn(RelevanceTuningLogic.actions, 'setSearchSettings'); + + RelevanceTuningLogic.actions.addBoostValue('foo', 1); + + expect(RelevanceTuningLogic.actions.setSearchSettings).toHaveBeenCalledWith( + searchSettingsWithBoost({ + factor: 1, + type: 'functional', + value: ['', ''], + }) + ); + }); + + it('will still work if the boost index is out of range', () => { + mount({ + searchSettings: searchSettingsWithBoost({ + factor: 1, + type: 'functional', + value: ['a', ''], + }), + }); + jest.spyOn(RelevanceTuningLogic.actions, 'setSearchSettings'); + + RelevanceTuningLogic.actions.addBoostValue('foo', 10); + + expect(RelevanceTuningLogic.actions.setSearchSettings).toHaveBeenCalledWith( + searchSettingsWithBoost({ + factor: 1, + type: 'functional', + value: ['a', ''], + }) + ); + }); + }); + + describe('removeBoostValue', () => { + it('will remove a boost value', () => { + mount({ + searchSettings: searchSettingsWithBoost({ + factor: 1, + type: 'functional', + value: ['a', 'b', 'c'], + }), + }); + jest.spyOn(RelevanceTuningLogic.actions, 'setSearchSettings'); + + RelevanceTuningLogic.actions.removeBoostValue('foo', 1, 1); + + expect(RelevanceTuningLogic.actions.setSearchSettings).toHaveBeenCalledWith( + searchSettingsWithBoost({ + factor: 1, + type: 'functional', + value: ['a', 'c'], + }) + ); + }); + + it('will do nothing if boost values do not exist', () => { + mount({ + searchSettings: searchSettingsWithBoost({ + factor: 1, + type: 'functional', + }), + }); + jest.spyOn(RelevanceTuningLogic.actions, 'setSearchSettings'); + + RelevanceTuningLogic.actions.removeBoostValue('foo', 1, 1); + + expect(RelevanceTuningLogic.actions.setSearchSettings).not.toHaveBeenCalled(); + }); + }); + + describe('updateBoostSelectOption', () => { + it('will update the boost', () => { + mount({ + searchSettings: searchSettingsWithBoost({ + factor: 1, + type: 'functional', + }), + }); + jest.spyOn(RelevanceTuningLogic.actions, 'setSearchSettings'); + + RelevanceTuningLogic.actions.updateBoostSelectOption('foo', 1, 'function', 'exponential'); + + expect(RelevanceTuningLogic.actions.setSearchSettings).toHaveBeenCalledWith( + searchSettingsWithBoost({ + factor: 1, + type: 'functional', + function: 'exponential', + }) + ); + }); + + it('can also update operation', () => { + mount({ + searchSettings: searchSettingsWithBoost({ + factor: 1, + type: 'functional', + }), + }); + jest.spyOn(RelevanceTuningLogic.actions, 'setSearchSettings'); + + RelevanceTuningLogic.actions.updateBoostSelectOption('foo', 1, 'operation', 'add'); + + expect(RelevanceTuningLogic.actions.setSearchSettings).toHaveBeenCalledWith( + searchSettingsWithBoost({ + factor: 1, + type: 'functional', + operation: 'add', + }) + ); + }); + }); + + describe('updateSearchValue', () => { + it('should update the query then update search results', () => { + mount(); + jest.spyOn(RelevanceTuningLogic.actions, 'setSearchQuery'); + jest.spyOn(RelevanceTuningLogic.actions, 'getSearchResults'); + + RelevanceTuningLogic.actions.updateSearchValue('foo'); + + expect(RelevanceTuningLogic.actions.setSearchQuery).toHaveBeenCalledWith('foo'); + expect(RelevanceTuningLogic.actions.getSearchResults).toHaveBeenCalled(); + }); + }); }); describe('selectors', () => { @@ -253,24 +1140,6 @@ describe('RelevanceTuningLogic', () => { }); expect(RelevanceTuningLogic.values.filteredSchemaFields).toEqual(['bar', 'baz']); }); - - it('should return all schema fields if there is no filter applied', () => { - mount({ - filterTerm: '', - schema: { - id: 'string', - foo: 'string', - bar: 'string', - baz: 'string', - }, - }); - expect(RelevanceTuningLogic.values.filteredSchemaFields).toEqual([ - 'id', - 'foo', - 'bar', - 'baz', - ]); - }); }); describe('filteredSchemaFieldsWithConflicts', () => { diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_logic.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_logic.ts index d4ec5e37f6ce5..cd3d8b5686cc0 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_logic.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_logic.ts @@ -6,10 +6,28 @@ */ import { kea, MakeLogicType } from 'kea'; +import { omit, cloneDeep, isEmpty } from 'lodash'; +import { setSuccessMessage, flashAPIErrors } from '../../../shared/flash_messages'; +import { HttpLogic } from '../../../shared/http'; import { Schema, SchemaConflicts } from '../../../shared/types'; -import { SearchSettings } from './types'; +import { EngineLogic } from '../engine'; +import { Result } from '../result/types'; + +import { + UPDATE_SUCCESS_MESSAGE, + RESET_CONFIRMATION_MESSAGE, + DELETE_SUCCESS_MESSAGE, + DELETE_CONFIRMATION_MESSAGE, +} from './constants'; +import { BaseBoost, Boost, BoostType, SearchSettings } from './types'; +import { + filterIfTerm, + parseBoostCenter, + removeBoostStateProps, + normalizeBoostValues, +} from './utils'; interface RelevanceTuningProps { searchSettings: SearchSettings; @@ -22,15 +40,60 @@ interface RelevanceTuningActions { setSearchSettings(searchSettings: SearchSettings): { searchSettings: SearchSettings }; setFilterValue(value: string): string; setSearchQuery(value: string): string; - setSearchResults(searchResults: object[]): object[]; + setSearchResults(searchResults: Result[]): Result[]; setResultsLoading(resultsLoading: boolean): boolean; clearSearchResults(): void; resetSearchSettingsState(): void; dismissSchemaConflictCallout(): void; + initializeRelevanceTuning(): void; + getSearchResults(): void; + setSearchSettingsResponse(searchSettings: SearchSettings): { searchSettings: SearchSettings }; + onSearchSettingsSuccess(searchSettings: SearchSettings): { searchSettings: SearchSettings }; + onSearchSettingsError(): void; + updateSearchSettings(): void; + resetSearchSettings(): void; + toggleSearchField(name: string, disableField: boolean): { name: string; disableField: boolean }; + updateFieldWeight(name: string, weight: number): { name: string; weight: number }; + addBoost(name: string, type: BoostType): { name: string; type: BoostType }; + deleteBoost(name: string, index: number): { name: string; index: number }; + updateBoostFactor( + name: string, + index: number, + factor: number + ): { name: string; index: number; factor: number }; + updateBoostValue( + name: string, + boostIndex: number, + valueIndex: number, + value: string + ): { name: string; boostIndex: number; valueIndex: number; value: string }; + updateBoostCenter( + name: string, + boostIndex: number, + value: string | number + ): { name: string; boostIndex: number; value: string | number }; + addBoostValue(name: string, boostIndex: number): { name: string; boostIndex: number }; + removeBoostValue( + name: string, + boostIndex: number, + valueIndex: number + ): { name: string; boostIndex: number; valueIndex: number }; + updateBoostSelectOption( + name: string, + boostIndex: number, + optionType: keyof BaseBoost, + value: string + ): { + name: string; + boostIndex: number; + optionType: keyof BaseBoost; + value: string; + }; + updateSearchValue(query: string): string; } interface RelevanceTuningValues { - searchSettings: Partial; + searchSettings: SearchSettings; schema: Schema; schemaFields: string[]; schemaFieldsWithConflicts: string[]; @@ -43,15 +106,10 @@ interface RelevanceTuningValues { query: string; unsavedChanges: boolean; dataLoading: boolean; - searchResults: object[] | null; + searchResults: Result[] | null; resultsLoading: boolean; } -// If the user hasn't entered a filter, then we can skip filtering the array entirely -const filterIfTerm = (array: string[], filterTerm: string): string[] => { - return filterTerm === '' ? array : array.filter((item) => item.includes(filterTerm)); -}; - export const RelevanceTuningLogic = kea< MakeLogicType >({ @@ -66,13 +124,47 @@ export const RelevanceTuningLogic = kea< clearSearchResults: true, resetSearchSettingsState: true, dismissSchemaConflictCallout: true, + initializeRelevanceTuning: true, + getSearchResults: true, + setSearchSettingsResponse: (searchSettings) => ({ + searchSettings, + }), + onSearchSettingsSuccess: (searchSettings) => ({ searchSettings }), + onSearchSettingsError: () => true, + updateSearchSettings: true, + resetSearchSettings: true, + toggleSearchField: (name, disableField) => ({ name, disableField }), + updateFieldWeight: (name, weight) => ({ name, weight }), + addBoost: (name, type) => ({ name, type }), + deleteBoost: (name, index) => ({ name, index }), + updateBoostFactor: (name, index, factor) => ({ name, index, factor }), + updateBoostValue: (name, boostIndex, valueIndex, value) => ({ + name, + boostIndex, + valueIndex, + value, + }), + updateBoostCenter: (name, boostIndex, value) => ({ name, boostIndex, value }), + addBoostValue: (name, boostIndex) => ({ name, boostIndex }), + removeBoostValue: (name, boostIndex, valueIndex) => ({ name, boostIndex, valueIndex }), + updateBoostSelectOption: (name, boostIndex, optionType, value) => ({ + name, + boostIndex, + optionType, + value, + }), + updateSearchValue: (query) => query, }), reducers: () => ({ searchSettings: [ - {}, + { + search_fields: {}, + boosts: {}, + }, { onInitializeRelevanceTuning: (_, { searchSettings }) => searchSettings, setSearchSettings: (_, { searchSettings }) => searchSettings, + setSearchSettingsResponse: (_, { searchSettings }) => searchSettings, }, ], schema: [ @@ -109,6 +201,7 @@ export const RelevanceTuningLogic = kea< false, { setSearchSettings: () => true, + setSearchSettingsResponse: () => false, }, ], @@ -155,4 +248,268 @@ export const RelevanceTuningLogic = kea< (schema: Schema): boolean => Object.keys(schema).length >= 2, ], }), + listeners: ({ actions, values }) => ({ + initializeRelevanceTuning: async () => { + const { http } = HttpLogic.values; + const { engineName } = EngineLogic.values; + + const url = `/api/app_search/engines/${engineName}/search_settings/details`; + + try { + const response = await http.get(url); + actions.onInitializeRelevanceTuning({ + ...response, + searchSettings: { + ...response.searchSettings, + boosts: normalizeBoostValues(response.searchSettings.boosts), + }, + }); + } catch (e) { + flashAPIErrors(e); + } + }, + getSearchResults: async (_, breakpoint) => { + await breakpoint(250); + + const query = values.query; + if (!query) return actions.clearSearchResults(); + + const { engineName } = EngineLogic.values; + const { http } = HttpLogic.values; + const { search_fields: searchFields, boosts } = removeBoostStateProps(values.searchSettings); + const url = `/api/app_search/engines/${engineName}/search_settings_search`; + + actions.setResultsLoading(true); + + try { + const response = await http.post(url, { + query: { + query, + }, + body: JSON.stringify({ + boosts: isEmpty(boosts) ? undefined : boosts, + search_fields: isEmpty(searchFields) ? undefined : searchFields, + }), + }); + + actions.setSearchResults(response.results); + } catch (e) { + flashAPIErrors(e); + } + }, + setSearchSettings: () => { + actions.getSearchResults(); + }, + onSearchSettingsSuccess: ({ searchSettings }) => { + actions.setSearchSettingsResponse(searchSettings); + actions.getSearchResults(); + window.scrollTo(0, 0); + }, + onSearchSettingsError: () => { + window.scrollTo(0, 0); + }, + updateSearchSettings: async () => { + const { http } = HttpLogic.values; + const { engineName } = EngineLogic.values; + + const url = `/api/app_search/engines/${engineName}/search_settings`; + + try { + const response = await http.put(url, { + body: JSON.stringify(removeBoostStateProps(values.searchSettings)), + }); + setSuccessMessage(UPDATE_SUCCESS_MESSAGE); + actions.onSearchSettingsSuccess(response); + } catch (e) { + flashAPIErrors(e); + actions.onSearchSettingsError(); + } + }, + resetSearchSettings: async () => { + if (window.confirm(RESET_CONFIRMATION_MESSAGE)) { + const { http } = HttpLogic.values; + const { engineName } = EngineLogic.values; + + const url = `/api/app_search/engines/${engineName}/search_settings/reset`; + + try { + const response = await http.post(url); + setSuccessMessage(DELETE_SUCCESS_MESSAGE); + actions.onSearchSettingsSuccess(response); + } catch (e) { + flashAPIErrors(e); + actions.onSearchSettingsError(); + } + } + }, + toggleSearchField: ({ name, disableField }) => { + const { searchSettings } = values; + + const searchFields = disableField + ? omit(searchSettings.search_fields, name) + : { ...searchSettings.search_fields, [name]: { weight: 1 } }; + + actions.setSearchSettings({ + ...searchSettings, + search_fields: searchFields, + }); + }, + updateFieldWeight: ({ name, weight }) => { + const { searchSettings } = values; + const { search_fields: searchFields } = searchSettings; + + actions.setSearchSettings({ + ...searchSettings, + search_fields: { + ...searchFields, + [name]: { + ...searchFields[name], + weight: Math.round(weight * 10) / 10, + }, + }, + }); + }, + addBoost: ({ name, type }) => { + const { searchSettings } = values; + const { boosts } = searchSettings; + const emptyBoost = { type, factor: 1, newBoost: true }; + let boostArray; + + if (Array.isArray(boosts[name])) { + boostArray = boosts[name].slice(); + boostArray.push(emptyBoost); + } else { + boostArray = [emptyBoost]; + } + + actions.setSearchSettings({ + ...searchSettings, + boosts: { + ...boosts, + [name]: boostArray, + }, + }); + }, + deleteBoost: ({ name, index }) => { + if (window.confirm(DELETE_CONFIRMATION_MESSAGE)) { + const { searchSettings } = values; + const { boosts } = searchSettings; + const boostsRemoved = boosts[name].slice(); + boostsRemoved.splice(index, 1); + const updatedBoosts = { ...boosts }; + + if (boostsRemoved.length > 0) { + updatedBoosts[name] = boostsRemoved; + } else { + delete updatedBoosts[name]; + } + + actions.setSearchSettings({ + ...searchSettings, + boosts: updatedBoosts, + }); + } + }, + updateBoostFactor: ({ name, index, factor }) => { + const { searchSettings } = values; + const { boosts } = searchSettings; + const updatedBoosts = cloneDeep(boosts[name]); + updatedBoosts[index].factor = Math.round(factor * 10) / 10; + + actions.setSearchSettings({ + ...searchSettings, + boosts: { + ...boosts, + [name]: updatedBoosts, + }, + }); + }, + updateBoostValue: ({ name, boostIndex, valueIndex, value }) => { + const { searchSettings } = values; + const { boosts } = searchSettings; + const updatedBoosts: Boost[] = cloneDeep(boosts[name]); + const existingValue = updatedBoosts[boostIndex].value; + if (existingValue === undefined) { + updatedBoosts[boostIndex].value = [value]; + } else { + existingValue[valueIndex] = value; + } + + actions.setSearchSettings({ + ...searchSettings, + boosts: { + ...boosts, + [name]: updatedBoosts, + }, + }); + }, + updateBoostCenter: ({ name, boostIndex, value }) => { + const { searchSettings } = values; + const { boosts } = searchSettings; + const updatedBoosts = cloneDeep(boosts[name]); + const fieldType = values.schema[name]; + updatedBoosts[boostIndex].center = parseBoostCenter(fieldType, value); + + actions.setSearchSettings({ + ...searchSettings, + boosts: { + ...boosts, + [name]: updatedBoosts, + }, + }); + }, + addBoostValue: ({ name, boostIndex }) => { + const { searchSettings } = values; + const { boosts } = searchSettings; + const updatedBoosts = cloneDeep(boosts[name]); + const updatedBoost = updatedBoosts[boostIndex]; + if (updatedBoost) { + updatedBoost.value = Array.isArray(updatedBoost.value) ? updatedBoost.value : ['']; + updatedBoost.value.push(''); + } + + actions.setSearchSettings({ + ...searchSettings, + boosts: { + ...boosts, + [name]: updatedBoosts, + }, + }); + }, + removeBoostValue: ({ name, boostIndex, valueIndex }) => { + const { searchSettings } = values; + const { boosts } = searchSettings; + const updatedBoosts = cloneDeep(boosts[name]); + const boostValue = updatedBoosts[boostIndex].value; + + if (boostValue === undefined) return; + + boostValue.splice(valueIndex, 1); + actions.setSearchSettings({ + ...searchSettings, + boosts: { + ...boosts, + [name]: updatedBoosts, + }, + }); + }, + updateBoostSelectOption: ({ name, boostIndex, optionType, value }) => { + const { searchSettings } = values; + const { boosts } = searchSettings; + const updatedBoosts = cloneDeep(boosts[name]); + updatedBoosts[boostIndex][optionType] = value; + + actions.setSearchSettings({ + ...searchSettings, + boosts: { + ...boosts, + [name]: updatedBoosts, + }, + }); + }, + updateSearchValue: (query) => { + actions.setSearchQuery(query); + actions.getSearchResults(); + }, + }), }); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/types.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/types.ts index 25187df89d64c..a1ed9797b9f5a 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/types.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/types.ts @@ -7,17 +7,31 @@ export type BoostType = 'value' | 'functional' | 'proximity'; -export interface Boost { - type: BoostType; +export interface BaseBoost { operation?: string; function?: string; +} + +// A boost that comes from the server, before we normalize it has a much looser schema +export interface RawBoost extends BaseBoost { + type: BoostType; newBoost?: boolean; center?: string | number; - value?: string | number | string[] | number[]; + value?: string | number | boolean | object | Array; factor: number; } +// We normalize raw boosts to make them safer and easier to work with +export interface Boost extends RawBoost { + value?: string[]; +} export interface SearchSettings { boosts: Record; - search_fields: object; + search_fields: Record< + string, + { + weight: number; + } + >; + result_fields?: object; } diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/utils.test.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/utils.test.ts new file mode 100644 index 0000000000000..a6598bf991c13 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/utils.test.ts @@ -0,0 +1,147 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { BoostType } from './types'; +import { + filterIfTerm, + normalizeBoostValues, + removeBoostStateProps, + parseBoostCenter, +} from './utils'; + +describe('filterIfTerm', () => { + it('will filter a list of strings to a list of strings containing the specified string', () => { + expect(filterIfTerm(['jalepeno', 'no', 'not', 'panorama', 'truck'], 'no')).toEqual([ + 'jalepeno', + 'no', + 'not', + 'panorama', + ]); + }); + + it('will not filter at all if an empty string is provided', () => { + expect(filterIfTerm(['jalepeno', 'no', 'not', 'panorama', 'truck'], '')).toEqual([ + 'jalepeno', + 'no', + 'not', + 'panorama', + 'truck', + ]); + }); +}); + +describe('removeBoostStateProps', () => { + it('will remove the newBoost flag from boosts within the provided searchSettings object', () => { + const searchSettings = { + boosts: { + foo: [ + { + type: 'value' as BoostType, + factor: 5, + newBoost: true, + }, + ], + }, + search_fields: { + foo: { + weight: 1, + }, + }, + }; + expect(removeBoostStateProps(searchSettings)).toEqual({ + ...searchSettings, + boosts: { + foo: [ + { + type: 'value' as BoostType, + factor: 5, + }, + ], + }, + }); + }); +}); + +describe('parseBoostCenter', () => { + it('should parse a boost center', () => { + expect(parseBoostCenter('text', 5)).toEqual(5); + expect(parseBoostCenter('text', '4')).toEqual('4'); + expect(parseBoostCenter('number', 5)).toEqual(5); + expect(parseBoostCenter('number', '5')).toEqual(5); + }); +}); + +describe('normalizeBoostValues', () => { + const boosts = { + foo: [ + { + type: 'value' as BoostType, + factor: 9.5, + value: 1, + }, + { + type: 'value' as BoostType, + factor: 9.5, + value: '1', + }, + { + type: 'value' as BoostType, + factor: 9.5, + value: [1], + }, + { + type: 'value' as BoostType, + factor: 9.5, + value: ['1'], + }, + { + type: 'value' as BoostType, + factor: 9.5, + value: [ + '1', + 1, + '2', + 2, + true, + { + b: 'a', + }, + {}, + ], + }, + ], + bar: [ + { + type: 'proximity' as BoostType, + factor: 9.5, + }, + ], + sp_def: [ + { + type: 'functional' as BoostType, + factor: 5, + }, + ], + }; + + it('converts all value types to string for consistency', () => { + expect(normalizeBoostValues(boosts)).toEqual({ + bar: [{ factor: 9.5, type: 'proximity' }], + foo: [ + { factor: 9.5, type: 'value', value: ['1'] }, + { factor: 9.5, type: 'value', value: ['1'] }, + { factor: 9.5, type: 'value', value: ['1'] }, + { factor: 9.5, type: 'value', value: ['1'] }, + { + factor: 9.5, + type: 'value', + value: ['1', '1', '2', '2', 'true', '[object Object]', '[object Object]'], + }, + ], + sp_def: [{ type: 'functional', factor: 5 }], + }); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/utils.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/utils.ts new file mode 100644 index 0000000000000..e2fd0f0bbd656 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/utils.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { cloneDeep, omit } from 'lodash'; + +import { NUMBER } from '../../../shared/constants/field_types'; +import { SchemaTypes } from '../../../shared/types'; + +import { RawBoost, Boost, SearchSettings, BoostType } from './types'; + +// If the user hasn't entered a filter, then we can skip filtering the array entirely +export const filterIfTerm = (array: string[], filterTerm: string): string[] => { + return filterTerm === '' ? array : array.filter((item) => item.includes(filterTerm)); +}; + +export const removeBoostStateProps = (searchSettings: SearchSettings) => { + const updatedSettings = cloneDeep(searchSettings); + const { boosts } = updatedSettings; + const keys = Object.keys(boosts); + keys.forEach((key) => boosts[key].forEach((boost) => delete boost.newBoost)); + + return updatedSettings; +}; + +export const parseBoostCenter = (fieldType: SchemaTypes, value: string | number) => { + // Leave non-numeric fields alone + if (fieldType === NUMBER) { + const floatValue = parseFloat(value as string); + return isNaN(floatValue) ? value : floatValue; + } + return value; +}; + +const toArray = (v: T | T[]): T[] => (Array.isArray(v) ? v : [v]); +const toString = (v1: T) => String(v1); + +const normalizeBoostValue = (boost: RawBoost): Boost => { + if (!boost.hasOwnProperty('value')) { + // Can't simply do `return boost` here as TS can't infer the correct type + return omit(boost, 'value'); + } + + return { + ...boost, + type: boost.type as BoostType, + value: toArray(boost.value).map(toString), + }; +}; + +// Data may have been set to invalid types directly via the public App Search API. Since these are invalid, we don't want to deal +// with them as valid types in the UI. For that reason, we stringify all values here, as the data comes in. +// Additionally, values can be in single values or in arrays. +export const normalizeBoostValues = (boosts: Record): Record => + Object.entries(boosts).reduce((newBoosts, [fieldName, boostList]) => { + return { + ...newBoosts, + [fieldName]: boostList.map(normalizeBoostValue), + }; + }, {}); From b18e4ec9b8638113e27c2974fd20af2c6df8b7b7 Mon Sep 17 00:00:00 2001 From: gchaps <33642766+gchaps@users.noreply.github.com> Date: Wed, 10 Feb 2021 13:05:33 -0800 Subject: [PATCH 04/13] [DOCS] Uses variable to refer to query profiler (#90976) --- docs/dev-tools/searchprofiler/getting-started.asciidoc | 2 +- docs/user/dev-tools.asciidoc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/dev-tools/searchprofiler/getting-started.asciidoc b/docs/dev-tools/searchprofiler/getting-started.asciidoc index 7cd54db5562b7..ad73d03bcbfd8 100644 --- a/docs/dev-tools/searchprofiler/getting-started.asciidoc +++ b/docs/dev-tools/searchprofiler/getting-started.asciidoc @@ -2,7 +2,7 @@ [[profiler-getting-started]] === Getting Started -The {searchprofiler} is automatically enabled in {kib}. Open the main menu, click *Dev Tools*, then click *Search Profiler* +The {searchprofiler} is automatically enabled in {kib}. Open the main menu, click *Dev Tools*, then click *{searchprofiler}* to get started. {searchprofiler} displays the names of the indices searched, the shards in each index, diff --git a/docs/user/dev-tools.asciidoc b/docs/user/dev-tools.asciidoc index 0ee7fbc741e00..0c5bef489dd01 100644 --- a/docs/user/dev-tools.asciidoc +++ b/docs/user/dev-tools.asciidoc @@ -15,7 +15,7 @@ a| <> | Interact with the REST API of Elasticsearch, including sending requests and viewing API documentation. -a| <> +a| <> | Inspect and analyze your search queries. From 4cd0548f482876e2a0822926d985137d59a8f7e8 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Wed, 10 Feb 2021 16:42:00 -0500 Subject: [PATCH 05/13] [CI] Fix auto-backport condditions so that it doesn't trigger for other labels (#91042) --- .github/workflows/backport.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index f64b9e95fbaab..238a21161b129 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -9,9 +9,14 @@ on: jobs: backport: name: Backport PR - if: github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'auto-backport') + if: | + github.event.pull_request.merged == true + && contains(github.event.pull_request.labels.*.name, 'auto-backport') + && ( + (github.event.action == 'labeled' && github.event.label.name == 'auto-backport') + || (github.event.action == 'closed') + ) runs-on: ubuntu-latest - steps: - name: 'Get backport config' run: | From c92af5a4d5bf3bab6b1dbbf5eb7250111b001aa3 Mon Sep 17 00:00:00 2001 From: John Schulz Date: Wed, 10 Feb 2021 17:04:01 -0500 Subject: [PATCH 06/13] [Fleet] Restrict integration changes for managed policies (#90675) ## Summary - [x] Integrations cannot be added ~~, unless with a force flag~~ - [x] API - [x] UI - [x] tests - [x] Integrations cannot be removed ~~, unless with a force flag~~ - [x] API - [x] UI - [x] tests closes https://github.com/elastic/kibana/issues/90445 refs https://github.com/elastic/kibana/issues/89617 ### Cannot add integrations to managed policy Screen Shot 2021-02-08 at 1 56 32 PM ### Cannot delete integrations from managed policy Screen Shot 2021-02-08 at 3 05 16 PM ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../server/routes/package_policy/handlers.ts | 5 +- .../fleet/server/services/agent_policy.ts | 14 +- .../fleet/server/services/package_policy.ts | 30 +++-- .../test/fleet_api_integration/apis/index.js | 1 + .../apis/package_policy/create.ts | 48 ++++++- .../apis/package_policy/delete.ts | 127 ++++++++++++++++++ .../apis/package_policy/update.ts | 48 ++++++- 7 files changed, 257 insertions(+), 16 deletions(-) create mode 100644 x-pack/test/fleet_api_integration/apis/package_policy/delete.ts diff --git a/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts b/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts index ef0c34ee56393..6b35f74b3febc 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts @@ -79,11 +79,10 @@ export const createPackagePolicyHandler: RequestHandler< const esClient = context.core.elasticsearch.client.asCurrentUser; const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; const user = (await appContextService.getSecurity()?.authc.getCurrentUser(request)) || undefined; - let newData = { ...request.body }; try { - newData = await packagePolicyService.runExternalCallbacks( + const newData = await packagePolicyService.runExternalCallbacks( 'packagePolicyCreate', - newData, + { ...request.body }, context, request ); diff --git a/x-pack/plugins/fleet/server/services/agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policy.ts index 9800ddf95f7b2..31e9a63175d18 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.ts @@ -36,7 +36,11 @@ import { FleetServerPolicy, AGENT_POLICY_INDEX, } from '../../common'; -import { AgentPolicyNameExistsError, AgentPolicyDeletionError } from '../errors'; +import { + AgentPolicyNameExistsError, + AgentPolicyDeletionError, + IngestManagerError, +} from '../errors'; import { createAgentPolicyAction, listAgents } from './agents'; import { packagePolicyService } from './package_policy'; import { outputService } from './output'; @@ -382,6 +386,10 @@ class AgentPolicyService { throw new Error('Agent policy not found'); } + if (oldAgentPolicy.is_managed) { + throw new IngestManagerError(`Cannot update integrations of managed policy ${id}`); + } + return await this._update( soClient, esClient, @@ -409,6 +417,10 @@ class AgentPolicyService { throw new Error('Agent policy not found'); } + if (oldAgentPolicy.is_managed) { + throw new IngestManagerError(`Cannot remove integrations of managed policy ${id}`); + } + return await this._update( soClient, esClient, diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index 4b04014b20969..8d1ac90f3ec15 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -25,6 +25,7 @@ import { doesAgentPolicyAlreadyIncludePackage, } from '../../common'; import { PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '../constants'; +import { IngestManagerError, ingestErrorToResponseOptions } from '../errors'; import { NewPackagePolicy, UpdatePackagePolicy, @@ -63,15 +64,20 @@ class PackagePolicyService { const parentAgentPolicy = await agentPolicyService.get(soClient, packagePolicy.policy_id); if (!parentAgentPolicy) { throw new Error('Agent policy not found'); - } else { - if ( - (parentAgentPolicy.package_policies as PackagePolicy[]).find( - (siblingPackagePolicy) => siblingPackagePolicy.name === packagePolicy.name - ) - ) { - throw new Error('There is already a package with the same name on this agent policy'); - } } + if (parentAgentPolicy.is_managed) { + throw new IngestManagerError( + `Cannot add integrations to managed policy ${parentAgentPolicy.id}` + ); + } + if ( + (parentAgentPolicy.package_policies as PackagePolicy[]).find( + (siblingPackagePolicy) => siblingPackagePolicy.name === packagePolicy.name + ) + ) { + throw new Error('There is already a package with the same name on this agent policy'); + } + // Add ids to stream const packagePolicyId = options?.id || uuid.v4(); let inputs: PackagePolicyInput[] = packagePolicy.inputs.map((input) => @@ -285,6 +291,9 @@ class PackagePolicyService { if (!parentAgentPolicy) { throw new Error('Agent policy not found'); } else { + if (parentAgentPolicy.is_managed) { + throw new IngestManagerError(`Cannot update integrations of managed policy ${id}`); + } if ( (parentAgentPolicy.package_policies as PackagePolicy[]).find( (siblingPackagePolicy) => @@ -295,7 +304,7 @@ class PackagePolicyService { } } - let inputs = await restOfPackagePolicy.inputs.map((input) => + let inputs = restOfPackagePolicy.inputs.map((input) => assignStreamIdToInput(oldPackagePolicy.id, input) ); @@ -363,10 +372,11 @@ class PackagePolicyService { name: packagePolicy.name, success: true, }); - } catch (e) { + } catch (error) { result.push({ id, success: false, + ...ingestErrorToResponseOptions(error), }); } } diff --git a/x-pack/test/fleet_api_integration/apis/index.js b/x-pack/test/fleet_api_integration/apis/index.js index 8c66db9c418ea..44431795a34ba 100644 --- a/x-pack/test/fleet_api_integration/apis/index.js +++ b/x-pack/test/fleet_api_integration/apis/index.js @@ -37,6 +37,7 @@ export default function ({ loadTestFile }) { loadTestFile(require.resolve('./package_policy/create')); loadTestFile(require.resolve('./package_policy/update')); loadTestFile(require.resolve('./package_policy/get')); + loadTestFile(require.resolve('./package_policy/delete')); // Agent policies loadTestFile(require.resolve('./agent_policy/index')); diff --git a/x-pack/test/fleet_api_integration/apis/package_policy/create.ts b/x-pack/test/fleet_api_integration/apis/package_policy/create.ts index 8e339bc78b087..c9c871e280f16 100644 --- a/x-pack/test/fleet_api_integration/apis/package_policy/create.ts +++ b/x-pack/test/fleet_api_integration/apis/package_policy/create.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - +import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; import { warnAndSkipTest } from '../../helpers'; @@ -39,6 +39,52 @@ export default function ({ getService }: FtrProviderContext) { .send({ agentPolicyId }); }); + it('should fail for managed agent policies', async function () { + if (server.enabled) { + // get a managed policy + const { + body: { item: managedPolicy }, + } = await supertest + .post(`/api/fleet/agent_policies`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: `Managed policy from ${Date.now()}`, + namespace: 'default', + is_managed: true, + }); + + // try to add an integration to the managed policy + const { body } = await supertest + .post(`/api/fleet/package_policies`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'filetest-1', + description: '', + namespace: 'default', + policy_id: managedPolicy.id, + enabled: true, + output_id: '', + inputs: [], + package: { + name: 'filetest', + title: 'For File Tests', + version: '0.1.0', + }, + }) + .expect(400); + + expect(body.statusCode).to.be(400); + expect(body.message).to.contain('Cannot add integrations to managed policy'); + + // delete policy we just made + await supertest.post(`/api/fleet/agent_policies/delete`).set('kbn-xsrf', 'xxxx').send({ + agentPolicyId: managedPolicy.id, + }); + } else { + warnAndSkipTest(this, log); + } + }); + it('should work with valid values', async function () { if (server.enabled) { await supertest diff --git a/x-pack/test/fleet_api_integration/apis/package_policy/delete.ts b/x-pack/test/fleet_api_integration/apis/package_policy/delete.ts new file mode 100644 index 0000000000000..e64ba8580d145 --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/package_policy/delete.ts @@ -0,0 +1,127 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; +import { skipIfNoDockerRegistry } from '../../helpers'; + +export default function (providerContext: FtrProviderContext) { + const { getService } = providerContext; + const supertest = getService('supertest'); + + // use function () {} and not () => {} here + // because `this` has to point to the Mocha context + // see https://mochajs.org/#arrow-functions + + describe('Package Policy - delete', async function () { + skipIfNoDockerRegistry(providerContext); + let agentPolicy: any; + let packagePolicy: any; + + before(async function () { + let agentPolicyResponse = await supertest + .post(`/api/fleet/agent_policies`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Test policy', + namespace: 'default', + is_managed: false, + }); + + // if one already exists, re-use that + if (agentPolicyResponse.body.statusCode === 409) { + const errorRegex = /^agent policy \'(?[\w,\-]+)\' already exists/i; + const result = errorRegex.exec(agentPolicyResponse.body.message); + if (result?.groups?.id) { + agentPolicyResponse = await supertest + .put(`/api/fleet/agent_policies/${result.groups.id}`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Test policy', + namespace: 'default', + is_managed: false, + }); + } + } + agentPolicy = agentPolicyResponse.body.item; + + const { body: packagePolicyResponse } = await supertest + .post(`/api/fleet/package_policies`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'filetest-1', + description: '', + namespace: 'default', + policy_id: agentPolicy.id, + enabled: true, + output_id: '', + inputs: [], + package: { + name: 'filetest', + title: 'For File Tests', + version: '0.1.0', + }, + }); + packagePolicy = packagePolicyResponse.item; + }); + + after(async function () { + await supertest + .post(`/api/fleet/agent_policies/delete`) + .set('kbn-xsrf', 'xxxx') + .send({ agentPolicyId: agentPolicy.id }); + + await supertest + .post(`/api/fleet/package_policies/delete`) + .set('kbn-xsrf', 'xxxx') + .send({ packagePolicyIds: [packagePolicy.id] }); + }); + + it('should fail on managed agent policies', async function () { + // update existing policy to managed + await supertest + .put(`/api/fleet/agent_policies/${agentPolicy.id}`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: agentPolicy.name, + namespace: agentPolicy.namespace, + is_managed: true, + }) + .expect(200); + + // try to delete + const { body: results } = await supertest + .post(`/api/fleet/package_policies/delete`) + .set('kbn-xsrf', 'xxxx') + .send({ packagePolicyIds: [packagePolicy.id] }) + .expect(200); + + // delete always succeeds (returns 200) with Array<{success: boolean}> + expect(Array.isArray(results)); + expect(results.length).to.be(1); + expect(results[0].success).to.be(false); + expect(results[0].body.message).to.contain('Cannot remove integrations of managed policy'); + + // revert existing policy to unmanaged + await supertest + .put(`/api/fleet/agent_policies/${agentPolicy.id}`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: agentPolicy.name, + namespace: agentPolicy.namespace, + is_managed: false, + }) + .expect(200); + }); + + it('should work for unmanaged policies', async function () { + await supertest + .post(`/api/fleet/package_policies/delete`) + .set('kbn-xsrf', 'xxxx') + .send({ packagePolicyIds: [packagePolicy.id] }); + }); + }); +} diff --git a/x-pack/test/fleet_api_integration/apis/package_policy/update.ts b/x-pack/test/fleet_api_integration/apis/package_policy/update.ts index e0dc1a5d96b4b..9a70c6ad004dd 100644 --- a/x-pack/test/fleet_api_integration/apis/package_policy/update.ts +++ b/x-pack/test/fleet_api_integration/apis/package_policy/update.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - +import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; import { skipIfNoDockerRegistry } from '../../helpers'; @@ -21,6 +21,7 @@ export default function (providerContext: FtrProviderContext) { describe('Package Policy - update', async function () { skipIfNoDockerRegistry(providerContext); let agentPolicyId: string; + let managedAgentPolicyId: string; let packagePolicyId: string; let packagePolicyId2: string; @@ -35,8 +36,30 @@ export default function (providerContext: FtrProviderContext) { name: 'Test policy', namespace: 'default', }); + agentPolicyId = agentPolicyResponse.item.id; + const { body: managedAgentPolicyResponse } = await supertest + .post(`/api/fleet/agent_policies`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Test managed policy', + namespace: 'default', + is_managed: true, + }); + + // if one already exists, re-use that + const managedExists = managedAgentPolicyResponse.statusCode === 409; + if (managedExists) { + const errorRegex = /^agent policy \'(?[\w,\-]+)\' already exists/i; + const result = errorRegex.exec(managedAgentPolicyResponse.message); + if (result?.groups?.id) { + managedAgentPolicyId = result.groups.id; + } + } else { + managedAgentPolicyId = managedAgentPolicyResponse.item.id; + } + const { body: packagePolicyResponse } = await supertest .post(`/api/fleet/package_policies`) .set('kbn-xsrf', 'xxxx') @@ -83,6 +106,29 @@ export default function (providerContext: FtrProviderContext) { .send({ agentPolicyId }); }); + it('should fail on managed agent policies', async function () { + const { body } = await supertest + .put(`/api/fleet/package_policies/${packagePolicyId}`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'filetest-1', + description: '', + namespace: 'updated_namespace', + policy_id: managedAgentPolicyId, + enabled: true, + output_id: '', + inputs: [], + package: { + name: 'filetest', + title: 'For File Tests', + version: '0.1.0', + }, + }) + .expect(400); + + expect(body.message).to.contain('Cannot update integrations of managed policy'); + }); + it('should work with valid values', async function () { await supertest .put(`/api/fleet/package_policies/${packagePolicyId}`) From 591bcc1c7135c744fd1d8e26dff064cf610b37f5 Mon Sep 17 00:00:00 2001 From: Spencer Date: Wed, 10 Feb 2021 14:13:49 -0800 Subject: [PATCH 07/13] [jest/ci] remove max-old-space-size override to use 4gb default (#91020) Co-authored-by: spalger --- test/scripts/test/jest_unit.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/scripts/test/jest_unit.sh b/test/scripts/test/jest_unit.sh index 1442a0f728727..fd1166b07f322 100755 --- a/test/scripts/test/jest_unit.sh +++ b/test/scripts/test/jest_unit.sh @@ -2,7 +2,5 @@ source src/dev/ci_setup/setup_env.sh -export NODE_OPTIONS="--max-old-space-size=2048" - checks-reporter-with-killswitch "Jest Unit Tests" \ node scripts/jest --ci --verbose --maxWorkers=8 From 9fe8ccce477afbeb540e92b72dd72a02ea0a688d Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 10 Feb 2021 22:40:21 +0000 Subject: [PATCH 08/13] chore(NA): move the instruction to remove yarn global bazelisk package into the first place on install bazel tools (#91026) --- packages/kbn-pm/dist/index.js | 8 ++++---- packages/kbn-pm/src/utils/bazel/install_tools.ts | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/kbn-pm/dist/index.js b/packages/kbn-pm/dist/index.js index d939e7b3000fa..375ad634cbc15 100644 --- a/packages/kbn-pm/dist/index.js +++ b/packages/kbn-pm/dist/index.js @@ -48147,13 +48147,13 @@ async function installBazelTools(repoRootPath) { const bazeliskVersion = await readBazelToolsVersionFile(repoRootPath, '.bazeliskversion'); const bazelVersion = await readBazelToolsVersionFile(repoRootPath, '.bazelversion'); // Check what globals are installed - _log__WEBPACK_IMPORTED_MODULE_4__["log"].debug(`[bazel_tools] verify if bazelisk is installed`); // Test if bazelisk is already installed in the correct version + _log__WEBPACK_IMPORTED_MODULE_4__["log"].debug(`[bazel_tools] verify if bazelisk is installed`); // Check if we need to remove bazelisk from yarn - const isBazeliskPkgInstalled = await isBazeliskInstalled(bazeliskVersion); // Test if bazel bin is available + await tryRemoveBazeliskFromYarnGlobal(); // Test if bazelisk is already installed in the correct version - const isBazelBinAlreadyAvailable = await isBazelBinAvailable(); // Check if we need to remove bazelisk from yarn + const isBazeliskPkgInstalled = await isBazeliskInstalled(bazeliskVersion); // Test if bazel bin is available - await tryRemoveBazeliskFromYarnGlobal(); // Install bazelisk if not installed + const isBazelBinAlreadyAvailable = await isBazelBinAvailable(); // Install bazelisk if not installed if (!isBazeliskPkgInstalled || !isBazelBinAlreadyAvailable) { _log__WEBPACK_IMPORTED_MODULE_4__["log"].info(`[bazel_tools] installing Bazel tools`); diff --git a/packages/kbn-pm/src/utils/bazel/install_tools.ts b/packages/kbn-pm/src/utils/bazel/install_tools.ts index b547c2bc141bd..93acbe09b4eab 100644 --- a/packages/kbn-pm/src/utils/bazel/install_tools.ts +++ b/packages/kbn-pm/src/utils/bazel/install_tools.ts @@ -83,15 +83,15 @@ export async function installBazelTools(repoRootPath: string) { // Check what globals are installed log.debug(`[bazel_tools] verify if bazelisk is installed`); + // Check if we need to remove bazelisk from yarn + await tryRemoveBazeliskFromYarnGlobal(); + // Test if bazelisk is already installed in the correct version const isBazeliskPkgInstalled = await isBazeliskInstalled(bazeliskVersion); // Test if bazel bin is available const isBazelBinAlreadyAvailable = await isBazelBinAvailable(); - // Check if we need to remove bazelisk from yarn - await tryRemoveBazeliskFromYarnGlobal(); - // Install bazelisk if not installed if (!isBazeliskPkgInstalled || !isBazelBinAlreadyAvailable) { log.info(`[bazel_tools] installing Bazel tools`); From 8c4af6fc5eab965b614b09306cb0f7316ca4847e Mon Sep 17 00:00:00 2001 From: Brandon Kobel Date: Wed, 10 Feb 2021 15:32:29 -0800 Subject: [PATCH 09/13] Removing the code plugin entirely for 8.0 (#77940) * Removing the code app entirely for 8.0 * Updating plugin list docs * Using a test plugin for the code_coverage integration tests * Fix borked test. Co-authored-by: Elastic Machine Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Tre' Seymour --- docs/developer/plugin-list.asciidoc | 4 -- .../fixtures/test_plugin/kibana.json | 6 +++ .../fixtures/test_plugin/server/index.ts | 13 +++++++ .../fixtures/test_plugin/server/plugin.ts | 23 +++++++++++ .../integration_tests/mocks/CODEOWNERS | 2 +- .../integration_tests/team_assignment.test.js | 7 +--- x-pack/plugins/code/kibana.json | 8 ---- x-pack/plugins/code/server/config.ts | 14 ------- x-pack/plugins/code/server/index.ts | 14 ------- x-pack/plugins/code/server/plugin.test.ts | 39 ------------------- x-pack/plugins/code/server/plugin.ts | 34 ---------------- 11 files changed, 45 insertions(+), 119 deletions(-) create mode 100644 src/dev/code_coverage/ingest_coverage/integration_tests/fixtures/test_plugin/kibana.json create mode 100644 src/dev/code_coverage/ingest_coverage/integration_tests/fixtures/test_plugin/server/index.ts create mode 100644 src/dev/code_coverage/ingest_coverage/integration_tests/fixtures/test_plugin/server/plugin.ts delete mode 100644 x-pack/plugins/code/kibana.json delete mode 100644 x-pack/plugins/code/server/config.ts delete mode 100644 x-pack/plugins/code/server/index.ts delete mode 100644 x-pack/plugins/code/server/plugin.test.ts delete mode 100644 x-pack/plugins/code/server/plugin.ts diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc index 263addc98ee62..613f2d0fbf20c 100644 --- a/docs/developer/plugin-list.asciidoc +++ b/docs/developer/plugin-list.asciidoc @@ -318,10 +318,6 @@ Failure to have auth enabled in Kibana will make for a broken UI. UI-based error |The cloud plugin adds cloud specific features to Kibana. -|{kib-repo}blob/{branch}/x-pack/plugins/code[code] -|WARNING: Missing README. - - |{kib-repo}blob/{branch}/x-pack/plugins/console_extensions/README.md[consoleExtensions] |This plugin provides autocomplete definitions of licensed APIs to the OSS Console plugin. diff --git a/src/dev/code_coverage/ingest_coverage/integration_tests/fixtures/test_plugin/kibana.json b/src/dev/code_coverage/ingest_coverage/integration_tests/fixtures/test_plugin/kibana.json new file mode 100644 index 0000000000000..cbb214b575701 --- /dev/null +++ b/src/dev/code_coverage/ingest_coverage/integration_tests/fixtures/test_plugin/kibana.json @@ -0,0 +1,6 @@ +{ + "id": "codeCoverageTestPlugin", + "version": "kibana", + "server": true, + "ui": false +} diff --git a/src/dev/code_coverage/ingest_coverage/integration_tests/fixtures/test_plugin/server/index.ts b/src/dev/code_coverage/ingest_coverage/integration_tests/fixtures/test_plugin/server/index.ts new file mode 100644 index 0000000000000..5499a33fbf739 --- /dev/null +++ b/src/dev/code_coverage/ingest_coverage/integration_tests/fixtures/test_plugin/server/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { Plugin } from './plugin'; + +export function plugin() { + return new Plugin(); +} diff --git a/src/dev/code_coverage/ingest_coverage/integration_tests/fixtures/test_plugin/server/plugin.ts b/src/dev/code_coverage/ingest_coverage/integration_tests/fixtures/test_plugin/server/plugin.ts new file mode 100644 index 0000000000000..d4704ba05b59c --- /dev/null +++ b/src/dev/code_coverage/ingest_coverage/integration_tests/fixtures/test_plugin/server/plugin.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { CoreSetup } from 'kibana/server'; + +export class Plugin { + constructor() {} + + public setup(core: CoreSetup) {} + + public start() { + // called after all plugins are set up + } + + public stop() { + // called when plugin is torn down during Kibana's shutdown sequence + } +} diff --git a/src/dev/code_coverage/ingest_coverage/integration_tests/mocks/CODEOWNERS b/src/dev/code_coverage/ingest_coverage/integration_tests/mocks/CODEOWNERS index 1822c3fd95e34..77b2202820350 100644 --- a/src/dev/code_coverage/ingest_coverage/integration_tests/mocks/CODEOWNERS +++ b/src/dev/code_coverage/ingest_coverage/integration_tests/mocks/CODEOWNERS @@ -3,4 +3,4 @@ # For more info, see https://help.github.com/articles/about-codeowners/ # App -/x-pack/plugins/code/ @elastic/kibana-tre +/src/dev/code_coverage/ingest_coverage/integration_tests/fixtures/test_plugin @elastic/kibana-tre diff --git a/src/dev/code_coverage/ingest_coverage/integration_tests/team_assignment.test.js b/src/dev/code_coverage/ingest_coverage/integration_tests/team_assignment.test.js index 8fc34d29103b3..8fe61ed76a923 100644 --- a/src/dev/code_coverage/ingest_coverage/integration_tests/team_assignment.test.js +++ b/src/dev/code_coverage/ingest_coverage/integration_tests/team_assignment.test.js @@ -39,11 +39,8 @@ describe('Team Assignment', () => { const { stdout } = await execa('grep', ['tre', teamAssignmentsPath], { cwd: ROOT_DIR }); const lines = stdout.split('\n').filter((line) => !line.includes('/target')); expect(lines).toEqual([ - 'x-pack/plugins/code/jest.config.js kibana-tre', - 'x-pack/plugins/code/server/config.ts kibana-tre', - 'x-pack/plugins/code/server/index.ts kibana-tre', - 'x-pack/plugins/code/server/plugin.test.ts kibana-tre', - 'x-pack/plugins/code/server/plugin.ts kibana-tre', + 'src/dev/code_coverage/ingest_coverage/integration_tests/fixtures/test_plugin/server/index.ts kibana-tre', + 'src/dev/code_coverage/ingest_coverage/integration_tests/fixtures/test_plugin/server/plugin.ts kibana-tre', ]); }); }); diff --git a/x-pack/plugins/code/kibana.json b/x-pack/plugins/code/kibana.json deleted file mode 100644 index 815bc147d3cfe..0000000000000 --- a/x-pack/plugins/code/kibana.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "id": "code", - "version": "8.0.0", - "kibanaVersion": "kibana", - "configPath": ["xpack", "code"], - "server": true, - "ui": false -} diff --git a/x-pack/plugins/code/server/config.ts b/x-pack/plugins/code/server/config.ts deleted file mode 100644 index 2cc3e78c0b962..0000000000000 --- a/x-pack/plugins/code/server/config.ts +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { schema } from '@kbn/config-schema'; - -const createCodeConfigSchema = () => { - return schema.any({ defaultValue: {} }); -}; - -export const CodeConfigSchema = createCodeConfigSchema(); diff --git a/x-pack/plugins/code/server/index.ts b/x-pack/plugins/code/server/index.ts deleted file mode 100644 index ccea83ca1ff9f..0000000000000 --- a/x-pack/plugins/code/server/index.ts +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { PluginInitializerContext } from 'src/core/server'; -import { CodeConfigSchema } from './config'; -import { CodePlugin } from './plugin'; - -export const config = { schema: CodeConfigSchema }; -export const plugin = (initializerContext: PluginInitializerContext) => - new CodePlugin(initializerContext); diff --git a/x-pack/plugins/code/server/plugin.test.ts b/x-pack/plugins/code/server/plugin.test.ts deleted file mode 100644 index 512658ca4da82..0000000000000 --- a/x-pack/plugins/code/server/plugin.test.ts +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { coreMock } from '../../../../src/core/server/mocks'; - -import { CodePlugin } from './plugin'; - -describe('Code Plugin', () => { - describe('setup()', () => { - it('does not log deprecation message if no xpack.code.* configurations are set', async () => { - const context = coreMock.createPluginInitializerContext(); - const plugin = new CodePlugin(context); - - await plugin.setup(); - - expect(context.logger.get).not.toHaveBeenCalled(); - }); - - it('logs deprecation message if any xpack.code.* configurations are set', async () => { - const context = coreMock.createPluginInitializerContext({ - foo: 'bar', - }); - const warn = jest.fn(); - context.logger.get = jest.fn().mockReturnValue({ warn }); - const plugin = new CodePlugin(context); - - await plugin.setup(); - - expect(context.logger.get).toHaveBeenCalledWith('config', 'deprecation'); - expect(warn.mock.calls[0][0]).toMatchInlineSnapshot( - `"The experimental app \\"Code\\" has been removed from Kibana. Remove all xpack.code.* configurations from kibana.yml so Kibana does not fail to start up in the next major version."` - ); - }); - }); -}); diff --git a/x-pack/plugins/code/server/plugin.ts b/x-pack/plugins/code/server/plugin.ts deleted file mode 100644 index eb7481d12387d..0000000000000 --- a/x-pack/plugins/code/server/plugin.ts +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { TypeOf } from '@kbn/config-schema'; -import { PluginInitializerContext, Plugin } from 'src/core/server'; -import { CodeConfigSchema } from './config'; - -/** - * Represents Code Plugin instance that will be managed by the Kibana plugin system. - */ -export class CodePlugin implements Plugin { - constructor(private readonly initializerContext: PluginInitializerContext) {} - - public async setup() { - const config = this.initializerContext.config.get>(); - - if (config && Object.keys(config).length > 0) { - this.initializerContext.logger - .get('config', 'deprecation') - .warn( - 'The experimental app "Code" has been removed from Kibana. Remove all xpack.code.* ' + - 'configurations from kibana.yml so Kibana does not fail to start up in the next major version.' - ); - } - } - - public start() {} - - public stop() {} -} From b11b8b8c9b69bc0bb2c54f388d7bbae0737c297b Mon Sep 17 00:00:00 2001 From: Garrett Spong Date: Wed, 10 Feb 2021 20:20:55 -0700 Subject: [PATCH 10/13] [Security Solution][Detections] Adds list plugin Saved Objects to Security feature privilege (#90895) ## Summary Add's the list plugins Saved Objects (`exception-list` and `exception-list-agnostic`) to the `Security` feature privilege. Resolves https://github.com/elastic/kibana/issues/90715 ### Test Instructions Load pre-packaged roles/users, and ensure only those with the Kibana Space privilege `Security:All` have the ability to create/edit rules and exception lists (space-aware/agnostic). Users with `Security:Read` should only be able to view rules/exception lists. Pre-packaged security roles should no longer be granted the `Saved Objects Management` feature privilege, and this feature privilege should no longer be required to use any of the Detections features. To add test users: t1_analyst (`"siem": ["read"]`): ``` bash cd x-pack/plugins/security_solution/server/lib/detection_engine/scripts/ ./roles_users/t1_analyst/post_detections_role.sh roles_users/t1_analyst/detections_role.json ./roles_users/t1_analyst/post_detections_user.sh roles_users/t1_analyst/detections_user.json ``` hunter (`"siem": ["all"]`): ``` bash cd x-pack/plugins/security_solution/server/lib/detection_engine/scripts/ ./roles_users/t1_analyst/post_detections_role.sh roles_users/hunter/detections_role.json ./roles_users/t1_analyst/post_detections_user.sh roles_users/hunter/detections_user.json ``` Note: Be sure to remove these users after testing if using a public cluster. ### Checklist Delete any items that are not applicable to this PR. - [X] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials -- `docs` label added, will work with @jmikell821 on doc changes - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../detections_admin/detections_role.json | 3 +- .../scripts/roles_users/hunter/README.md | 1 - .../roles_users/hunter/detections_role.json | 3 +- .../platform_engineer/detections_role.json | 3 +- .../roles_users/reader/detections_role.json | 3 +- .../rule_author/detections_role.json | 3 +- .../soc_manager/detections_role.json | 3 +- .../t1_analyst/detections_role.json | 5 +- .../t2_analyst/detections_role.json | 5 +- .../security_solution/server/plugin.ts | 4 + .../roles_users_utils/index.ts | 1 - .../tests/create_exceptions.ts | 785 +++++++++--------- .../security_and_spaces/tests/create_rules.ts | 350 ++++---- 13 files changed, 616 insertions(+), 553 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/detections_admin/detections_role.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/detections_admin/detections_role.json index 357b8cde8ad10..6c9b4e2cba49c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/detections_admin/detections_role.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/detections_admin/detections_role.json @@ -26,8 +26,7 @@ "siem": ["all"], "actions": ["read"], "builtInAlerts": ["all"], - "dev_tools": ["all"], - "savedObjectsManagement": ["all"] + "dev_tools": ["all"] }, "spaces": ["*"] } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/hunter/README.md b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/hunter/README.md index f0060fb006e32..1344c5bbb0891 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/hunter/README.md +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/hunter/README.md @@ -2,7 +2,6 @@ This user can CRUD rules and signals. The main difference here is the user has ```json "builtInAlerts": ["all"], -"savedObjectsManagement": ["all"] ``` privileges whereas the T1 and T2 have "read" privileges which prevents them from creating rules diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/hunter/detections_role.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/hunter/detections_role.json index f5482643fb268..119fe5421c86c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/hunter/detections_role.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/hunter/detections_role.json @@ -30,8 +30,7 @@ "ml": ["read"], "siem": ["all"], "actions": ["read"], - "builtInAlerts": ["all"], - "savedObjectsManagement": ["all"] + "builtInAlerts": ["all"] }, "spaces": ["*"] } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/platform_engineer/detections_role.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/platform_engineer/detections_role.json index 75001292242c3..17dbd90d17925 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/platform_engineer/detections_role.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/platform_engineer/detections_role.json @@ -30,8 +30,7 @@ "ml": ["all"], "siem": ["all"], "actions": ["all"], - "builtInAlerts": ["all"], - "savedObjectsManagement": ["all"] + "builtInAlerts": ["all"] }, "spaces": ["*"] } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/reader/detections_role.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/reader/detections_role.json index de2aa18386188..289aeca24d45e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/reader/detections_role.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/reader/detections_role.json @@ -24,8 +24,7 @@ "ml": ["read"], "siem": ["read"], "actions": ["read"], - "builtInAlerts": ["read"], - "savedObjectsManagement": ["read"] + "builtInAlerts": ["read"] }, "spaces": ["*"] } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/rule_author/detections_role.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/rule_author/detections_role.json index da69643f3c2d3..0db8359c57764 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/rule_author/detections_role.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/rule_author/detections_role.json @@ -28,8 +28,7 @@ "ml": ["read"], "siem": ["all"], "actions": ["read"], - "builtInAlerts": ["all"], - "savedObjectsManagement": ["all"] + "builtInAlerts": ["all"] }, "spaces": ["*"] } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/soc_manager/detections_role.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/soc_manager/detections_role.json index a6cb64ef83ba7..6962701ae5be3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/soc_manager/detections_role.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/soc_manager/detections_role.json @@ -28,8 +28,7 @@ "ml": ["read"], "siem": ["all"], "actions": ["all"], - "builtInAlerts": ["all"], - "savedObjectsManagement": ["all"] + "builtInAlerts": ["all"] }, "spaces": ["*"] } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/t1_analyst/detections_role.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/t1_analyst/detections_role.json index 10b0ffc9d9890..07827069dbc73 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/t1_analyst/detections_role.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/t1_analyst/detections_role.json @@ -21,10 +21,9 @@ { "feature": { "ml": ["read"], - "siem": ["all"], + "siem": ["read"], "actions": ["read"], - "builtInAlerts": ["read"], - "savedObjectsManagement": ["read"] + "builtInAlerts": ["read"] }, "spaces": ["*"] } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/t2_analyst/detections_role.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/t2_analyst/detections_role.json index 58a069e03985c..f554c916c6684 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/t2_analyst/detections_role.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/t2_analyst/detections_role.json @@ -23,10 +23,9 @@ { "feature": { "ml": ["read"], - "siem": ["all"], + "siem": ["read"], "actions": ["read"], - "builtInAlerts": ["read"], - "savedObjectsManagement": ["read"] + "builtInAlerts": ["read"] }, "spaces": ["*"] } diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 6e03d81a7d356..164ccfd738919 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -219,6 +219,8 @@ export class Plugin implements IPlugin; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_exceptions.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_exceptions.ts index 1ae6aa80b219f..e8beef3e58a43 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_exceptions.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_exceptions.ts @@ -14,7 +14,10 @@ import { deleteAllExceptions } from '../../../lists_api_integration/utils'; import { RulesSchema } from '../../../../plugins/security_solution/common/detection_engine/schemas/response'; import { getCreateExceptionListMinimalSchemaMock } from '../../../../plugins/lists/common/schemas/request/create_exception_list_schema.mock'; import { CreateExceptionListItemSchema } from '../../../../plugins/lists/common'; -import { EXCEPTION_LIST_URL } from '../../../../plugins/lists/common/constants'; +import { + EXCEPTION_LIST_ITEM_URL, + EXCEPTION_LIST_URL, +} from '../../../../plugins/lists/common/constants'; import { DETECTION_ENGINE_RULES_URL } from '../../../../plugins/security_solution/common/constants'; import { FtrProviderContext } from '../../common/ftr_provider_context'; @@ -37,10 +40,13 @@ import { findImmutableRuleById, getPrePackagedRulesStatus, } from '../../utils'; +import { ROLES } from '../../../../plugins/security_solution/common/test'; +import { createUserAndRole, deleteUserAndRole } from '../roles_users_utils'; // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); + const supertestWithoutAuth = getService('supertestWithoutAuth'); const esArchiver = getService('esArchiver'); const es = getService('es'); @@ -58,129 +64,19 @@ export default ({ getService }: FtrProviderContext) => { await esArchiver.unload('auditbeat/hosts'); }); - it('should create a single rule with a rule_id and add an exception list to the rule', async () => { - const { - body: { id, list_id, namespace_type, type }, - } = await supertest - .post(EXCEPTION_LIST_URL) - .set('kbn-xsrf', 'true') - .send(getCreateExceptionListMinimalSchemaMock()) - .expect(200); - - const ruleWithException: CreateRulesSchema = { - ...getSimpleRule(), - exceptions_list: [ - { - id, - list_id, - namespace_type, - type, - }, - ], - }; - - const rule = await createRule(supertest, ruleWithException); - const expected: Partial = { - ...getSimpleRuleOutput(), - exceptions_list: [ - { - id, - list_id, - namespace_type, - type, - }, - ], - }; - const bodyToCompare = removeServerGeneratedProperties(rule); - expect(bodyToCompare).to.eql(expected); - }); - - it('should create a single rule with an exception list and validate it ran successfully', async () => { - const { - body: { id, list_id, namespace_type, type }, - } = await supertest - .post(EXCEPTION_LIST_URL) - .set('kbn-xsrf', 'true') - .send(getCreateExceptionListMinimalSchemaMock()) - .expect(200); - - const ruleWithException: CreateRulesSchema = { - ...getSimpleRule(), - enabled: true, - exceptions_list: [ - { - id, - list_id, - namespace_type, - type, - }, - ], - }; - - const rule = await createRule(supertest, ruleWithException); - await waitForRuleSuccessOrStatus(supertest, rule.id); - const bodyToCompare = removeServerGeneratedProperties(rule); + describe('elastic admin', () => { + it('should create a single rule with a rule_id and add an exception list to the rule', async () => { + const { + body: { id, list_id, namespace_type, type }, + } = await supertest + .post(EXCEPTION_LIST_URL) + .set('kbn-xsrf', 'true') + .send(getCreateExceptionListMinimalSchemaMock()) + .expect(200); - const expected: Partial = { - ...getSimpleRuleOutput(), - enabled: true, - exceptions_list: [ - { - id, - list_id, - namespace_type, - type, - }, - ], - }; - expect(bodyToCompare).to.eql(expected); - }); - - it('should allow removing an exception list from an immutable rule through patch', async () => { - await installPrePackagedRules(supertest); - - // Rule id of "9a1a2dae-0b5f-4c3d-8305-a268d404c306" is from the file: - // x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/elastic_endpoint.json - // This rule has an existing exceptions_list that we are going to use - const immutableRule = await getRule(supertest, '9a1a2dae-0b5f-4c3d-8305-a268d404c306'); - expect(immutableRule.exceptions_list.length).greaterThan(0); // make sure we have at least one exceptions_list - - // remove the exceptions list as a user is allowed to remove it from an immutable rule - await supertest - .patch(DETECTION_ENGINE_RULES_URL) - .set('kbn-xsrf', 'true') - .send({ rule_id: '9a1a2dae-0b5f-4c3d-8305-a268d404c306', exceptions_list: [] }) - .expect(200); - - const immutableRuleSecondTime = await getRule( - supertest, - '9a1a2dae-0b5f-4c3d-8305-a268d404c306' - ); - expect(immutableRuleSecondTime.exceptions_list.length).to.eql(0); - }); - - it('should allow adding a second exception list to an immutable rule through patch', async () => { - await installPrePackagedRules(supertest); - - const { id, list_id, namespace_type, type } = await createExceptionList( - supertest, - getCreateExceptionListMinimalSchemaMock() - ); - - // Rule id of "9a1a2dae-0b5f-4c3d-8305-a268d404c306" is from the file: - // x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/elastic_endpoint.json - // This rule has an existing exceptions_list that we are going to use - const immutableRule = await getRule(supertest, '9a1a2dae-0b5f-4c3d-8305-a268d404c306'); - expect(immutableRule.exceptions_list.length).greaterThan(0); // make sure we have at least one - - // add a second exceptions list as a user is allowed to add a second list to an immutable rule - await supertest - .patch(DETECTION_ENGINE_RULES_URL) - .set('kbn-xsrf', 'true') - .send({ - rule_id: '9a1a2dae-0b5f-4c3d-8305-a268d404c306', + const ruleWithException: CreateRulesSchema = { + ...getSimpleRule(), exceptions_list: [ - ...immutableRule.exceptions_list, { id, list_id, @@ -188,64 +84,11 @@ export default ({ getService }: FtrProviderContext) => { type, }, ], - }) - .expect(200); - - const immutableRuleSecondTime = await getRule( - supertest, - '9a1a2dae-0b5f-4c3d-8305-a268d404c306' - ); - - expect(immutableRuleSecondTime.exceptions_list.length).to.eql(2); - }); - - it('should override any updates to pre-packaged rules if the user removes the exception list through the API but the new version of a rule has an exception list again', async () => { - await installPrePackagedRules(supertest); - - // Rule id of "9a1a2dae-0b5f-4c3d-8305-a268d404c306" is from the file: - // x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/elastic_endpoint.json - // This rule has an existing exceptions_list that we are going to use - const immutableRule = await getRule(supertest, '9a1a2dae-0b5f-4c3d-8305-a268d404c306'); - expect(immutableRule.exceptions_list.length).greaterThan(0); // make sure we have at least one - - await supertest - .patch(DETECTION_ENGINE_RULES_URL) - .set('kbn-xsrf', 'true') - .send({ rule_id: '9a1a2dae-0b5f-4c3d-8305-a268d404c306', exceptions_list: [] }) - .expect(200); - - await downgradeImmutableRule(es, '9a1a2dae-0b5f-4c3d-8305-a268d404c306'); - await installPrePackagedRules(supertest); - const immutableRuleSecondTime = await getRule( - supertest, - '9a1a2dae-0b5f-4c3d-8305-a268d404c306' - ); - - // We should have a length of 1 and it should be the same as our original before we tried to remove it using patch - expect(immutableRuleSecondTime.exceptions_list.length).to.eql(1); - expect(immutableRuleSecondTime.exceptions_list).to.eql(immutableRule.exceptions_list); - }); + }; - it('should merge back an exceptions_list if it was removed from the immutable rule through PATCH', async () => { - await installPrePackagedRules(supertest); - - const { id, list_id, namespace_type, type } = await createExceptionList( - supertest, - getCreateExceptionListMinimalSchemaMock() - ); - - // Rule id of "9a1a2dae-0b5f-4c3d-8305-a268d404c306" is from the file: - // x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/elastic_endpoint.json - // This rule has an existing exceptions_list that we are going to ensure does not stomp on our existing rule - const immutableRule = await getRule(supertest, '9a1a2dae-0b5f-4c3d-8305-a268d404c306'); - expect(immutableRule.exceptions_list.length).greaterThan(0); // make sure we have at least one - - // remove the exception list and only have a single list that is not an endpoint_list - await supertest - .patch(DETECTION_ENGINE_RULES_URL) - .set('kbn-xsrf', 'true') - .send({ - rule_id: '9a1a2dae-0b5f-4c3d-8305-a268d404c306', + const rule = await createRule(supertest, ruleWithException); + const expected: Partial = { + ...getSimpleRuleOutput(), exceptions_list: [ { id, @@ -254,70 +97,24 @@ export default ({ getService }: FtrProviderContext) => { type, }, ], - }) - .expect(200); - - await downgradeImmutableRule(es, '9a1a2dae-0b5f-4c3d-8305-a268d404c306'); - await installPrePackagedRules(supertest); - const immutableRuleSecondTime = await getRule( - supertest, - '9a1a2dae-0b5f-4c3d-8305-a268d404c306' - ); - - expect(immutableRuleSecondTime.exceptions_list).to.eql([ - ...immutableRule.exceptions_list, - { - id, - list_id, - namespace_type, - type, - }, - ]); - }); - - it('should NOT add an extra exceptions_list that already exists on a rule during an upgrade', async () => { - await installPrePackagedRules(supertest); - - // Rule id of "9a1a2dae-0b5f-4c3d-8305-a268d404c306" is from the file: - // x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/elastic_endpoint.json - // This rule has an existing exceptions_list that we are going to ensure does not stomp on our existing rule - const immutableRule = await getRule(supertest, '9a1a2dae-0b5f-4c3d-8305-a268d404c306'); - expect(immutableRule.exceptions_list.length).greaterThan(0); // make sure we have at least one - - await downgradeImmutableRule(es, '9a1a2dae-0b5f-4c3d-8305-a268d404c306'); - await installPrePackagedRules(supertest); - - const immutableRuleSecondTime = await getRule( - supertest, - '9a1a2dae-0b5f-4c3d-8305-a268d404c306' - ); + }; + const bodyToCompare = removeServerGeneratedProperties(rule); + expect(bodyToCompare).to.eql(expected); + }); - // The installed rule should have both the original immutable exceptions list back and the - // new list the user added. - expect(immutableRuleSecondTime.exceptions_list).to.eql([...immutableRule.exceptions_list]); - }); + it('should create a single rule with an exception list and validate it ran successfully', async () => { + const { + body: { id, list_id, namespace_type, type }, + } = await supertest + .post(EXCEPTION_LIST_URL) + .set('kbn-xsrf', 'true') + .send(getCreateExceptionListMinimalSchemaMock()) + .expect(200); - it('should NOT allow updates to pre-packaged rules to overwrite existing exception based rules when the user adds an additional exception list', async () => { - await installPrePackagedRules(supertest); - - const { id, list_id, namespace_type, type } = await createExceptionList( - supertest, - getCreateExceptionListMinimalSchemaMock() - ); - - // Rule id of "9a1a2dae-0b5f-4c3d-8305-a268d404c306" is from the file: - // x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/elastic_endpoint.json - // This rule has an existing exceptions_list that we are going to ensure does not stomp on our existing rule - const immutableRule = await getRule(supertest, '9a1a2dae-0b5f-4c3d-8305-a268d404c306'); - - // add a second exceptions list as a user is allowed to add a second list to an immutable rule - await supertest - .patch(DETECTION_ENGINE_RULES_URL) - .set('kbn-xsrf', 'true') - .send({ - rule_id: '9a1a2dae-0b5f-4c3d-8305-a268d404c306', + const ruleWithException: CreateRulesSchema = { + ...getSimpleRule(), + enabled: true, exceptions_list: [ - ...immutableRule.exceptions_list, { id, list_id, @@ -325,99 +122,16 @@ export default ({ getService }: FtrProviderContext) => { type, }, ], - }) - .expect(200); - - await downgradeImmutableRule(es, '9a1a2dae-0b5f-4c3d-8305-a268d404c306'); - await installPrePackagedRules(supertest); - const immutableRuleSecondTime = await getRule( - supertest, - '9a1a2dae-0b5f-4c3d-8305-a268d404c306' - ); - - // It should be the same as what the user added originally - expect(immutableRuleSecondTime.exceptions_list).to.eql([ - ...immutableRule.exceptions_list, - { - id, - list_id, - namespace_type, - type, - }, - ]); - }); + }; - it('should not remove any exceptions added to a pre-packaged/immutable rule during an update if that rule has no existing exception lists', async () => { - await installPrePackagedRules(supertest); - - // Create a new exception list - const { id, list_id, namespace_type, type } = await createExceptionList( - supertest, - getCreateExceptionListMinimalSchemaMock() - ); - - // Rule id of "eb079c62-4481-4d6e-9643-3ca499df7aaa" is from the file: - // x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/external_alerts.json - // since this rule does not have existing exceptions_list that we are going to use for tests - const immutableRule = await getRule(supertest, 'eb079c62-4481-4d6e-9643-3ca499df7aaa'); - expect(immutableRule.exceptions_list.length).eql(0); // make sure we have no exceptions_list - - // add a second exceptions list as a user is allowed to add a second list to an immutable rule - await supertest - .patch(DETECTION_ENGINE_RULES_URL) - .set('kbn-xsrf', 'true') - .send({ - rule_id: 'eb079c62-4481-4d6e-9643-3ca499df7aaa', - exceptions_list: [ - { - id, - list_id, - namespace_type, - type, - }, - ], - }) - .expect(200); - - await downgradeImmutableRule(es, 'eb079c62-4481-4d6e-9643-3ca499df7aaa'); - await installPrePackagedRules(supertest); - const immutableRuleSecondTime = await getRule( - supertest, - 'eb079c62-4481-4d6e-9643-3ca499df7aaa' - ); - - expect(immutableRuleSecondTime.exceptions_list).to.eql([ - { - id, - list_id, - namespace_type, - type, - }, - ]); - }); + const rule = await createRule(supertest, ruleWithException); + await waitForRuleSuccessOrStatus(supertest, rule.id); + const bodyToCompare = removeServerGeneratedProperties(rule); - it('should not change the immutable tags when adding a second exception list to an immutable rule through patch', async () => { - await installPrePackagedRules(supertest); - - const { id, list_id, namespace_type, type } = await createExceptionList( - supertest, - getCreateExceptionListMinimalSchemaMock() - ); - - // Rule id of "9a1a2dae-0b5f-4c3d-8305-a268d404c306" is from the file: - // x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/elastic_endpoint.json - // This rule has an existing exceptions_list that we are going to use - const immutableRule = await getRule(supertest, '9a1a2dae-0b5f-4c3d-8305-a268d404c306'); - expect(immutableRule.exceptions_list.length).greaterThan(0); // make sure we have at least one - - // add a second exceptions list as a user is allowed to add a second list to an immutable rule - await supertest - .patch(DETECTION_ENGINE_RULES_URL) - .set('kbn-xsrf', 'true') - .send({ - rule_id: '9a1a2dae-0b5f-4c3d-8305-a268d404c306', + const expected: Partial = { + ...getSimpleRuleOutput(), + enabled: true, exceptions_list: [ - ...immutableRule.exceptions_list, { id, list_id, @@ -425,52 +139,381 @@ export default ({ getService }: FtrProviderContext) => { type, }, ], - }) - .expect(200); + }; + expect(bodyToCompare).to.eql(expected); + }); + + it('should allow removing an exception list from an immutable rule through patch', async () => { + await installPrePackagedRules(supertest); + + // Rule id of "9a1a2dae-0b5f-4c3d-8305-a268d404c306" is from the file: + // x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/elastic_endpoint.json + // This rule has an existing exceptions_list that we are going to use + const immutableRule = await getRule(supertest, '9a1a2dae-0b5f-4c3d-8305-a268d404c306'); + expect(immutableRule.exceptions_list.length).greaterThan(0); // make sure we have at least one exceptions_list + + // remove the exceptions list as a user is allowed to remove it from an immutable rule + await supertest + .patch(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send({ rule_id: '9a1a2dae-0b5f-4c3d-8305-a268d404c306', exceptions_list: [] }) + .expect(200); + + const immutableRuleSecondTime = await getRule( + supertest, + '9a1a2dae-0b5f-4c3d-8305-a268d404c306' + ); + expect(immutableRuleSecondTime.exceptions_list.length).to.eql(0); + }); + + it('should allow adding a second exception list to an immutable rule through patch', async () => { + await installPrePackagedRules(supertest); + + const { id, list_id, namespace_type, type } = await createExceptionList( + supertest, + getCreateExceptionListMinimalSchemaMock() + ); + + // Rule id of "9a1a2dae-0b5f-4c3d-8305-a268d404c306" is from the file: + // x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/elastic_endpoint.json + // This rule has an existing exceptions_list that we are going to use + const immutableRule = await getRule(supertest, '9a1a2dae-0b5f-4c3d-8305-a268d404c306'); + expect(immutableRule.exceptions_list.length).greaterThan(0); // make sure we have at least one + + // add a second exceptions list as a user is allowed to add a second list to an immutable rule + await supertest + .patch(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send({ + rule_id: '9a1a2dae-0b5f-4c3d-8305-a268d404c306', + exceptions_list: [ + ...immutableRule.exceptions_list, + { + id, + list_id, + namespace_type, + type, + }, + ], + }) + .expect(200); + + const immutableRuleSecondTime = await getRule( + supertest, + '9a1a2dae-0b5f-4c3d-8305-a268d404c306' + ); + + expect(immutableRuleSecondTime.exceptions_list.length).to.eql(2); + }); + + it('should override any updates to pre-packaged rules if the user removes the exception list through the API but the new version of a rule has an exception list again', async () => { + await installPrePackagedRules(supertest); + + // Rule id of "9a1a2dae-0b5f-4c3d-8305-a268d404c306" is from the file: + // x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/elastic_endpoint.json + // This rule has an existing exceptions_list that we are going to use + const immutableRule = await getRule(supertest, '9a1a2dae-0b5f-4c3d-8305-a268d404c306'); + expect(immutableRule.exceptions_list.length).greaterThan(0); // make sure we have at least one + + await supertest + .patch(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send({ rule_id: '9a1a2dae-0b5f-4c3d-8305-a268d404c306', exceptions_list: [] }) + .expect(200); + + await downgradeImmutableRule(es, '9a1a2dae-0b5f-4c3d-8305-a268d404c306'); + await installPrePackagedRules(supertest); + const immutableRuleSecondTime = await getRule( + supertest, + '9a1a2dae-0b5f-4c3d-8305-a268d404c306' + ); + + // We should have a length of 1 and it should be the same as our original before we tried to remove it using patch + expect(immutableRuleSecondTime.exceptions_list.length).to.eql(1); + expect(immutableRuleSecondTime.exceptions_list).to.eql(immutableRule.exceptions_list); + }); - const body = await findImmutableRuleById(supertest, '9a1a2dae-0b5f-4c3d-8305-a268d404c306'); - expect(body.data.length).to.eql(1); // should have only one length to the data set, otherwise we have duplicates or the tags were removed and that is incredibly bad. + it('should merge back an exceptions_list if it was removed from the immutable rule through PATCH', async () => { + await installPrePackagedRules(supertest); - const bodyToCompare = removeServerGeneratedProperties(body.data[0]); - expect(bodyToCompare.rule_id).to.eql(immutableRule.rule_id); // Rule id should not change with a a patch - expect(bodyToCompare.immutable).to.eql(immutableRule.immutable); // Immutable should always stay the same which is true and never flip to false. - expect(bodyToCompare.version).to.eql(immutableRule.version); // The version should never update on a patch + const { id, list_id, namespace_type, type } = await createExceptionList( + supertest, + getCreateExceptionListMinimalSchemaMock() + ); + + // Rule id of "9a1a2dae-0b5f-4c3d-8305-a268d404c306" is from the file: + // x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/elastic_endpoint.json + // This rule has an existing exceptions_list that we are going to ensure does not stomp on our existing rule + const immutableRule = await getRule(supertest, '9a1a2dae-0b5f-4c3d-8305-a268d404c306'); + expect(immutableRule.exceptions_list.length).greaterThan(0); // make sure we have at least one + + // remove the exception list and only have a single list that is not an endpoint_list + await supertest + .patch(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send({ + rule_id: '9a1a2dae-0b5f-4c3d-8305-a268d404c306', + exceptions_list: [ + { + id, + list_id, + namespace_type, + type, + }, + ], + }) + .expect(200); + + await downgradeImmutableRule(es, '9a1a2dae-0b5f-4c3d-8305-a268d404c306'); + await installPrePackagedRules(supertest); + const immutableRuleSecondTime = await getRule( + supertest, + '9a1a2dae-0b5f-4c3d-8305-a268d404c306' + ); + + expect(immutableRuleSecondTime.exceptions_list).to.eql([ + ...immutableRule.exceptions_list, + { + id, + list_id, + namespace_type, + type, + }, + ]); + }); + + it('should NOT add an extra exceptions_list that already exists on a rule during an upgrade', async () => { + await installPrePackagedRules(supertest); + + // Rule id of "9a1a2dae-0b5f-4c3d-8305-a268d404c306" is from the file: + // x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/elastic_endpoint.json + // This rule has an existing exceptions_list that we are going to ensure does not stomp on our existing rule + const immutableRule = await getRule(supertest, '9a1a2dae-0b5f-4c3d-8305-a268d404c306'); + expect(immutableRule.exceptions_list.length).greaterThan(0); // make sure we have at least one + + await downgradeImmutableRule(es, '9a1a2dae-0b5f-4c3d-8305-a268d404c306'); + await installPrePackagedRules(supertest); + + const immutableRuleSecondTime = await getRule( + supertest, + '9a1a2dae-0b5f-4c3d-8305-a268d404c306' + ); + + // The installed rule should have both the original immutable exceptions list back and the + // new list the user added. + expect(immutableRuleSecondTime.exceptions_list).to.eql([ + ...immutableRule.exceptions_list, + ]); + }); + + it('should NOT allow updates to pre-packaged rules to overwrite existing exception based rules when the user adds an additional exception list', async () => { + await installPrePackagedRules(supertest); + + const { id, list_id, namespace_type, type } = await createExceptionList( + supertest, + getCreateExceptionListMinimalSchemaMock() + ); + + // Rule id of "9a1a2dae-0b5f-4c3d-8305-a268d404c306" is from the file: + // x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/elastic_endpoint.json + // This rule has an existing exceptions_list that we are going to ensure does not stomp on our existing rule + const immutableRule = await getRule(supertest, '9a1a2dae-0b5f-4c3d-8305-a268d404c306'); + + // add a second exceptions list as a user is allowed to add a second list to an immutable rule + await supertest + .patch(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send({ + rule_id: '9a1a2dae-0b5f-4c3d-8305-a268d404c306', + exceptions_list: [ + ...immutableRule.exceptions_list, + { + id, + list_id, + namespace_type, + type, + }, + ], + }) + .expect(200); + + await downgradeImmutableRule(es, '9a1a2dae-0b5f-4c3d-8305-a268d404c306'); + await installPrePackagedRules(supertest); + const immutableRuleSecondTime = await getRule( + supertest, + '9a1a2dae-0b5f-4c3d-8305-a268d404c306' + ); + + // It should be the same as what the user added originally + expect(immutableRuleSecondTime.exceptions_list).to.eql([ + ...immutableRule.exceptions_list, + { + id, + list_id, + namespace_type, + type, + }, + ]); + }); + + it('should not remove any exceptions added to a pre-packaged/immutable rule during an update if that rule has no existing exception lists', async () => { + await installPrePackagedRules(supertest); + + // Create a new exception list + const { id, list_id, namespace_type, type } = await createExceptionList( + supertest, + getCreateExceptionListMinimalSchemaMock() + ); + + // Rule id of "eb079c62-4481-4d6e-9643-3ca499df7aaa" is from the file: + // x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/external_alerts.json + // since this rule does not have existing exceptions_list that we are going to use for tests + const immutableRule = await getRule(supertest, 'eb079c62-4481-4d6e-9643-3ca499df7aaa'); + expect(immutableRule.exceptions_list.length).eql(0); // make sure we have no exceptions_list + + // add a second exceptions list as a user is allowed to add a second list to an immutable rule + await supertest + .patch(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send({ + rule_id: 'eb079c62-4481-4d6e-9643-3ca499df7aaa', + exceptions_list: [ + { + id, + list_id, + namespace_type, + type, + }, + ], + }) + .expect(200); + + await downgradeImmutableRule(es, 'eb079c62-4481-4d6e-9643-3ca499df7aaa'); + await installPrePackagedRules(supertest); + const immutableRuleSecondTime = await getRule( + supertest, + 'eb079c62-4481-4d6e-9643-3ca499df7aaa' + ); + + expect(immutableRuleSecondTime.exceptions_list).to.eql([ + { + id, + list_id, + namespace_type, + type, + }, + ]); + }); + + it('should not change the immutable tags when adding a second exception list to an immutable rule through patch', async () => { + await installPrePackagedRules(supertest); + + const { id, list_id, namespace_type, type } = await createExceptionList( + supertest, + getCreateExceptionListMinimalSchemaMock() + ); + + // Rule id of "9a1a2dae-0b5f-4c3d-8305-a268d404c306" is from the file: + // x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/elastic_endpoint.json + // This rule has an existing exceptions_list that we are going to use + const immutableRule = await getRule(supertest, '9a1a2dae-0b5f-4c3d-8305-a268d404c306'); + expect(immutableRule.exceptions_list.length).greaterThan(0); // make sure we have at least one + + // add a second exceptions list as a user is allowed to add a second list to an immutable rule + await supertest + .patch(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send({ + rule_id: '9a1a2dae-0b5f-4c3d-8305-a268d404c306', + exceptions_list: [ + ...immutableRule.exceptions_list, + { + id, + list_id, + namespace_type, + type, + }, + ], + }) + .expect(200); + + const body = await findImmutableRuleById( + supertest, + '9a1a2dae-0b5f-4c3d-8305-a268d404c306' + ); + expect(body.data.length).to.eql(1); // should have only one length to the data set, otherwise we have duplicates or the tags were removed and that is incredibly bad. + + const bodyToCompare = removeServerGeneratedProperties(body.data[0]); + expect(bodyToCompare.rule_id).to.eql(immutableRule.rule_id); // Rule id should not change with a a patch + expect(bodyToCompare.immutable).to.eql(immutableRule.immutable); // Immutable should always stay the same which is true and never flip to false. + expect(bodyToCompare.version).to.eql(immutableRule.version); // The version should never update on a patch + }); + + it('should not change count of prepacked rules when adding a second exception list to an immutable rule through patch. If this fails, suspect the immutable tags are not staying on the rule correctly.', async () => { + await installPrePackagedRules(supertest); + + const { id, list_id, namespace_type, type } = await createExceptionList( + supertest, + getCreateExceptionListMinimalSchemaMock() + ); + + // Rule id of "9a1a2dae-0b5f-4c3d-8305-a268d404c306" is from the file: + // x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/elastic_endpoint.json + // This rule has an existing exceptions_list that we are going to use + const immutableRule = await getRule(supertest, '9a1a2dae-0b5f-4c3d-8305-a268d404c306'); + expect(immutableRule.exceptions_list.length).greaterThan(0); // make sure we have at least one + + // add a second exceptions list as a user is allowed to add a second list to an immutable rule + await supertest + .patch(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send({ + rule_id: '9a1a2dae-0b5f-4c3d-8305-a268d404c306', + exceptions_list: [ + ...immutableRule.exceptions_list, + { + id, + list_id, + namespace_type, + type, + }, + ], + }) + .expect(200); + + const status = await getPrePackagedRulesStatus(supertest); + expect(status.rules_not_installed).to.eql(0); + }); }); - it('should not change count of prepacked rules when adding a second exception list to an immutable rule through patch. If this fails, suspect the immutable tags are not staying on the rule correctly.', async () => { - await installPrePackagedRules(supertest); - - const { id, list_id, namespace_type, type } = await createExceptionList( - supertest, - getCreateExceptionListMinimalSchemaMock() - ); - - // Rule id of "9a1a2dae-0b5f-4c3d-8305-a268d404c306" is from the file: - // x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/elastic_endpoint.json - // This rule has an existing exceptions_list that we are going to use - const immutableRule = await getRule(supertest, '9a1a2dae-0b5f-4c3d-8305-a268d404c306'); - expect(immutableRule.exceptions_list.length).greaterThan(0); // make sure we have at least one - - // add a second exceptions list as a user is allowed to add a second list to an immutable rule - await supertest - .patch(DETECTION_ENGINE_RULES_URL) - .set('kbn-xsrf', 'true') - .send({ - rule_id: '9a1a2dae-0b5f-4c3d-8305-a268d404c306', - exceptions_list: [ - ...immutableRule.exceptions_list, - { - id, - list_id, - namespace_type, - type, - }, - ], - }) - .expect(200); + describe('t1_analyst', () => { + const role = ROLES.t1_analyst; + + beforeEach(async () => { + await createUserAndRole(getService, role); + }); - const status = await getPrePackagedRulesStatus(supertest); - expect(status.rules_not_installed).to.eql(0); + afterEach(async () => { + await deleteUserAndRole(getService, role); + }); + + it('should NOT be able to create an exception list', async () => { + await supertestWithoutAuth + .post(EXCEPTION_LIST_ITEM_URL) + .auth(role, 'changeme') + .set('kbn-xsrf', 'true') + .send(getCreateExceptionListItemMinimalSchemaMock()) + .expect(403); + }); + + it('should NOT be able to create an exception list item', async () => { + await supertestWithoutAuth + .post(EXCEPTION_LIST_ITEM_URL) + .auth(role, 'changeme') + .set('kbn-xsrf', 'true') + .send(getCreateExceptionListItemMinimalSchemaMock()) + .expect(403); + }); }); describe('tests with auditbeat data', () => { diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_rules.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_rules.ts index 30d734b0e0262..a735eba6693fe 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_rules.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_rules.ts @@ -30,10 +30,13 @@ import { getRuleForSignalTesting, getRuleForSignalTestingWithTimestampOverride, } from '../../utils'; +import { ROLES } from '../../../../plugins/security_solution/common/test'; +import { createUserAndRole, deleteUserAndRole } from '../roles_users_utils'; // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); + const supertestWithoutAuth = getService('supertestWithoutAuth'); const esArchiver = getService('esArchiver'); describe('create_rules', () => { @@ -65,186 +68,209 @@ export default ({ getService }: FtrProviderContext) => { await esArchiver.unload('auditbeat/hosts'); }); - it('should create a single rule with a rule_id', async () => { - const { body } = await supertest - .post(DETECTION_ENGINE_RULES_URL) - .set('kbn-xsrf', 'true') - .send(getSimpleRule()) - .expect(200); - - const bodyToCompare = removeServerGeneratedProperties(body); - expect(bodyToCompare).to.eql(getSimpleRuleOutput()); - }); - - /* - This test is to ensure no future regressions introduced by the following scenario - a call to updateApiKey was invalidating the api key used by the - rule while the rule was executing, or even before it executed, - on the first rule run. - this pr https://github.com/elastic/kibana/pull/68184 - fixed this by finding the true source of a bug that required the manual - api key update, and removed the call to that function. - - When the api key is updated before / while the rule is executing, the alert - executor no longer has access to a service to update the rule status - saved object in Elasticsearch. Because of this, we cannot set the rule into - a 'failure' state, so the user ends up seeing 'going to run' as that is the - last status set for the rule before it erupts in an error that cannot be - recorded inside of the executor. - - This adds an e2e test for the backend to catch that in case - this pops up again elsewhere. - */ - it('should create a single rule with a rule_id and validate it ran successfully', async () => { - const simpleRule = getRuleForSignalTesting(['auditbeat-*']); - const { body } = await supertest - .post(DETECTION_ENGINE_RULES_URL) - .set('kbn-xsrf', 'true') - .send(simpleRule) - .expect(200); - - await waitForRuleSuccessOrStatus(supertest, body.id); - - const { body: statusBody } = await supertest - .post(DETECTION_ENGINE_RULES_STATUS_URL) - .set('kbn-xsrf', 'true') - .send({ ids: [body.id] }) - .expect(200); + describe('elastic admin', () => { + it('should create a single rule with a rule_id', async () => { + const { body } = await supertest + .post(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send(getSimpleRule()) + .expect(200); - expect(statusBody[body.id].current_status.status).to.eql('succeeded'); - }); + const bodyToCompare = removeServerGeneratedProperties(body); + expect(bodyToCompare).to.eql(getSimpleRuleOutput()); + }); - it('should create a single rule with a rule_id and an index pattern that does not match anything available and fail the rule', async () => { - const simpleRule = getRuleForSignalTesting(['does-not-exist-*']); - const { body } = await supertest - .post(DETECTION_ENGINE_RULES_URL) - .set('kbn-xsrf', 'true') - .send(simpleRule) - .expect(200); + /* + This test is to ensure no future regressions introduced by the following scenario + a call to updateApiKey was invalidating the api key used by the + rule while the rule was executing, or even before it executed, + on the first rule run. + this pr https://github.com/elastic/kibana/pull/68184 + fixed this by finding the true source of a bug that required the manual + api key update, and removed the call to that function. + + When the api key is updated before / while the rule is executing, the alert + executor no longer has access to a service to update the rule status + saved object in Elasticsearch. Because of this, we cannot set the rule into + a 'failure' state, so the user ends up seeing 'going to run' as that is the + last status set for the rule before it erupts in an error that cannot be + recorded inside of the executor. + + This adds an e2e test for the backend to catch that in case + this pops up again elsewhere. + */ + it('should create a single rule with a rule_id and validate it ran successfully', async () => { + const simpleRule = getRuleForSignalTesting(['auditbeat-*']); + const { body } = await supertest + .post(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send(simpleRule) + .expect(200); + + await waitForRuleSuccessOrStatus(supertest, body.id); + + const { body: statusBody } = await supertest + .post(DETECTION_ENGINE_RULES_STATUS_URL) + .set('kbn-xsrf', 'true') + .send({ ids: [body.id] }) + .expect(200); + + expect(statusBody[body.id].current_status.status).to.eql('succeeded'); + }); - await waitForRuleSuccessOrStatus(supertest, body.id, 'failed'); + it('should create a single rule with a rule_id and an index pattern that does not match anything available and fail the rule', async () => { + const simpleRule = getRuleForSignalTesting(['does-not-exist-*']); + const { body } = await supertest + .post(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send(simpleRule) + .expect(200); + + await waitForRuleSuccessOrStatus(supertest, body.id, 'failed'); + + const { body: statusBody } = await supertest + .post(DETECTION_ENGINE_RULES_STATUS_URL) + .set('kbn-xsrf', 'true') + .send({ ids: [body.id] }) + .expect(200); + + expect(statusBody[body.id].current_status.status).to.eql('failed'); + expect(statusBody[body.id].current_status.last_failure_message).to.eql( + 'The following index patterns did not match any indices: ["does-not-exist-*"]' + ); + }); - const { body: statusBody } = await supertest - .post(DETECTION_ENGINE_RULES_STATUS_URL) - .set('kbn-xsrf', 'true') - .send({ ids: [body.id] }) - .expect(200); + it('should create a single rule with a rule_id and an index pattern that does not match anything and an index pattern that does and the rule should be successful', async () => { + const simpleRule = getRuleForSignalTesting(['does-not-exist-*', 'auditbeat-*']); + const { body } = await supertest + .post(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send(simpleRule) + .expect(200); - expect(statusBody[body.id].current_status.status).to.eql('failed'); - expect(statusBody[body.id].current_status.last_failure_message).to.eql( - 'The following index patterns did not match any indices: ["does-not-exist-*"]' - ); - }); + await waitForRuleSuccessOrStatus(supertest, body.id, 'succeeded'); - it('should create a single rule with a rule_id and an index pattern that does not match anything and an index pattern that does and the rule should be successful', async () => { - const simpleRule = getRuleForSignalTesting(['does-not-exist-*', 'auditbeat-*']); - const { body } = await supertest - .post(DETECTION_ENGINE_RULES_URL) - .set('kbn-xsrf', 'true') - .send(simpleRule) - .expect(200); + const { body: statusBody } = await supertest + .post(DETECTION_ENGINE_RULES_STATUS_URL) + .set('kbn-xsrf', 'true') + .send({ ids: [body.id] }) + .expect(200); - await waitForRuleSuccessOrStatus(supertest, body.id, 'succeeded'); - - const { body: statusBody } = await supertest - .post(DETECTION_ENGINE_RULES_STATUS_URL) - .set('kbn-xsrf', 'true') - .send({ ids: [body.id] }) - .expect(200); + expect(statusBody[body.id].current_status.status).to.eql('succeeded'); + }); - expect(statusBody[body.id].current_status.status).to.eql('succeeded'); - }); + it('should create a single rule without an input index', async () => { + const rule: CreateRulesSchema = { + name: 'Simple Rule Query', + description: 'Simple Rule Query', + enabled: true, + risk_score: 1, + rule_id: 'rule-1', + severity: 'high', + type: 'query', + query: 'user.name: root or user.name: admin', + }; + const expected = { + actions: [], + author: [], + created_by: 'elastic', + description: 'Simple Rule Query', + enabled: true, + false_positives: [], + from: 'now-6m', + immutable: false, + interval: '5m', + rule_id: 'rule-1', + language: 'kuery', + output_index: '.siem-signals-default', + max_signals: 100, + risk_score: 1, + risk_score_mapping: [], + name: 'Simple Rule Query', + query: 'user.name: root or user.name: admin', + references: [], + severity: 'high', + severity_mapping: [], + updated_by: 'elastic', + tags: [], + to: 'now', + type: 'query', + threat: [], + throttle: 'no_actions', + exceptions_list: [], + version: 1, + }; + + const { body } = await supertest + .post(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send(rule) + .expect(200); + + const bodyToCompare = removeServerGeneratedProperties(body); + expect(bodyToCompare).to.eql(expected); + }); - it('should create a single rule without an input index', async () => { - const rule: CreateRulesSchema = { - name: 'Simple Rule Query', - description: 'Simple Rule Query', - enabled: true, - risk_score: 1, - rule_id: 'rule-1', - severity: 'high', - type: 'query', - query: 'user.name: root or user.name: admin', - }; - const expected = { - actions: [], - author: [], - created_by: 'elastic', - description: 'Simple Rule Query', - enabled: true, - false_positives: [], - from: 'now-6m', - immutable: false, - interval: '5m', - rule_id: 'rule-1', - language: 'kuery', - output_index: '.siem-signals-default', - max_signals: 100, - risk_score: 1, - risk_score_mapping: [], - name: 'Simple Rule Query', - query: 'user.name: root or user.name: admin', - references: [], - severity: 'high', - severity_mapping: [], - updated_by: 'elastic', - tags: [], - to: 'now', - type: 'query', - threat: [], - throttle: 'no_actions', - exceptions_list: [], - version: 1, - }; + it('should create a single rule without a rule_id', async () => { + const { body } = await supertest + .post(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send(getSimpleRuleWithoutRuleId()) + .expect(200); - const { body } = await supertest - .post(DETECTION_ENGINE_RULES_URL) - .set('kbn-xsrf', 'true') - .send(rule) - .expect(200); + const bodyToCompare = removeServerGeneratedPropertiesIncludingRuleId(body); + expect(bodyToCompare).to.eql(getSimpleRuleOutputWithoutRuleId()); + }); - const bodyToCompare = removeServerGeneratedProperties(body); - expect(bodyToCompare).to.eql(expected); - }); + it('should create a single Machine Learning rule', async () => { + const { body } = await supertest + .post(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send(getSimpleMlRule()) + .expect(200); - it('should create a single rule without a rule_id', async () => { - const { body } = await supertest - .post(DETECTION_ENGINE_RULES_URL) - .set('kbn-xsrf', 'true') - .send(getSimpleRuleWithoutRuleId()) - .expect(200); + const bodyToCompare = removeServerGeneratedProperties(body); + expect(bodyToCompare).to.eql(getSimpleMlRuleOutput()); + }); - const bodyToCompare = removeServerGeneratedPropertiesIncludingRuleId(body); - expect(bodyToCompare).to.eql(getSimpleRuleOutputWithoutRuleId()); + it('should cause a 409 conflict if we attempt to create the same rule_id twice', async () => { + await supertest + .post(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send(getSimpleRule()) + .expect(200); + + const { body } = await supertest + .post(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send(getSimpleRule()) + .expect(409); + + expect(body).to.eql({ + message: 'rule_id: "rule-1" already exists', + status_code: 409, + }); + }); }); - it('should create a single Machine Learning rule', async () => { - const { body } = await supertest - .post(DETECTION_ENGINE_RULES_URL) - .set('kbn-xsrf', 'true') - .send(getSimpleMlRule()) - .expect(200); + describe('t1_analyst', () => { + const role = ROLES.t1_analyst; - const bodyToCompare = removeServerGeneratedProperties(body); - expect(bodyToCompare).to.eql(getSimpleMlRuleOutput()); - }); - - it('should cause a 409 conflict if we attempt to create the same rule_id twice', async () => { - await supertest - .post(DETECTION_ENGINE_RULES_URL) - .set('kbn-xsrf', 'true') - .send(getSimpleRule()) - .expect(200); + beforeEach(async () => { + await createUserAndRole(getService, role); + }); - const { body } = await supertest - .post(DETECTION_ENGINE_RULES_URL) - .set('kbn-xsrf', 'true') - .send(getSimpleRule()) - .expect(409); + afterEach(async () => { + await deleteUserAndRole(getService, role); + }); - expect(body).to.eql({ - message: 'rule_id: "rule-1" already exists', - status_code: 409, + it('should NOT be able to create a rule', async () => { + await supertestWithoutAuth + .post(DETECTION_ENGINE_RULES_URL) + .auth(role, 'changeme') + .set('kbn-xsrf', 'true') + .send(getSimpleRule()) + .expect(403); }); }); }); From f9f8562a539335d58b8a730ff46dacdcfc5b01a4 Mon Sep 17 00:00:00 2001 From: Frank Hassanabad Date: Wed, 10 Feb 2021 21:27:33 -0700 Subject: [PATCH 11/13] Fixes track_total_hits in the body not having an effect when using search strategy (#91068) ## Summary Moves `track_total_hits` from body messages of our queries into the params section of our queries. Several of our `track_total_hits: false` were not taking effect and instead were being set to `track_total_hits: true` when being executed within the Kibana search strategy vs. previously when they were regular Elasticsearch queries and always took effect. When teams port over their searches to the search strategies provided by Kibana, they are required to move any and all `track_total_hits` from their `body` sections of their code into the `params` part of their code. The reason for this is that the search strategy maintains a backwards compatibility with earlier versions of searches before Elasticsearch introduced the `track_total_hits`. However, the code does not detect if you put the `track_total_hits` in your body, it only checks the params section and forces it to `true` if it is not found in the params section. If the search strategy does not see a `track_total_hits` within the params section of the query, it will force add one and that one will override any within the body of the query. For example, if you had a `track_total_hits` in your body and not in the params section, then search strategy would execute the query like so: ```ts GET someindex-*/_search?track_total_hits=true { // some query here "track_total_hits": false } ``` The forced parameter of `?track_total_hits=true` overrides the `track_total_hits: false` within the body of your query regardless of what the `track_total_hits` is set to and you always get the true. This bug has existed since 7.10.0 when we ported over queries to search strategy. You can see the code which sets this parameter if you do not here for master, 7.11, 7.10: https://github.com/elastic/kibana/blob/master/src/plugins/data/server/search/es_search/request_utils.ts#L31 https://github.com/elastic/kibana/blob/7.11/src/plugins/data/server/search/es_search/request_utils.ts#L31 https://github.com/elastic/kibana/blob/7.10/src/plugins/data/server/search/es_search/get_default_search_params.ts#L42 Comments about the behavior from 7.10: https://github.com/elastic/kibana/pull/75728#pullrequestreview-479367296 When running this code you can open dev tools and inspect the data and now notice when the total hits does not get set vs. before when it was getting set: before fix where total shows up for queries with `track_total_hits` in the body: event_view_before after fix where total no longer shows up for queries with `track_total_hits` moved to the params section: event_view_after ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../server/lib/hosts/query.detail_host.dsl.ts | 2 +- .../server/lib/hosts/query.hosts.dsl.ts | 2 +- .../server/lib/hosts/query.last_first_seen_host.dsl.ts | 2 +- .../factory/hosts/all/__mocks__/index.ts | 4 ++-- .../factory/hosts/all/query.all_hosts.dsl.ts | 2 +- .../factory/hosts/details/__mocks__/index.ts | 4 ++-- .../factory/hosts/details/query.host_details.dsl.ts | 2 +- .../query.hosts_kpi_authentications.dsl.ts | 2 +- .../hosts/kpi/hosts/query.hosts_kpi_hosts.dsl.ts | 2 +- .../kpi/unique_ips/query.hosts_kpi_unique_ips.dsl.ts | 2 +- .../factory/hosts/last_first_seen/__mocks__/index.ts | 4 ++-- .../last_first_seen/query.last_first_seen_host.dsl.ts | 2 +- .../factory/hosts/overview/__mocks__/index.ts | 4 ++-- .../factory/hosts/overview/query.overview_host.dsl.ts | 2 +- .../factory/matrix_histogram/__mocks__/index.ts | 10 +++++----- .../factory/matrix_histogram/alerts/__mocks__/index.ts | 2 +- .../alerts/query.alerts_histogram.dsl.ts | 2 +- .../matrix_histogram/anomalies/__mocks__/index.ts | 2 +- .../anomalies/query.anomalies_histogram.dsl.ts | 2 +- .../authentications/__mocks__/index.ts | 2 +- .../query.authentications_histogram.dsl.ts | 2 +- .../factory/matrix_histogram/events/__mocks__/index.ts | 6 +++--- .../events/query.events_histogram.dsl.ts | 2 +- .../factory/network/details/__mocks__/index.ts | 4 ++-- .../network/details/query.details_network.dsl.ts | 2 +- .../network/kpi/dns/query.network_kpi_dns.dsl.ts | 2 +- .../query.network_kpi_network_events.dsl.ts | 2 +- .../query.network_kpi_tls_handshakes.dsl.ts | 2 +- .../unique_flows/query.network_kpi_unique_flows.dsl.ts | 2 +- .../query.network_kpi_unique_private_ips.dsl.ts | 2 +- .../factory/network/overview/__mocks__/index.ts | 2 +- .../network/overview/query.overview_network.dsl.ts | 2 +- .../factory/network/tls/__mocks__/index.ts | 4 ++-- .../factory/network/tls/query.tls_network.dsl.ts | 2 +- .../factory/network/users/__mocks__/index.ts | 4 ++-- .../factory/network/users/query.users_network.dsl.ts | 2 +- .../query.events_last_event_time.dsl.ts | 6 +++--- 37 files changed, 52 insertions(+), 52 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/hosts/query.detail_host.dsl.ts b/x-pack/plugins/security_solution/server/lib/hosts/query.detail_host.dsl.ts index 2c1c39259aae3..4dd5a86e46bf6 100644 --- a/x-pack/plugins/security_solution/server/lib/hosts/query.detail_host.dsl.ts +++ b/x-pack/plugins/security_solution/server/lib/hosts/query.detail_host.dsl.ts @@ -39,13 +39,13 @@ export const buildHostOverviewQuery = ({ allowNoIndices: true, index: defaultIndex, ignoreUnavailable: true, + track_total_hits: false, body: { aggregations: { ...buildFieldsTermAggregation(esFields.filter((field) => !['@timestamp'].includes(field))), }, query: { bool: { filter } }, size: 0, - track_total_hits: false, }, }; diff --git a/x-pack/plugins/security_solution/server/lib/hosts/query.hosts.dsl.ts b/x-pack/plugins/security_solution/server/lib/hosts/query.hosts.dsl.ts index d83b4c9f9fd80..16c53aa6a85eb 100644 --- a/x-pack/plugins/security_solution/server/lib/hosts/query.hosts.dsl.ts +++ b/x-pack/plugins/security_solution/server/lib/hosts/query.hosts.dsl.ts @@ -44,6 +44,7 @@ export const buildHostsQuery = ({ allowNoIndices: true, index: defaultIndex, ignoreUnavailable: true, + track_total_hits: false, body: { ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), aggregations: { @@ -72,7 +73,6 @@ export const buildHostsQuery = ({ }, query: { bool: { filter } }, size: 0, - track_total_hits: false, }, }; diff --git a/x-pack/plugins/security_solution/server/lib/hosts/query.last_first_seen_host.dsl.ts b/x-pack/plugins/security_solution/server/lib/hosts/query.last_first_seen_host.dsl.ts index e7e9ec48fc534..a047be8ed2674 100644 --- a/x-pack/plugins/security_solution/server/lib/hosts/query.last_first_seen_host.dsl.ts +++ b/x-pack/plugins/security_solution/server/lib/hosts/query.last_first_seen_host.dsl.ts @@ -19,6 +19,7 @@ export const buildLastFirstSeenHostQuery = ({ allowNoIndices: true, index: defaultIndex, ignoreUnavailable: true, + track_total_hits: false, body: { ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), aggregations: { @@ -27,7 +28,6 @@ export const buildLastFirstSeenHostQuery = ({ }, query: { bool: { filter } }, size: 0, - track_total_hits: false, }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/__mocks__/index.ts index 96082ee1b4be8..b6a5435a0e046 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/__mocks__/index.ts @@ -621,6 +621,7 @@ export const formattedSearchStrategyResponse = { 'winlogbeat-*', ], ignoreUnavailable: true, + track_total_hits: false, body: { docvalue_fields: mockOptions.docValueFields, aggregations: { @@ -656,7 +657,6 @@ export const formattedSearchStrategyResponse = { }, }, size: 0, - track_total_hits: false, }, }, null, @@ -782,6 +782,7 @@ export const mockBuckets: HostAggEsItem = { export const expectedDsl = { allowNoIndices: true, + track_total_hits: false, body: { aggregations: { host_count: { cardinality: { field: 'host.name' } }, @@ -817,7 +818,6 @@ export const expectedDsl = { }, docvalue_fields: mockOptions.docValueFields, size: 0, - track_total_hits: false, }, ignoreUnavailable: true, index: [ diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/query.all_hosts.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/query.all_hosts.dsl.ts index 5196eaa257444..08c9711794978 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/query.all_hosts.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/query.all_hosts.dsl.ts @@ -43,6 +43,7 @@ export const buildHostsQuery = ({ allowNoIndices: true, index: defaultIndex, ignoreUnavailable: true, + track_total_hits: false, body: { ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), aggregations: { @@ -71,7 +72,6 @@ export const buildHostsQuery = ({ }, query: { bool: { filter } }, size: 0, - track_total_hits: false, }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/__mocks__/index.ts index 9c3380191507c..7561682e070fc 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/__mocks__/index.ts @@ -1311,6 +1311,7 @@ export const formattedSearchStrategyResponse = { 'winlogbeat-*', ], ignoreUnavailable: true, + track_total_hits: false, body: { aggregations: { host_architecture: { @@ -1387,7 +1388,6 @@ export const formattedSearchStrategyResponse = { }, }, size: 0, - track_total_hits: false, }, }, null, @@ -1410,6 +1410,7 @@ export const expectedDsl = { 'winlogbeat-*', ], ignoreUnavailable: true, + track_total_hits: false, body: { aggregations: { host_architecture: { @@ -1645,6 +1646,5 @@ export const expectedDsl = { }, }, size: 0, - track_total_hits: false, }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/query.host_details.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/query.host_details.dsl.ts index fa720825bb3f9..f340e4d905666 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/query.host_details.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/query.host_details.dsl.ts @@ -36,13 +36,13 @@ export const buildHostDetailsQuery = ({ allowNoIndices: true, index: defaultIndex, ignoreUnavailable: true, + track_total_hits: false, body: { aggregations: { ...buildFieldsTermAggregation(esFields.filter((field) => !['@timestamp'].includes(field))), }, query: { bool: { filter } }, size: 0, - track_total_hits: false, }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/authentications/query.hosts_kpi_authentications.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/authentications/query.hosts_kpi_authentications.dsl.ts index 455eeed5ba80f..a5c82688e01ba 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/authentications/query.hosts_kpi_authentications.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/authentications/query.hosts_kpi_authentications.dsl.ts @@ -41,6 +41,7 @@ export const buildHostsKpiAuthenticationsQuery = ({ index: defaultIndex, allowNoIndices: true, ignoreUnavailable: true, + track_total_hits: false, body: { aggs: { authentication_success: { @@ -94,7 +95,6 @@ export const buildHostsKpiAuthenticationsQuery = ({ }, }, size: 0, - track_total_hits: false, }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/hosts/query.hosts_kpi_hosts.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/hosts/query.hosts_kpi_hosts.dsl.ts index 21e862e3858d0..0e0cbd8a2649d 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/hosts/query.hosts_kpi_hosts.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/hosts/query.hosts_kpi_hosts.dsl.ts @@ -30,6 +30,7 @@ export const buildHostsKpiHostsQuery = ({ index: defaultIndex, allowNoIndices: true, ignoreUnavailable: true, + track_total_hits: false, body: { aggregations: { hosts: { @@ -57,7 +58,6 @@ export const buildHostsKpiHostsQuery = ({ }, }, size: 0, - track_total_hits: false, }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/unique_ips/query.hosts_kpi_unique_ips.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/unique_ips/query.hosts_kpi_unique_ips.dsl.ts index 815a2644355eb..a702982ab8253 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/unique_ips/query.hosts_kpi_unique_ips.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/unique_ips/query.hosts_kpi_unique_ips.dsl.ts @@ -30,6 +30,7 @@ export const buildHostsKpiUniqueIpsQuery = ({ index: defaultIndex, allowNoIndices: true, ignoreUnavailable: true, + track_total_hits: false, body: { aggregations: { unique_source_ips: { @@ -75,7 +76,6 @@ export const buildHostsKpiUniqueIpsQuery = ({ }, }, size: 0, - track_total_hits: false, }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/last_first_seen/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/last_first_seen/__mocks__/index.ts index b43727e977a12..0cad31bffb2a1 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/last_first_seen/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/last_first_seen/__mocks__/index.ts @@ -69,6 +69,7 @@ export const formattedSearchStrategyResponse = { 'winlogbeat-*', ], ignoreUnavailable: true, + track_total_hits: false, body: { aggregations: { firstSeen: { min: { field: '@timestamp' } }, @@ -76,7 +77,6 @@ export const formattedSearchStrategyResponse = { }, query: { bool: { filter: [{ term: { 'host.name': 'siem-kibana' } }] } }, size: 0, - track_total_hits: false, }, }, null, @@ -100,6 +100,7 @@ export const expectedDsl = { 'winlogbeat-*', ], ignoreUnavailable: true, + track_total_hits: false, body: { aggregations: { firstSeen: { min: { field: '@timestamp' } }, @@ -107,6 +108,5 @@ export const expectedDsl = { }, query: { bool: { filter: [{ term: { 'host.name': 'siem-kibana' } }] } }, size: 0, - track_total_hits: false, }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/last_first_seen/query.last_first_seen_host.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/last_first_seen/query.last_first_seen_host.dsl.ts index f14727f94b30a..d601a5905dd6e 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/last_first_seen/query.last_first_seen_host.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/last_first_seen/query.last_first_seen_host.dsl.ts @@ -20,6 +20,7 @@ export const buildFirstLastSeenHostQuery = ({ allowNoIndices: true, index: defaultIndex, ignoreUnavailable: true, + track_total_hits: false, body: { ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), aggregations: { @@ -28,7 +29,6 @@ export const buildFirstLastSeenHostQuery = ({ }, query: { bool: { filter } }, size: 0, - track_total_hits: false, }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/overview/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/overview/__mocks__/index.ts index 1105914fa5d7f..987754420430d 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/overview/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/overview/__mocks__/index.ts @@ -127,6 +127,7 @@ export const formattedSearchStrategyResponse = { 'winlogbeat-*', ], ignoreUnavailable: true, + track_total_hits: false, body: { aggregations: { auditd_count: { filter: { term: { 'event.module': 'auditd' } } }, @@ -299,7 +300,6 @@ export const formattedSearchStrategyResponse = { }, }, size: 0, - track_total_hits: false, }, }, null, @@ -339,6 +339,7 @@ export const expectedDsl = { 'winlogbeat-*', ], ignoreUnavailable: true, + track_total_hits: false, body: { aggregations: { auditd_count: { filter: { term: { 'event.module': 'auditd' } } }, @@ -511,6 +512,5 @@ export const expectedDsl = { }, }, size: 0, - track_total_hits: false, }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/overview/query.overview_host.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/overview/query.overview_host.dsl.ts index d7c9b2b25f35e..2c237ab75bcbb 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/overview/query.overview_host.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/overview/query.overview_host.dsl.ts @@ -31,6 +31,7 @@ export const buildOverviewHostQuery = ({ allowNoIndices: true, index: defaultIndex, ignoreUnavailable: true, + track_total_hits: false, body: { aggregations: { auditd_count: { @@ -289,7 +290,6 @@ export const buildOverviewHostQuery = ({ }, }, size: 0, - track_total_hits: false, }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/__mocks__/index.ts index b43bd7e378fa6..07ae64bc63f19 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/__mocks__/index.ts @@ -42,6 +42,7 @@ export const formattedAlertsSearchStrategyResponse: MatrixHistogramStrategyRespo ], allowNoIndices: true, ignoreUnavailable: true, + track_total_hits: true, body: { aggregations: { alertsGroup: { @@ -113,7 +114,6 @@ export const formattedAlertsSearchStrategyResponse: MatrixHistogramStrategyRespo }, }, size: 0, - track_total_hits: true, }, }, null, @@ -127,6 +127,7 @@ export const formattedAlertsSearchStrategyResponse: MatrixHistogramStrategyRespo export const expectedDsl = { allowNoIndices: true, + track_total_hits: false, body: { aggregations: { host_count: { cardinality: { field: 'host.name' } }, @@ -161,7 +162,6 @@ export const expectedDsl = { }, }, size: 0, - track_total_hits: false, }, ignoreUnavailable: true, index: [ @@ -208,6 +208,7 @@ export const formattedAnomaliesSearchStrategyResponse: MatrixHistogramStrategyRe ], allowNoIndices: true, ignoreUnavailable: true, + track_total_hits: true, body: { aggs: { anomalyActionGroup: { @@ -258,7 +259,6 @@ export const formattedAnomaliesSearchStrategyResponse: MatrixHistogramStrategyRe }, }, size: 0, - track_total_hits: true, }, }, null, @@ -390,6 +390,7 @@ export const formattedAuthenticationsSearchStrategyResponse: MatrixHistogramStra ], allowNoIndices: true, ignoreUnavailable: true, + track_total_hits: true, body: { aggregations: { eventActionGroup: { @@ -429,7 +430,6 @@ export const formattedAuthenticationsSearchStrategyResponse: MatrixHistogramStra }, }, size: 0, - track_total_hits: true, }, }, null, @@ -956,6 +956,7 @@ export const formattedEventsSearchStrategyResponse: MatrixHistogramStrategyRespo ], allowNoIndices: true, ignoreUnavailable: true, + track_total_hits: true, body: { aggregations: { eventActionGroup: { @@ -994,7 +995,6 @@ export const formattedEventsSearchStrategyResponse: MatrixHistogramStrategyRespo }, }, size: 0, - track_total_hits: true, }, }, null, diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/alerts/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/alerts/__mocks__/index.ts index 74b7e8b18028b..86006c3155447 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/alerts/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/alerts/__mocks__/index.ts @@ -36,6 +36,7 @@ export const expectedDsl = { ], allowNoIndices: true, ignoreUnavailable: true, + track_total_hits: true, body: { aggregations: { alertsGroup: { @@ -104,6 +105,5 @@ export const expectedDsl = { }, }, size: 0, - track_total_hits: true, }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/alerts/query.alerts_histogram.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/alerts/query.alerts_histogram.dsl.ts index 7dd867b19f284..54ee066b64119 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/alerts/query.alerts_histogram.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/alerts/query.alerts_histogram.dsl.ts @@ -85,6 +85,7 @@ export const buildAlertsHistogramQuery = ({ index: defaultIndex, allowNoIndices: true, ignoreUnavailable: true, + track_total_hits: true, body: { aggregations: getHistogramAggregation(), query: { @@ -93,7 +94,6 @@ export const buildAlertsHistogramQuery = ({ }, }, size: 0, - track_total_hits: true, }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/anomalies/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/anomalies/__mocks__/index.ts index 561e2fb1f0058..81da78a132084 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/anomalies/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/anomalies/__mocks__/index.ts @@ -36,6 +36,7 @@ export const expectedDsl = { ], allowNoIndices: true, ignoreUnavailable: true, + track_total_hits: true, body: { aggs: { anomalyActionGroup: { @@ -83,6 +84,5 @@ export const expectedDsl = { }, }, size: 0, - track_total_hits: true, }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/anomalies/query.anomalies_histogram.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/anomalies/query.anomalies_histogram.dsl.ts index 34e5831b52b92..78fc0a30d0477 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/anomalies/query.anomalies_histogram.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/anomalies/query.anomalies_histogram.dsl.ts @@ -66,6 +66,7 @@ export const buildAnomaliesHistogramQuery = ({ index: defaultIndex, allowNoIndices: true, ignoreUnavailable: true, + track_total_hits: true, body: { aggs: getHistogramAggregation(), query: { @@ -74,7 +75,6 @@ export const buildAnomaliesHistogramQuery = ({ }, }, size: 0, - track_total_hits: true, }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/__mocks__/index.ts index 169f1569adc37..5cf667a0085fa 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/__mocks__/index.ts @@ -35,6 +35,7 @@ export const expectedDsl = { ], allowNoIndices: true, ignoreUnavailable: true, + track_total_hits: true, body: { aggregations: { eventActionGroup: { @@ -74,6 +75,5 @@ export const expectedDsl = { }, }, size: 0, - track_total_hits: true, }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/query.authentications_histogram.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/query.authentications_histogram.dsl.ts index 4a208f2ab341e..8661fff574b4a 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/query.authentications_histogram.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/query.authentications_histogram.dsl.ts @@ -78,6 +78,7 @@ export const buildAuthenticationsHistogramQuery = ({ index: defaultIndex, allowNoIndices: true, ignoreUnavailable: true, + track_total_hits: true, body: { aggregations: getHistogramAggregation(), query: { @@ -86,7 +87,6 @@ export const buildAuthenticationsHistogramQuery = ({ }, }, size: 0, - track_total_hits: true, }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/events/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/events/__mocks__/index.ts index 312c0d528f20b..0bf1118835414 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/events/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/events/__mocks__/index.ts @@ -40,6 +40,7 @@ export const expectedDsl = { ], allowNoIndices: true, ignoreUnavailable: true, + track_total_hits: true, body: { aggregations: { eventActionGroup: { @@ -78,7 +79,6 @@ export const expectedDsl = { }, }, size: 0, - track_total_hits: true, }, }; @@ -94,6 +94,7 @@ export const expectedThresholdDsl = { ], allowNoIndices: true, ignoreUnavailable: true, + track_total_hits: true, body: { aggregations: { eventActionGroup: { @@ -132,7 +133,6 @@ export const expectedThresholdDsl = { }, }, size: 0, - track_total_hits: true, }, }; @@ -148,6 +148,7 @@ export const expectedThresholdMissingFieldDsl = { ], allowNoIndices: true, ignoreUnavailable: true, + track_total_hits: true, body: { aggregations: { eventActionGroup: { @@ -187,6 +188,5 @@ export const expectedThresholdMissingFieldDsl = { }, }, size: 0, - track_total_hits: true, }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/events/query.events_histogram.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/events/query.events_histogram.dsl.ts index aa1e1d47c87c6..04b428f9de89e 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/events/query.events_histogram.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/events/query.events_histogram.dsl.ts @@ -97,6 +97,7 @@ export const buildEventsHistogramQuery = ({ index: defaultIndex, allowNoIndices: true, ignoreUnavailable: true, + track_total_hits: true, body: { aggregations: getHistogramAggregation(), query: { @@ -105,7 +106,6 @@ export const buildEventsHistogramQuery = ({ }, }, size: 0, - track_total_hits: true, }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/__mocks__/index.ts index 46d9c23321a8f..1cea4c3eb63bc 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/__mocks__/index.ts @@ -314,6 +314,7 @@ export const formattedSearchStrategyResponse = { 'winlogbeat-*', ], ignoreUnavailable: true, + track_total_hits: false, body: { docvalue_fields: mockOptions.docValueFields, aggs: { @@ -390,7 +391,6 @@ export const formattedSearchStrategyResponse = { }, query: { bool: { should: [] } }, size: 0, - track_total_hits: false, }, }, null, @@ -455,6 +455,7 @@ export const expectedDsl = { 'winlogbeat-*', ], ignoreUnavailable: true, + track_total_hits: false, body: { aggs: { source: { @@ -521,6 +522,5 @@ export const expectedDsl = { docvalue_fields: mockOptions.docValueFields, query: { bool: { should: [] } }, size: 0, - track_total_hits: false, }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/query.details_network.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/query.details_network.dsl.ts index b20de12624db4..d1d0c44d9b61b 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/query.details_network.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/query.details_network.dsl.ts @@ -106,6 +106,7 @@ export const buildNetworkDetailsQuery = ({ allowNoIndices: true, index: defaultIndex, ignoreUnavailable: true, + track_total_hits: false, body: { ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), aggs: { @@ -119,7 +120,6 @@ export const buildNetworkDetailsQuery = ({ }, }, size: 0, - track_total_hits: false, }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/dns/query.network_kpi_dns.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/dns/query.network_kpi_dns.dsl.ts index 0c4379fa89fd8..1d1aa50cc3ee2 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/dns/query.network_kpi_dns.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/dns/query.network_kpi_dns.dsl.ts @@ -58,6 +58,7 @@ export const buildDnsQuery = ({ index: defaultIndex, allowNoIndices: true, ignoreUnavailable: true, + track_total_hits: true, body: { query: { bool: { @@ -65,7 +66,6 @@ export const buildDnsQuery = ({ }, }, size: 0, - track_total_hits: true, }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/network_events/query.network_kpi_network_events.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/network_events/query.network_kpi_network_events.dsl.ts index 7222519bb0ac0..3d5607c8b443a 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/network_events/query.network_kpi_network_events.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/network_events/query.network_kpi_network_events.dsl.ts @@ -32,6 +32,7 @@ export const buildNetworkEventsQuery = ({ index: defaultIndex, allowNoIndices: true, ignoreUnavailable: true, + track_total_hits: true, body: { query: { bool: { @@ -39,7 +40,6 @@ export const buildNetworkEventsQuery = ({ }, }, size: 0, - track_total_hits: true, }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/tls_handshakes/query.network_kpi_tls_handshakes.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/tls_handshakes/query.network_kpi_tls_handshakes.dsl.ts index d8d27a8ad7e35..0a826938e95b8 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/tls_handshakes/query.network_kpi_tls_handshakes.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/tls_handshakes/query.network_kpi_tls_handshakes.dsl.ts @@ -58,6 +58,7 @@ export const buildTlsHandshakeQuery = ({ index: defaultIndex, allowNoIndices: true, ignoreUnavailable: true, + track_total_hits: true, body: { query: { bool: { @@ -65,7 +66,6 @@ export const buildTlsHandshakeQuery = ({ }, }, size: 0, - track_total_hits: true, }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/unique_flows/query.network_kpi_unique_flows.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/unique_flows/query.network_kpi_unique_flows.dsl.ts index 13a404ec3720b..ec8de30cfff85 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/unique_flows/query.network_kpi_unique_flows.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/unique_flows/query.network_kpi_unique_flows.dsl.ts @@ -32,6 +32,7 @@ export const buildUniqueFlowsQuery = ({ index: defaultIndex, allowNoIndices: true, ignoreUnavailable: true, + track_total_hits: false, body: { aggregations: { unique_flow_id: { @@ -46,7 +47,6 @@ export const buildUniqueFlowsQuery = ({ }, }, size: 0, - track_total_hits: false, }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/unique_private_ips/query.network_kpi_unique_private_ips.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/unique_private_ips/query.network_kpi_unique_private_ips.dsl.ts index e12ccf5b7889b..590e7117826d7 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/unique_private_ips/query.network_kpi_unique_private_ips.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/unique_private_ips/query.network_kpi_unique_private_ips.dsl.ts @@ -87,6 +87,7 @@ export const buildUniquePrivateIpsQuery = ({ allowNoIndices: true, index: defaultIndex, ignoreUnavailable: true, + track_total_hits: false, body: { aggregations: { ...getAggs('source'), @@ -98,7 +99,6 @@ export const buildUniquePrivateIpsQuery = ({ }, }, size: 0, - track_total_hits: false, }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/overview/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/overview/__mocks__/index.ts index 79ad6489558de..fcb30be7a403d 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/overview/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/overview/__mocks__/index.ts @@ -111,6 +111,7 @@ export const formattedSearchStrategyResponse = { 'winlogbeat-*', ], ignoreUnavailable: true, + track_total_hits: false, body: { aggregations: { unique_flow_count: { filter: { term: { type: 'flow' } } }, @@ -182,7 +183,6 @@ export const formattedSearchStrategyResponse = { }, }, size: 0, - track_total_hits: false, }, }, null, diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/overview/query.overview_network.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/overview/query.overview_network.dsl.ts index c5e2892bd9f82..7e35ae2fd4308 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/overview/query.overview_network.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/overview/query.overview_network.dsl.ts @@ -31,6 +31,7 @@ export const buildOverviewNetworkQuery = ({ allowNoIndices: true, index: defaultIndex, ignoreUnavailable: true, + track_total_hits: false, body: { aggregations: { unique_flow_count: { @@ -99,7 +100,6 @@ export const buildOverviewNetworkQuery = ({ }, }, size: 0, - track_total_hits: false, }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/__mocks__/index.ts index 5028e4a27c93e..16750acc5adee 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/__mocks__/index.ts @@ -69,6 +69,7 @@ export const formattedSearchStrategyResponse = { 'winlogbeat-*', ], ignoreUnavailable: true, + track_total_hits: false, body: { aggs: { count: { cardinality: { field: 'tls.server.hash.sha1' } }, @@ -99,7 +100,6 @@ export const formattedSearchStrategyResponse = { }, }, size: 0, - track_total_hits: false, }, }, null, @@ -123,6 +123,7 @@ export const expectedDsl = { 'winlogbeat-*', ], ignoreUnavailable: true, + track_total_hits: false, body: { aggs: { count: { cardinality: { field: 'tls.server.hash.sha1' } }, @@ -153,6 +154,5 @@ export const expectedDsl = { }, }, size: 0, - track_total_hits: false, }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/query.tls_network.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/query.tls_network.dsl.ts index ff5fe20f685f1..be60b33ae2d22 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/query.tls_network.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/query.tls_network.dsl.ts @@ -78,6 +78,7 @@ export const buildNetworkTlsQuery = ({ allowNoIndices: true, index: defaultIndex, ignoreUnavailable: true, + track_total_hits: false, body: { aggs: { ...getAggs(querySize, sort), @@ -88,7 +89,6 @@ export const buildNetworkTlsQuery = ({ }, }, size: 0, - track_total_hits: false, }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/users/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/users/__mocks__/index.ts index 252f165f11ad9..3837afabe5799 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/users/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/users/__mocks__/index.ts @@ -129,6 +129,7 @@ export const formattedSearchStrategyResponse = { 'winlogbeat-*', ], ignoreUnavailable: true, + track_total_hits: false, body: { aggs: { user_count: { cardinality: { field: 'user.name' } }, @@ -160,7 +161,6 @@ export const formattedSearchStrategyResponse = { }, }, size: 0, - track_total_hits: false, }, }, null, @@ -174,6 +174,7 @@ export const formattedSearchStrategyResponse = { export const expectedDsl = { allowNoIndices: true, + track_total_hits: false, body: { aggs: { user_count: { cardinality: { field: 'user.name' } }, @@ -205,7 +206,6 @@ export const expectedDsl = { }, }, size: 0, - track_total_hits: false, }, ignoreUnavailable: true, index: [ diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/users/query.users_network.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/users/query.users_network.dsl.ts index 57cb6093ae355..2b02b25292a32 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/users/query.users_network.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/users/query.users_network.dsl.ts @@ -37,6 +37,7 @@ export const buildUsersQuery = ({ allowNoIndices: true, index: defaultIndex, ignoreUnavailable: true, + track_total_hits: false, body: { aggs: { user_count: { @@ -84,7 +85,6 @@ export const buildUsersQuery = ({ }, }, size: 0, - track_total_hits: false, }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/last_event_time/query.events_last_event_time.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/last_event_time/query.events_last_event_time.dsl.ts index 1e7b531d7fcf1..ccc156af84922 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/last_event_time/query.events_last_event_time.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/last_event_time/query.events_last_event_time.dsl.ts @@ -40,6 +40,7 @@ export const buildLastEventTimeQuery = ({ allowNoIndices: true, index: indicesToQuery.network, ignoreUnavailable: true, + track_total_hits: false, body: { ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), aggregations: { @@ -47,7 +48,6 @@ export const buildLastEventTimeQuery = ({ }, query: { bool: { should: getIpDetailsFilter(details.ip) } }, size: 0, - track_total_hits: false, }, }; } @@ -58,6 +58,7 @@ export const buildLastEventTimeQuery = ({ allowNoIndices: true, index: indicesToQuery.hosts, ignoreUnavailable: true, + track_total_hits: false, body: { ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), aggregations: { @@ -65,7 +66,6 @@ export const buildLastEventTimeQuery = ({ }, query: { bool: { filter: getHostDetailsFilter(details.hostName) } }, size: 0, - track_total_hits: false, }, }; } @@ -76,6 +76,7 @@ export const buildLastEventTimeQuery = ({ allowNoIndices: true, index: indicesToQuery[indexKey], ignoreUnavailable: true, + track_total_hits: false, body: { ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), aggregations: { @@ -83,7 +84,6 @@ export const buildLastEventTimeQuery = ({ }, query: { match_all: {} }, size: 0, - track_total_hits: false, }, }; default: From d9abaa180b15ef72f18a9ad03a4395c0a9601873 Mon Sep 17 00:00:00 2001 From: Nathan L Smith Date: Wed, 10 Feb 2021 23:34:36 -0600 Subject: [PATCH 12/13] Don't clean when running e2e tests (#91057) I don't think this is necessary, and since it's run before `bootstrap`, the Bazel tools aren't installed so it fails silently. Example: https://apm-ci.elastic.co/blue/organizations/jenkins/apm-ui%2Fapm-ui-e2e-tests-mbp%2FPR-89647/detail/PR-89647/21/pipeline/124/ Should fix APM E2E failures. --- x-pack/plugins/apm/e2e/ci/prepare-kibana.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/apm/e2e/ci/prepare-kibana.sh b/x-pack/plugins/apm/e2e/ci/prepare-kibana.sh index f383dd6d16f7f..9e6198bcc526d 100755 --- a/x-pack/plugins/apm/e2e/ci/prepare-kibana.sh +++ b/x-pack/plugins/apm/e2e/ci/prepare-kibana.sh @@ -1,13 +1,13 @@ #!/usr/bin/env bash -set -ex +set -e E2E_DIR=x-pack/plugins/apm/e2e -echo "1/2 Install dependencies ..." +echo "1/2 Install dependencies..." # shellcheck disable=SC1091 source src/dev/ci_setup/setup_env.sh true -yarn kbn clean && yarn kbn bootstrap +yarn kbn bootstrap -echo "2/2 Start Kibana ..." +echo "2/2 Start Kibana..." ## Might help to avoid FATAL ERROR: Ineffective mark-compacts near heap limit Allocation failed - JavaScript heap out of memory export NODE_OPTIONS="--max-old-space-size=4096" nohup node ./scripts/kibana --no-base-path --no-watch --dev --no-dev-config --config ${E2E_DIR}/ci/kibana.e2e.yml > ${E2E_DIR}/kibana.log 2>&1 & From c0a974aa0e048af7c5829172c41aa32d860961c4 Mon Sep 17 00:00:00 2001 From: Spencer Date: Wed, 10 Feb 2021 21:38:06 -0800 Subject: [PATCH 13/13] [ts/build_ts_refs] add support for --clean flag (#91060) Co-authored-by: spalger --- scripts/build_ts_refs.js | 2 +- src/dev/typescript/build_refs.ts | 35 ------------ src/dev/typescript/build_ts_refs.ts | 24 ++++++++ src/dev/typescript/build_ts_refs_cli.ts | 37 ++++++++++++ src/dev/typescript/concurrent_map.ts | 28 ++++++++++ src/dev/typescript/index.ts | 1 + src/dev/typescript/project.ts | 15 +---- src/dev/typescript/run_type_check_cli.ts | 4 +- src/dev/typescript/ts_configfile.ts | 71 ++++++++++++++++++++++++ 9 files changed, 166 insertions(+), 51 deletions(-) delete mode 100644 src/dev/typescript/build_refs.ts create mode 100644 src/dev/typescript/build_ts_refs.ts create mode 100644 src/dev/typescript/build_ts_refs_cli.ts create mode 100644 src/dev/typescript/concurrent_map.ts create mode 100644 src/dev/typescript/ts_configfile.ts diff --git a/scripts/build_ts_refs.js b/scripts/build_ts_refs.js index 3222e0e90797b..a4ee6ec491ef1 100644 --- a/scripts/build_ts_refs.js +++ b/scripts/build_ts_refs.js @@ -7,4 +7,4 @@ */ require('../src/setup_node_env'); -require('../src/dev/typescript/build_refs').runBuildRefs(); +require('../src/dev/typescript').runBuildRefsCli(); diff --git a/src/dev/typescript/build_refs.ts b/src/dev/typescript/build_refs.ts deleted file mode 100644 index 77d6eb2abc612..0000000000000 --- a/src/dev/typescript/build_refs.ts +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import execa from 'execa'; -import { run, ToolingLog } from '@kbn/dev-utils'; - -export async function buildAllRefs(log: ToolingLog) { - await buildRefs(log, 'tsconfig.refs.json'); -} - -async function buildRefs(log: ToolingLog, projectPath: string) { - try { - log.debug(`Building TypeScript projects refs for ${projectPath}...`); - await execa(require.resolve('typescript/bin/tsc'), ['-b', projectPath, '--pretty']); - } catch (e) { - log.error(e); - process.exit(1); - } -} - -export async function runBuildRefs() { - run( - async ({ log }) => { - await buildAllRefs(log); - }, - { - description: 'Build TypeScript projects', - } - ); -} diff --git a/src/dev/typescript/build_ts_refs.ts b/src/dev/typescript/build_ts_refs.ts new file mode 100644 index 0000000000000..2e25827996e45 --- /dev/null +++ b/src/dev/typescript/build_ts_refs.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; + +import execa from 'execa'; +import { ToolingLog, REPO_ROOT } from '@kbn/dev-utils'; + +export const REF_CONFIG_PATHS = [Path.resolve(REPO_ROOT, 'tsconfig.refs.json')]; + +export async function buildAllTsRefs(log: ToolingLog) { + for (const path of REF_CONFIG_PATHS) { + const relative = Path.relative(REPO_ROOT, path); + log.debug(`Building TypeScript projects refs for ${relative}...`); + await execa(require.resolve('typescript/bin/tsc'), ['-b', relative, '--pretty'], { + cwd: REPO_ROOT, + }); + } +} diff --git a/src/dev/typescript/build_ts_refs_cli.ts b/src/dev/typescript/build_ts_refs_cli.ts new file mode 100644 index 0000000000000..1f7bf18b5012d --- /dev/null +++ b/src/dev/typescript/build_ts_refs_cli.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { run } from '@kbn/dev-utils'; +import del from 'del'; + +import { buildAllTsRefs, REF_CONFIG_PATHS } from './build_ts_refs'; +import { getOutputsDeep } from './ts_configfile'; +import { concurrentMap } from './concurrent_map'; + +export async function runBuildRefsCli() { + run( + async ({ log, flags }) => { + if (flags.clean) { + const outDirs = getOutputsDeep(REF_CONFIG_PATHS); + log.info('deleting', outDirs.length, 'ts output directories'); + await concurrentMap(100, outDirs, (outDir) => del(outDir)); + } + + await buildAllTsRefs(log); + }, + { + description: 'Build TypeScript projects', + flags: { + boolean: ['clean'], + }, + log: { + defaultLevel: 'debug', + }, + } + ); +} diff --git a/src/dev/typescript/concurrent_map.ts b/src/dev/typescript/concurrent_map.ts new file mode 100644 index 0000000000000..793630ab85a55 --- /dev/null +++ b/src/dev/typescript/concurrent_map.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Rx from 'rxjs'; +import { mergeMap, toArray, map } from 'rxjs/operators'; +import { lastValueFrom } from '@kbn/std'; + +export async function concurrentMap( + concurrency: number, + arr: T[], + fn: (item: T, i: number) => Promise +): Promise { + return await lastValueFrom( + Rx.from(arr).pipe( + // execute items in parallel based on concurrency + mergeMap(async (item, index) => ({ index, result: await fn(item, index) }), concurrency), + // collect the results into an array + toArray(), + // sort items back into order and return array of just results + map((list) => list.sort((a, b) => a.index - b.index).map(({ result }) => result)) + ) + ); +} diff --git a/src/dev/typescript/index.ts b/src/dev/typescript/index.ts index 934c032152faf..34ecd76a994db 100644 --- a/src/dev/typescript/index.ts +++ b/src/dev/typescript/index.ts @@ -11,3 +11,4 @@ export { filterProjectsByFlag } from './projects'; export { getTsProjectForAbsolutePath } from './get_ts_project_for_absolute_path'; export { execInProjects } from './exec_in_projects'; export { runTypeCheckCli } from './run_type_check_cli'; +export * from './build_ts_refs_cli'; diff --git a/src/dev/typescript/project.ts b/src/dev/typescript/project.ts index 4eb8036d9c902..8d92284e49637 100644 --- a/src/dev/typescript/project.ts +++ b/src/dev/typescript/project.ts @@ -6,14 +6,13 @@ * Side Public License, v 1. */ -import { readFileSync } from 'fs'; import { basename, dirname, relative, resolve } from 'path'; import { IMinimatch, Minimatch } from 'minimatch'; -import { parseConfigFileTextToJson } from 'typescript'; - import { REPO_ROOT } from '@kbn/utils'; +import { parseTsConfig } from './ts_configfile'; + function makeMatchers(directory: string, patterns: string[]) { return patterns.map( (pattern) => @@ -23,16 +22,6 @@ function makeMatchers(directory: string, patterns: string[]) { ); } -function parseTsConfig(path: string) { - const { error, config } = parseConfigFileTextToJson(path, readFileSync(path, 'utf8')); - - if (error) { - throw error; - } - - return config; -} - function testMatchers(matchers: IMinimatch[], path: string) { return matchers.some((matcher) => matcher.match(path)); } diff --git a/src/dev/typescript/run_type_check_cli.ts b/src/dev/typescript/run_type_check_cli.ts index 50f725891753b..f95c230f44b9e 100644 --- a/src/dev/typescript/run_type_check_cli.ts +++ b/src/dev/typescript/run_type_check_cli.ts @@ -13,7 +13,7 @@ import getopts from 'getopts'; import { execInProjects } from './exec_in_projects'; import { filterProjectsByFlag } from './projects'; -import { buildAllRefs } from './build_refs'; +import { buildAllTsRefs } from './build_ts_refs'; export async function runTypeCheckCli() { const extraFlags: string[] = []; @@ -69,7 +69,7 @@ export async function runTypeCheckCli() { process.exit(); } - await buildAllRefs(log); + await buildAllTsRefs(log); const tscArgs = [ // composite project cannot be used with --noEmit diff --git a/src/dev/typescript/ts_configfile.ts b/src/dev/typescript/ts_configfile.ts new file mode 100644 index 0000000000000..7998edcf80bcf --- /dev/null +++ b/src/dev/typescript/ts_configfile.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Fs from 'fs'; +import Path from 'path'; + +import { parseConfigFileTextToJson } from 'typescript'; + +// yes, this is just `any`, but I'm hoping that TypeScript will give us some help here eventually +type TsConfigFile = ReturnType['config']; + +export function parseTsConfig(tsConfigPath: string): TsConfigFile { + const { error, config } = parseConfigFileTextToJson( + tsConfigPath, + Fs.readFileSync(tsConfigPath, 'utf8') + ); + + if (error) { + throw error; + } + + return config; +} + +export function getOutputsDeep(tsConfigPaths: string[]) { + const tsConfigs = new Map(); + + const read = (path: string) => { + const cached = tsConfigs.get(path); + if (cached) { + return cached; + } + + const config = parseTsConfig(path); + tsConfigs.set(path, config); + return config; + }; + + const outputDirs: string[] = []; + const seen = new Set(); + + const traverse = (path: string) => { + const config = read(path); + if (seen.has(config)) { + return; + } + seen.add(config); + + const dirname = Path.dirname(path); + const relativeOutDir: string | undefined = config.compilerOptions?.outDir; + if (relativeOutDir) { + outputDirs.push(Path.resolve(dirname, relativeOutDir)); + } + + const refs: undefined | Array<{ path: string }> = config.references; + for (const ref of refs ?? []) { + traverse(Path.resolve(dirname, ref.path)); + } + }; + + for (const path of tsConfigPaths) { + traverse(path); + } + + return outputDirs; +}