From 5f844bfb6a39e17010d8a331ea5b0338693b5f36 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Tue, 24 Nov 2020 16:59:18 +0100 Subject: [PATCH 01/49] update geckodriver to 0.28 (#84085) --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 39149c801da46..af80102641dbb 100644 --- a/package.json +++ b/package.json @@ -654,7 +654,7 @@ "file-loader": "^4.2.0", "file-saver": "^1.3.8", "formsy-react": "^1.1.5", - "geckodriver": "^1.20.0", + "geckodriver": "^1.21.0", "glob-watcher": "5.0.3", "graphql-code-generator": "^0.18.2", "graphql-codegen-add": "^0.18.2", diff --git a/yarn.lock b/yarn.lock index d20c7f78c9792..8d47d3e843788 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14255,10 +14255,10 @@ gaze@^1.0.0, gaze@^1.1.0: dependencies: globule "^1.0.0" -geckodriver@^1.20.0: - version "1.20.0" - resolved "https://registry.yarnpkg.com/geckodriver/-/geckodriver-1.20.0.tgz#cd16edb177b88e31affcb54b18a238cae88950a7" - integrity sha512-5nVF4ixR+ZGhVsc4udnVihA9RmSlO6guPV1d2HqxYsgAOUNh0HfzxbzG7E49w4ilXq/CSu87x9yWvrsOstrADQ== +geckodriver@^1.21.0: + version "1.21.0" + resolved "https://registry.yarnpkg.com/geckodriver/-/geckodriver-1.21.0.tgz#1f04780ebfb451ffd08fa8fddc25cc26e37ac4a2" + integrity sha512-NamdJwGIWpPiafKQIvGman95BBi/SBqHddRXAnIEpFNFCFToTW0sEA0nUckMKCBNn1DVIcLfULfyFq/sTn9bkA== dependencies: adm-zip "0.4.16" bluebird "3.7.2" From a12bb044382ada8f557590851943a1568f24c58f Mon Sep 17 00:00:00 2001 From: Scotty Bollinger Date: Tue, 24 Nov 2020 10:16:33 -0600 Subject: [PATCH 02/49] [Workplace Search] Initial rendering of Org Sources (#84164) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix broken routes Didn’t have a way to test these when created * Get context from global state No need to do this in 2 places now. There was a race condition where the default logic value for `isOrganization` was set to `false` We don’t need a useEffect call here because the value is synchronous and has no side effects. Calling the method directly fixes the race condition. * Add the ‘path’ to the logic files for easier debugging * Add SourceSubNav component * Flip routes to match new convention It was decided by Product that instead of keying off of `/org` to determine context, that we would now flip it where we key of provate with `/p`. This means that /sources is now organization where before it was personal * Convert routers to use children instead of props This aligns with App Search and allows for easier telemtry and breadcrumbs * Add breadcrumbs and basic telemetry * Add in and refactor subnavigation As a part of this commit, the approach for rendering subnavs was refactored to align with App Search. There was a bug where some components weren’t rendering properly because the SourceLogic and GroupsLogic files were never unmounting. The reason for this is the subnav components use their respective logic files to get the IDs needed for rendering the subnav links. That is, SourceSubNav would call SourceLogic to get the ID to render the links and would stay rendered for the duration of the user’s time in the app. The result is that users would leave the source details page and navigate to add a new source and the logic file would never reset to a loading state and the UI would break. The fix was to borrow from the pattern App Search uses and pass the subnavs as props. Because App Search only uses a single engines subnav, they only needed one prop. We use multiple props for each subnav. Also, the subnav should not be rendered on the root routes (/sources, /p/sources, and /groups) so conditionals were added to only render the subnavs when not on those root routes. * Add FlashMessages * Fix some failed tests Missed this in first commit * Update SourceIcon to use EuiIcon Before this change, the legacy styles were not ported over. This gives a uniform size for both wrapped and unwrapped icons. The icons are a bit smaller on the add source page but Eui has lowered it’s largest size ‘xxl’ and we would need to write manual overrides. IMO the change is not significant enough to override. * Fix broken icons * Replace legacy div with component The eui.css file in ent-search is no longer up to date with current EUI and this was broken. The best fix was to use the component that renders as expected * Add base styles for Sources More in a future PR but this makes the majority of things look correct. * Cleanup Fix some type errors and rename constants * Couple more failing tests We have multiple `Layouts` now with the new subnavs * Fix prepare routes Like the first commit, missed these when porting over routes with no UI. * Clean up the desgin of the source connect screen The columns were way off in Kibana * Remove ORG_PATH const No longer needed since ‘/org’ is gone --- .../components/layout/nav.tsx | 13 +- .../shared/assets/source_icons/index.ts | 5 + .../shared/source_icon/source_icon.scss | 21 +++ .../shared/source_icon/source_icon.test.tsx | 6 +- .../shared/source_icon/source_icon.tsx | 13 +- .../workplace_search/constants.ts | 12 ++ .../workplace_search/index.test.tsx | 4 +- .../applications/workplace_search/index.tsx | 44 ++++-- .../workplace_search/routes.test.tsx | 6 +- .../applications/workplace_search/routes.ts | 24 ++-- .../add_source/configured_sources_list.tsx | 5 +- .../add_source/connect_instance.tsx | 4 +- .../components/source_sub_nav.tsx | 59 ++++++++ .../views/content_sources/index.ts | 1 + .../views/content_sources/source_logic.ts | 3 +- .../views/content_sources/source_router.tsx | 62 ++++---- .../views/content_sources/sources.scss | 23 +++ .../views/content_sources/sources_logic.ts | 1 + .../views/content_sources/sources_router.tsx | 133 ++++++++++-------- .../groups/components/group_manager_modal.tsx | 4 +- .../views/overview/onboarding_steps.test.tsx | 4 +- .../views/overview/onboarding_steps.tsx | 4 +- .../views/overview/organization_stats.tsx | 4 +- .../routes/workplace_search/sources.test.ts | 16 +-- .../server/routes/workplace_search/sources.ts | 18 +-- 25 files changed, 332 insertions(+), 157 deletions(-) create mode 100644 x-pack/plugins/enterprise_search/public/applications/workplace_search/components/shared/source_icon/source_icon.scss create mode 100644 x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/source_sub_nav.tsx create mode 100644 x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/sources.scss diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/layout/nav.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/layout/nav.tsx index 6fa6698e6b6ba..de6c75d60189e 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/layout/nav.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/layout/nav.tsx @@ -11,11 +11,9 @@ import { WORKPLACE_SEARCH_PLUGIN } from '../../../../../common/constants'; import { getWorkplaceSearchUrl } from '../../../shared/enterprise_search_url'; import { SideNav, SideNavLink } from '../../../shared/layout'; -import { GroupSubNav } from '../../views/groups/components/group_sub_nav'; import { NAV } from '../../constants'; import { - ORG_SOURCES_PATH, SOURCES_PATH, SECURITY_PATH, ROLE_MAPPINGS_PATH, @@ -23,17 +21,22 @@ import { ORG_SETTINGS_PATH, } from '../../routes'; -export const WorkplaceSearchNav: React.FC = () => { +interface Props { + sourcesSubNav?: React.ReactNode; + groupsSubNav?: React.ReactNode; +} + +export const WorkplaceSearchNav: React.FC = ({ sourcesSubNav, groupsSubNav }) => { // TODO: icons return ( {NAV.OVERVIEW} - + {NAV.SOURCES} - }> + {NAV.GROUPS} diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/shared/assets/source_icons/index.ts b/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/shared/assets/source_icons/index.ts index 5f93694da09b8..2ac3f518e4e11 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/shared/assets/source_icons/index.ts +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/shared/assets/source_icons/index.ts @@ -30,22 +30,27 @@ import zendesk from './zendesk.svg'; export const images = { box, confluence, + confluenceCloud: confluence, + confluenceServer: confluence, crawler, custom, drive, dropbox, github, + githubEnterpriseServer: github, gmail, googleDrive, google, jira, jiraServer, + jiraCloud: jira, loadingSmall, office365, oneDrive, outlook, people, salesforce, + salesforceSandbox: salesforce, serviceNow, sharePoint, slack, diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/shared/source_icon/source_icon.scss b/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/shared/source_icon/source_icon.scss new file mode 100644 index 0000000000000..b04d5b8bc218f --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/shared/source_icon/source_icon.scss @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +.wrapped-icon { + width: 30px; + height: 30px; + overflow: hidden; + margin-right: 4px; + position: relative; + display: flex; + justify-content: center; + align-items: center; + + img { + max-width: 100%; + max-height: 100%; + } +} diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/shared/source_icon/source_icon.test.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/shared/source_icon/source_icon.test.tsx index c17b89c93a28b..4007f7a69f77a 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/shared/source_icon/source_icon.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/shared/source_icon/source_icon.test.tsx @@ -7,19 +7,21 @@ import React from 'react'; import { shallow } from 'enzyme'; +import { EuiIcon } from '@elastic/eui'; + import { SourceIcon } from './'; describe('SourceIcon', () => { it('renders unwrapped icon', () => { const wrapper = shallow(); - expect(wrapper.find('img')).toHaveLength(1); + expect(wrapper.find(EuiIcon)).toHaveLength(1); expect(wrapper.find('.user-group-source')).toHaveLength(0); }); it('renders wrapped icon', () => { const wrapper = shallow(); - expect(wrapper.find('.user-group-source')).toHaveLength(1); + expect(wrapper.find('.wrapped-icon')).toHaveLength(1); }); }); diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/shared/source_icon/source_icon.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/shared/source_icon/source_icon.tsx index dec9e25fe2440..1af5420a164be 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/shared/source_icon/source_icon.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/shared/source_icon/source_icon.tsx @@ -8,6 +8,10 @@ import React from 'react'; import { camelCase } from 'lodash'; +import { EuiIcon } from '@elastic/eui'; + +import './source_icon.scss'; + import { images } from '../assets/source_icons'; import { imagesFull } from '../assets/sources_full_bleed'; @@ -27,14 +31,15 @@ export const SourceIcon: React.FC = ({ fullBleed = false, }) => { const icon = ( - {name} ); return wrapped ? ( -
+
{icon}
) : ( diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/constants.ts b/x-pack/plugins/enterprise_search/public/applications/workplace_search/constants.ts index 1846115d73900..327ee7b30582b 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/constants.ts +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/constants.ts @@ -25,15 +25,27 @@ export const NAV = { 'xpack.enterpriseSearch.workplaceSearch.nav.groups.sourcePrioritization', { defaultMessage: 'Source Prioritization' } ), + CONTENT: i18n.translate('xpack.enterpriseSearch.workplaceSearch.nav.content', { + defaultMessage: 'Content', + }), ROLE_MAPPINGS: i18n.translate('xpack.enterpriseSearch.workplaceSearch.nav.roleMappings', { defaultMessage: 'Role Mappings', }), SECURITY: i18n.translate('xpack.enterpriseSearch.workplaceSearch.nav.security', { defaultMessage: 'Security', }), + SCHEMA: i18n.translate('xpack.enterpriseSearch.workplaceSearch.nav.schema', { + defaultMessage: 'Schema', + }), + DISPLAY_SETTINGS: i18n.translate('xpack.enterpriseSearch.workplaceSearch.nav.displaySettings', { + defaultMessage: 'Display Settings', + }), SETTINGS: i18n.translate('xpack.enterpriseSearch.workplaceSearch.nav.settings', { defaultMessage: 'Settings', }), + ADD_SOURCE: i18n.translate('xpack.enterpriseSearch.workplaceSearch.nav.addSource', { + defaultMessage: 'Add Source', + }), PERSONAL_DASHBOARD: i18n.translate( 'xpack.enterpriseSearch.workplaceSearch.nav.personalDashboard', { diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/index.test.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/index.test.tsx index 5f1e2dd18d3b6..20b15bcfc45ca 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/index.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/index.test.tsx @@ -57,7 +57,7 @@ describe('WorkplaceSearchConfigured', () => { it('renders layout and header actions', () => { const wrapper = shallow(); - expect(wrapper.find(Layout).prop('readOnlyMode')).toBeFalsy(); + expect(wrapper.find(Layout).first().prop('readOnlyMode')).toBeFalsy(); expect(wrapper.find(Overview)).toHaveLength(1); expect(mockKibanaValues.renderHeaderActions).toHaveBeenCalledWith(WorkplaceSearchHeaderActions); }); @@ -90,6 +90,6 @@ describe('WorkplaceSearchConfigured', () => { const wrapper = shallow(); - expect(wrapper.find(Layout).prop('readOnlyMode')).toEqual(true); + expect(wrapper.find(Layout).first().prop('readOnlyMode')).toEqual(true); }); }); diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/index.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/index.tsx index 776cae24dfdfb..562a2ffb32888 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/index.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/index.tsx @@ -16,13 +16,17 @@ import { AppLogic } from './app_logic'; import { Layout } from '../shared/layout'; import { WorkplaceSearchNav, WorkplaceSearchHeaderActions } from './components/layout'; -import { GROUPS_PATH, SETUP_GUIDE_PATH } from './routes'; +import { GROUPS_PATH, SETUP_GUIDE_PATH, SOURCES_PATH, PERSONAL_SOURCES_PATH } from './routes'; import { SetupGuide } from './views/setup_guide'; import { ErrorState } from './views/error_state'; import { NotFound } from '../shared/not_found'; import { Overview } from './views/overview'; import { GroupsRouter } from './views/groups'; +import { SourcesRouter } from './views/content_sources'; + +import { GroupSubNav } from './views/groups/components/group_sub_nav'; +import { SourceSubNav } from './views/content_sources/components/source_sub_nav'; export const WorkplaceSearch: React.FC = (props) => { const { config } = useValues(KibanaLogic); @@ -37,6 +41,10 @@ export const WorkplaceSearchConfigured: React.FC = (props) => { const { pathname } = useLocation(); + // We don't want so show the subnavs on the container root pages. + const showSourcesSubnav = pathname !== SOURCES_PATH && pathname !== PERSONAL_SOURCES_PATH; + const showGroupsSubnav = pathname !== GROUPS_PATH; + /** * Personal dashboard urls begin with /p/ * EX: http://localhost:5601/app/enterprise_search/workplace_search/p/sources @@ -45,6 +53,7 @@ export const WorkplaceSearchConfigured: React.FC = (props) => { // TODO: Once auth is figured out, we need to have a check for the equivilent of `isAdmin`. const isOrganization = !pathname.match(personalSourceUrlRegex); + setContext(isOrganization); useEffect(() => { if (!hasInitialized) { @@ -53,10 +62,6 @@ export const WorkplaceSearchConfigured: React.FC = (props) => { } }, [hasInitialized]); - useEffect(() => { - setContext(isOrganization); - }, [isOrganization]); - return ( @@ -65,19 +70,32 @@ export const WorkplaceSearchConfigured: React.FC = (props) => { {errorConnecting ? : } + + } />} + restrictWidth + readOnlyMode={readOnlyMode} + > + + + + + } />} + restrictWidth + readOnlyMode={readOnlyMode} + > + + + } restrictWidth readOnlyMode={readOnlyMode}> {errorConnecting ? ( ) : ( - - - - - - - - + + + )} diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/routes.test.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/routes.test.tsx index d03c0abb441b9..3fddcf3b77fe4 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/routes.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/routes.test.tsx @@ -12,7 +12,7 @@ import { EuiLink } from '@elastic/eui'; import { getContentSourcePath, SOURCES_PATH, - ORG_SOURCES_PATH, + PERSONAL_SOURCES_PATH, SOURCE_DETAILS_PATH, } from './routes'; @@ -26,13 +26,13 @@ describe('getContentSourcePath', () => { const wrapper = shallow(); const path = wrapper.find(EuiLink).prop('href'); - expect(path).toEqual(`${ORG_SOURCES_PATH}/123`); + expect(path).toEqual(`${SOURCES_PATH}/123`); }); it('should format user route', () => { const wrapper = shallow(); const path = wrapper.find(EuiLink).prop('href'); - expect(path).toEqual(`${SOURCES_PATH}/123`); + expect(path).toEqual(`${PERSONAL_SOURCES_PATH}/123`); }); }); diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/routes.ts b/x-pack/plugins/enterprise_search/public/applications/workplace_search/routes.ts index e41a043911dc9..3ec22ede888ab 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/routes.ts +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/routes.ts @@ -44,21 +44,21 @@ export const CUSTOM_API_DOCS_URL = `${DOCS_PREFIX}/workplace-search-custom-sourc export const CUSTOM_API_DOCUMENT_PERMISSIONS_DOCS_URL = `${CUSTOM_SOURCE_DOCS_URL}#custom-api-source-document-level-access-control`; export const ENT_SEARCH_LICENSE_MANAGEMENT = `${ENT_SEARCH_DOCS_PREFIX}/license-management.html`; -export const ORG_PATH = '/org'; +export const PERSONAL_PATH = '/p'; -export const ROLE_MAPPINGS_PATH = `${ORG_PATH}/role-mappings`; +export const ROLE_MAPPINGS_PATH = '/role-mappings'; export const ROLE_MAPPING_PATH = `${ROLE_MAPPINGS_PATH}/:roleId`; export const ROLE_MAPPING_NEW_PATH = `${ROLE_MAPPINGS_PATH}/new`; -export const USERS_PATH = `${ORG_PATH}/users`; -export const SECURITY_PATH = `${ORG_PATH}/security`; +export const USERS_PATH = '/users'; +export const SECURITY_PATH = '/security'; export const GROUPS_PATH = '/groups'; export const GROUP_PATH = `${GROUPS_PATH}/:groupId`; export const GROUP_SOURCE_PRIORITIZATION_PATH = `${GROUPS_PATH}/:groupId/source_prioritization`; export const SOURCES_PATH = '/sources'; -export const ORG_SOURCES_PATH = `${ORG_PATH}${SOURCES_PATH}`; +export const PERSONAL_SOURCES_PATH = `${PERSONAL_PATH}${SOURCES_PATH}`; export const SOURCE_ADDED_PATH = `${SOURCES_PATH}/added`; export const ADD_SOURCE_PATH = `${SOURCES_PATH}/add`; @@ -81,7 +81,7 @@ export const ADD_SLACK_PATH = `${SOURCES_PATH}/add/slack`; export const ADD_ZENDESK_PATH = `${SOURCES_PATH}/add/zendesk`; export const ADD_CUSTOM_PATH = `${SOURCES_PATH}/add/custom`; -export const PERSONAL_SETTINGS_PATH = '/settings'; +export const PERSONAL_SETTINGS_PATH = `${PERSONAL_PATH}/settings`; export const SOURCE_DETAILS_PATH = `${SOURCES_PATH}/:sourceId`; export const SOURCE_CONTENT_PATH = `${SOURCES_PATH}/:sourceId/content`; @@ -93,7 +93,7 @@ export const REINDEX_JOB_PATH = `${SOURCES_PATH}/:sourceId/schema-errors/:active export const DISPLAY_SETTINGS_SEARCH_RESULT_PATH = `${SOURCE_DISPLAY_SETTINGS_PATH}/`; export const DISPLAY_SETTINGS_RESULT_DETAIL_PATH = `${SOURCE_DISPLAY_SETTINGS_PATH}/result-detail`; -export const ORG_SETTINGS_PATH = `${ORG_PATH}/settings`; +export const ORG_SETTINGS_PATH = '/settings'; export const ORG_SETTINGS_CUSTOMIZE_PATH = `${ORG_SETTINGS_PATH}/customize`; export const ORG_SETTINGS_CONNECTORS_PATH = `${ORG_SETTINGS_PATH}/connectors`; export const ORG_SETTINGS_OAUTH_APPLICATION_PATH = `${ORG_SETTINGS_PATH}/oauth`; @@ -120,9 +120,9 @@ export const getContentSourcePath = ( path: string, sourceId: string, isOrganization: boolean -): string => generatePath(isOrganization ? ORG_PATH + path : path, { sourceId }); -export const getGroupPath = (groupId: string) => generatePath(GROUP_PATH, { groupId }); -export const getGroupSourcePrioritizationPath = (groupId: string) => +): string => generatePath(isOrganization ? path : `${PERSONAL_PATH}${path}`, { sourceId }); +export const getGroupPath = (groupId: string): string => generatePath(GROUP_PATH, { groupId }); +export const getGroupSourcePrioritizationPath = (groupId: string): string => `${GROUPS_PATH}/${groupId}/source_prioritization`; -export const getSourcesPath = (path: string, isOrganization: boolean) => - isOrganization ? `${ORG_PATH}${path}` : path; +export const getSourcesPath = (path: string, isOrganization: boolean): string => + isOrganization ? path : `${PERSONAL_PATH}${path}`; diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/add_source/configured_sources_list.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/add_source/configured_sources_list.tsx index a95d5ca75b0b6..fbd053f9b8374 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/add_source/configured_sources_list.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/add_source/configured_sources_list.tsx @@ -13,6 +13,7 @@ import { EuiFlexGrid, EuiFlexGroup, EuiFlexItem, + EuiPanel, EuiSpacer, EuiText, EuiTitle, @@ -57,7 +58,7 @@ export const ConfiguredSourcesList: React.FC = ({ {sources.map(({ name, serviceType, addPath, connected, accountContextOnly }, i) => ( -
+ = ({ )} -
+ ))} diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/add_source/connect_instance.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/add_source/connect_instance.tsx index ad183181b4eca..f9123ab4e1cca 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/add_source/connect_instance.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/add_source/connect_instance.tsx @@ -240,13 +240,13 @@ export const ConnectInstance: React.FC = ({ gutterSize="xl" responsive={false} > - + {header} {featureBadgeGroup()} {descriptionBlock} {formFields} - + diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/source_sub_nav.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/source_sub_nav.tsx new file mode 100644 index 0000000000000..cc68a62b9555d --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/source_sub_nav.tsx @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { useValues } from 'kea'; + +import { AppLogic } from '../../../app_logic'; +import { NAV, CUSTOM_SERVICE_TYPE } from '../../../constants'; + +import { SourceLogic } from '../source_logic'; + +import { SideNavLink } from '../../../../shared/layout'; + +import { + getContentSourcePath, + SOURCE_DETAILS_PATH, + SOURCE_CONTENT_PATH, + SOURCE_SCHEMAS_PATH, + SOURCE_DISPLAY_SETTINGS_PATH, + SOURCE_SETTINGS_PATH, +} from '../../../routes'; + +export const SourceSubNav: React.FC = () => { + const { isOrganization } = useValues(AppLogic); + const { + contentSource: { id, serviceType }, + } = useValues(SourceLogic); + + if (!id) return null; + + const isCustom = serviceType === CUSTOM_SERVICE_TYPE; + + return ( + <> + + {NAV.OVERVIEW} + + + {NAV.CONTENT} + + {isCustom && ( + <> + + {NAV.SCHEMA} + + + {NAV.DISPLAY_SETTINGS} + + + )} + + {NAV.SETTINGS} + + + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/index.ts b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/index.ts index 0ef2099968f10..f447751e96594 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/index.ts +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/index.ts @@ -5,3 +5,4 @@ */ export { Overview } from './components/overview'; +export { SourcesRouter } from './sources_router'; diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/source_logic.ts b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/source_logic.ts index 0a11da02dc789..51b5735f01045 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/source_logic.ts +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/source_logic.ts @@ -146,6 +146,7 @@ interface PreContentSourceResponse { } export const SourceLogic = kea>({ + path: ['enterprise_search', 'workplace_search', 'source_logic'], actions: { onInitializeSource: (contentSource: ContentSourceFullData) => contentSource, onUpdateSourceName: (name: string) => name, @@ -601,7 +602,7 @@ export const SourceLogic = kea>({ try { const response = await HttpLogic.values.http.post(route, { - body: JSON.stringify({ params }), + body: JSON.stringify({ ...params }), }); actions.setCustomSourceData(response); successCallback(); diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/source_router.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/source_router.tsx index b8b8e6e1040a1..7161e613247cd 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/source_router.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/source_router.tsx @@ -13,6 +13,11 @@ import { Route, Switch, useHistory, useParams } from 'react-router-dom'; import { EuiButton, EuiCallOut, EuiSpacer } from '@elastic/eui'; +import { SetWorkplaceSearchChrome as SetPageChrome } from '../../../shared/kibana_chrome'; +import { SendWorkplaceSearchTelemetry as SendTelemetry } from '../../../shared/telemetry'; + +import { NAV } from '../../constants'; + import { ENT_SEARCH_LICENSE_MANAGEMENT, REINDEX_JOB_PATH, @@ -99,39 +104,42 @@ export const SourceRouter: React.FC = () => { {/* TODO: Figure out with design how to make this look better */} {pageHeader} - - + + + + + + + + + + {isCustomSource && ( - + + + + + )} {isCustomSource && ( - + + + + + )} {isCustomSource && ( - + + + + + )} - + + + + + ); diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/sources.scss b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/sources.scss new file mode 100644 index 0000000000000..fb0cecc181487 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/sources.scss @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +.source-grid-configured { + + .source-card-configured { + padding: 8px; + + &__icon { + width: 2em; + height: 2em; + } + + &__not-connected-tooltip { + position: relative; + top: 3px; + left: 4px; + } + } +} diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/sources_logic.ts b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/sources_logic.ts index 600b5871fc499..1757f2a6414f7 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/sources_logic.ts +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/sources_logic.ts @@ -78,6 +78,7 @@ interface ISourcesServerResponse { } export const SourcesLogic = kea>({ + path: ['enterprise_search', 'workplace_search', 'sources_logic'], actions: { setServerSourceStatuses: (statuses: ContentSourceStatus[]) => statuses, onInitializeSources: (serverResponse: ISourcesServerResponse) => serverResponse, diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/sources_router.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/sources_router.tsx index e4f15286145f9..9f96a13e272d2 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/sources_router.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/sources_router.tsx @@ -10,18 +10,23 @@ import { Location } from 'history'; import { useActions, useValues } from 'kea'; import { Redirect, Route, Switch, useLocation } from 'react-router-dom'; +import { SetWorkplaceSearchChrome as SetPageChrome } from '../../../shared/kibana_chrome'; +import { SendWorkplaceSearchTelemetry as SendTelemetry } from '../../../shared/telemetry'; + import { LicensingLogic } from '../../../../applications/shared/licensing'; +import { NAV } from '../../constants'; import { ADD_SOURCE_PATH, SOURCE_ADDED_PATH, SOURCE_DETAILS_PATH, - ORG_PATH, - ORG_SOURCES_PATH, + PERSONAL_SOURCES_PATH, SOURCES_PATH, getSourcesPath, } from '../../routes'; +import { FlashMessages } from '../../../shared/flash_messages'; + import { AppLogic } from '../../app_logic'; import { staticSourceData } from './source_data'; import { SourcesLogic } from './sources_logic'; @@ -32,12 +37,15 @@ import { OrganizationSources } from './organization_sources'; import { PrivateSources } from './private_sources'; import { SourceRouter } from './source_router'; +import './sources.scss'; + export const SourcesRouter: React.FC = () => { const { pathname } = useLocation() as Location; const { hasPlatinumLicense } = useValues(LicensingLogic); const { resetSourcesState } = useActions(SourcesLogic); const { account: { canCreatePersonalSources }, + isOrganization, } = useValues(AppLogic); /** @@ -48,61 +56,76 @@ export const SourcesRouter: React.FC = () => { resetSourcesState(); }, [pathname]); - const isOrgRoute = pathname.includes(ORG_PATH); - return ( - - - - {staticSourceData.map(({ addPath, accountContextOnly }, i) => ( - - !hasPlatinumLicense && accountContextOnly ? ( - + <> + + + + + + + + + + + + + {staticSourceData.map(({ addPath, accountContextOnly, name }, i) => ( + + + {!hasPlatinumLicense && accountContextOnly ? ( + ) : ( - ) - } - /> - ))} - {staticSourceData.map(({ addPath }, i) => ( - } - /> - ))} - {staticSourceData.map(({ addPath }, i) => ( - } - /> - ))} - {staticSourceData.map(({ addPath, configuration: { needsConfiguration } }, i) => { - if (needsConfiguration) - return ( - } - /> - ); - })} - {canCreatePersonalSources ? ( - - ) : ( - - )} - : - - - + )} + + ))} + {staticSourceData.map(({ addPath, name }, i) => ( + + + + + ))} + {staticSourceData.map(({ addPath, name }, i) => ( + + + + + ))} + {staticSourceData.map(({ addPath, name, configuration: { needsConfiguration } }, i) => { + if (needsConfiguration) + return ( + + + + + ); + })} + {canCreatePersonalSources ? ( + + + + + + ) : ( + + )} + + + + + + + + + + + + + ); }; diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/groups/components/group_manager_modal.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/groups/components/group_manager_modal.tsx index c0f8bf57989ca..cbfb22915c4eb 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/groups/components/group_manager_modal.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/groups/components/group_manager_modal.tsx @@ -29,7 +29,7 @@ import { import { EuiButtonTo } from '../../../../shared/react_router_helpers'; import { Group } from '../../../types'; -import { ORG_SOURCES_PATH } from '../../../routes'; +import { SOURCES_PATH } from '../../../routes'; import noSharedSourcesIcon from '../../../assets/share_circle.svg'; @@ -96,7 +96,7 @@ export const GroupManagerModal: React.FC = ({ const handleSelectAll = () => selectAll(allSelected ? [] : allItems); const sourcesButton = ( - + {ADD_SOURCE_BUTTON_TEXT} ); diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_steps.test.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_steps.test.tsx index 268e4f8da445a..64dc5149decd5 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_steps.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_steps.test.tsx @@ -11,7 +11,7 @@ import { setMockValues } from './__mocks__'; import React from 'react'; import { shallow } from 'enzyme'; -import { ORG_SOURCES_PATH, USERS_PATH } from '../../routes'; +import { SOURCES_PATH, USERS_PATH } from '../../routes'; import { OnboardingSteps, OrgNameOnboarding } from './onboarding_steps'; import { OnboardingCard } from './onboarding_card'; @@ -32,7 +32,7 @@ describe('OnboardingSteps', () => { const wrapper = shallow(); expect(wrapper.find(OnboardingCard)).toHaveLength(1); - expect(wrapper.find(OnboardingCard).prop('actionPath')).toBe(ORG_SOURCES_PATH); + expect(wrapper.find(OnboardingCard).prop('actionPath')).toBe(SOURCES_PATH); expect(wrapper.find(OnboardingCard).prop('description')).toBe( 'Add shared sources for your organization to start searching.' ); diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_steps.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_steps.tsx index ed5136a6f7a4e..4957324aa6bd7 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_steps.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_steps.tsx @@ -24,7 +24,7 @@ import { import sharedSourcesIcon from '../../components/shared/assets/source_icons/share_circle.svg'; import { TelemetryLogic } from '../../../shared/telemetry'; import { getWorkplaceSearchUrl } from '../../../shared/enterprise_search_url'; -import { ORG_SOURCES_PATH, USERS_PATH, ORG_SETTINGS_PATH } from '../../routes'; +import { SOURCES_PATH, USERS_PATH, ORG_SETTINGS_PATH } from '../../routes'; import { ContentSection } from '../../components/shared/content_section'; @@ -75,7 +75,7 @@ export const OnboardingSteps: React.FC = () => { const accountsPath = !isFederatedAuth && (canCreateInvitations || isCurated) ? USERS_PATH : undefined; - const sourcesPath = canCreateContentSources || isCurated ? ORG_SOURCES_PATH : undefined; + const sourcesPath = canCreateContentSources || isCurated ? SOURCES_PATH : undefined; const SOURCES_CARD_DESCRIPTION = i18n.translate( 'xpack.enterpriseSearch.workplaceSearch.sourcesOnboardingCard.description', diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/organization_stats.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/organization_stats.tsx index 6614ac58b0744..06c620ad384e6 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/organization_stats.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/organization_stats.tsx @@ -12,7 +12,7 @@ import { FormattedMessage } from '@kbn/i18n/react'; import { i18n } from '@kbn/i18n'; import { ContentSection } from '../../components/shared/content_section'; -import { ORG_SOURCES_PATH, USERS_PATH } from '../../routes'; +import { SOURCES_PATH, USERS_PATH } from '../../routes'; import { AppLogic } from '../../app_logic'; import { OverviewLogic } from './overview_logic'; @@ -43,7 +43,7 @@ export const OrganizationStats: React.FC = () => { { defaultMessage: 'Shared sources' } )} count={sourcesCount} - actionPath={ORG_SOURCES_PATH} + actionPath={SOURCES_PATH} /> {!isFederatedAuth && ( <> diff --git a/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.test.ts b/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.test.ts index 9cf491b79fd24..22e2deaace1dc 100644 --- a/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.test.ts +++ b/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.test.ts @@ -328,10 +328,8 @@ describe('sources routes', () => { const mockRequest = { params: { id: '123' }, body: { - query: { - content_source: { - name: 'foo', - }, + content_source: { + name: 'foo', }, }, }; @@ -406,7 +404,7 @@ describe('sources routes', () => { mockRouter.callRoute(mockRequest); expect(mockRequestHandler.createRequest).toHaveBeenCalledWith({ - path: '/ws/pre_content_sources/zendesk', + path: '/ws/sources/zendesk/prepare', }); }); }); @@ -732,10 +730,8 @@ describe('sources routes', () => { const mockRequest = { params: { id: '123' }, body: { - query: { - content_source: { - name: 'foo', - }, + content_source: { + name: 'foo', }, }, }; @@ -810,7 +806,7 @@ describe('sources routes', () => { mockRouter.callRoute(mockRequest); expect(mockRequestHandler.createRequest).toHaveBeenCalledWith({ - path: '/ws/org/pre_content_sources/zendesk', + path: '/ws/org/sources/zendesk/prepare', }); }); }); diff --git a/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.ts b/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.ts index bdd048438dae5..24473388c03b1 100644 --- a/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.ts +++ b/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.ts @@ -200,10 +200,8 @@ export function registerAccountSourceSettingsRoute({ path: '/api/workplace_search/account/sources/{id}/settings', validate: { body: schema.object({ - query: schema.object({ - content_source: schema.object({ - name: schema.string(), - }), + content_source: schema.object({ + name: schema.string(), }), }), params: schema.object({ @@ -256,7 +254,7 @@ export function registerAccountPrepareSourcesRoute({ }, async (context, request, response) => { return enterpriseSearchRequestHandler.createRequest({ - path: `/ws/pre_content_sources/${request.params.service_type}`, + path: `/ws/sources/${request.params.service_type}/prepare`, })(context, request, response); } ); @@ -372,7 +370,7 @@ export function registerOrgCreateSourceRoute({ login: schema.maybe(schema.string()), password: schema.maybe(schema.string()), organizations: schema.maybe(schema.arrayOf(schema.string())), - indexPermissions: schema.boolean(), + indexPermissions: schema.maybe(schema.boolean()), }), }, }, @@ -462,10 +460,8 @@ export function registerOrgSourceSettingsRoute({ path: '/api/workplace_search/org/sources/{id}/settings', validate: { body: schema.object({ - query: schema.object({ - content_source: schema.object({ - name: schema.string(), - }), + content_source: schema.object({ + name: schema.string(), }), }), params: schema.object({ @@ -518,7 +514,7 @@ export function registerOrgPrepareSourcesRoute({ }, async (context, request, response) => { return enterpriseSearchRequestHandler.createRequest({ - path: `/ws/org/pre_content_sources/${request.params.service_type}`, + path: `/ws/org/sources/${request.params.service_type}/prepare`, })(context, request, response); } ); From 6ef6c0fa4deeace16f47aaf08195b2c10278150b Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Tue, 24 Nov 2020 19:19:06 +0300 Subject: [PATCH 03/49] TSVB should use "histogram:maxBars" and "histogram:barTarget" settings for auto instead of a default 100 buckets (#83628) * TSVB needs a "tsvb:max_buckets" target setting for auto instead of a default 120 buckets Closes: #54012 * remove calculate_auto * max bars -> Level of detail * rename allowLevelofDetail * fix PR comment * Update constants.ts * Update src/plugins/vis_type_timeseries/public/application/components/index_pattern.js Co-authored-by: Wylie Conlon * create LEVEL_OF_DETAIL_MIN_BUCKETS constant * calcAutoIntervalLessThan -> search.aggs.calcAutoIntervalLessThan Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Wylie Conlon --- ...ibana-plugin-plugins-data-server.search.md | 1 + .../data/common/search/aggs/buckets/index.ts | 1 + src/plugins/data/server/index.ts | 2 + src/plugins/data/server/server.api.md | 32 ++--- .../vis_type_timeseries/common/constants.ts | 2 +- .../vis_type_timeseries/common/vis_schema.ts | 2 + .../application/components/index_pattern.js | 115 ++++++++++++++++-- .../components/lib/get_interval.js | 3 +- .../components/panel_config/timeseries.js | 1 + .../components/vis_editor_visualization.js | 2 +- .../components/vis_types/timeseries/config.js | 2 +- .../annotations/get_request_params.js | 8 +- .../vis_data/get_interval_and_timefield.js | 14 ++- .../server/lib/vis_data/get_table_data.js | 8 +- .../lib/vis_data/helpers/calculate_auto.js | 90 -------------- .../lib/vis_data/helpers/get_bucket_size.js | 15 +-- .../vis_data/helpers/get_bucket_size.test.js | 18 ++- ...imerange.test.js => get_timerange.test.ts} | 8 +- .../{get_timerange.js => get_timerange.ts} | 15 +-- .../annotations/date_histogram.js | 10 +- .../request_processors/annotations/query.js | 12 +- .../series/date_histogram.js | 23 +++- .../series/date_histogram.test.js | 70 ++++++++--- .../series/metric_buckets.js | 13 +- .../series/metric_buckets.test.js | 84 +++++++------ .../series/positive_rate.js | 13 +- .../series/positive_rate.test.js | 9 +- .../series/sibling_buckets.js | 6 +- .../series/sibling_buckets.test.js | 10 +- .../table/date_histogram.js | 16 ++- .../table/metric_buckets.js | 12 +- .../request_processors/table/positive_rate.js | 12 +- .../table/sibling_buckets.js | 12 +- .../series/build_request_body.test.ts | 3 +- .../lib/vis_data/series/get_request_params.js | 9 +- 35 files changed, 424 insertions(+), 229 deletions(-) delete mode 100644 src/plugins/vis_type_timeseries/server/lib/vis_data/helpers/calculate_auto.js rename src/plugins/vis_type_timeseries/server/lib/vis_data/helpers/{get_timerange.test.js => get_timerange.test.ts} (92%) rename src/plugins/vis_type_timeseries/server/lib/vis_data/helpers/{get_timerange.js => get_timerange.ts} (75%) diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.search.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.search.md index e2a71a7badd4d..77abcacd7704a 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.search.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.search.md @@ -52,6 +52,7 @@ search: { siblingPipelineType: string; termsAggFilter: string[]; toAbsoluteDates: typeof toAbsoluteDates; + calcAutoIntervalLessThan: typeof calcAutoIntervalLessThan; }; getRequestInspectorStats: typeof getRequestInspectorStats; getResponseInspectorStats: typeof getResponseInspectorStats; diff --git a/src/plugins/data/common/search/aggs/buckets/index.ts b/src/plugins/data/common/search/aggs/buckets/index.ts index b16242e519872..04a748bfb1965 100644 --- a/src/plugins/data/common/search/aggs/buckets/index.ts +++ b/src/plugins/data/common/search/aggs/buckets/index.ts @@ -35,3 +35,4 @@ export * from './lib/ip_range'; export * from './migrate_include_exclude_format'; export * from './significant_terms'; export * from './terms'; +export * from './lib/time_buckets/calc_auto_interval'; diff --git a/src/plugins/data/server/index.ts b/src/plugins/data/server/index.ts index 9d85caa624e7a..b3fe412152c9d 100644 --- a/src/plugins/data/server/index.ts +++ b/src/plugins/data/server/index.ts @@ -196,6 +196,7 @@ import { includeTotalLoaded, toKibanaSearchResponse, getTotalLoaded, + calcAutoIntervalLessThan, } from '../common'; export { @@ -282,6 +283,7 @@ export const search = { siblingPipelineType, termsAggFilter, toAbsoluteDates, + calcAutoIntervalLessThan, }, getRequestInspectorStats, getResponseInspectorStats, diff --git a/src/plugins/data/server/server.api.md b/src/plugins/data/server/server.api.md index 6583651e074c3..6870ad5e2402f 100644 --- a/src/plugins/data/server/server.api.md +++ b/src/plugins/data/server/server.api.md @@ -1084,6 +1084,7 @@ export const search: { siblingPipelineType: string; termsAggFilter: string[]; toAbsoluteDates: typeof toAbsoluteDates; + calcAutoIntervalLessThan: typeof calcAutoIntervalLessThan; }; getRequestInspectorStats: typeof getRequestInspectorStats; getResponseInspectorStats: typeof getResponseInspectorStats; @@ -1246,21 +1247,22 @@ export function usageProvider(core: CoreSetup_2): SearchUsage; // src/plugins/data/server/index.ts:111:26 - (ae-forgotten-export) The symbol "TruncateFormat" needs to be exported by the entry point index.d.ts // src/plugins/data/server/index.ts:137:27 - (ae-forgotten-export) The symbol "isFilterable" needs to be exported by the entry point index.d.ts // src/plugins/data/server/index.ts:137:27 - (ae-forgotten-export) The symbol "isNestedField" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:253:20 - (ae-forgotten-export) The symbol "getRequestInspectorStats" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:253:20 - (ae-forgotten-export) The symbol "getResponseInspectorStats" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:253:20 - (ae-forgotten-export) The symbol "tabifyAggResponse" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:253:20 - (ae-forgotten-export) The symbol "tabifyGetColumns" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:268:5 - (ae-forgotten-export) The symbol "getTotalLoaded" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:269:5 - (ae-forgotten-export) The symbol "toSnakeCase" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:273:1 - (ae-forgotten-export) The symbol "CidrMask" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:274:1 - (ae-forgotten-export) The symbol "dateHistogramInterval" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:283:1 - (ae-forgotten-export) The symbol "InvalidEsCalendarIntervalError" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:284:1 - (ae-forgotten-export) The symbol "InvalidEsIntervalFormatError" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:285:1 - (ae-forgotten-export) The symbol "Ipv4Address" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:289:1 - (ae-forgotten-export) The symbol "isValidEsInterval" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:290:1 - (ae-forgotten-export) The symbol "isValidInterval" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:294:1 - (ae-forgotten-export) The symbol "propFilter" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:297:1 - (ae-forgotten-export) The symbol "toAbsoluteDates" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:254:20 - (ae-forgotten-export) The symbol "getRequestInspectorStats" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:254:20 - (ae-forgotten-export) The symbol "getResponseInspectorStats" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:254:20 - (ae-forgotten-export) The symbol "tabifyAggResponse" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:254:20 - (ae-forgotten-export) The symbol "tabifyGetColumns" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:269:5 - (ae-forgotten-export) The symbol "getTotalLoaded" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:270:5 - (ae-forgotten-export) The symbol "toSnakeCase" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:274:1 - (ae-forgotten-export) The symbol "CidrMask" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:275:1 - (ae-forgotten-export) The symbol "dateHistogramInterval" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:284:1 - (ae-forgotten-export) The symbol "InvalidEsCalendarIntervalError" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:285:1 - (ae-forgotten-export) The symbol "InvalidEsIntervalFormatError" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:286:1 - (ae-forgotten-export) The symbol "Ipv4Address" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:290:1 - (ae-forgotten-export) The symbol "isValidEsInterval" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:291:1 - (ae-forgotten-export) The symbol "isValidInterval" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:295:1 - (ae-forgotten-export) The symbol "propFilter" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:298:1 - (ae-forgotten-export) The symbol "toAbsoluteDates" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:299:1 - (ae-forgotten-export) The symbol "calcAutoIntervalLessThan" needs to be exported by the entry point index.d.ts // src/plugins/data/server/index_patterns/index_patterns_service.ts:58:14 - (ae-forgotten-export) The symbol "IndexPatternsService" needs to be exported by the entry point index.d.ts // src/plugins/data/server/plugin.ts:88:66 - (ae-forgotten-export) The symbol "DataEnhancements" needs to be exported by the entry point index.d.ts // src/plugins/data/server/search/types.ts:104:5 - (ae-forgotten-export) The symbol "ISearchStartSearchSource" needs to be exported by the entry point index.d.ts diff --git a/src/plugins/vis_type_timeseries/common/constants.ts b/src/plugins/vis_type_timeseries/common/constants.ts index 4f24bc273e265..bfcb5e8e15b9d 100644 --- a/src/plugins/vis_type_timeseries/common/constants.ts +++ b/src/plugins/vis_type_timeseries/common/constants.ts @@ -19,7 +19,7 @@ export const MAX_BUCKETS_SETTING = 'metrics:max_buckets'; export const INDEXES_SEPARATOR = ','; - +export const AUTO_INTERVAL = 'auto'; export const ROUTES = { VIS_DATA: '/api/metrics/vis/data', }; diff --git a/src/plugins/vis_type_timeseries/common/vis_schema.ts b/src/plugins/vis_type_timeseries/common/vis_schema.ts index 7f17a9c44298a..a90fa752ad7dc 100644 --- a/src/plugins/vis_type_timeseries/common/vis_schema.ts +++ b/src/plugins/vis_type_timeseries/common/vis_schema.ts @@ -175,6 +175,7 @@ export const seriesItems = schema.object({ separate_axis: numberIntegerOptional, seperate_axis: numberIntegerOptional, series_index_pattern: stringOptionalNullable, + series_max_bars: numberIntegerOptional, series_time_field: stringOptionalNullable, series_interval: stringOptionalNullable, series_drop_last_bucket: numberIntegerOptional, @@ -229,6 +230,7 @@ export const panel = schema.object({ ignore_global_filters: numberOptional, ignore_global_filter: numberOptional, index_pattern: stringRequired, + max_bars: numberIntegerOptional, interval: stringRequired, isModelInvalid: schema.maybe(schema.boolean()), legend_position: stringOptionalNullable, diff --git a/src/plugins/vis_type_timeseries/public/application/components/index_pattern.js b/src/plugins/vis_type_timeseries/public/application/components/index_pattern.js index 85f31285df69b..e976519dfe635 100644 --- a/src/plugins/vis_type_timeseries/public/application/components/index_pattern.js +++ b/src/plugins/vis_type_timeseries/public/application/components/index_pattern.js @@ -19,7 +19,7 @@ import { get } from 'lodash'; import PropTypes from 'prop-types'; -import React, { useContext } from 'react'; +import React, { useContext, useCallback } from 'react'; import { htmlIdGenerator, EuiFieldText, @@ -27,7 +27,10 @@ import { EuiFlexItem, EuiFormRow, EuiComboBox, + EuiRange, + EuiIconTip, EuiText, + EuiFormLabel, } from '@elastic/eui'; import { FieldSelect } from './aggs/field_select'; import { createSelectHandler } from './lib/create_select_handler'; @@ -35,19 +38,20 @@ import { createTextHandler } from './lib/create_text_handler'; import { YesNo } from './yes_no'; import { KBN_FIELD_TYPES } from '../../../../../plugins/data/public'; import { FormValidationContext } from '../contexts/form_validation_context'; -import { - isGteInterval, - validateReInterval, - isAutoInterval, - AUTO_INTERVAL, -} from './lib/get_interval'; +import { isGteInterval, validateReInterval, isAutoInterval } from './lib/get_interval'; import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; import { TIME_RANGE_DATA_MODES, TIME_RANGE_MODE_KEY } from '../../../common/timerange_data_modes'; import { PANEL_TYPES } from '../../../common/panel_types'; import { isTimerangeModeEnabled } from '../lib/check_ui_restrictions'; import { VisDataContext } from '../contexts/vis_data_context'; +import { getUISettings } from '../../services'; +import { AUTO_INTERVAL } from '../../../common/constants'; +import { UI_SETTINGS } from '../../../../data/common'; const RESTRICT_FIELDS = [KBN_FIELD_TYPES.DATE]; +const LEVEL_OF_DETAIL_STEPS = 10; +const LEVEL_OF_DETAIL_MIN_BUCKETS = 1; const validateIntervalValue = (intervalValue) => { const isAutoOrGteInterval = isGteInterval(intervalValue) || isAutoInterval(intervalValue); @@ -65,15 +69,36 @@ const htmlId = htmlIdGenerator(); const isEntireTimeRangeActive = (model, isTimeSeries) => !isTimeSeries && model[TIME_RANGE_MODE_KEY] === TIME_RANGE_DATA_MODES.ENTIRE_TIME_RANGE; -export const IndexPattern = ({ fields, prefix, onChange, disabled, model: _model }) => { +export const IndexPattern = ({ + fields, + prefix, + onChange, + disabled, + model: _model, + allowLevelofDetail, +}) => { + const config = getUISettings(); + const handleSelectChange = createSelectHandler(onChange); const handleTextChange = createTextHandler(onChange); + const timeFieldName = `${prefix}time_field`; const indexPatternName = `${prefix}index_pattern`; const intervalName = `${prefix}interval`; + const maxBarsName = `${prefix}max_bars`; const dropBucketName = `${prefix}drop_last_bucket`; const updateControlValidity = useContext(FormValidationContext); const uiRestrictions = get(useContext(VisDataContext), 'uiRestrictions'); + const maxBarsUiSettings = config.get(UI_SETTINGS.HISTOGRAM_MAX_BARS); + + const handleMaxBarsChange = useCallback( + ({ target }) => { + onChange({ + [maxBarsName]: Math.max(LEVEL_OF_DETAIL_MIN_BUCKETS, target.value), + }); + }, + [onChange, maxBarsName] + ); const timeRangeOptions = [ { @@ -97,10 +122,12 @@ export const IndexPattern = ({ fields, prefix, onChange, disabled, model: _model [indexPatternName]: '*', [intervalName]: AUTO_INTERVAL, [dropBucketName]: 1, + [maxBarsName]: config.get(UI_SETTINGS.HISTOGRAM_BAR_TARGET), [TIME_RANGE_MODE_KEY]: timeRangeOptions[0].value, }; const model = { ...defaults, ..._model }; + const isDefaultIndexPatternUsed = model.default_index_pattern && !model[indexPatternName]; const intervalValidation = validateIntervalValue(model[intervalName]); const selectedTimeRangeOption = timeRangeOptions.find( @@ -229,6 +256,77 @@ export const IndexPattern = ({ fields, prefix, onChange, disabled, model: _model + {allowLevelofDetail && ( + + + + {' '} + + } + type="questionInCircle" + /> + + } + > + + + + + + + + + + + + + + + + + + + )}
); }; @@ -245,4 +343,5 @@ IndexPattern.propTypes = { prefix: PropTypes.string, disabled: PropTypes.bool, className: PropTypes.string, + allowLevelofDetail: PropTypes.bool, }; diff --git a/src/plugins/vis_type_timeseries/public/application/components/lib/get_interval.js b/src/plugins/vis_type_timeseries/public/application/components/lib/get_interval.js index c1d484765f4cb..f54d52620e67a 100644 --- a/src/plugins/vis_type_timeseries/public/application/components/lib/get_interval.js +++ b/src/plugins/vis_type_timeseries/public/application/components/lib/get_interval.js @@ -22,8 +22,7 @@ import { get } from 'lodash'; import { search } from '../../../../../../plugins/data/public'; const { parseEsInterval } = search.aggs; import { GTE_INTERVAL_RE } from '../../../../common/interval_regexp'; - -export const AUTO_INTERVAL = 'auto'; +import { AUTO_INTERVAL } from '../../../../common/constants'; export const unitLookup = { s: i18n.translate('visTypeTimeseries.getInterval.secondsLabel', { defaultMessage: 'seconds' }), diff --git a/src/plugins/vis_type_timeseries/public/application/components/panel_config/timeseries.js b/src/plugins/vis_type_timeseries/public/application/components/panel_config/timeseries.js index 03da52b10f08b..180411dd13a3d 100644 --- a/src/plugins/vis_type_timeseries/public/application/components/panel_config/timeseries.js +++ b/src/plugins/vis_type_timeseries/public/application/components/panel_config/timeseries.js @@ -193,6 +193,7 @@ class TimeseriesPanelConfigUi extends Component { fields={this.props.fields} model={this.props.model} onChange={this.props.onChange} + allowLevelofDetail={true} /> diff --git a/src/plugins/vis_type_timeseries/public/application/components/vis_editor_visualization.js b/src/plugins/vis_type_timeseries/public/application/components/vis_editor_visualization.js index 9742d817f7c0d..7893d5ba6d15e 100644 --- a/src/plugins/vis_type_timeseries/public/application/components/vis_editor_visualization.js +++ b/src/plugins/vis_type_timeseries/public/application/components/vis_editor_visualization.js @@ -26,8 +26,8 @@ import { convertIntervalIntoUnit, isAutoInterval, isGteInterval, - AUTO_INTERVAL, } from './lib/get_interval'; +import { AUTO_INTERVAL } from '../../../common/constants'; import { PANEL_TYPES } from '../../../common/panel_types'; const MIN_CHART_HEIGHT = 300; diff --git a/src/plugins/vis_type_timeseries/public/application/components/vis_types/timeseries/config.js b/src/plugins/vis_type_timeseries/public/application/components/vis_types/timeseries/config.js index 59277257c0c94..25561cfe1dc04 100644 --- a/src/plugins/vis_type_timeseries/public/application/components/vis_types/timeseries/config.js +++ b/src/plugins/vis_type_timeseries/public/application/components/vis_types/timeseries/config.js @@ -554,7 +554,7 @@ export const TimeseriesConfig = injectI18n(function (props) { {...props} prefix="series_" disabled={!model.override_index_pattern} - with-interval={true} + allowLevelofDetail={true} /> diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/annotations/get_request_params.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/annotations/get_request_params.js index d11e9316c959b..1b2334c7dea94 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/annotations/get_request_params.js +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/annotations/get_request_params.js @@ -19,6 +19,7 @@ import { buildAnnotationRequest } from './build_request_body'; import { getEsShardTimeout } from '../helpers/get_es_shard_timeout'; import { getIndexPatternObject } from '../helpers/get_index_pattern'; +import { UI_SETTINGS } from '../../../../../data/common'; export async function getAnnotationRequestParams( req, @@ -27,6 +28,7 @@ export async function getAnnotationRequestParams( esQueryConfig, capabilities ) { + const uiSettings = req.getUiSettingsService(); const esShardTimeout = await getEsShardTimeout(req); const indexPattern = annotation.index_pattern; const { indexPatternObject, indexPatternString } = await getIndexPatternObject(req, indexPattern); @@ -36,7 +38,11 @@ export async function getAnnotationRequestParams( annotation, esQueryConfig, indexPatternObject, - capabilities + capabilities, + { + maxBarsUiSettings: await uiSettings.get(UI_SETTINGS.HISTOGRAM_MAX_BARS), + barTargetUiSettings: await uiSettings.get(UI_SETTINGS.HISTOGRAM_BAR_TARGET), + } ); return { diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/get_interval_and_timefield.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/get_interval_and_timefield.js index 82a2ef66cb1c0..9714b551ea82f 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/get_interval_and_timefield.js +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/get_interval_and_timefield.js @@ -17,6 +17,8 @@ * under the License. */ +import { AUTO_INTERVAL } from '../../../common/constants'; + const DEFAULT_TIME_FIELD = '@timestamp'; export function getIntervalAndTimefield(panel, series = {}, indexPatternObject) { @@ -26,10 +28,18 @@ export function getIntervalAndTimefield(panel, series = {}, indexPatternObject) (series.override_index_pattern && series.series_time_field) || panel.time_field || getDefaultTimeField(); - const interval = (series.override_index_pattern && series.series_interval) || panel.interval; + + let interval = panel.interval; + let maxBars = panel.max_bars; + + if (series.override_index_pattern) { + interval = series.series_interval; + maxBars = series.series_max_bars; + } return { timeField, - interval, + interval: interval || AUTO_INTERVAL, + maxBars, }; } diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/get_table_data.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/get_table_data.js index 3791eb229db5b..eaaa5a9605b4b 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/get_table_data.js +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/get_table_data.js @@ -22,6 +22,7 @@ import { get } from 'lodash'; import { processBucket } from './table/process_bucket'; import { getEsQueryConfig } from './helpers/get_es_query_uisettings'; import { getIndexPatternObject } from './helpers/get_index_pattern'; +import { UI_SETTINGS } from '../../../../data/common'; export async function getTableData(req, panel) { const panelIndexPattern = panel.index_pattern; @@ -39,7 +40,12 @@ export async function getTableData(req, panel) { }; try { - const body = buildRequestBody(req, panel, esQueryConfig, indexPatternObject, capabilities); + const uiSettings = req.getUiSettingsService(); + const body = buildRequestBody(req, panel, esQueryConfig, indexPatternObject, capabilities, { + maxBarsUiSettings: await uiSettings.get(UI_SETTINGS.HISTOGRAM_MAX_BARS), + barTargetUiSettings: await uiSettings.get(UI_SETTINGS.HISTOGRAM_BAR_TARGET), + }); + const [resp] = await searchStrategy.search(req, [ { body, diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/helpers/calculate_auto.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/helpers/calculate_auto.js deleted file mode 100644 index 0c3555adff1a6..0000000000000 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/helpers/calculate_auto.js +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import moment from 'moment'; -const d = moment.duration; - -const roundingRules = [ - [d(500, 'ms'), d(100, 'ms')], - [d(5, 'second'), d(1, 'second')], - [d(7.5, 'second'), d(5, 'second')], - [d(15, 'second'), d(10, 'second')], - [d(45, 'second'), d(30, 'second')], - [d(3, 'minute'), d(1, 'minute')], - [d(9, 'minute'), d(5, 'minute')], - [d(20, 'minute'), d(10, 'minute')], - [d(45, 'minute'), d(30, 'minute')], - [d(2, 'hour'), d(1, 'hour')], - [d(6, 'hour'), d(3, 'hour')], - [d(24, 'hour'), d(12, 'hour')], - [d(1, 'week'), d(1, 'd')], - [d(3, 'week'), d(1, 'week')], - [d(1, 'year'), d(1, 'month')], - [Infinity, d(1, 'year')], -]; - -const revRoundingRules = roundingRules.slice(0).reverse(); - -function find(rules, check, last) { - function pick(buckets, duration) { - const target = duration / buckets; - let lastResp = null; - - for (let i = 0; i < rules.length; i++) { - const rule = rules[i]; - const resp = check(rule[0], rule[1], target); - - if (resp == null) { - if (!last) continue; - if (lastResp) return lastResp; - break; - } - - if (!last) return resp; - lastResp = resp; - } - - // fallback to just a number of milliseconds, ensure ms is >= 1 - const ms = Math.max(Math.floor(target), 1); - return moment.duration(ms, 'ms'); - } - - return (buckets, duration) => { - const interval = pick(buckets, duration); - if (interval) return moment.duration(interval._data); - }; -} - -export const calculateAuto = { - near: find( - revRoundingRules, - function near(bound, interval, target) { - if (bound > target) return interval; - }, - true - ), - - lessThan: find(revRoundingRules, function lessThan(_bound, interval, target) { - if (interval < target) return interval; - }), - - atLeast: find(revRoundingRules, function atLeast(_bound, interval, target) { - if (interval <= target) return interval; - }), -}; diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/helpers/get_bucket_size.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/helpers/get_bucket_size.js index c021ba3cebc66..4384da58fb569 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/helpers/get_bucket_size.js +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/helpers/get_bucket_size.js @@ -17,15 +17,15 @@ * under the License. */ -import { calculateAuto } from './calculate_auto'; import { getUnitValue, parseInterval, convertIntervalToUnit, ASCENDING_UNIT_ORDER, } from './unit_to_seconds'; -import { getTimerangeDuration } from './get_timerange'; +import { getTimerange } from './get_timerange'; import { INTERVAL_STRING_RE, GTE_INTERVAL_RE } from '../../../../common/interval_regexp'; +import { search } from '../../../../../data/server'; const calculateBucketData = (timeInterval, capabilities) => { let intervalString = capabilities @@ -65,14 +65,15 @@ const calculateBucketData = (timeInterval, capabilities) => { }; }; -const calculateBucketSizeForAutoInterval = (req) => { - const duration = getTimerangeDuration(req); +const calculateBucketSizeForAutoInterval = (req, maxBars) => { + const { from, to } = getTimerange(req); + const timerange = to.valueOf() - from.valueOf(); - return calculateAuto.near(100, duration).asSeconds(); + return search.aggs.calcAutoIntervalLessThan(maxBars, timerange).asSeconds(); }; -export const getBucketSize = (req, interval, capabilities) => { - const bucketSize = calculateBucketSizeForAutoInterval(req); +export const getBucketSize = (req, interval, capabilities, maxBars) => { + const bucketSize = calculateBucketSizeForAutoInterval(req, maxBars); let intervalString = `${bucketSize}s`; const gteAutoMatch = Boolean(interval) && interval.match(GTE_INTERVAL_RE); diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/helpers/get_bucket_size.test.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/helpers/get_bucket_size.test.js index 99bef2de6b72d..8810ccd406be4 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/helpers/get_bucket_size.test.js +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/helpers/get_bucket_size.test.js @@ -30,37 +30,43 @@ describe('getBucketSize', () => { }; test('returns auto calculated buckets', () => { - const result = getBucketSize(req, 'auto'); + const result = getBucketSize(req, 'auto', undefined, 100); + expect(result).toHaveProperty('bucketSize', 30); expect(result).toHaveProperty('intervalString', '30s'); }); test('returns overridden buckets (1s)', () => { - const result = getBucketSize(req, '1s'); + const result = getBucketSize(req, '1s', undefined, 100); + expect(result).toHaveProperty('bucketSize', 1); expect(result).toHaveProperty('intervalString', '1s'); }); test('returns overridden buckets (10m)', () => { - const result = getBucketSize(req, '10m'); + const result = getBucketSize(req, '10m', undefined, 100); + expect(result).toHaveProperty('bucketSize', 600); expect(result).toHaveProperty('intervalString', '10m'); }); test('returns overridden buckets (1d)', () => { - const result = getBucketSize(req, '1d'); + const result = getBucketSize(req, '1d', undefined, 100); + expect(result).toHaveProperty('bucketSize', 86400); expect(result).toHaveProperty('intervalString', '1d'); }); test('returns overridden buckets (>=2d)', () => { - const result = getBucketSize(req, '>=2d'); + const result = getBucketSize(req, '>=2d', undefined, 100); + expect(result).toHaveProperty('bucketSize', 86400 * 2); expect(result).toHaveProperty('intervalString', '2d'); }); test('returns overridden buckets (>=10s)', () => { - const result = getBucketSize(req, '>=10s'); + const result = getBucketSize(req, '>=10s', undefined, 100); + expect(result).toHaveProperty('bucketSize', 30); expect(result).toHaveProperty('intervalString', '30s'); }); diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/helpers/get_timerange.test.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/helpers/get_timerange.test.ts similarity index 92% rename from src/plugins/vis_type_timeseries/server/lib/vis_data/helpers/get_timerange.test.js rename to src/plugins/vis_type_timeseries/server/lib/vis_data/helpers/get_timerange.test.ts index 1a1b12c651992..183ce50dd4a09 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/helpers/get_timerange.test.js +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/helpers/get_timerange.test.ts @@ -17,20 +17,22 @@ * under the License. */ -import { getTimerange } from './get_timerange'; import moment from 'moment'; +import { getTimerange } from './get_timerange'; +import { ReqFacade, VisPayload } from '../../..'; describe('getTimerange(req)', () => { test('should return a moment object for to and from', () => { - const req = { + const req = ({ payload: { timerange: { min: '2017-01-01T00:00:00Z', max: '2017-01-01T01:00:00Z', }, }, - }; + } as unknown) as ReqFacade; const { from, to } = getTimerange(req); + expect(moment.isMoment(from)).toEqual(true); expect(moment.isMoment(to)).toEqual(true); expect(moment.utc('2017-01-01T00:00:00Z').isSame(from)).toEqual(true); diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/helpers/get_timerange.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/helpers/get_timerange.ts similarity index 75% rename from src/plugins/vis_type_timeseries/server/lib/vis_data/helpers/get_timerange.js rename to src/plugins/vis_type_timeseries/server/lib/vis_data/helpers/get_timerange.ts index 682befe9ab050..54f3110b45808 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/helpers/get_timerange.js +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/helpers/get_timerange.ts @@ -17,19 +17,14 @@ * under the License. */ -import moment from 'moment'; +import { utc } from 'moment'; +import { ReqFacade, VisPayload } from '../../..'; -export const getTimerange = (req) => { +export const getTimerange = (req: ReqFacade) => { const { min, max } = req.payload.timerange; return { - from: moment.utc(min), - to: moment.utc(max), + from: utc(min), + to: utc(max), }; }; - -export const getTimerangeDuration = (req) => { - const { from, to } = getTimerange(req); - - return moment.duration(to.valueOf() - from.valueOf(), 'ms'); -}; diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/annotations/date_histogram.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/annotations/date_histogram.js index 4b611e46f1588..617a75f6bd59f 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/annotations/date_histogram.js +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/annotations/date_histogram.js @@ -29,11 +29,17 @@ export function dateHistogram( annotation, esQueryConfig, indexPatternObject, - capabilities + capabilities, + { barTargetUiSettings } ) { return (next) => (doc) => { const timeField = annotation.time_field; - const { bucketSize, intervalString } = getBucketSize(req, 'auto', capabilities); + const { bucketSize, intervalString } = getBucketSize( + req, + 'auto', + capabilities, + barTargetUiSettings + ); const { from, to } = getTimerange(req); const timezone = capabilities.searchTimezone; diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/annotations/query.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/annotations/query.js index 127687bf11fe9..cf02f601ea5ff 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/annotations/query.js +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/annotations/query.js @@ -21,10 +21,18 @@ import { getBucketSize } from '../../helpers/get_bucket_size'; import { getTimerange } from '../../helpers/get_timerange'; import { esQuery } from '../../../../../../data/server'; -export function query(req, panel, annotation, esQueryConfig, indexPattern, capabilities) { +export function query( + req, + panel, + annotation, + esQueryConfig, + indexPattern, + capabilities, + { barTargetUiSettings } +) { return (next) => (doc) => { const timeField = annotation.time_field; - const { bucketSize } = getBucketSize(req, 'auto', capabilities); + const { bucketSize } = getBucketSize(req, 'auto', capabilities, barTargetUiSettings); const { from, to } = getTimerange(req); doc.size = 0; diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/date_histogram.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/date_histogram.js index f1e58b8e4af2a..98c683bda1fdb 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/date_histogram.js +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/date_histogram.js @@ -25,10 +25,27 @@ import { isLastValueTimerangeMode } from '../../helpers/get_timerange_mode'; import { search } from '../../../../../../../plugins/data/server'; const { dateHistogramInterval } = search.aggs; -export function dateHistogram(req, panel, series, esQueryConfig, indexPatternObject, capabilities) { +export function dateHistogram( + req, + panel, + series, + esQueryConfig, + indexPatternObject, + capabilities, + { maxBarsUiSettings, barTargetUiSettings } +) { return (next) => (doc) => { - const { timeField, interval } = getIntervalAndTimefield(panel, series, indexPatternObject); - const { bucketSize, intervalString } = getBucketSize(req, interval, capabilities); + const { timeField, interval, maxBars } = getIntervalAndTimefield( + panel, + series, + indexPatternObject + ); + const { bucketSize, intervalString } = getBucketSize( + req, + interval, + capabilities, + maxBars ? Math.min(maxBarsUiSettings, maxBars) : barTargetUiSettings + ); const getDateHistogramForLastBucketMode = () => { const { from, to } = offsetTime(req, series.offset_time); diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/date_histogram.test.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/date_histogram.test.js index 45cad1195fc78..aa95a79a62796 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/date_histogram.test.js +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/date_histogram.test.js @@ -27,6 +27,7 @@ describe('dateHistogram(req, panel, series)', () => { let capabilities; let config; let indexPatternObject; + let uiSettings; beforeEach(() => { req = { @@ -50,19 +51,29 @@ describe('dateHistogram(req, panel, series)', () => { }; indexPatternObject = {}; capabilities = new DefaultSearchCapabilities(req); + uiSettings = { maxBarsUiSettings: 100, barTargetUiSettings: 50 }; }); test('calls next when finished', () => { const next = jest.fn(); - dateHistogram(req, panel, series, config, indexPatternObject, capabilities)(next)({}); + dateHistogram(req, panel, series, config, indexPatternObject, capabilities, uiSettings)(next)( + {} + ); expect(next.mock.calls.length).toEqual(1); }); test('returns valid date histogram', () => { const next = (doc) => doc; - const doc = dateHistogram(req, panel, series, config, indexPatternObject, capabilities)(next)( - {} - ); + const doc = dateHistogram( + req, + panel, + series, + config, + indexPatternObject, + capabilities, + uiSettings + )(next)({}); + expect(doc).toEqual({ aggs: { test: { @@ -94,9 +105,16 @@ describe('dateHistogram(req, panel, series)', () => { test('returns valid date histogram (offset by 1h)', () => { series.offset_time = '1h'; const next = (doc) => doc; - const doc = dateHistogram(req, panel, series, config, indexPatternObject, capabilities)(next)( - {} - ); + const doc = dateHistogram( + req, + panel, + series, + config, + indexPatternObject, + capabilities, + uiSettings + )(next)({}); + expect(doc).toEqual({ aggs: { test: { @@ -131,9 +149,16 @@ describe('dateHistogram(req, panel, series)', () => { series.series_time_field = 'timestamp'; series.series_interval = '20s'; const next = (doc) => doc; - const doc = dateHistogram(req, panel, series, config, indexPatternObject, capabilities)(next)( - {} - ); + const doc = dateHistogram( + req, + panel, + series, + config, + indexPatternObject, + capabilities, + uiSettings + )(next)({}); + expect(doc).toEqual({ aggs: { test: { @@ -168,9 +193,15 @@ describe('dateHistogram(req, panel, series)', () => { panel.type = 'timeseries'; const next = (doc) => doc; - const doc = dateHistogram(req, panel, series, config, indexPatternObject, capabilities)(next)( - {} - ); + const doc = dateHistogram( + req, + panel, + series, + config, + indexPatternObject, + capabilities, + uiSettings + )(next)({}); expect(doc.aggs.test.aggs.timeseries.auto_date_histogram).toBeUndefined(); expect(doc.aggs.test.aggs.timeseries.date_histogram).toBeDefined(); @@ -180,9 +211,16 @@ describe('dateHistogram(req, panel, series)', () => { panel.time_range_mode = 'entire_time_range'; const next = (doc) => doc; - const doc = dateHistogram(req, panel, series, config, indexPatternObject, capabilities)(next)( - {} - ); + const doc = dateHistogram( + req, + panel, + series, + config, + indexPatternObject, + capabilities, + uiSettings + )(next)({}); + expect(doc).toEqual({ aggs: { test: { diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/metric_buckets.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/metric_buckets.js index 800145dac5468..023ee054a5e13 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/metric_buckets.js +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/metric_buckets.js @@ -21,10 +21,19 @@ import { getBucketSize } from '../../helpers/get_bucket_size'; import { bucketTransform } from '../../helpers/bucket_transform'; import { getIntervalAndTimefield } from '../../get_interval_and_timefield'; -export function metricBuckets(req, panel, series, esQueryConfig, indexPatternObject, capabilities) { +export function metricBuckets( + req, + panel, + series, + esQueryConfig, + indexPatternObject, + capabilities, + { barTargetUiSettings } +) { return (next) => (doc) => { const { interval } = getIntervalAndTimefield(panel, series, indexPatternObject); - const { intervalString } = getBucketSize(req, interval, capabilities); + const { intervalString } = getBucketSize(req, interval, capabilities, barTargetUiSettings); + series.metrics .filter((row) => !/_bucket$/.test(row.type) && !/^series/.test(row.type)) .forEach((metric) => { diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/metric_buckets.test.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/metric_buckets.test.js index 1ac4329b60f82..2154d2257815b 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/metric_buckets.test.js +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/metric_buckets.test.js @@ -20,56 +20,64 @@ import { metricBuckets } from './metric_buckets'; describe('metricBuckets(req, panel, series)', () => { - let panel; - let series; - let req; + let metricBucketsProcessor; + beforeEach(() => { - panel = { - time_field: 'timestamp', - }; - series = { - id: 'test', - split_mode: 'terms', - terms_size: 10, - terms_field: 'host', - metrics: [ - { - id: 'metric-1', - type: 'max', - field: 'io', - }, - { - id: 'metric-2', - type: 'derivative', - field: 'metric-1', - unit: '1s', - }, - { - id: 'metric-3', - type: 'avg_bucket', - field: 'metric-2', - }, - ], - }; - req = { - payload: { - timerange: { - min: '2017-01-01T00:00:00Z', - max: '2017-01-01T01:00:00Z', + metricBucketsProcessor = metricBuckets( + { + payload: { + timerange: { + min: '2017-01-01T00:00:00Z', + max: '2017-01-01T01:00:00Z', + }, }, }, - }; + { + time_field: 'timestamp', + }, + { + id: 'test', + split_mode: 'terms', + terms_size: 10, + terms_field: 'host', + metrics: [ + { + id: 'metric-1', + type: 'max', + field: 'io', + }, + { + id: 'metric-2', + type: 'derivative', + field: 'metric-1', + unit: '1s', + }, + { + id: 'metric-3', + type: 'avg_bucket', + field: 'metric-2', + }, + ], + }, + {}, + {}, + undefined, + { + barTargetUiSettings: 50, + } + ); }); test('calls next when finished', () => { const next = jest.fn(); - metricBuckets(req, panel, series)(next)({}); + metricBucketsProcessor(next)({}); expect(next.mock.calls.length).toEqual(1); }); test('returns metric aggs', () => { const next = (doc) => doc; - const doc = metricBuckets(req, panel, series)(next)({}); + const doc = metricBucketsProcessor(next)({}); + expect(doc).toEqual({ aggs: { test: { diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/positive_rate.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/positive_rate.js index 4a79ec2295877..c16e0fd3aaf15 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/positive_rate.js +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/positive_rate.js @@ -57,10 +57,19 @@ export const createPositiveRate = (doc, intervalString, aggRoot) => (metric) => overwrite(doc, `${aggRoot}.timeseries.aggs.${metric.id}`, positiveOnlyBucket); }; -export function positiveRate(req, panel, series, esQueryConfig, indexPatternObject, capabilities) { +export function positiveRate( + req, + panel, + series, + esQueryConfig, + indexPatternObject, + capabilities, + { barTargetUiSettings } +) { return (next) => (doc) => { const { interval } = getIntervalAndTimefield(panel, series, indexPatternObject); - const { intervalString } = getBucketSize(req, interval, capabilities); + const { intervalString } = getBucketSize(req, interval, capabilities, barTargetUiSettings); + if (series.metrics.some(filter)) { series.metrics .filter(filter) diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/positive_rate.test.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/positive_rate.test.js index 7c0f43adf02f5..d891fc01bb266 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/positive_rate.test.js +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/positive_rate.test.js @@ -22,6 +22,8 @@ describe('positiveRate(req, panel, series)', () => { let panel; let series; let req; + let uiSettings; + beforeEach(() => { panel = { time_field: 'timestamp', @@ -48,17 +50,20 @@ describe('positiveRate(req, panel, series)', () => { }, }, }; + uiSettings = { + barTargetUiSettings: 50, + }; }); test('calls next when finished', () => { const next = jest.fn(); - positiveRate(req, panel, series)(next)({}); + positiveRate(req, panel, series, {}, {}, undefined, uiSettings)(next)({}); expect(next.mock.calls.length).toEqual(1); }); test('returns positive rate aggs', () => { const next = (doc) => doc; - const doc = positiveRate(req, panel, series)(next)({}); + const doc = positiveRate(req, panel, series, {}, {}, undefined, uiSettings)(next)({}); expect(doc).toEqual({ aggs: { test: { diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/sibling_buckets.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/sibling_buckets.js index f2b58822e68b6..f69473b613d1b 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/sibling_buckets.js +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/sibling_buckets.js @@ -28,11 +28,13 @@ export function siblingBuckets( series, esQueryConfig, indexPatternObject, - capabilities + capabilities, + { barTargetUiSettings } ) { return (next) => (doc) => { const { interval } = getIntervalAndTimefield(panel, series, indexPatternObject); - const { bucketSize } = getBucketSize(req, interval, capabilities); + const { bucketSize } = getBucketSize(req, interval, capabilities, barTargetUiSettings); + series.metrics .filter((row) => /_bucket$/.test(row.type)) .forEach((metric) => { diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/sibling_buckets.test.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/sibling_buckets.test.js index 8f84023ce0c75..48714e83341ea 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/sibling_buckets.test.js +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/sibling_buckets.test.js @@ -23,6 +23,8 @@ describe('siblingBuckets(req, panel, series)', () => { let panel; let series; let req; + let uiSettings; + beforeEach(() => { panel = { time_field: 'timestamp', @@ -53,17 +55,21 @@ describe('siblingBuckets(req, panel, series)', () => { }, }, }; + uiSettings = { + barTargetUiSettings: 50, + }; }); test('calls next when finished', () => { const next = jest.fn(); - siblingBuckets(req, panel, series)(next)({}); + siblingBuckets(req, panel, series, {}, {}, undefined, uiSettings)(next)({}); expect(next.mock.calls.length).toEqual(1); }); test('returns sibling aggs', () => { const next = (doc) => doc; - const doc = siblingBuckets(req, panel, series)(next)({}); + const doc = siblingBuckets(req, panel, series, {}, {}, undefined, uiSettings)(next)({}); + expect(doc).toEqual({ aggs: { test: { diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/table/date_histogram.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/table/date_histogram.js index 947e48ed2cab2..ba65e583cc094 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/table/date_histogram.js +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/table/date_histogram.js @@ -26,7 +26,14 @@ import { calculateAggRoot } from './calculate_agg_root'; import { search } from '../../../../../../../plugins/data/server'; const { dateHistogramInterval } = search.aggs; -export function dateHistogram(req, panel, esQueryConfig, indexPatternObject, capabilities) { +export function dateHistogram( + req, + panel, + esQueryConfig, + indexPatternObject, + capabilities, + { barTargetUiSettings } +) { return (next) => (doc) => { const { timeField, interval } = getIntervalAndTimefield(panel, {}, indexPatternObject); const meta = { @@ -34,7 +41,12 @@ export function dateHistogram(req, panel, esQueryConfig, indexPatternObject, cap }; const getDateHistogramForLastBucketMode = () => { - const { bucketSize, intervalString } = getBucketSize(req, interval, capabilities); + const { bucketSize, intervalString } = getBucketSize( + req, + interval, + capabilities, + barTargetUiSettings + ); const { from, to } = getTimerange(req); const timezone = capabilities.searchTimezone; diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/table/metric_buckets.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/table/metric_buckets.js index ba2c09e93e7e6..fe6a8b537d64b 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/table/metric_buckets.js +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/table/metric_buckets.js @@ -23,10 +23,18 @@ import { bucketTransform } from '../../helpers/bucket_transform'; import { getIntervalAndTimefield } from '../../get_interval_and_timefield'; import { calculateAggRoot } from './calculate_agg_root'; -export function metricBuckets(req, panel, esQueryConfig, indexPatternObject) { +export function metricBuckets( + req, + panel, + esQueryConfig, + indexPatternObject, + capabilities, + { barTargetUiSettings } +) { return (next) => (doc) => { const { interval } = getIntervalAndTimefield(panel, {}, indexPatternObject); - const { intervalString } = getBucketSize(req, interval); + const { intervalString } = getBucketSize(req, interval, capabilities, barTargetUiSettings); + panel.series.forEach((column) => { const aggRoot = calculateAggRoot(doc, column); column.metrics diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/table/positive_rate.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/table/positive_rate.js index b219f84deef80..6cf165d124e26 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/table/positive_rate.js +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/table/positive_rate.js @@ -22,10 +22,18 @@ import { getIntervalAndTimefield } from '../../get_interval_and_timefield'; import { calculateAggRoot } from './calculate_agg_root'; import { createPositiveRate, filter } from '../series/positive_rate'; -export function positiveRate(req, panel, esQueryConfig, indexPatternObject) { +export function positiveRate( + req, + panel, + esQueryConfig, + indexPatternObject, + capabilities, + { barTargetUiSettings } +) { return (next) => (doc) => { const { interval } = getIntervalAndTimefield(panel, {}, indexPatternObject); - const { intervalString } = getBucketSize(req, interval); + const { intervalString } = getBucketSize(req, interval, capabilities, barTargetUiSettings); + panel.series.forEach((column) => { const aggRoot = calculateAggRoot(doc, column); column.metrics.filter(filter).forEach(createPositiveRate(doc, intervalString, aggRoot)); diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/table/sibling_buckets.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/table/sibling_buckets.js index 1b14ffe34a947..ba08b18256dec 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/table/sibling_buckets.js +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/table/sibling_buckets.js @@ -23,10 +23,18 @@ import { bucketTransform } from '../../helpers/bucket_transform'; import { getIntervalAndTimefield } from '../../get_interval_and_timefield'; import { calculateAggRoot } from './calculate_agg_root'; -export function siblingBuckets(req, panel, esQueryConfig, indexPatternObject) { +export function siblingBuckets( + req, + panel, + esQueryConfig, + indexPatternObject, + capabilities, + { barTargetUiSettings } +) { return (next) => (doc) => { const { interval } = getIntervalAndTimefield(panel, {}, indexPatternObject); - const { bucketSize } = getBucketSize(req, interval); + const { bucketSize } = getBucketSize(req, interval, capabilities, barTargetUiSettings); + panel.series.forEach((column) => { const aggRoot = calculateAggRoot(doc, column); column.metrics diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/series/build_request_body.test.ts b/src/plugins/vis_type_timeseries/server/lib/vis_data/series/build_request_body.test.ts index 0c75e6ef1c5bd..6b2ef320d54b7 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/series/build_request_body.test.ts +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/series/build_request_body.test.ts @@ -97,7 +97,8 @@ describe('buildRequestBody(req)', () => { series, config, indexPatternObject, - capabilities + capabilities, + { barTargetUiSettings: 50 } ); expect(doc).toEqual({ diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/series/get_request_params.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/series/get_request_params.js index 4c653ea49e7c6..3804b1407b086 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/series/get_request_params.js +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/series/get_request_params.js @@ -19,18 +19,25 @@ import { buildRequestBody } from './build_request_body'; import { getEsShardTimeout } from '../helpers/get_es_shard_timeout'; import { getIndexPatternObject } from '../helpers/get_index_pattern'; +import { UI_SETTINGS } from '../../../../../data/common'; export async function getSeriesRequestParams(req, panel, series, esQueryConfig, capabilities) { + const uiSettings = req.getUiSettingsService(); const indexPattern = (series.override_index_pattern && series.series_index_pattern) || panel.index_pattern; const { indexPatternObject, indexPatternString } = await getIndexPatternObject(req, indexPattern); + const request = buildRequestBody( req, panel, series, esQueryConfig, indexPatternObject, - capabilities + capabilities, + { + maxBarsUiSettings: await uiSettings.get(UI_SETTINGS.HISTOGRAM_MAX_BARS), + barTargetUiSettings: await uiSettings.get(UI_SETTINGS.HISTOGRAM_BAR_TARGET), + } ); const esShardTimeout = await getEsShardTimeout(req); From e892b03173a83ebb001f228c2215b160012b9667 Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Tue, 24 Nov 2020 08:24:26 -0800 Subject: [PATCH 04/49] [build] Provide ARM build of RE2 (#84163) Signed-off-by: Tyler Smalley --- .../build/tasks/patch_native_modules_task.ts | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/dev/build/tasks/patch_native_modules_task.ts b/src/dev/build/tasks/patch_native_modules_task.ts index c3011fa80988c..b6eda2dbfd560 100644 --- a/src/dev/build/tasks/patch_native_modules_task.ts +++ b/src/dev/build/tasks/patch_native_modules_task.ts @@ -46,15 +46,30 @@ const packages: Package[] = [ destinationPath: 'node_modules/re2/build/Release/re2.node', extractMethod: 'gunzip', archives: { - darwin: { + 'darwin-x64': { url: 'https://github.com/uhop/node-re2/releases/download/1.15.4/darwin-x64-72.gz', sha256: '983106049bb86e21b7f823144b2b83e3f1408217401879b3cde0312c803512c9', }, - linux: { + 'linux-x64': { url: 'https://github.com/uhop/node-re2/releases/download/1.15.4/linux-x64-72.gz', sha256: '8b6692037f7b0df24dabc9c9b039038d1c3a3110f62121616b406c482169710a', }, - win32: { + + // ARM build is currently done manually as Github Actions used in upstream project + // do not natively support an ARM target. + + // From a AWS Graviton instance: + // * checkout the node-re2 project, + // * install Node using the same minor used by Kibana + // * npm install, which will also create a build + // * gzip -c build/Release/re2.node > linux-arm64-72.gz + // * upload to kibana-ci-proxy-cache bucket + 'linux-arm64': { + url: + 'https://storage.googleapis.com/kibana-ci-proxy-cache/node-re2/uhop/node-re2/releases/download/1.15.4/linux-arm64-72.gz', + sha256: '5942353ec9cf46a39199818d474f7af137cfbb1bc5727047fe22f31f36602a7e', + }, + 'win32-x64': { url: 'https://github.com/uhop/node-re2/releases/download/1.15.4/win32-x64-72.gz', sha256: '0a6991e693577160c3e9a3f196bd2518368c52d920af331a1a183313e0175604', }, @@ -84,7 +99,7 @@ async function patchModule( `Can't patch ${pkg.name}'s native module, we were expecting version ${pkg.version} and found ${installedVersion}` ); } - const platformName = platform.getName(); + const platformName = platform.getNodeArch(); const archive = pkg.archives[platformName]; const archiveName = path.basename(archive.url); const downloadPath = config.resolveFromRepo(DOWNLOAD_DIRECTORY, pkg.name, archiveName); From 24f262b9ca74f8e3e219ea417c2cd3889696f08c Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Tue, 24 Nov 2020 16:29:32 +0000 Subject: [PATCH 05/49] [ML] Space permision checks for job deletion (#83871) * [ML] Space permision checks for job deletion * updating spaces dependency * updating endpoint comments * adding delete job capabilities check * small change based on review * improving permissions checks * renaming function and endpoint Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../plugins/ml/common/types/capabilities.ts | 2 +- .../plugins/ml/common/types/saved_objects.ts | 9 ++ x-pack/plugins/ml/server/lib/spaces_utils.ts | 24 +++- .../models/data_recognizer/data_recognizer.ts | 4 +- x-pack/plugins/ml/server/plugin.ts | 5 +- x-pack/plugins/ml/server/routes/apidoc.json | 1 + .../plugins/ml/server/routes/saved_objects.ts | 56 ++++++++- .../ml/server/routes/schemas/saved_objects.ts | 4 + .../ml/server/saved_objects/authorization.ts | 3 +- .../plugins/ml/server/saved_objects/checks.ts | 108 +++++++++++++++++- .../ml/server/saved_objects/service.ts | 17 ++- x-pack/plugins/ml/server/types.ts | 5 + 12 files changed, 223 insertions(+), 15 deletions(-) diff --git a/x-pack/plugins/ml/common/types/capabilities.ts b/x-pack/plugins/ml/common/types/capabilities.ts index d708cd56b78df..91020eee26602 100644 --- a/x-pack/plugins/ml/common/types/capabilities.ts +++ b/x-pack/plugins/ml/common/types/capabilities.ts @@ -123,7 +123,7 @@ export function getPluginPrivileges() { catalogue: [], savedObject: { all: [], - read: ['ml-job'], + read: [ML_SAVED_OBJECT_TYPE], }, api: apmUserMlCapabilitiesKeys.map((k) => `ml:${k}`), ui: apmUserMlCapabilitiesKeys, diff --git a/x-pack/plugins/ml/common/types/saved_objects.ts b/x-pack/plugins/ml/common/types/saved_objects.ts index 9f4d402ec1759..d6c9ad758e8c6 100644 --- a/x-pack/plugins/ml/common/types/saved_objects.ts +++ b/x-pack/plugins/ml/common/types/saved_objects.ts @@ -27,3 +27,12 @@ export interface InitializeSavedObjectResponse { success: boolean; error?: any; } + +export interface DeleteJobCheckResponse { + [jobId: string]: DeleteJobPermission; +} + +export interface DeleteJobPermission { + canDelete: boolean; + canUntag: boolean; +} diff --git a/x-pack/plugins/ml/server/lib/spaces_utils.ts b/x-pack/plugins/ml/server/lib/spaces_utils.ts index b96fe6f2d1eb6..ecff3b8124cf5 100644 --- a/x-pack/plugins/ml/server/lib/spaces_utils.ts +++ b/x-pack/plugins/ml/server/lib/spaces_utils.ts @@ -7,6 +7,7 @@ import { Legacy } from 'kibana'; import { KibanaRequest } from '../../../../../src/core/server'; import { SpacesPluginStart } from '../../../spaces/server'; +import { PLUGIN_ID } from '../../common/constants/app'; export type RequestFacade = KibanaRequest | Legacy.Request; @@ -22,19 +23,34 @@ export function spacesUtilsProvider( const space = await (await getSpacesPlugin()).spacesService.getActiveSpace( request instanceof KibanaRequest ? request : KibanaRequest.from(request) ); - return space.disabledFeatures.includes('ml') === false; + return space.disabledFeatures.includes(PLUGIN_ID) === false; } - async function getAllSpaces(): Promise { + async function getAllSpaces() { if (getSpacesPlugin === undefined) { return null; } const client = (await getSpacesPlugin()).spacesService.createSpacesClient( request instanceof KibanaRequest ? request : KibanaRequest.from(request) ); - const spaces = await client.getAll(); + return await client.getAll(); + } + + async function getAllSpaceIds(): Promise { + const spaces = await getAllSpaces(); + if (spaces === null) { + return null; + } return spaces.map((s) => s.id); } - return { isMlEnabledInSpace, getAllSpaces }; + async function getMlSpaceIds(): Promise { + const spaces = await getAllSpaces(); + if (spaces === null) { + return null; + } + return spaces.filter((s) => s.disabledFeatures.includes(PLUGIN_ID) === false).map((s) => s.id); + } + + return { isMlEnabledInSpace, getAllSpaces, getAllSpaceIds, getMlSpaceIds }; } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts index f875788d50c5e..aeaf13ebf954e 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts +++ b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts @@ -1095,7 +1095,9 @@ export class DataRecognizer { job.config.analysis_limits.model_memory_limit = modelMemoryLimit; } } catch (error) { - mlLog.warn(`Data recognizer could not estimate model memory limit ${error.body}`); + mlLog.warn( + `Data recognizer could not estimate model memory limit ${JSON.stringify(error.body)}` + ); } } diff --git a/x-pack/plugins/ml/server/plugin.ts b/x-pack/plugins/ml/server/plugin.ts index 5e103dbc1806a..e48983c1c5365 100644 --- a/x-pack/plugins/ml/server/plugin.ts +++ b/x-pack/plugins/ml/server/plugin.ts @@ -178,7 +178,10 @@ export class MlServerPlugin notificationRoutes(routeInit); resultsServiceRoutes(routeInit); jobValidationRoutes(routeInit, this.version); - savedObjectsRoutes(routeInit); + savedObjectsRoutes(routeInit, { + getSpaces, + resolveMlCapabilities, + }); systemRoutes(routeInit, { getSpaces, cloud: plugins.cloud, diff --git a/x-pack/plugins/ml/server/routes/apidoc.json b/x-pack/plugins/ml/server/routes/apidoc.json index c157ae9e8200f..5672824f3d040 100644 --- a/x-pack/plugins/ml/server/routes/apidoc.json +++ b/x-pack/plugins/ml/server/routes/apidoc.json @@ -150,6 +150,7 @@ "AssignJobsToSpaces", "RemoveJobsFromSpaces", "JobsSpaces", + "DeleteJobCheck", "TrainedModels", "GetTrainedModel", diff --git a/x-pack/plugins/ml/server/routes/saved_objects.ts b/x-pack/plugins/ml/server/routes/saved_objects.ts index 1c9c975b66269..3ba69b0d6b505 100644 --- a/x-pack/plugins/ml/server/routes/saved_objects.ts +++ b/x-pack/plugins/ml/server/routes/saved_objects.ts @@ -5,14 +5,18 @@ */ import { wrapError } from '../client/error_wrapper'; -import { RouteInitialization } from '../types'; +import { RouteInitialization, SavedObjectsRouteDeps } from '../types'; import { checksFactory, repairFactory } from '../saved_objects'; -import { jobsAndSpaces, repairJobObjects } from './schemas/saved_objects'; +import { jobsAndSpaces, repairJobObjects, jobTypeSchema } from './schemas/saved_objects'; +import { jobIdsSchema } from './schemas/job_service_schema'; /** * Routes for job saved object management */ -export function savedObjectsRoutes({ router, routeGuard }: RouteInitialization) { +export function savedObjectsRoutes( + { router, routeGuard }: RouteInitialization, + { getSpaces, resolveMlCapabilities }: SavedObjectsRouteDeps +) { /** * @apiGroup JobSavedObjects * @@ -220,4 +224,50 @@ export function savedObjectsRoutes({ router, routeGuard }: RouteInitialization) } }) ); + + /** + * @apiGroup JobSavedObjects + * + * @api {get} /api/ml/saved_objects/delete_job_check Check whether user can delete a job + * @apiName DeleteJobCheck + * @apiDescription Check the user's ability to delete jobs. Returns whether they are able + * to fully delete the job and whether they are able to remove it from + * the current space. + * + * @apiSchema (body) jobIdsSchema (params) jobTypeSchema + * + */ + router.post( + { + path: '/api/ml/saved_objects/can_delete_job/{jobType}', + validate: { + params: jobTypeSchema, + body: jobIdsSchema, + }, + options: { + tags: ['access:ml:canGetJobs', 'access:ml:canGetDataFrameAnalytics'], + }, + }, + routeGuard.fullLicenseAPIGuard(async ({ request, response, jobSavedObjectService, client }) => { + try { + const { jobType } = request.params; + const { jobIds }: { jobIds: string[] } = request.body; + + const { canDeleteJobs } = checksFactory(client, jobSavedObjectService); + const body = await canDeleteJobs( + request, + jobType, + jobIds, + getSpaces !== undefined, + resolveMlCapabilities + ); + + return response.ok({ + body, + }); + } catch (e) { + return response.customError(wrapError(e)); + } + }) + ); } diff --git a/x-pack/plugins/ml/server/routes/schemas/saved_objects.ts b/x-pack/plugins/ml/server/routes/schemas/saved_objects.ts index d7385f6468f46..6b8c64714a82c 100644 --- a/x-pack/plugins/ml/server/routes/schemas/saved_objects.ts +++ b/x-pack/plugins/ml/server/routes/schemas/saved_objects.ts @@ -13,3 +13,7 @@ export const jobsAndSpaces = schema.object({ }); export const repairJobObjects = schema.object({ simulate: schema.maybe(schema.boolean()) }); + +export const jobTypeSchema = schema.object({ + jobType: schema.string(), +}); diff --git a/x-pack/plugins/ml/server/saved_objects/authorization.ts b/x-pack/plugins/ml/server/saved_objects/authorization.ts index 815ff29ae010c..958ee2091f11e 100644 --- a/x-pack/plugins/ml/server/saved_objects/authorization.ts +++ b/x-pack/plugins/ml/server/saved_objects/authorization.ts @@ -6,6 +6,7 @@ import { KibanaRequest } from 'kibana/server'; import type { SecurityPluginSetup } from '../../../security/server'; +import { ML_SAVED_OBJECT_TYPE } from '../../common/types/saved_objects'; export function authorizationProvider(authorization: SecurityPluginSetup['authz']) { async function authorizationCheck(request: KibanaRequest) { @@ -18,7 +19,7 @@ export function authorizationProvider(authorization: SecurityPluginSetup['authz' request ); const createMLJobAuthorizationAction = authorization.actions.savedObject.get( - 'ml-job', + ML_SAVED_OBJECT_TYPE, 'create' ); const canCreateGlobally = ( diff --git a/x-pack/plugins/ml/server/saved_objects/checks.ts b/x-pack/plugins/ml/server/saved_objects/checks.ts index 51269599105da..f682999cd5966 100644 --- a/x-pack/plugins/ml/server/saved_objects/checks.ts +++ b/x-pack/plugins/ml/server/saved_objects/checks.ts @@ -4,13 +4,15 @@ * you may not use this file except in compliance with the Elastic License. */ -import { IScopedClusterClient } from 'kibana/server'; +import Boom from '@hapi/boom'; +import { IScopedClusterClient, KibanaRequest } from 'kibana/server'; import type { JobSavedObjectService } from './service'; -import { JobType } from '../../common/types/saved_objects'; +import { JobType, DeleteJobCheckResponse } from '../../common/types/saved_objects'; import { Job } from '../../common/types/anomaly_detection_jobs'; import { Datafeed } from '../../common/types/anomaly_detection_jobs'; import { DataFrameAnalyticsConfig } from '../../common/types/data_frame_analytics'; +import { ResolveMlCapabilities } from '../../common/types/capabilities'; interface JobSavedObjectStatus { jobId: string; @@ -154,5 +156,105 @@ export function checksFactory( }; } - return { checkStatus }; + async function canDeleteJobs( + request: KibanaRequest, + jobType: JobType, + jobIds: string[], + spacesEnabled: boolean, + resolveMlCapabilities: ResolveMlCapabilities + ) { + if (jobType !== 'anomaly-detector' && jobType !== 'data-frame-analytics') { + throw Boom.badRequest('Job type must be "anomaly-detector" or "data-frame-analytics"'); + } + + const mlCapabilities = await resolveMlCapabilities(request); + if (mlCapabilities === null) { + throw Boom.internal('mlCapabilities is not defined'); + } + + if ( + (jobType === 'anomaly-detector' && mlCapabilities.canDeleteJob === false) || + (jobType === 'data-frame-analytics' && mlCapabilities.canDeleteDataFrameAnalytics === false) + ) { + // user does not have access to delete jobs. + return jobIds.reduce((results, jobId) => { + results[jobId] = { + canDelete: false, + canUntag: false, + }; + return results; + }, {} as DeleteJobCheckResponse); + } + + if (spacesEnabled === false) { + // spaces are disabled, delete only no untagging + return jobIds.reduce((results, jobId) => { + results[jobId] = { + canDelete: true, + canUntag: false, + }; + return results; + }, {} as DeleteJobCheckResponse); + } + const canCreateGlobalJobs = await jobSavedObjectService.canCreateGlobalJobs(request); + + const jobObjects = await Promise.all( + jobIds.map((id) => jobSavedObjectService.getJobObject(jobType, id)) + ); + + return jobIds.reduce((results, jobId) => { + const jobObject = jobObjects.find((j) => j?.attributes.job_id === jobId); + if (jobObject === undefined || jobObject.namespaces === undefined) { + // job saved object not found + results[jobId] = { + canDelete: false, + canUntag: false, + }; + return results; + } + + const { namespaces } = jobObject; + const isGlobalJob = namespaces.includes('*'); + + // job is in * space, user can see all spaces - delete and no option to untag + if (canCreateGlobalJobs && isGlobalJob) { + results[jobId] = { + canDelete: true, + canUntag: false, + }; + return results; + } + + // job is in * space, user cannot see all spaces - no untagging, no deleting + if (isGlobalJob) { + results[jobId] = { + canDelete: false, + canUntag: false, + }; + return results; + } + + // jobs with are in individual spaces can only be untagged + // from current space if the job is in more than 1 space + const canUntag = namespaces.length > 1; + + // job is in individual spaces, user cannot see all of them - untag only, no delete + if (namespaces.includes('?')) { + results[jobId] = { + canDelete: false, + canUntag, + }; + return results; + } + + // job is individual spaces, user can see all of them - delete and option to untag + results[jobId] = { + canDelete: true, + canUntag, + }; + return results; + }, {} as DeleteJobCheckResponse); + } + + return { checkStatus, canDeleteJobs }; } diff --git a/x-pack/plugins/ml/server/saved_objects/service.ts b/x-pack/plugins/ml/server/saved_objects/service.ts index ecaf0869d196c..bfc5b165fe555 100644 --- a/x-pack/plugins/ml/server/saved_objects/service.ts +++ b/x-pack/plugins/ml/server/saved_objects/service.ts @@ -5,7 +5,12 @@ */ import RE2 from 're2'; -import { KibanaRequest, SavedObjectsClientContract, SavedObjectsFindOptions } from 'kibana/server'; +import { + KibanaRequest, + SavedObjectsClientContract, + SavedObjectsFindOptions, + SavedObjectsFindResult, +} from 'kibana/server'; import type { SecurityPluginSetup } from '../../../security/server'; import { JobType, ML_SAVED_OBJECT_TYPE } from '../../common/types/saved_objects'; import { MLJobNotFound } from '../lib/ml_client'; @@ -133,6 +138,15 @@ export function jobSavedObjectServiceFactory( return await _getJobObjects(jobType, undefined, undefined, currentSpaceOnly); } + async function getJobObject( + jobType: JobType, + jobId: string, + currentSpaceOnly: boolean = true + ): Promise | undefined> { + const [jobObject] = await _getJobObjects(jobType, jobId, undefined, currentSpaceOnly); + return jobObject; + } + async function getAllJobObjectsForAllSpaces(jobType?: JobType) { await isMlReady(); const filterObject: JobObjectFilter = {}; @@ -307,6 +321,7 @@ export function jobSavedObjectServiceFactory( return { getAllJobObjects, + getJobObject, createAnomalyDetectionJob, createDataFrameAnalyticsJob, deleteAnomalyDetectionJob, diff --git a/x-pack/plugins/ml/server/types.ts b/x-pack/plugins/ml/server/types.ts index df40f5a26b0f3..780a4284312e7 100644 --- a/x-pack/plugins/ml/server/types.ts +++ b/x-pack/plugins/ml/server/types.ts @@ -31,6 +31,11 @@ export interface SystemRouteDeps { resolveMlCapabilities: ResolveMlCapabilities; } +export interface SavedObjectsRouteDeps { + getSpaces?: () => Promise; + resolveMlCapabilities: ResolveMlCapabilities; +} + export interface PluginsSetup { cloud: CloudSetup; features: FeaturesPluginSetup; From 5e183dd46d9e901827aa8c5246e72bd5e4092067 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com> Date: Tue, 24 Nov 2020 11:57:23 -0500 Subject: [PATCH 06/49] [Security Solution][Resolver] Allow a configurable entity_id field (#81679) * Trying to flesh out new tree route * Working on the descendants query * Almost working descendants * Possible solution for aggs * Working aggregations extraction * Working on the ancestry array for descendants * Making changes to the unique id for ancestr * Implementing ancestry funcitonality * Deleting the multiple edges * Fleshing out the descendants loop for levels * Writing tests for ancestors and descendants * Fixing type errors and writing more tests * Renaming validation variable and deprecating old tree routes * Renaming tree integration test file * Adding some integration tests * Fixing ancestry to handle multiple nodes in the request and writing more tests * Adding more tests * Renaming new tree to handler file * Renaming new tree directory * Adding more unit tests * Using doc value fields and working on types * Adding comments and more tests * Fixing timestamp test issue * Adding more comments * Fixing timestamp test issue take 2 * Adding id, parent, and name fields to the top level response * Fixing generator start and end time generation * Adding more comments * Revert "Fixing generator start and end time generation" This reverts commit 9e9abf68a6612f25ef9c9c85645f2e1bf72c9359. * Adding test for time Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../common/endpoint/generate_data.test.ts | 26 +- .../common/endpoint/generate_data.ts | 38 +- .../common/endpoint/schema/resolver.ts | 42 +- .../common/endpoint/types/index.ts | 50 + .../server/endpoint/routes/resolver.ts | 31 +- .../server/endpoint/routes/resolver/tree.ts | 7 +- .../endpoint/routes/resolver/tree/handler.ts | 28 + .../resolver/tree/queries/descendants.ts | 206 +++++ .../routes/resolver/tree/queries/lifecycle.ts | 101 ++ .../routes/resolver/tree/queries/stats.ts | 139 +++ .../routes/resolver/tree/utils/fetch.test.ts | 707 ++++++++++++++ .../routes/resolver/tree/utils/fetch.ts | 334 +++++++ .../routes/resolver/tree/utils/index.ts | 62 ++ .../apis/resolver/common.ts | 361 +++++++- .../apis/resolver/index.ts | 1 + .../apis/resolver/tree.ts | 867 ++++++++++++------ .../apis/resolver/tree_entity_id.ts | 375 ++++++++ 17 files changed, 3082 insertions(+), 293 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/handler.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/descendants.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/lifecycle.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/stats.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.test.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/index.ts create mode 100644 x-pack/test/security_solution_endpoint_api_int/apis/resolver/tree_entity_id.ts diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts index 66119e098238e..ec82f4795158e 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts @@ -27,7 +27,6 @@ interface Node { } describe('data generator data streams', () => { - // these tests cast the result of the generate methods so that we can specifically compare the `data_stream` fields it('creates a generator with default data streams', () => { const generator = new EndpointDocGenerator('seed'); expect(generator.generateHostMetadata().data_stream).toEqual({ @@ -268,6 +267,31 @@ describe('data generator', () => { } }; + it('sets the start and end times correctly', () => { + const startOfEpoch = new Date(0); + let startTime = new Date(timestampSafeVersion(tree.allEvents[0]) ?? startOfEpoch); + expect(startTime).not.toEqual(startOfEpoch); + let endTime = new Date(timestampSafeVersion(tree.allEvents[0]) ?? startOfEpoch); + expect(startTime).not.toEqual(startOfEpoch); + + for (const event of tree.allEvents) { + const currentEventTime = new Date(timestampSafeVersion(event) ?? startOfEpoch); + expect(currentEventTime).not.toEqual(startOfEpoch); + expect(tree.startTime.getTime()).toBeLessThanOrEqual(currentEventTime.getTime()); + expect(tree.endTime.getTime()).toBeGreaterThanOrEqual(currentEventTime.getTime()); + if (currentEventTime < startTime) { + startTime = currentEventTime; + } + + if (currentEventTime > endTime) { + endTime = currentEventTime; + } + } + expect(startTime).toEqual(tree.startTime); + expect(endTime).toEqual(tree.endTime); + expect(endTime.getTime() - startTime.getTime()).toBeGreaterThanOrEqual(0); + }); + it('creates related events in ascending order', () => { // the order should not change since it should already be in ascending order const relatedEventsAsc = _.cloneDeep(tree.origin.relatedEvents).sort( diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts index a4bdc4fc59a7c..3c508bed5b2f1 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts @@ -317,6 +317,8 @@ export interface Tree { * All events from children, ancestry, origin, and the alert in a single array */ allEvents: Event[]; + startTime: Date; + endTime: Date; } export interface TreeOptions { @@ -718,6 +720,35 @@ export class EndpointDocGenerator { }; } + private static getStartEndTimes(events: Event[]): { startTime: Date; endTime: Date } { + let startTime: number; + let endTime: number; + if (events.length > 0) { + startTime = timestampSafeVersion(events[0]) ?? new Date().getTime(); + endTime = startTime; + } else { + startTime = new Date().getTime(); + endTime = startTime; + } + + for (const event of events) { + const eventTimestamp = timestampSafeVersion(event); + if (eventTimestamp !== undefined) { + if (eventTimestamp < startTime) { + startTime = eventTimestamp; + } + + if (eventTimestamp > endTime) { + endTime = eventTimestamp; + } + } + } + return { + startTime: new Date(startTime), + endTime: new Date(endTime), + }; + } + /** * This generates a full resolver tree and keeps the entire tree in memory. This is useful for tests that want * to compare results from elasticsearch with the actual events created by this generator. Because all the events @@ -815,12 +846,17 @@ export class EndpointDocGenerator { const childrenByParent = groupNodesByParent(childrenNodes); const levels = createLevels(childrenByParent, [], childrenByParent.get(origin.id)); + const allEvents = [...ancestry, ...children]; + const { startTime, endTime } = EndpointDocGenerator.getStartEndTimes(allEvents); + return { children: childrenNodes, ancestry: ancestryNodes, - allEvents: [...ancestry, ...children], + allEvents, origin, childrenLevels: levels, + startTime, + endTime, }; } diff --git a/x-pack/plugins/security_solution/common/endpoint/schema/resolver.ts b/x-pack/plugins/security_solution/common/endpoint/schema/resolver.ts index 1dd5668b3177a..6777b1dabbd53 100644 --- a/x-pack/plugins/security_solution/common/endpoint/schema/resolver.ts +++ b/x-pack/plugins/security_solution/common/endpoint/schema/resolver.ts @@ -7,9 +7,9 @@ import { schema } from '@kbn/config-schema'; /** - * Used to validate GET requests for a complete resolver tree. + * Used to validate GET requests for a complete resolver tree centered around an entity_id. */ -export const validateTree = { +export const validateTreeEntityID = { params: schema.object({ id: schema.string({ minLength: 1 }) }), query: schema.object({ children: schema.number({ defaultValue: 200, min: 0, max: 10000 }), @@ -23,6 +23,44 @@ export const validateTree = { }), }; +/** + * Used to validate GET requests for a complete resolver tree. + */ +export const validateTree = { + body: schema.object({ + /** + * If the ancestry field is specified this field will be ignored + * + * If the ancestry field is specified we have a much more performant way of retrieving levels so let's not limit + * the number of levels that come back in that scenario. We could still limit it, but what we'd likely have to do + * is get all the levels back like we normally do with the ancestry array, bucket them together by level, and then + * remove the levels that exceeded the requested number which seems kind of wasteful. + */ + descendantLevels: schema.number({ defaultValue: 20, min: 0, max: 1000 }), + descendants: schema.number({ defaultValue: 1000, min: 0, max: 10000 }), + // if the ancestry array isn't specified allowing 200 might be too high + ancestors: schema.number({ defaultValue: 200, min: 0, max: 10000 }), + timerange: schema.object({ + from: schema.string(), + to: schema.string(), + }), + schema: schema.object({ + // the ancestry field is optional + ancestry: schema.maybe(schema.string({ minLength: 1 })), + id: schema.string({ minLength: 1 }), + name: schema.maybe(schema.string({ minLength: 1 })), + parent: schema.string({ minLength: 1 }), + }), + // only allowing strings and numbers for node IDs because Elasticsearch only allows those types for collapsing: + // https://www.elastic.co/guide/en/elasticsearch/reference/current/collapse-search-results.html + // We use collapsing in our Elasticsearch queries for the tree api + nodes: schema.arrayOf(schema.oneOf([schema.string({ minLength: 1 }), schema.number()]), { + minSize: 1, + }), + indexPatterns: schema.arrayOf(schema.string(), { minSize: 1 }), + }), +}; + /** * Used to validate POST requests for `/resolver/events` api. */ diff --git a/x-pack/plugins/security_solution/common/endpoint/types/index.ts b/x-pack/plugins/security_solution/common/endpoint/types/index.ts index e7d060b463aba..cd5c60e2698cb 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types/index.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types/index.ts @@ -78,6 +78,56 @@ export interface EventStats { byCategory: Record; } +/** + * Represents the object structure of a returned document when using doc value fields to filter the fields + * returned in a document from an Elasticsearch query. + * + * Here is an example: + * + * { + * "_index": ".ds-logs-endpoint.events.process-default-000001", + * "_id": "bc7brnUBxO0aE7QcCVHo", + * "_score": null, + * "fields": { <----------- The FieldsObject represents this portion + * "@timestamp": [ + * "2020-11-09T21:13:25.246Z" + * ], + * "process.name": "explorer.exe", + * "process.parent.entity_id": [ + * "0i17c2m22c" + * ], + * "process.Ext.ancestry": [ <------------ Notice that the keys are flattened + * "0i17c2m22c", + * "2z9j8dlx72", + * "oj61pr6g62", + * "x0leonbrc9" + * ], + * "process.entity_id": [ + * "6k8waczi22" + * ] + * }, + * "sort": [ + * 0, + * 1604956405246 + * ] + * } + */ +export interface FieldsObject { + [key: string]: ECSField; +} + +/** + * A node in a resolver graph. + */ +export interface ResolverNode { + data: FieldsObject; + id: string | number; + // the very root node might not have the parent field defined + parent?: string | number; + name?: string; + stats: EventStats; +} + /** * Statistical information for a node in a resolver tree. */ diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver.ts index b5d657fe55a1f..42a69d7b1e964 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver.ts @@ -7,16 +7,18 @@ import { IRouter } from 'kibana/server'; import { EndpointAppContext } from '../types'; import { - validateTree, + validateTreeEntityID, validateEvents, validateChildren, validateAncestry, validateAlerts, validateEntities, + validateTree, } from '../../../common/endpoint/schema/resolver'; import { handleChildren } from './resolver/children'; import { handleAncestry } from './resolver/ancestry'; -import { handleTree } from './resolver/tree'; +import { handleTree as handleTreeEntityID } from './resolver/tree'; +import { handleTree } from './resolver/tree/handler'; import { handleAlerts } from './resolver/alerts'; import { handleEntities } from './resolver/entity'; import { handleEvents } from './resolver/events'; @@ -24,6 +26,15 @@ import { handleEvents } from './resolver/events'; export function registerResolverRoutes(router: IRouter, endpointAppContext: EndpointAppContext) { const log = endpointAppContext.logFactory.get('resolver'); + router.post( + { + path: '/api/endpoint/resolver/tree', + validate: validateTree, + options: { authRequired: true }, + }, + handleTree(log) + ); + router.post( { path: '/api/endpoint/resolver/events', @@ -33,6 +44,9 @@ export function registerResolverRoutes(router: IRouter, endpointAppContext: Endp handleEvents(log) ); + /** + * @deprecated will be removed because it is not used + */ router.post( { path: '/api/endpoint/resolver/{id}/alerts', @@ -42,6 +56,9 @@ export function registerResolverRoutes(router: IRouter, endpointAppContext: Endp handleAlerts(log, endpointAppContext) ); + /** + * @deprecated use the /resolver/tree api instead + */ router.get( { path: '/api/endpoint/resolver/{id}/children', @@ -51,6 +68,9 @@ export function registerResolverRoutes(router: IRouter, endpointAppContext: Endp handleChildren(log, endpointAppContext) ); + /** + * @deprecated use the /resolver/tree api instead + */ router.get( { path: '/api/endpoint/resolver/{id}/ancestry', @@ -60,13 +80,16 @@ export function registerResolverRoutes(router: IRouter, endpointAppContext: Endp handleAncestry(log, endpointAppContext) ); + /** + * @deprecated use the /resolver/tree api instead + */ router.get( { path: '/api/endpoint/resolver/{id}', - validate: validateTree, + validate: validateTreeEntityID, options: { authRequired: true }, }, - handleTree(log, endpointAppContext) + handleTreeEntityID(log, endpointAppContext) ); /** diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree.ts index 02cddc3ddcf6e..08cb9b56bf64c 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree.ts @@ -7,14 +7,17 @@ import { RequestHandler, Logger } from 'kibana/server'; import { TypeOf } from '@kbn/config-schema'; import { eventsIndexPattern, alertsIndexPattern } from '../../../../common/endpoint/constants'; -import { validateTree } from '../../../../common/endpoint/schema/resolver'; +import { validateTreeEntityID } from '../../../../common/endpoint/schema/resolver'; import { Fetcher } from './utils/fetch'; import { EndpointAppContext } from '../../types'; export function handleTree( log: Logger, endpointAppContext: EndpointAppContext -): RequestHandler, TypeOf> { +): RequestHandler< + TypeOf, + TypeOf +> { return async (context, req, res) => { try { const client = context.core.elasticsearch.legacy.client; diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/handler.ts new file mode 100644 index 0000000000000..8c62cf8762981 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/handler.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { RequestHandler, Logger } from 'kibana/server'; +import { TypeOf } from '@kbn/config-schema'; +import { validateTree } from '../../../../../common/endpoint/schema/resolver'; +import { Fetcher } from './utils/fetch'; + +export function handleTree( + log: Logger +): RequestHandler> { + return async (context, req, res) => { + try { + const client = context.core.elasticsearch.client; + const fetcher = new Fetcher(client); + const body = await fetcher.tree(req.body); + return res.ok({ + body, + }); + } catch (err) { + log.warn(err); + return res.internalError({ body: 'Error retrieving tree.' }); + } + }; +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/descendants.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/descendants.ts new file mode 100644 index 0000000000000..405429cc24191 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/descendants.ts @@ -0,0 +1,206 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { SearchResponse } from 'elasticsearch'; +import { ApiResponse } from '@elastic/elasticsearch'; +import { IScopedClusterClient } from 'src/core/server'; +import { FieldsObject } from '../../../../../../common/endpoint/types'; +import { JsonObject, JsonValue } from '../../../../../../../../../src/plugins/kibana_utils/common'; +import { NodeID, Schema, Timerange, docValueFields } from '../utils/index'; + +interface DescendantsParams { + schema: Schema; + indexPatterns: string | string[]; + timerange: Timerange; +} + +/** + * Builds a query for retrieving descendants of a node. + */ +export class DescendantsQuery { + private readonly schema: Schema; + private readonly indexPatterns: string | string[]; + private readonly timerange: Timerange; + private readonly docValueFields: JsonValue[]; + constructor({ schema, indexPatterns, timerange }: DescendantsParams) { + this.docValueFields = docValueFields(schema); + this.schema = schema; + this.indexPatterns = indexPatterns; + this.timerange = timerange; + } + + private query(nodes: NodeID[], size: number): JsonObject { + return { + _source: false, + docvalue_fields: this.docValueFields, + size, + collapse: { + field: this.schema.id, + }, + sort: [{ '@timestamp': 'asc' }], + query: { + bool: { + filter: [ + { + range: { + '@timestamp': { + gte: this.timerange.from, + lte: this.timerange.to, + format: 'strict_date_optional_time', + }, + }, + }, + { + terms: { [this.schema.parent]: nodes }, + }, + { + exists: { + field: this.schema.id, + }, + }, + { + exists: { + field: this.schema.parent, + }, + }, + { + term: { 'event.category': 'process' }, + }, + { + term: { 'event.kind': 'event' }, + }, + ], + }, + }, + }; + } + + private queryWithAncestryArray(nodes: NodeID[], ancestryField: string, size: number): JsonObject { + return { + _source: false, + docvalue_fields: this.docValueFields, + size, + collapse: { + field: this.schema.id, + }, + sort: [ + { + _script: { + type: 'number', + script: { + /** + * This script is used to sort the returned documents in a breadth first order so that we return all of + * a single level of nodes before returning the next level of nodes. This is needed because using the + * ancestry array could result in the search going deep before going wide depending on when the nodes + * spawned their children. If a node spawns a child before it's sibling is spawned then the child would + * be found before the sibling because by default the sort was on timestamp ascending. + */ + source: ` + Map ancestryToIndex = [:]; + List sourceAncestryArray = params._source.${ancestryField}; + int length = sourceAncestryArray.length; + for (int i = 0; i < length; i++) { + ancestryToIndex[sourceAncestryArray[i]] = i; + } + for (String id : params.ids) { + def index = ancestryToIndex[id]; + if (index != null) { + return index; + } + } + return -1; + `, + params: { + ids: nodes, + }, + }, + }, + }, + { '@timestamp': 'asc' }, + ], + query: { + bool: { + filter: [ + { + range: { + '@timestamp': { + gte: this.timerange.from, + lte: this.timerange.to, + format: 'strict_date_optional_time', + }, + }, + }, + { + terms: { + [ancestryField]: nodes, + }, + }, + { + exists: { + field: this.schema.id, + }, + }, + { + exists: { + field: this.schema.parent, + }, + }, + { + exists: { + field: ancestryField, + }, + }, + { + term: { 'event.category': 'process' }, + }, + { + term: { 'event.kind': 'event' }, + }, + ], + }, + }, + }; + } + + /** + * Searches for descendant nodes matching the specified IDs. + * + * @param client for making requests to Elasticsearch + * @param nodes the unique IDs to search for in Elasticsearch + * @param limit the upper limit of documents to returned + */ + async search( + client: IScopedClusterClient, + nodes: NodeID[], + limit: number + ): Promise { + if (nodes.length <= 0) { + return []; + } + + let response: ApiResponse>; + if (this.schema.ancestry) { + response = await client.asCurrentUser.search({ + body: this.queryWithAncestryArray(nodes, this.schema.ancestry, limit), + index: this.indexPatterns, + }); + } else { + response = await client.asCurrentUser.search({ + body: this.query(nodes, limit), + index: this.indexPatterns, + }); + } + + /** + * The returned values will look like: + * [ + * { 'schema_id_value': , 'schema_parent_value': } + * ] + * + * So the schema fields are flattened ('process.parent.entity_id') + */ + return response.body.hits.hits.map((hit) => hit.fields); + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/lifecycle.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/lifecycle.ts new file mode 100644 index 0000000000000..606a4538ba88c --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/lifecycle.ts @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { SearchResponse } from 'elasticsearch'; +import { ApiResponse } from '@elastic/elasticsearch'; +import { IScopedClusterClient } from 'src/core/server'; +import { FieldsObject } from '../../../../../../common/endpoint/types'; +import { JsonObject, JsonValue } from '../../../../../../../../../src/plugins/kibana_utils/common'; +import { NodeID, Schema, Timerange, docValueFields } from '../utils/index'; + +interface LifecycleParams { + schema: Schema; + indexPatterns: string | string[]; + timerange: Timerange; +} + +/** + * Builds a query for retrieving descendants of a node. + */ +export class LifecycleQuery { + private readonly schema: Schema; + private readonly indexPatterns: string | string[]; + private readonly timerange: Timerange; + private readonly docValueFields: JsonValue[]; + constructor({ schema, indexPatterns, timerange }: LifecycleParams) { + this.docValueFields = docValueFields(schema); + this.schema = schema; + this.indexPatterns = indexPatterns; + this.timerange = timerange; + } + + private query(nodes: NodeID[]): JsonObject { + return { + _source: false, + docvalue_fields: this.docValueFields, + size: nodes.length, + collapse: { + field: this.schema.id, + }, + sort: [{ '@timestamp': 'asc' }], + query: { + bool: { + filter: [ + { + range: { + '@timestamp': { + gte: this.timerange.from, + lte: this.timerange.to, + format: 'strict_date_optional_time', + }, + }, + }, + { + terms: { [this.schema.id]: nodes }, + }, + { + exists: { + field: this.schema.id, + }, + }, + { + term: { 'event.category': 'process' }, + }, + { + term: { 'event.kind': 'event' }, + }, + ], + }, + }, + }; + } + + /** + * Searches for lifecycle events matching the specified node IDs. + * + * @param client for making requests to Elasticsearch + * @param nodes the unique IDs to search for in Elasticsearch + */ + async search(client: IScopedClusterClient, nodes: NodeID[]): Promise { + if (nodes.length <= 0) { + return []; + } + + const response: ApiResponse> = await client.asCurrentUser.search({ + body: this.query(nodes), + index: this.indexPatterns, + }); + + /** + * The returned values will look like: + * [ + * { 'schema_id_value': , 'schema_parent_value': } + * ] + * + * So the schema fields are flattened ('process.parent.entity_id') + */ + return response.body.hits.hits.map((hit) => hit.fields); + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/stats.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/stats.ts new file mode 100644 index 0000000000000..33dcdce8987f5 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/stats.ts @@ -0,0 +1,139 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { SearchResponse } from 'elasticsearch'; +import { ApiResponse } from '@elastic/elasticsearch'; +import { IScopedClusterClient } from 'src/core/server'; +import { JsonObject } from '../../../../../../../../../src/plugins/kibana_utils/common'; +import { EventStats } from '../../../../../../common/endpoint/types'; +import { NodeID, Schema, Timerange } from '../utils/index'; + +interface AggBucket { + key: string; + doc_count: number; +} + +interface CategoriesAgg extends AggBucket { + /** + * The reason categories is optional here is because if no data was returned in the query the categories aggregation + * will not be defined on the response (because it's a sub aggregation). + */ + categories?: { + buckets?: AggBucket[]; + }; +} + +interface StatsParams { + schema: Schema; + indexPatterns: string | string[]; + timerange: Timerange; +} + +/** + * Builds a query for retrieving descendants of a node. + */ +export class StatsQuery { + private readonly schema: Schema; + private readonly indexPatterns: string | string[]; + private readonly timerange: Timerange; + constructor({ schema, indexPatterns, timerange }: StatsParams) { + this.schema = schema; + this.indexPatterns = indexPatterns; + this.timerange = timerange; + } + + private query(nodes: NodeID[]): JsonObject { + return { + size: 0, + query: { + bool: { + filter: [ + { + range: { + '@timestamp': { + gte: this.timerange.from, + lte: this.timerange.to, + format: 'strict_date_optional_time', + }, + }, + }, + { + terms: { [this.schema.id]: nodes }, + }, + { + term: { 'event.kind': 'event' }, + }, + { + bool: { + must_not: { + term: { + 'event.category': 'process', + }, + }, + }, + }, + ], + }, + }, + aggs: { + ids: { + terms: { field: this.schema.id, size: nodes.length }, + aggs: { + categories: { + terms: { field: 'event.category', size: 1000 }, + }, + }, + }, + }, + }; + } + + private static getEventStats(catAgg: CategoriesAgg): EventStats { + const total = catAgg.doc_count; + if (!catAgg.categories?.buckets) { + return { + total, + byCategory: {}, + }; + } + + const byCategory: Record = catAgg.categories.buckets.reduce( + (cummulative: Record, bucket: AggBucket) => ({ + ...cummulative, + [bucket.key]: bucket.doc_count, + }), + {} + ); + return { + total, + byCategory, + }; + } + + /** + * Returns the related event statistics for a set of nodes. + * @param client used to make requests to Elasticsearch + * @param nodes an array of unique IDs representing nodes in a resolver graph + */ + async search(client: IScopedClusterClient, nodes: NodeID[]): Promise> { + if (nodes.length <= 0) { + return {}; + } + + // leaving unknown here because we don't actually need the hits part of the body + const response: ApiResponse> = await client.asCurrentUser.search({ + body: this.query(nodes), + index: this.indexPatterns, + }); + + return response.body.aggregations?.ids?.buckets.reduce( + (cummulative: Record, bucket: CategoriesAgg) => ({ + ...cummulative, + [bucket.key]: StatsQuery.getEventStats(bucket), + }), + {} + ); + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.test.ts new file mode 100644 index 0000000000000..8105f1125d01d --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.test.ts @@ -0,0 +1,707 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + Fetcher, + getAncestryAsArray, + getIDField, + getLeafNodes, + getNameField, + getParentField, + TreeOptions, +} from './fetch'; +import { LifecycleQuery } from '../queries/lifecycle'; +import { DescendantsQuery } from '../queries/descendants'; +import { StatsQuery } from '../queries/stats'; +import { IScopedClusterClient } from 'src/core/server'; +import { elasticsearchServiceMock } from 'src/core/server/mocks'; +import { FieldsObject, ResolverNode } from '../../../../../../common/endpoint/types'; +import { Schema } from './index'; + +jest.mock('../queries/descendants'); +jest.mock('../queries/lifecycle'); +jest.mock('../queries/stats'); + +function formatResponse(results: FieldsObject[], schema: Schema): ResolverNode[] { + return results.map((node) => { + return { + id: getIDField(node, schema) ?? '', + parent: getParentField(node, schema), + name: getNameField(node, schema), + data: node, + stats: { + total: 0, + byCategory: {}, + }, + }; + }); +} + +describe('fetcher test', () => { + const schemaIDParent = { + id: 'id', + parent: 'parent', + }; + + const schemaIDParentAncestry = { + id: 'id', + parent: 'parent', + ancestry: 'ancestry', + }; + + const schemaIDParentName = { + id: 'id', + parent: 'parent', + name: 'name', + }; + + let client: jest.Mocked; + beforeAll(() => { + StatsQuery.prototype.search = jest.fn().mockImplementation(async () => { + return {}; + }); + }); + beforeEach(() => { + client = elasticsearchServiceMock.createScopedClusterClient(); + }); + + describe('descendants', () => { + it('correctly exists loop when the search returns no results', async () => { + DescendantsQuery.prototype.search = jest.fn().mockImplementationOnce(async () => { + return []; + }); + const options: TreeOptions = { + descendantLevels: 1, + descendants: 5, + ancestors: 0, + timerange: { + from: '', + to: '', + }, + schema: { + id: '', + parent: '', + }, + indexPatterns: [''], + nodes: ['a'], + }; + const fetcher = new Fetcher(client); + expect(await fetcher.tree(options)).toEqual([]); + }); + + it('exists the loop when the options specify no descendants', async () => { + const options: TreeOptions = { + descendantLevels: 0, + descendants: 0, + ancestors: 0, + timerange: { + from: '', + to: '', + }, + schema: { + id: '', + parent: '', + }, + indexPatterns: [''], + nodes: ['a'], + }; + + const fetcher = new Fetcher(client); + expect(await fetcher.tree(options)).toEqual([]); + }); + + it('returns the correct results without the ancestry defined', async () => { + /** + . + └── 0 + ├── 1 + │ └── 2 + └── 3 + ├── 4 + └── 5 + */ + const level1 = [ + { + id: '1', + parent: '0', + }, + { + id: '3', + parent: '0', + }, + ]; + const level2 = [ + { + id: '2', + parent: '1', + }, + + { + id: '4', + parent: '3', + }, + { + id: '5', + parent: '3', + }, + ]; + DescendantsQuery.prototype.search = jest + .fn() + .mockImplementationOnce(async () => { + return level1; + }) + .mockImplementationOnce(async () => { + return level2; + }); + const options: TreeOptions = { + descendantLevels: 2, + descendants: 5, + ancestors: 0, + timerange: { + from: '', + to: '', + }, + schema: schemaIDParent, + indexPatterns: [''], + nodes: ['0'], + }; + + const fetcher = new Fetcher(client); + expect(await fetcher.tree(options)).toEqual( + formatResponse([...level1, ...level2], schemaIDParent) + ); + }); + }); + + describe('ancestors', () => { + it('correctly exits loop when the search returns no results', async () => { + LifecycleQuery.prototype.search = jest.fn().mockImplementationOnce(async () => { + return []; + }); + const options: TreeOptions = { + descendantLevels: 0, + descendants: 0, + ancestors: 5, + timerange: { + from: '', + to: '', + }, + schema: { + id: '', + parent: '', + }, + indexPatterns: [''], + nodes: ['a'], + }; + const fetcher = new Fetcher(client); + expect(await fetcher.tree(options)).toEqual([]); + }); + + it('correctly exits loop when the options specify no ancestors', async () => { + LifecycleQuery.prototype.search = jest.fn().mockImplementationOnce(async () => { + throw new Error('should not have called this'); + }); + const options: TreeOptions = { + descendantLevels: 0, + descendants: 0, + ancestors: 0, + timerange: { + from: '', + to: '', + }, + schema: { + id: '', + parent: '', + }, + indexPatterns: [''], + nodes: ['a'], + }; + const fetcher = new Fetcher(client); + expect(await fetcher.tree(options)).toEqual([]); + }); + + it('correctly returns the ancestors when the number of levels has been reached', async () => { + LifecycleQuery.prototype.search = jest + .fn() + .mockImplementationOnce(async () => { + return [ + { + id: '3', + parent: '2', + }, + ]; + }) + .mockImplementationOnce(async () => { + return [ + { + id: '2', + parent: '1', + }, + ]; + }); + const options: TreeOptions = { + descendantLevels: 0, + descendants: 0, + ancestors: 2, + timerange: { + from: '', + to: '', + }, + schema: schemaIDParent, + indexPatterns: [''], + nodes: ['3'], + }; + const fetcher = new Fetcher(client); + expect(await fetcher.tree(options)).toEqual( + formatResponse( + [ + { id: '3', parent: '2' }, + { id: '2', parent: '1' }, + ], + schemaIDParent + ) + ); + }); + + it('correctly adds name field to response', async () => { + LifecycleQuery.prototype.search = jest + .fn() + .mockImplementationOnce(async () => { + return [ + { + id: '3', + parent: '2', + }, + ]; + }) + .mockImplementationOnce(async () => { + return [ + { + id: '2', + parent: '1', + }, + ]; + }); + const options: TreeOptions = { + descendantLevels: 0, + descendants: 0, + ancestors: 2, + timerange: { + from: '', + to: '', + }, + schema: schemaIDParentName, + indexPatterns: [''], + nodes: ['3'], + }; + const fetcher = new Fetcher(client); + expect(await fetcher.tree(options)).toEqual( + formatResponse( + [ + { id: '3', parent: '2' }, + { id: '2', parent: '1' }, + ], + schemaIDParentName + ) + ); + }); + + it('correctly returns the ancestors with ancestry arrays', async () => { + const node3 = { + ancestry: ['2', '1'], + id: '3', + parent: '2', + }; + + const node1 = { + ancestry: ['0'], + id: '1', + parent: '0', + }; + + const node2 = { + ancestry: ['1', '0'], + id: '2', + parent: '1', + }; + LifecycleQuery.prototype.search = jest + .fn() + .mockImplementationOnce(async () => { + return [node3]; + }) + .mockImplementationOnce(async () => { + return [node1, node2]; + }); + const options: TreeOptions = { + descendantLevels: 0, + descendants: 0, + ancestors: 3, + timerange: { + from: '', + to: '', + }, + schema: schemaIDParentAncestry, + indexPatterns: [''], + nodes: ['3'], + }; + const fetcher = new Fetcher(client); + expect(await fetcher.tree(options)).toEqual( + formatResponse([node3, node1, node2], schemaIDParentAncestry) + ); + }); + }); + + describe('retrieving leaf nodes', () => { + it('correctly identifies the leaf nodes in a response without the ancestry field', () => { + /** + . + └── 0 + ├── 1 + ├── 2 + └── 3 + */ + const results = [ + { + id: '1', + parent: '0', + }, + { + id: '2', + parent: '0', + }, + { + id: '3', + parent: '0', + }, + ]; + const leaves = getLeafNodes(results, ['0'], { id: 'id', parent: 'parent' }); + expect(leaves).toStrictEqual(['1', '2', '3']); + }); + + it('correctly ignores nodes without the proper fields', () => { + /** + . + └── 0 + ├── 1 + ├── 2 + */ + const results = [ + { + id: '1', + parent: '0', + }, + { + id: '2', + parent: '0', + }, + { + idNotReal: '3', + parentNotReal: '0', + }, + ]; + const leaves = getLeafNodes(results, ['0'], { id: 'id', parent: 'parent' }); + expect(leaves).toStrictEqual(['1', '2']); + }); + + it('returns an empty response when the proper fields are not defined', () => { + const results = [ + { + id: '1', + parentNotReal: '0', + }, + { + id: '2', + parentNotReal: '0', + }, + { + idNotReal: '3', + parent: '0', + }, + ]; + const leaves = getLeafNodes(results, ['0'], { id: 'id', parent: 'parent' }); + expect(leaves).toStrictEqual([]); + }); + + describe('with the ancestry field defined', () => { + it('correctly identifies the leaf nodes in a response with the ancestry field', () => { + /** + . + ├── 1 + │ └── 2 + └── 3 + */ + const results = [ + { + id: '1', + parent: '0', + ancestry: ['0', 'a'], + }, + { + id: '2', + parent: '1', + ancestry: ['1', '0'], + }, + { + id: '3', + parent: '0', + ancestry: ['0', 'a'], + }, + ]; + const leaves = getLeafNodes(results, ['0'], { + id: 'id', + parent: 'parent', + ancestry: 'ancestry', + }); + expect(leaves).toStrictEqual(['2']); + }); + + it('falls back to using parent field if it cannot find the ancestry field', () => { + /** + . + ├── 1 + │ └── 2 + └── 3 + */ + const results = [ + { + id: '1', + parent: '0', + ancestryNotValid: ['0', 'a'], + }, + { + id: '2', + parent: '1', + }, + { + id: '3', + parent: '0', + }, + ]; + const leaves = getLeafNodes(results, ['0'], { + id: 'id', + parent: 'parent', + ancestry: 'ancestry', + }); + expect(leaves).toStrictEqual(['1', '3']); + }); + + it('correctly identifies the leaf nodes with a tree with multiple leaves', () => { + /** + . + └── 0 + ├── 1 + │ └── 2 + └── 3 + ├── 4 + └── 5 + */ + const results = [ + { + id: '1', + parent: '0', + ancestry: ['0', 'a'], + }, + { + id: '2', + parent: '1', + ancestry: ['1', '0'], + }, + { + id: '3', + parent: '0', + ancestry: ['0', 'a'], + }, + { + id: '4', + parent: '3', + ancestry: ['3', '0'], + }, + { + id: '5', + parent: '3', + ancestry: ['3', '0'], + }, + ]; + const leaves = getLeafNodes(results, ['0'], { + id: 'id', + parent: 'parent', + ancestry: 'ancestry', + }); + expect(leaves).toStrictEqual(['2', '4', '5']); + }); + + it('correctly identifies the leaf nodes with multiple queried nodes', () => { + /** + . + ├── 0 + │ ├── 1 + │ │ └── 2 + │ └── 3 + │ ├── 4 + │ └── 5 + └── a + └── b + ├── c + └── d + */ + const results = [ + { + id: '1', + parent: '0', + ancestry: ['0'], + }, + { + id: '2', + parent: '1', + ancestry: ['1', '0'], + }, + { + id: '3', + parent: '0', + ancestry: ['0'], + }, + { + id: '4', + parent: '3', + ancestry: ['3', '0'], + }, + { + id: '5', + parent: '3', + ancestry: ['3', '0'], + }, + { + id: 'b', + parent: 'a', + ancestry: ['a'], + }, + { + id: 'c', + parent: 'b', + ancestry: ['b', 'a'], + }, + { + id: 'd', + parent: 'b', + ancestry: ['b', 'a'], + }, + ]; + const leaves = getLeafNodes(results, ['0', 'a'], { + id: 'id', + parent: 'parent', + ancestry: 'ancestry', + }); + expect(leaves).toStrictEqual(['2', '4', '5', 'c', 'd']); + }); + + it('correctly identifies the leaf nodes with an unbalanced tree', () => { + /** + . + ├── 0 + │ ├── 1 + │ │ └── 2 + │ └── 3 + │ ├── 4 + │ └── 5 + └── a + └── b + */ + const results = [ + { + id: '1', + parent: '0', + ancestry: ['0'], + }, + { + id: '2', + parent: '1', + ancestry: ['1', '0'], + }, + { + id: '3', + parent: '0', + ancestry: ['0'], + }, + { + id: '4', + parent: '3', + ancestry: ['3', '0'], + }, + { + id: '5', + parent: '3', + ancestry: ['3', '0'], + }, + { + id: 'b', + parent: 'a', + ancestry: ['a'], + }, + ]; + const leaves = getLeafNodes(results, ['0', 'a'], { + id: 'id', + parent: 'parent', + ancestry: 'ancestry', + }); + // the reason b is not identified here is because the ancestry array + // size is 2, which means that if b had a descendant, then it would have been found + // using our query which found 2, 4, 5. So either we hit the size limit or there are no + // children of b + expect(leaves).toStrictEqual(['2', '4', '5']); + }); + }); + }); + + describe('getIDField', () => { + it('returns undefined if the field does not exist', () => { + expect(getIDField({}, { id: 'a', parent: 'b' })).toBeUndefined(); + }); + + it('returns the first value if the field is an array', () => { + expect(getIDField({ 'a.b': ['1', '2'] }, { id: 'a.b', parent: 'b' })).toStrictEqual('1'); + }); + }); + + describe('getParentField', () => { + it('returns undefined if the field does not exist', () => { + expect(getParentField({}, { id: 'a', parent: 'b' })).toBeUndefined(); + }); + + it('returns the first value if the field is an array', () => { + expect(getParentField({ 'a.b': ['1', '2'] }, { id: 'z', parent: 'a.b' })).toStrictEqual('1'); + }); + }); + + describe('getAncestryAsArray', () => { + it('returns an empty array if the field does not exist', () => { + expect(getAncestryAsArray({}, { id: 'a', parent: 'b', ancestry: 'z' })).toStrictEqual([]); + }); + + it('returns the full array if the field exists', () => { + expect( + getAncestryAsArray({ 'a.b': ['1', '2'] }, { id: 'z', parent: 'f', ancestry: 'a.b' }) + ).toStrictEqual(['1', '2']); + }); + + it('returns a built array using the parent field if ancestry field is empty', () => { + expect( + getAncestryAsArray( + { 'aParent.bParent': ['1', '2'], ancestry: [] }, + { id: 'z', parent: 'aParent.bParent', ancestry: 'ancestry' } + ) + ).toStrictEqual(['1']); + }); + + it('returns a built array using the parent field if ancestry field does not exist', () => { + expect( + getAncestryAsArray( + { 'aParent.bParent': '1' }, + { id: 'z', parent: 'aParent.bParent', ancestry: 'ancestry' } + ) + ).toStrictEqual(['1']); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.ts new file mode 100644 index 0000000000000..eaecad6c47970 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.ts @@ -0,0 +1,334 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { IScopedClusterClient } from 'kibana/server'; +import { + firstNonNullValue, + values, +} from '../../../../../../common/endpoint/models/ecs_safety_helpers'; +import { ECSField, ResolverNode, FieldsObject } from '../../../../../../common/endpoint/types'; +import { DescendantsQuery } from '../queries/descendants'; +import { Schema, NodeID } from './index'; +import { LifecycleQuery } from '../queries/lifecycle'; +import { StatsQuery } from '../queries/stats'; + +/** + * The query parameters passed in from the request. These define the limits for the ES requests for retrieving the + * resolver tree. + */ +export interface TreeOptions { + descendantLevels: number; + descendants: number; + ancestors: number; + timerange: { + from: string; + to: string; + }; + schema: Schema; + nodes: NodeID[]; + indexPatterns: string[]; +} + +/** + * Handles retrieving nodes of a resolver tree. + */ +export class Fetcher { + constructor(private readonly client: IScopedClusterClient) {} + + /** + * This method retrieves the ancestors and descendants of a resolver tree. + * + * @param options the options for retrieving the structure of the tree. + */ + public async tree(options: TreeOptions): Promise { + const treeParts = await Promise.all([ + this.retrieveAncestors(options), + this.retrieveDescendants(options), + ]); + + const tree = treeParts.reduce((results, partArray) => { + results.push(...partArray); + return results; + }, []); + + return this.formatResponse(tree, options); + } + + private async formatResponse( + treeNodes: FieldsObject[], + options: TreeOptions + ): Promise { + const statsIDs: NodeID[] = []; + for (const node of treeNodes) { + const id = getIDField(node, options.schema); + if (id) { + statsIDs.push(id); + } + } + + const query = new StatsQuery({ + indexPatterns: options.indexPatterns, + schema: options.schema, + timerange: options.timerange, + }); + + const eventStats = await query.search(this.client, statsIDs); + const statsNodes: ResolverNode[] = []; + for (const node of treeNodes) { + const id = getIDField(node, options.schema); + const parent = getParentField(node, options.schema); + const name = getNameField(node, options.schema); + + // at this point id should never be undefined, it should be enforced by the Elasticsearch query + // but let's check anyway + if (id !== undefined) { + statsNodes.push({ + id, + parent, + name, + data: node, + stats: eventStats[id] ?? { total: 0, byCategory: {} }, + }); + } + } + return statsNodes; + } + + private static getNextAncestorsToFind( + results: FieldsObject[], + schema: Schema, + levelsLeft: number + ): NodeID[] { + const nodesByID = results.reduce((accMap: Map, result: FieldsObject) => { + const id = getIDField(result, schema); + if (id) { + accMap.set(id, result); + } + return accMap; + }, new Map()); + + const nodes: NodeID[] = []; + // Find all the nodes that don't have their parent in the result set, we will use these + // nodes to find the additional ancestry + for (const result of results) { + const parent = getParentField(result, schema); + if (parent) { + const parentNode = nodesByID.get(parent); + if (!parentNode) { + // it's ok if the nodes array is larger than the levelsLeft because the query + // will have the size set to the levelsLeft which will restrict the number of results + nodes.push(...getAncestryAsArray(result, schema).slice(0, levelsLeft)); + } + } + } + return nodes; + } + + private async retrieveAncestors(options: TreeOptions): Promise { + const ancestors: FieldsObject[] = []; + const query = new LifecycleQuery({ + schema: options.schema, + indexPatterns: options.indexPatterns, + timerange: options.timerange, + }); + + let nodes = options.nodes; + let numLevelsLeft = options.ancestors; + + while (numLevelsLeft > 0) { + const results: FieldsObject[] = await query.search(this.client, nodes); + if (results.length <= 0) { + return ancestors; + } + + /** + * This array (this.ancestry.ancestors) is the accumulated ancestors of the node of interest. This array is different + * from the ancestry array of a specific document. The order of this array is going to be weird, it will look like this + * [most distant ancestor...closer ancestor, next recursive call most distant ancestor...closer ancestor] + * + * Here is an example of why this happens + * Consider the following tree: + * A -> B -> C -> D -> E -> Origin + * Where A was spawn before B, which was before C, etc + * + * Let's assume the ancestry array limit is 2 so Origin's array would be: [E, D] + * E's ancestry array would be: [D, C] etc + * + * If a request comes in to retrieve all the ancestors in this tree, the accumulate results will be: + * [D, E, B, C, A] + * + * The first iteration would retrieve D and E in that order because they are sorted in ascending order by timestamp. + * The next iteration would get the ancestors of D (since that's the most distant ancestor from Origin) which are + * [B, C] + * The next iteration would get the ancestors of B which is A + * Hence: [D, E, B, C, A] + */ + ancestors.push(...results); + numLevelsLeft -= results.length; + nodes = Fetcher.getNextAncestorsToFind(results, options.schema, numLevelsLeft); + } + return ancestors; + } + + private async retrieveDescendants(options: TreeOptions): Promise { + const descendants: FieldsObject[] = []; + const query = new DescendantsQuery({ + schema: options.schema, + indexPatterns: options.indexPatterns, + timerange: options.timerange, + }); + + let nodes: NodeID[] = options.nodes; + let numNodesLeftToRequest: number = options.descendants; + let levelsLeftToRequest: number = options.descendantLevels; + // if the ancestry was specified then ignore the levels + while ( + numNodesLeftToRequest > 0 && + (options.schema.ancestry !== undefined || levelsLeftToRequest > 0) + ) { + const results: FieldsObject[] = await query.search(this.client, nodes, numNodesLeftToRequest); + if (results.length <= 0) { + return descendants; + } + + nodes = getLeafNodes(results, nodes, options.schema); + + numNodesLeftToRequest -= results.length; + levelsLeftToRequest -= 1; + descendants.push(...results); + } + + return descendants; + } +} + +/** + * This functions finds the leaf nodes for a given response from an Elasticsearch query. + * + * Exporting so it can be tested. + * + * @param results the doc values portion of the documents returned from an Elasticsearch query + * @param nodes an array of unique IDs that were used to find the returned documents + * @param schema the field definitions for how nodes are represented in the resolver graph + */ +export function getLeafNodes( + results: FieldsObject[], + nodes: Array, + schema: Schema +): NodeID[] { + let largestAncestryArray = 0; + const nodesToQueryNext: Map> = new Map(); + const queriedNodes = new Set(nodes); + const isDistantGrandchild = (event: FieldsObject) => { + const ancestry = getAncestryAsArray(event, schema); + return ancestry.length > 0 && queriedNodes.has(ancestry[ancestry.length - 1]); + }; + + for (const result of results) { + const ancestry = getAncestryAsArray(result, schema); + // This is to handle the following unlikely but possible scenario: + // if an alert was generated by the kernel process (parent process of all other processes) then + // the direct children of that process would only have an ancestry array of [parent_kernel], a single value in the array. + // The children of those children would have two values in their array [direct parent, parent_kernel] + // we need to determine which nodes are the most distant grandchildren of the queriedNodes because those should + // be used for the next query if more nodes should be retrieved. To generally determine the most distant grandchildren + // we can use the last entry in the ancestry array because of its ordering. The problem with that is in the scenario above + // the direct children of parent_kernel will also meet that criteria even though they are not actually the most + // distant grandchildren. To get around that issue we'll bucket all the nodes by the size of their ancestry array + // and then only return the nodes in the largest bucket because those should be the most distant grandchildren + // from the queried nodes that were passed in. + if (ancestry.length > largestAncestryArray) { + largestAncestryArray = ancestry.length; + } + + // a grandchild must have an array of > 0 and have it's last parent be in the set of previously queried nodes + // this is one of the furthest descendants from the queried nodes + if (isDistantGrandchild(result)) { + let levelOfNodes = nodesToQueryNext.get(ancestry.length); + if (!levelOfNodes) { + levelOfNodes = new Set(); + nodesToQueryNext.set(ancestry.length, levelOfNodes); + } + const nodeID = getIDField(result, schema); + if (nodeID) { + levelOfNodes.add(nodeID); + } + } + } + const nextNodes = nodesToQueryNext.get(largestAncestryArray); + + return nextNodes !== undefined ? Array.from(nextNodes) : []; +} + +/** + * Retrieves the unique ID field from a document. + * + * Exposed for testing. + * @param obj the doc value fields retrieved from a document returned by Elasticsearch + * @param schema the schema used for identifying connections between documents + */ +export function getIDField(obj: FieldsObject, schema: Schema): NodeID | undefined { + const id: ECSField = obj[schema.id]; + return firstNonNullValue(id); +} + +/** + * Retrieves the name field from a document. + * + * Exposed for testing. + * @param obj the doc value fields retrieved from a document returned by Elasticsearch + * @param schema the schema used for identifying connections between documents + */ +export function getNameField(obj: FieldsObject, schema: Schema): string | undefined { + if (!schema.name) { + return undefined; + } + + const name: ECSField = obj[schema.name]; + return String(firstNonNullValue(name)); +} + +/** + * Retrieves the unique parent ID field from a document. + * + * Exposed for testing. + * @param obj the doc value fields retrieved from a document returned by Elasticsearch + * @param schema the schema used for identifying connections between documents + */ +export function getParentField(obj: FieldsObject, schema: Schema): NodeID | undefined { + const parent: ECSField = obj[schema.parent]; + return firstNonNullValue(parent); +} + +function getAncestryField(obj: FieldsObject, schema: Schema): NodeID[] | undefined { + if (!schema.ancestry) { + return undefined; + } + + const ancestry: ECSField = obj[schema.ancestry]; + if (!ancestry) { + return undefined; + } + + return values(ancestry); +} + +/** + * Retrieves the ancestry array field if it exists. If it doesn't exist or if it is empty it reverts to + * creating an array using the parent field. If the parent field doesn't exist, it returns + * an empty array. + * + * Exposed for testing. + * @param obj the doc value fields retrieved from a document returned by Elasticsearch + * @param schema the schema used for identifying connections between documents + */ +export function getAncestryAsArray(obj: FieldsObject, schema: Schema): NodeID[] { + const ancestry = getAncestryField(obj, schema); + if (!ancestry || ancestry.length <= 0) { + const parentField = getParentField(obj, schema); + return parentField !== undefined ? [parentField] : []; + } + return ancestry; +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/index.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/index.ts new file mode 100644 index 0000000000000..21a49e268310b --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/index.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/** + * Represents a time range filter + */ +export interface Timerange { + from: string; + to: string; +} + +/** + * An array of unique IDs to identify nodes within the resolver tree. + */ +export type NodeID = string | number; + +/** + * The fields to use to identify nodes within a resolver tree. + */ +export interface Schema { + /** + * the ancestry field should be set to a field that contains an order array representing + * the ancestors of a node. + */ + ancestry?: string; + /** + * id represents the field to use as the unique ID for a node. + */ + id: string; + /** + * field to use for the name of the node + */ + name?: string; + /** + * parent represents the field that is the edge between two nodes. + */ + parent: string; +} + +/** + * Returns the doc value fields filter to use in queries to limit the number of fields returned in the + * query response. + * + * See for more info: https://www.elastic.co/guide/en/elasticsearch/reference/current/search-fields.html#docvalue-fields + * + * @param schema is the node schema information describing how relationships are formed between nodes + * in the resolver graph. + */ +export function docValueFields(schema: Schema): Array<{ field: string }> { + const filter = [{ field: '@timestamp' }, { field: schema.id }, { field: schema.parent }]; + if (schema.ancestry) { + filter.push({ field: schema.ancestry }); + } + + if (schema.name) { + filter.push({ field: schema.name }); + } + return filter; +} diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/common.ts b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/common.ts index 2c59863099ae7..b4e98d7d4b95e 100644 --- a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/common.ts +++ b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/common.ts @@ -5,16 +5,24 @@ */ import _ from 'lodash'; import expect from '@kbn/expect'; +import { firstNonNullValue } from '../../../../plugins/security_solution/common/endpoint/models/ecs_safety_helpers'; +import { + NodeID, + Schema, +} from '../../../../plugins/security_solution/server/endpoint/routes/resolver/tree/utils'; import { SafeResolverChildNode, SafeResolverLifecycleNode, SafeResolverEvent, ResolverNodeStats, + ResolverNode, } from '../../../../plugins/security_solution/common/endpoint/types'; import { parentEntityIDSafeVersion, entityIDSafeVersion, eventIDSafeVersion, + timestampSafeVersion, + timestampAsDateSafeVersion, } from '../../../../plugins/security_solution/common/endpoint/models/event'; import { Event, @@ -24,6 +32,344 @@ import { categoryMapping, } from '../../../../plugins/security_solution/common/endpoint/generate_data'; +const createLevels = ({ + descendantsByParent, + levels, + currentNodes, + schema, +}: { + descendantsByParent: Map>; + levels: Array>; + currentNodes: Map | undefined; + schema: Schema; +}): Array> => { + if (!currentNodes || currentNodes.size === 0) { + return levels; + } + levels.push(currentNodes); + const nextLevel: Map = new Map(); + for (const node of currentNodes.values()) { + const id = getID(node, schema); + const children = descendantsByParent.get(id); + if (children) { + for (const child of children.values()) { + const childID = getID(child, schema); + nextLevel.set(childID, child); + } + } + } + return createLevels({ descendantsByParent, levels, currentNodes: nextLevel, schema }); +}; + +interface TreeExpectation { + origin: NodeID; + nodeExpectations: NodeExpectations; +} + +interface NodeExpectations { + ancestors?: number; + descendants?: number; + descendantLevels?: number; +} + +interface APITree { + // entries closer to the beginning of the array are more direct parents of the origin aka + // ancestors[0] = the origin's parent, ancestors[1] = the origin's grandparent + ancestors: ResolverNode[]; + // if no ancestors were retrieved then the origin will be undefined + origin: ResolverNode | undefined; + descendantLevels: Array>; + nodeExpectations: NodeExpectations; +} + +/** + * Represents a utility structure for making it easier to perform expect calls on the response + * from the /tree api. This can represent multiple trees, since the tree api can return multiple trees. + */ +export interface APIResponse { + nodesByID: Map; + trees: Map; + allNodes: ResolverNode[]; +} + +/** + * Gets the ID field from a resolver node. Throws an error if the ID doesn't exist. + * + * @param node a resolver node + * @param schema the schema that was used to retrieve this resolver node + */ +export const getID = (node: ResolverNode | undefined, schema: Schema): NodeID => { + const id = firstNonNullValue(node?.data[schema.id]); + if (!id) { + throw new Error(`Unable to find id ${schema.id} in node: ${JSON.stringify(node)}`); + } + return id; +}; + +const getParentInternal = (node: ResolverNode | undefined, schema: Schema): NodeID | undefined => { + if (node) { + return firstNonNullValue(node?.data[schema.parent]); + } + return undefined; +}; + +/** + * Gets the parent ID field from a resolver node. Throws an error if the ID doesn't exist. + * + * @param node a resolver node + * @param schema the schema that was used to retrieve this resolver node + */ +export const getParent = (node: ResolverNode | undefined, schema: Schema): NodeID => { + const parent = getParentInternal(node, schema); + if (!parent) { + throw new Error(`Unable to find parent ${schema.parent} in node: ${JSON.stringify(node)}`); + } + return parent; +}; + +/** + * Reformats the tree's response to make it easier to perform testing on the results. + * + * @param treeExpectations the node IDs used to retrieve the trees and the expected number of ancestors/descendants in the + * resulting trees + * @param nodes the response from the tree api + * @param schema the schema used when calling the tree api + */ +const createTreeFromResponse = ( + treeExpectations: TreeExpectation[], + nodes: ResolverNode[], + schema: Schema +) => { + const nodesByID = new Map(); + const nodesByParent = new Map>(); + + for (const node of nodes) { + const id = getID(node, schema); + const parent = getParentInternal(node, schema); + + nodesByID.set(id, node); + + if (parent) { + let groupedChildren = nodesByParent.get(parent); + if (!groupedChildren) { + groupedChildren = new Map(); + nodesByParent.set(parent, groupedChildren); + } + + groupedChildren.set(id, node); + } + } + + const trees: Map = new Map(); + + for (const expectation of treeExpectations) { + const descendantLevels = createLevels({ + descendantsByParent: nodesByParent, + levels: [], + currentNodes: nodesByParent.get(expectation.origin), + schema, + }); + + const ancestors: ResolverNode[] = []; + const originNode = nodesByID.get(expectation.origin); + if (originNode) { + let currentID: NodeID | undefined = getParentInternal(originNode, schema); + // construct an array with all the ancestors from the response. We'll use this to verify that + // all the expected ancestors were returned in the response. + while (currentID !== undefined) { + const parentNode = nodesByID.get(currentID); + if (parentNode) { + ancestors.push(parentNode); + } + currentID = getParentInternal(parentNode, schema); + } + } + + trees.set(expectation.origin, { + ancestors, + origin: originNode, + descendantLevels, + nodeExpectations: expectation.nodeExpectations, + }); + } + + return { + nodesByID, + trees, + allNodes: nodes, + }; +}; + +const verifyAncestry = ({ + responseTrees, + schema, + genTree, +}: { + responseTrees: APIResponse; + schema: Schema; + genTree: Tree; +}) => { + const allGenNodes = new Map([ + ...genTree.ancestry, + ...genTree.children, + [genTree.origin.id, genTree.origin], + ]); + + for (const tree of responseTrees.trees.values()) { + if (tree.nodeExpectations.ancestors !== undefined) { + expect(tree.ancestors.length).to.be(tree.nodeExpectations.ancestors); + } + + if (tree.origin !== undefined) { + // make sure the origin node from the request exists in the generated data and has the same fields + const originID = getID(tree.origin, schema); + const originParentID = getParent(tree.origin, schema); + expect(tree.origin.id).to.be(originID); + expect(tree.origin.parent).to.be(originParentID); + expect(allGenNodes.get(String(originID))?.id).to.be(String(originID)); + expect(allGenNodes.get(String(originParentID))?.id).to.be(String(originParentID)); + expect(originID).to.be(entityIDSafeVersion(allGenNodes.get(String(originID))!.lifecycle[0])); + expect(originParentID).to.be( + parentEntityIDSafeVersion(allGenNodes.get(String(originID))!.lifecycle[0]) + ); + // make sure the lifecycle events are sorted by timestamp in ascending order because the + // event that will be returned that we need to compare to should be the earliest event + // found + const originLifecycleSorted = [...allGenNodes.get(String(originID))!.lifecycle].sort( + (a: Event, b: Event) => { + const aTime: number | undefined = timestampSafeVersion(a); + const bTime = timestampSafeVersion(b); + if (aTime !== undefined && bTime !== undefined) { + return aTime - bTime; + } else { + return 0; + } + } + ); + + const ts = timestampAsDateSafeVersion(tree.origin?.data); + expect(ts).to.not.be(undefined); + expect(ts).to.eql(timestampAsDateSafeVersion(originLifecycleSorted[0])); + } + + // check the constructed ancestors array to see if we're missing any nodes in the ancestry + for (let i = 0; i < tree.ancestors.length; i++) { + const id = getID(tree.ancestors[i], schema); + const parent = getParentInternal(tree.ancestors[i], schema); + // only compare to the parent if this is not the last entry in the array + if (i < tree.ancestors.length - 1) { + // the current node's parent ID should match the parent's ID field + expect(parent).to.be(getID(tree.ancestors[i + 1], schema)); + expect(parent).to.not.be(undefined); + expect(tree.ancestors[i].parent).to.not.be(undefined); + expect(tree.ancestors[i].parent).to.be(parent); + } + // the current node's ID must exist in the generated tree + expect(allGenNodes.get(String(id))?.id).to.be(id); + expect(tree.ancestors[i].id).to.be(id); + } + } +}; + +const verifyChildren = ({ + responseTrees, + schema, + genTree, +}: { + responseTrees: APIResponse; + schema: Schema; + genTree: Tree; +}) => { + const allGenNodes = new Map([ + ...genTree.ancestry, + ...genTree.children, + [genTree.origin.id, genTree.origin], + ]); + for (const tree of responseTrees.trees.values()) { + if (tree.nodeExpectations.descendantLevels !== undefined) { + expect(tree.nodeExpectations.descendantLevels).to.be(tree.descendantLevels.length); + } + let totalDescendants = 0; + + for (const level of tree.descendantLevels) { + for (const node of level.values()) { + totalDescendants += 1; + const id = getID(node, schema); + const parent = getParent(node, schema); + const genNode = allGenNodes.get(String(id)); + expect(id).to.be(node.id); + expect(parent).to.be(node.parent); + expect(node.parent).to.not.be(undefined); + // make sure the id field is the same in the returned node as the generated one + expect(id).to.be(entityIDSafeVersion(genNode!.lifecycle[0])); + // make sure the parent field is the same in the returned node as the generated one + expect(parent).to.be(parentEntityIDSafeVersion(genNode!.lifecycle[0])); + } + } + if (tree.nodeExpectations.descendants !== undefined) { + expect(tree.nodeExpectations.descendants).to.be(totalDescendants); + } + } +}; + +const verifyStats = ({ + responseTrees, + relatedEventsCategories, +}: { + responseTrees: APIResponse; + relatedEventsCategories: RelatedEventInfo[]; +}) => { + for (const node of responseTrees.allNodes) { + let totalExpEvents = 0; + for (const cat of relatedEventsCategories) { + const ecsCategories = categoryMapping[cat.category]; + if (Array.isArray(ecsCategories)) { + // if there are multiple ecs categories used to define a related event, the count for all of them should be the same + // and they should equal what is defined in the categories used to generate the related events + for (const ecsCat of ecsCategories) { + expect(node.stats.byCategory[ecsCat]).to.be(cat.count); + } + } else { + expect(node.stats.byCategory[ecsCategories]).to.be(cat.count); + } + + totalExpEvents += cat.count; + } + expect(node.stats.total).to.be(totalExpEvents); + } +}; + +/** + * Verify the ancestry of multiple trees. + * + * @param expectations array of expectations based on the origin that built a particular tree + * @param response the nodes returned from the api + * @param schema the schema fields passed to the tree api + * @param genTree the generated tree that was inserted in Elasticsearch that we are querying + * @param relatedEventsCategories an optional array to instruct the verification to check the stats + * on each node returned + */ +export const verifyTree = ({ + expectations, + response, + schema, + genTree, + relatedEventsCategories, +}: { + expectations: TreeExpectation[]; + response: ResolverNode[]; + schema: Schema; + genTree: Tree; + relatedEventsCategories?: RelatedEventInfo[]; +}) => { + const responseTrees = createTreeFromResponse(expectations, response, schema); + verifyAncestry({ responseTrees, schema, genTree }); + verifyChildren({ responseTrees, schema, genTree }); + if (relatedEventsCategories !== undefined) { + verifyStats({ responseTrees, relatedEventsCategories }); + } +}; + /** * Creates the ancestry array based on an array of events. The order of the ancestry array will match the order * of the events passed in. @@ -44,6 +390,7 @@ export const createAncestryArray = (events: Event[]) => { /** * Check that the given lifecycle is in the resolver tree's corresponding map * + * @deprecated use verifyTree * @param node a lifecycle node containing the start and end events for a node * @param nodeMap a map of entity_ids to nodes to look for the passed in `node` */ @@ -59,12 +406,13 @@ const expectLifecycleNodeInMap = ( /** * Verify that all the ancestor nodes are valid and optionally have parents. * + * @deprecated use verifyTree * @param ancestors an array of ancestors * @param tree the generated resolver tree as the source of truth * @param verifyLastParent a boolean indicating whether to check the last ancestor. If the ancestors array intentionally * does not contain all the ancestors, the last one will not have the parent */ -export const verifyAncestry = ( +export const checkAncestryFromEntityTreeAPI = ( ancestors: SafeResolverLifecycleNode[], tree: Tree, verifyLastParent: boolean @@ -114,6 +462,7 @@ export const verifyAncestry = ( /** * Retrieves the most distant ancestor in the given array. * + * @deprecated use verifyTree * @param ancestors an array of ancestor nodes */ export const retrieveDistantAncestor = (ancestors: SafeResolverLifecycleNode[]) => { @@ -137,12 +486,13 @@ export const retrieveDistantAncestor = (ancestors: SafeResolverLifecycleNode[]) /** * Verify that the children nodes are correct * + * @deprecated use verifyTree * @param children the children nodes * @param tree the generated resolver tree as the source of truth * @param numberOfParents an optional number to compare that are a certain number of parents in the children array * @param childrenPerParent an optional number to compare that there are a certain number of children for each parent */ -export const verifyChildren = ( +export const verifyChildrenFromEntityTreeAPI = ( children: SafeResolverChildNode[], tree: Tree, numberOfParents?: number, @@ -200,10 +550,11 @@ export const compareArrays = ( /** * Verifies that the stats received from ES for a node reflect the categories of events that the generator created. * + * @deprecated use verifyTree * @param relatedEvents the related events received for a particular node * @param categories the related event info used when generating the resolver tree */ -export const verifyStats = ( +export const verifyEntityTreeStats = ( stats: ResolverNodeStats | undefined, categories: RelatedEventInfo[], relatedAlerts: number @@ -225,12 +576,12 @@ export const verifyStats = ( totalExpEvents += cat.count; } expect(stats?.events.total).to.be(totalExpEvents); - expect(stats?.totalAlerts); }; /** * A helper function for verifying the stats information an array of nodes. * + * @deprecated use verifyTree * @param nodes an array of lifecycle nodes that should have a stats field defined * @param categories the related event info used when generating the resolver tree */ @@ -240,6 +591,6 @@ export const verifyLifecycleStats = ( relatedAlerts: number ) => { for (const node of nodes) { - verifyStats(node.stats, categories, relatedAlerts); + verifyEntityTreeStats(node.stats, categories, relatedAlerts); } }; diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/index.ts b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/index.ts index ecfc1ef5bb7f5..0ba5460f09d9d 100644 --- a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/index.ts +++ b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/index.ts @@ -12,6 +12,7 @@ export default function (providerContext: FtrProviderContext) { loadTestFile(require.resolve('./entity_id')); loadTestFile(require.resolve('./entity')); loadTestFile(require.resolve('./children')); + loadTestFile(require.resolve('./tree_entity_id')); loadTestFile(require.resolve('./tree')); loadTestFile(require.resolve('./alerts')); loadTestFile(require.resolve('./events')); diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/tree.ts b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/tree.ts index 7a95bf7bab883..646a666629ac9 100644 --- a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/tree.ts +++ b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/tree.ts @@ -4,31 +4,23 @@ * you may not use this file except in compliance with the Elastic License. */ import expect from '@kbn/expect'; +import { getNameField } from '../../../../plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch'; +import { Schema } from '../../../../plugins/security_solution/server/endpoint/routes/resolver/tree/utils'; +import { ResolverNode } from '../../../../plugins/security_solution/common/endpoint/types'; import { - SafeResolverAncestry, - SafeResolverChildren, - SafeResolverTree, - SafeLegacyEndpointEvent, -} from '../../../../plugins/security_solution/common/endpoint/types'; -import { parentEntityIDSafeVersion } from '../../../../plugins/security_solution/common/endpoint/models/event'; + parentEntityIDSafeVersion, + timestampSafeVersion, +} from '../../../../plugins/security_solution/common/endpoint/models/event'; import { FtrProviderContext } from '../../ftr_provider_context'; import { Tree, RelatedEventCategory, } from '../../../../plugins/security_solution/common/endpoint/generate_data'; import { Options, GeneratedTrees } from '../../services/resolver'; -import { - compareArrays, - verifyAncestry, - retrieveDistantAncestor, - verifyChildren, - verifyLifecycleStats, - verifyStats, -} from './common'; +import { verifyTree } from './common'; export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); - const esArchiver = getService('esArchiver'); const resolver = getService('resolverGenerator'); const relatedEventsToGen = [ @@ -52,322 +44,641 @@ export default function ({ getService }: FtrProviderContext) { ancestryArraySize: 2, }; + const schemaWithAncestry: Schema = { + ancestry: 'process.Ext.ancestry', + id: 'process.entity_id', + parent: 'process.parent.entity_id', + }; + + const schemaWithoutAncestry: Schema = { + id: 'process.entity_id', + parent: 'process.parent.entity_id', + }; + + const schemaWithName: Schema = { + id: 'process.entity_id', + parent: 'process.parent.entity_id', + name: 'process.name', + }; + describe('Resolver tree', () => { before(async () => { - await esArchiver.load('endpoint/resolver/api_feature'); resolverTrees = await resolver.createTrees(treeOptions); // we only requested a single alert so there's only 1 tree tree = resolverTrees.trees[0]; }); after(async () => { await resolver.deleteData(resolverTrees); - // this unload is for an endgame-* index so it does not use data streams - await esArchiver.unload('endpoint/resolver/api_feature'); }); - describe('ancestry events route', () => { - describe('legacy events', () => { - const endpointID = '5a0c957f-b8e7-4538-965e-57e8bb86ad3a'; - const entityID = '94042'; - - it('should return details for the root node', async () => { - const { body }: { body: SafeResolverAncestry } = await supertest - .get( - `/api/endpoint/resolver/${entityID}/ancestry?legacyEndpointID=${endpointID}&ancestors=5` - ) - .expect(200); - expect(body.ancestors[0].lifecycle.length).to.eql(2); - expect(body.ancestors.length).to.eql(2); - expect(body.nextAncestor).to.eql(null); - }); - - it('should have a populated next parameter', async () => { - const { body }: { body: SafeResolverAncestry } = await supertest - .get( - `/api/endpoint/resolver/${entityID}/ancestry?legacyEndpointID=${endpointID}&ancestors=0` - ) - .expect(200); - expect(body.nextAncestor).to.eql('94041'); + describe('ancestry events', () => { + it('should return the correct ancestor nodes for the tree', async () => { + const { body }: { body: ResolverNode[] } = await supertest + .post('/api/endpoint/resolver/tree') + .set('kbn-xsrf', 'xxx') + .send({ + descendants: 0, + descendantLevels: 0, + ancestors: 9, + schema: schemaWithAncestry, + nodes: [tree.origin.id], + timerange: { + from: tree.startTime.toISOString(), + to: tree.endTime.toISOString(), + }, + indexPatterns: ['logs-*'], + }) + .expect(200); + verifyTree({ + expectations: [{ origin: tree.origin.id, nodeExpectations: { ancestors: 5 } }], + response: body, + schema: schemaWithAncestry, + genTree: tree, }); + }); - it('should handle an ancestors param request', async () => { - let { body }: { body: SafeResolverAncestry } = await supertest - .get( - `/api/endpoint/resolver/${entityID}/ancestry?legacyEndpointID=${endpointID}&ancestors=0` - ) - .expect(200); - const next = body.nextAncestor; + it('should handle an invalid id', async () => { + const { body }: { body: ResolverNode[] } = await supertest + .post('/api/endpoint/resolver/tree') + .set('kbn-xsrf', 'xxx') + .send({ + descendants: 0, + descendantLevels: 0, + ancestors: 9, + schema: schemaWithAncestry, + nodes: ['bogus id'], + timerange: { + from: tree.startTime.toISOString(), + to: tree.endTime.toISOString(), + }, + indexPatterns: ['logs-*'], + }) + .expect(200); + expect(body).to.be.empty(); + }); - ({ body } = await supertest - .get( - `/api/endpoint/resolver/${next}/ancestry?legacyEndpointID=${endpointID}&ancestors=1` - ) - .expect(200)); - expect(body.ancestors[0].lifecycle.length).to.eql(1); - expect(body.nextAncestor).to.eql(null); + it('should return a subset of the ancestors', async () => { + const { body }: { body: ResolverNode[] } = await supertest + .post('/api/endpoint/resolver/tree') + .set('kbn-xsrf', 'xxx') + .send({ + descendants: 0, + descendantLevels: 0, + // 3 ancestors means 1 origin and 2 ancestors of the origin + ancestors: 3, + schema: schemaWithAncestry, + nodes: [tree.origin.id], + timerange: { + from: tree.startTime.toISOString(), + to: tree.endTime.toISOString(), + }, + indexPatterns: ['logs-*'], + }) + .expect(200); + verifyTree({ + expectations: [{ origin: tree.origin.id, nodeExpectations: { ancestors: 2 } }], + response: body, + schema: schemaWithAncestry, + genTree: tree, }); }); - describe('endpoint events', () => { - it('should return the origin node at the front of the array', async () => { - const { body }: { body: SafeResolverAncestry } = await supertest - .get(`/api/endpoint/resolver/${tree.origin.id}/ancestry?ancestors=9`) - .expect(200); - expect(body.ancestors[0].entityID).to.eql(tree.origin.id); + it('should return ancestors without the ancestry array', async () => { + const { body }: { body: ResolverNode[] } = await supertest + .post('/api/endpoint/resolver/tree') + .set('kbn-xsrf', 'xxx') + .send({ + descendants: 0, + descendantLevels: 0, + ancestors: 50, + schema: schemaWithoutAncestry, + nodes: [tree.origin.id], + timerange: { + from: tree.startTime.toISOString(), + to: tree.endTime.toISOString(), + }, + indexPatterns: ['logs-*'], + }) + .expect(200); + verifyTree({ + expectations: [{ origin: tree.origin.id, nodeExpectations: { ancestors: 5 } }], + response: body, + schema: schemaWithoutAncestry, + genTree: tree, }); + }); - it('should return details for the root node', async () => { - const { body }: { body: SafeResolverAncestry } = await supertest - .get(`/api/endpoint/resolver/${tree.origin.id}/ancestry?ancestors=9`) - .expect(200); - // the tree we generated had 5 ancestors + 1 origin node - expect(body.ancestors.length).to.eql(6); - expect(body.ancestors[0].entityID).to.eql(tree.origin.id); - verifyAncestry(body.ancestors, tree, true); - expect(body.nextAncestor).to.eql(null); + it('should respect the time range specified and only return the origin node', async () => { + const from = new Date( + timestampSafeVersion(tree.origin.lifecycle[0]) ?? new Date() + ).toISOString(); + const { body }: { body: ResolverNode[] } = await supertest + .post('/api/endpoint/resolver/tree') + .set('kbn-xsrf', 'xxx') + .send({ + descendants: 0, + descendantLevels: 0, + ancestors: 50, + schema: schemaWithoutAncestry, + nodes: [tree.origin.id], + timerange: { + from, + to: from, + }, + indexPatterns: ['logs-*'], + }) + .expect(200); + verifyTree({ + expectations: [{ origin: tree.origin.id, nodeExpectations: { ancestors: 0 } }], + response: body, + schema: schemaWithoutAncestry, + genTree: tree, }); + }); - it('should handle an invalid id', async () => { - const { body }: { body: SafeResolverAncestry } = await supertest - .get(`/api/endpoint/resolver/alskdjflasj/ancestry`) - .expect(200); - expect(body.ancestors).to.be.empty(); - expect(body.nextAncestor).to.eql(null); + it('should support returning multiple ancestor trees when multiple nodes are requested', async () => { + // There should be 2 levels of descendants under the origin, grab the bottom one, and the first node's id + const bottomMostDescendant = Array.from(tree.childrenLevels[1].values())[0].id; + const { body }: { body: ResolverNode[] } = await supertest + .post('/api/endpoint/resolver/tree') + .set('kbn-xsrf', 'xxx') + .send({ + descendants: 0, + descendantLevels: 0, + ancestors: 50, + schema: schemaWithoutAncestry, + nodes: [tree.origin.id, bottomMostDescendant], + timerange: { + from: tree.startTime.toISOString(), + to: tree.endTime.toISOString(), + }, + indexPatterns: ['logs-*'], + }) + .expect(200); + verifyTree({ + expectations: [ + // there are 5 ancestors above the origin + { origin: tree.origin.id, nodeExpectations: { ancestors: 5 } }, + // there are 2 levels below the origin so the bottom node's ancestry should be + // all the ancestors (5) + one level + the origin = 7 + { origin: bottomMostDescendant, nodeExpectations: { ancestors: 7 } }, + ], + response: body, + schema: schemaWithoutAncestry, + genTree: tree, }); + }); - it('should have a populated next parameter', async () => { - const { body }: { body: SafeResolverAncestry } = await supertest - .get(`/api/endpoint/resolver/${tree.origin.id}/ancestry?ancestors=2`) - .expect(200); - // it should have 2 ancestors + 1 origin - expect(body.ancestors.length).to.eql(3); - verifyAncestry(body.ancestors, tree, false); - const distantGrandparent = retrieveDistantAncestor(body.ancestors); - expect(body.nextAncestor).to.eql( - parentEntityIDSafeVersion(distantGrandparent.lifecycle[0]) - ); + it('should return a single ancestry when two nodes a the same level and from same parent are requested', async () => { + // there are 2 levels after the origin, let's get the first level, there will be three + // children so get the left and right most ones + const level0Nodes = Array.from(tree.childrenLevels[0].values()); + const leftNode = level0Nodes[0].id; + const rightNode = level0Nodes[2].id; + const { body }: { body: ResolverNode[] } = await supertest + .post('/api/endpoint/resolver/tree') + .set('kbn-xsrf', 'xxx') + .send({ + descendants: 0, + descendantLevels: 0, + ancestors: 50, + schema: schemaWithoutAncestry, + nodes: [leftNode, rightNode], + timerange: { + from: tree.startTime.toISOString(), + to: tree.endTime.toISOString(), + }, + indexPatterns: ['logs-*'], + }) + .expect(200); + verifyTree({ + expectations: [ + // We should be 1 level below the origin so the node's ancestry should be + // all the ancestors (5) + the origin = 6 + { origin: leftNode, nodeExpectations: { ancestors: 6 } }, + // these nodes should be at the same level so the ancestors should be the same number + { origin: rightNode, nodeExpectations: { ancestors: 6 } }, + ], + response: body, + schema: schemaWithoutAncestry, + genTree: tree, }); + }); - it('should handle multiple ancestor requests', async () => { - let { body }: { body: SafeResolverAncestry } = await supertest - .get(`/api/endpoint/resolver/${tree.origin.id}/ancestry?ancestors=3`) - .expect(200); - expect(body.ancestors.length).to.eql(4); - const next = body.nextAncestor; - - ({ body } = await supertest - .get(`/api/endpoint/resolver/${next}/ancestry?ancestors=1`) - .expect(200)); - expect(body.ancestors.length).to.eql(2); - verifyAncestry(body.ancestors, tree, true); - // the highest node in the generated tree will not have a parent ID which causes the server to return - // without setting the pagination so nextAncestor will be null - expect(body.nextAncestor).to.eql(null); - }); + it('should not return any nodes when the search index does not have any data', async () => { + const { body }: { body: ResolverNode[] } = await supertest + .post('/api/endpoint/resolver/tree') + .set('kbn-xsrf', 'xxx') + .send({ + descendants: 0, + descendantLevels: 0, + ancestors: 50, + schema: schemaWithoutAncestry, + nodes: [tree.origin.id], + timerange: { + from: tree.startTime.toISOString(), + to: tree.endTime.toISOString(), + }, + indexPatterns: ['metrics-*'], + }) + .expect(200); + expect(body).to.be.empty(); }); }); - describe('children route', () => { - describe('legacy events', () => { - const endpointID = '5a0c957f-b8e7-4538-965e-57e8bb86ad3a'; - const entityID = '94041'; - - it('returns child process lifecycle events', async () => { - const { body }: { body: SafeResolverChildren } = await supertest - .get(`/api/endpoint/resolver/${entityID}/children?legacyEndpointID=${endpointID}`) - .expect(200); - expect(body.childNodes.length).to.eql(1); - expect(body.childNodes[0].lifecycle.length).to.eql(2); - expect( - // for some reason the ts server doesn't think `endgame` exists even though we're using ResolverEvent - // here, so to avoid it complaining we'll just force it - (body.childNodes[0].lifecycle[0] as SafeLegacyEndpointEvent).endgame.unique_pid - ).to.eql(94042); + describe('descendant events', () => { + it('returns all descendants for the origin without using the ancestry field', async () => { + const { body }: { body: ResolverNode[] } = await supertest + .post('/api/endpoint/resolver/tree') + .set('kbn-xsrf', 'xxx') + .send({ + descendants: 100, + descendantLevels: 2, + ancestors: 0, + schema: schemaWithoutAncestry, + nodes: [tree.origin.id], + timerange: { + from: tree.startTime.toISOString(), + to: tree.endTime.toISOString(), + }, + indexPatterns: ['logs-*'], + }) + .expect(200); + verifyTree({ + expectations: [ + // there are 2 levels in the descendant part of the tree and 3 nodes for each + // descendant = 3 children for the origin + 3 children for each of the origin's children = 12 + { origin: tree.origin.id, nodeExpectations: { descendants: 12, descendantLevels: 2 } }, + ], + response: body, + schema: schemaWithoutAncestry, + genTree: tree, }); + }); - it('returns multiple levels of child process lifecycle events', async () => { - const { body }: { body: SafeResolverChildren } = await supertest - .get(`/api/endpoint/resolver/93802/children?legacyEndpointID=${endpointID}&children=10`) - .expect(200); - expect(body.childNodes.length).to.eql(10); - expect(body.nextChild).to.be(null); - expect(body.childNodes[0].lifecycle.length).to.eql(1); - expect( - // for some reason the ts server doesn't think `endgame` exists even though we're using ResolverEvent - // here, so to avoid it complaining we'll just force it - (body.childNodes[0].lifecycle[0] as SafeLegacyEndpointEvent).endgame.unique_pid - ).to.eql(93932); + it('returns all descendants for the origin using the ancestry field', async () => { + const { body }: { body: ResolverNode[] } = await supertest + .post('/api/endpoint/resolver/tree') + .set('kbn-xsrf', 'xxx') + .send({ + descendants: 100, + // should be ignored when using the ancestry array + descendantLevels: 0, + ancestors: 0, + schema: schemaWithAncestry, + nodes: [tree.origin.id], + timerange: { + from: tree.startTime.toISOString(), + to: tree.endTime.toISOString(), + }, + indexPatterns: ['logs-*'], + }) + .expect(200); + verifyTree({ + expectations: [ + // there are 2 levels in the descendant part of the tree and 3 nodes for each + // descendant = 3 children for the origin + 3 children for each of the origin's children = 12 + { origin: tree.origin.id, nodeExpectations: { descendants: 12, descendantLevels: 2 } }, + ], + response: body, + schema: schemaWithAncestry, + genTree: tree, }); + }); - it('returns no values when there is no more data', async () => { - let { body }: { body: SafeResolverChildren } = await supertest - .get( - // there should only be a single child for this node - `/api/endpoint/resolver/94041/children?legacyEndpointID=${endpointID}&children=1` - ) - .expect(200); - expect(body.nextChild).to.not.be(null); - - ({ body } = await supertest - .get( - `/api/endpoint/resolver/94041/children?legacyEndpointID=${endpointID}&afterChild=${body.nextChild}` - ) - .expect(200)); - expect(body.childNodes).be.empty(); - expect(body.nextChild).to.eql(null); - }); + it('should handle an invalid id', async () => { + const { body }: { body: ResolverNode[] } = await supertest + .post('/api/endpoint/resolver/tree') + .set('kbn-xsrf', 'xxx') + .send({ + descendants: 100, + descendantLevels: 100, + ancestors: 0, + schema: schemaWithAncestry, + nodes: ['bogus id'], + timerange: { + from: tree.startTime.toISOString(), + to: tree.endTime.toISOString(), + }, + indexPatterns: ['logs-*'], + }) + .expect(200); + expect(body).to.be.empty(); + }); - it('returns the first page of information when the cursor is invalid', async () => { - const { body }: { body: SafeResolverChildren } = await supertest - .get( - `/api/endpoint/resolver/${entityID}/children?legacyEndpointID=${endpointID}&afterChild=blah` - ) - .expect(200); - expect(body.childNodes.length).to.eql(1); - expect(body.nextChild).to.be(null); + it('returns a single generation of children', async () => { + // this gets a node should have 3 children which were created in succession so that the timestamps + // are ordered correctly to be retrieved in a single call + const childID = Array.from(tree.childrenLevels[0].values())[0].id; + const { body }: { body: ResolverNode[] } = await supertest + .post('/api/endpoint/resolver/tree') + .set('kbn-xsrf', 'xxx') + .send({ + descendants: 100, + descendantLevels: 1, + ancestors: 0, + schema: schemaWithoutAncestry, + nodes: [childID], + timerange: { + from: tree.startTime.toISOString(), + to: tree.endTime.toISOString(), + }, + indexPatterns: ['logs-*'], + }) + .expect(200); + verifyTree({ + expectations: [ + // a single generation should be three nodes + { origin: childID, nodeExpectations: { descendants: 3, descendantLevels: 1 } }, + ], + response: body, + schema: schemaWithoutAncestry, + genTree: tree, }); + }); - it('errors on invalid pagination values', async () => { - await supertest.get(`/api/endpoint/resolver/${entityID}/children?children=0`).expect(400); - await supertest - .get(`/api/endpoint/resolver/${entityID}/children?children=20000`) - .expect(400); - await supertest - .get(`/api/endpoint/resolver/${entityID}/children?children=-1`) - .expect(400); + it('should support returning multiple descendant trees when multiple nodes are requested', async () => { + // there are 2 levels after the origin, let's get the first level, there will be three + // children so get the left and right most ones + const level0Nodes = Array.from(tree.childrenLevels[0].values()); + const leftNodeID = level0Nodes[0].id; + const rightNodeID = level0Nodes[2].id; + const { body }: { body: ResolverNode[] } = await supertest + .post('/api/endpoint/resolver/tree') + .set('kbn-xsrf', 'xxx') + .send({ + descendants: 6, + descendantLevels: 0, + ancestors: 0, + schema: schemaWithAncestry, + nodes: [leftNodeID, rightNodeID], + timerange: { + from: tree.startTime.toISOString(), + to: tree.endTime.toISOString(), + }, + indexPatterns: ['logs-*'], + }) + .expect(200); + verifyTree({ + expectations: [ + { origin: leftNodeID, nodeExpectations: { descendantLevels: 1, descendants: 3 } }, + { origin: rightNodeID, nodeExpectations: { descendantLevels: 1, descendants: 3 } }, + ], + response: body, + schema: schemaWithAncestry, + genTree: tree, }); + }); - it('returns empty events without a matching entity id', async () => { - const { body }: { body: SafeResolverChildren } = await supertest - .get(`/api/endpoint/resolver/5555/children`) - .expect(200); - expect(body.nextChild).to.eql(null); - expect(body.childNodes).to.be.empty(); + it('should support returning multiple descendant trees when multiple nodes are requested at different levels', async () => { + const originParent = parentEntityIDSafeVersion(tree.origin.lifecycle[0]) ?? ''; + expect(originParent).to.not.be(''); + const originGrandparent = + parentEntityIDSafeVersion(tree.ancestry.get(originParent)!.lifecycle[0]) ?? ''; + expect(originGrandparent).to.not.be(''); + const { body }: { body: ResolverNode[] } = await supertest + .post('/api/endpoint/resolver/tree') + .set('kbn-xsrf', 'xxx') + .send({ + descendants: 2, + descendantLevels: 0, + ancestors: 0, + schema: schemaWithAncestry, + nodes: [tree.origin.id, originGrandparent], + timerange: { + from: tree.startTime.toISOString(), + to: tree.endTime.toISOString(), + }, + indexPatterns: ['logs-*'], + }) + .expect(200); + verifyTree({ + expectations: [ + { origin: tree.origin.id, nodeExpectations: { descendantLevels: 1, descendants: 1 } }, + // the origin's grandparent should only have the origin's parent as a descendant + { + origin: originGrandparent, + nodeExpectations: { descendantLevels: 1, descendants: 1 }, + }, + ], + response: body, + schema: schemaWithAncestry, + genTree: tree, }); + }); - it('returns empty events with an invalid endpoint id', async () => { - const { body }: { body: SafeResolverChildren } = await supertest - .get(`/api/endpoint/resolver/${entityID}/children?legacyEndpointID=foo`) - .expect(200); - expect(body.nextChild).to.eql(null); - expect(body.childNodes).to.be.empty(); + it('should support returning multiple descendant trees when multiple nodes are requested at different levels without ancestry field', async () => { + const originParent = parentEntityIDSafeVersion(tree.origin.lifecycle[0]) ?? ''; + expect(originParent).to.not.be(''); + const originGrandparent = + parentEntityIDSafeVersion(tree.ancestry.get(originParent)!.lifecycle[0]) ?? ''; + expect(originGrandparent).to.not.be(''); + const { body }: { body: ResolverNode[] } = await supertest + .post('/api/endpoint/resolver/tree') + .set('kbn-xsrf', 'xxx') + .send({ + descendants: 6, + descendantLevels: 1, + ancestors: 0, + schema: schemaWithoutAncestry, + nodes: [tree.origin.id, originGrandparent], + timerange: { + from: tree.startTime.toISOString(), + to: tree.endTime.toISOString(), + }, + indexPatterns: ['logs-*'], + }) + .expect(200); + verifyTree({ + expectations: [ + { origin: tree.origin.id, nodeExpectations: { descendantLevels: 1, descendants: 3 } }, + // the origin's grandparent should only have the origin's parent as a descendant + { + origin: originGrandparent, + nodeExpectations: { descendantLevels: 1, descendants: 1 }, + }, + ], + response: body, + schema: schemaWithoutAncestry, + genTree: tree, }); }); - describe('endpoint events', () => { - it('returns all children for the origin', async () => { - const { body }: { body: SafeResolverChildren } = await supertest - .get(`/api/endpoint/resolver/${tree.origin.id}/children?children=100`) - .expect(200); - // there are 2 levels in the children part of the tree and 3 nodes for each = - // 3 children for the origin + 3 children for each of the origin's children = 12 - expect(body.childNodes.length).to.eql(12); - // there will be 4 parents, the origin of the tree, and it's 3 children - verifyChildren(body.childNodes, tree, 4, 3); - expect(body.nextChild).to.eql(null); + it('should respect the time range specified and only return one descendant', async () => { + const level0Node = Array.from(tree.childrenLevels[0].values())[0]; + const end = new Date( + timestampSafeVersion(level0Node.lifecycle[0]) ?? new Date() + ).toISOString(); + const { body }: { body: ResolverNode[] } = await supertest + .post('/api/endpoint/resolver/tree') + .set('kbn-xsrf', 'xxx') + .send({ + descendants: 100, + descendantLevels: 5, + ancestors: 0, + schema: schemaWithoutAncestry, + nodes: [tree.origin.id], + timerange: { + from: tree.startTime.toISOString(), + to: end, + }, + indexPatterns: ['logs-*'], + }) + .expect(200); + verifyTree({ + expectations: [ + { origin: tree.origin.id, nodeExpectations: { descendantLevels: 1, descendants: 1 } }, + ], + response: body, + schema: schemaWithoutAncestry, + genTree: tree, }); + }); + }); - it('returns a single generation of children', async () => { - // this gets a node should have 3 children which were created in succession so that the timestamps - // are ordered correctly to be retrieved in a single call - const distantChildEntityID = Array.from(tree.childrenLevels[0].values())[0].id; - const { body }: { body: SafeResolverChildren } = await supertest - .get(`/api/endpoint/resolver/${distantChildEntityID}/children?children=3`) - .expect(200); - expect(body.childNodes.length).to.eql(3); - verifyChildren(body.childNodes, tree, 1, 3); - expect(body.nextChild).to.not.eql(null); + describe('ancestry and descendants', () => { + it('returns all descendants and ancestors without the ancestry field and they should have the name field', async () => { + const { body }: { body: ResolverNode[] } = await supertest + .post('/api/endpoint/resolver/tree') + .set('kbn-xsrf', 'xxx') + .send({ + descendants: 100, + descendantLevels: 10, + ancestors: 50, + schema: schemaWithName, + nodes: [tree.origin.id], + timerange: { + from: tree.startTime.toISOString(), + to: tree.endTime.toISOString(), + }, + indexPatterns: ['logs-*'], + }) + .expect(200); + verifyTree({ + expectations: [ + // there are 2 levels in the descendant part of the tree and 3 nodes for each + // descendant = 3 children for the origin + 3 children for each of the origin's children = 12 + { + origin: tree.origin.id, + nodeExpectations: { descendants: 12, descendantLevels: 2, ancestors: 5 }, + }, + ], + response: body, + schema: schemaWithName, + genTree: tree, + relatedEventsCategories: relatedEventsToGen, }); - it('paginates the children', async () => { - // this gets a node should have 3 children which were created in succession so that the timestamps - // are ordered correctly to be retrieved in a single call - const distantChildEntityID = Array.from(tree.childrenLevels[0].values())[0].id; - let { body }: { body: SafeResolverChildren } = await supertest - .get(`/api/endpoint/resolver/${distantChildEntityID}/children?children=1`) - .expect(200); - expect(body.childNodes.length).to.eql(1); - verifyChildren(body.childNodes, tree, 1, 1); - expect(body.nextChild).to.not.be(null); - - ({ body } = await supertest - .get( - `/api/endpoint/resolver/${distantChildEntityID}/children?children=2&afterChild=${body.nextChild}` - ) - .expect(200)); - expect(body.childNodes.length).to.eql(2); - verifyChildren(body.childNodes, tree, 1, 2); - expect(body.nextChild).to.not.be(null); + for (const node of body) { + expect(node.name).to.be(getNameField(node.data, schemaWithName)); + expect(node.name).to.not.be(undefined); + } + }); - ({ body } = await supertest - .get( - `/api/endpoint/resolver/${distantChildEntityID}/children?children=2&afterChild=${body.nextChild}` - ) - .expect(200)); - expect(body.childNodes.length).to.eql(0); - expect(body.nextChild).to.be(null); + it('returns all descendants and ancestors without the ancestry field', async () => { + const { body }: { body: ResolverNode[] } = await supertest + .post('/api/endpoint/resolver/tree') + .set('kbn-xsrf', 'xxx') + .send({ + descendants: 100, + descendantLevels: 10, + ancestors: 50, + schema: schemaWithoutAncestry, + nodes: [tree.origin.id], + timerange: { + from: tree.startTime.toISOString(), + to: tree.endTime.toISOString(), + }, + indexPatterns: ['logs-*'], + }) + .expect(200); + verifyTree({ + expectations: [ + // there are 2 levels in the descendant part of the tree and 3 nodes for each + // descendant = 3 children for the origin + 3 children for each of the origin's children = 12 + { + origin: tree.origin.id, + nodeExpectations: { descendants: 12, descendantLevels: 2, ancestors: 5 }, + }, + ], + response: body, + schema: schemaWithoutAncestry, + genTree: tree, + relatedEventsCategories: relatedEventsToGen, }); - it('gets all children in two queries', async () => { - // should get all the children of the origin - let { body }: { body: SafeResolverChildren } = await supertest - .get(`/api/endpoint/resolver/${tree.origin.id}/children?children=3`) - .expect(200); - expect(body.childNodes.length).to.eql(3); - verifyChildren(body.childNodes, tree); - expect(body.nextChild).to.not.be(null); - const firstNodes = [...body.childNodes]; - - ({ body } = await supertest - .get( - `/api/endpoint/resolver/${tree.origin.id}/children?children=10&afterChild=${body.nextChild}` - ) - .expect(200)); - expect(body.childNodes.length).to.eql(9); - // put all the results together and we should have all the children - verifyChildren([...firstNodes, ...body.childNodes], tree, 4, 3); - expect(body.nextChild).to.be(null); - }); + for (const node of body) { + expect(node.name).to.be(getNameField(node.data, schemaWithoutAncestry)); + expect(node.name).to.be(undefined); + } }); - }); - - describe('tree api', () => { - describe('legacy events', () => { - const endpointID = '5a0c957f-b8e7-4538-965e-57e8bb86ad3a'; - it('returns ancestors, events, children, and current process lifecycle', async () => { - const { body }: { body: SafeResolverTree } = await supertest - .get(`/api/endpoint/resolver/93933?legacyEndpointID=${endpointID}`) - .expect(200); - expect(body.ancestry.nextAncestor).to.equal(null); - expect(body.children.nextChild).to.equal(null); - expect(body.children.childNodes.length).to.equal(0); - expect(body.lifecycle.length).to.equal(2); + it('returns all descendants and ancestors with the ancestry field', async () => { + const { body }: { body: ResolverNode[] } = await supertest + .post('/api/endpoint/resolver/tree') + .set('kbn-xsrf', 'xxx') + .send({ + descendants: 100, + descendantLevels: 10, + ancestors: 50, + schema: schemaWithAncestry, + nodes: [tree.origin.id], + timerange: { + from: tree.startTime.toISOString(), + to: tree.endTime.toISOString(), + }, + indexPatterns: ['logs-*'], + }) + .expect(200); + verifyTree({ + expectations: [ + // there are 2 levels in the descendant part of the tree and 3 nodes for each + // descendant = 3 children for the origin + 3 children for each of the origin's children = 12 + { + origin: tree.origin.id, + nodeExpectations: { descendants: 12, descendantLevels: 2, ancestors: 5 }, + }, + ], + response: body, + schema: schemaWithAncestry, + genTree: tree, + relatedEventsCategories: relatedEventsToGen, }); - }); - - describe('endpoint events', () => { - it('returns a tree', async () => { - const { body }: { body: SafeResolverTree } = await supertest - .get( - `/api/endpoint/resolver/${tree.origin.id}?children=100&ancestors=5&events=5&alerts=5` - ) - .expect(200); - - expect(body.children.nextChild).to.equal(null); - expect(body.children.childNodes.length).to.equal(12); - verifyChildren(body.children.childNodes, tree, 4, 3); - verifyLifecycleStats(body.children.childNodes, relatedEventsToGen, relatedAlerts); - expect(body.ancestry.nextAncestor).to.equal(null); - verifyAncestry(body.ancestry.ancestors, tree, true); - verifyLifecycleStats(body.ancestry.ancestors, relatedEventsToGen, relatedAlerts); - - expect(body.relatedAlerts.nextAlert).to.equal(null); - compareArrays(tree.origin.relatedAlerts, body.relatedAlerts.alerts, true); + for (const node of body) { + expect(node.name).to.be(getNameField(node.data, schemaWithAncestry)); + expect(node.name).to.be(undefined); + } + }); - compareArrays(tree.origin.lifecycle, body.lifecycle, true); - verifyStats(body.stats, relatedEventsToGen, relatedAlerts); + it('returns an empty response when limits are zero', async () => { + const { body }: { body: ResolverNode[] } = await supertest + .post('/api/endpoint/resolver/tree') + .set('kbn-xsrf', 'xxx') + .send({ + descendants: 0, + descendantLevels: 0, + ancestors: 0, + schema: schemaWithAncestry, + nodes: [tree.origin.id], + timerange: { + from: tree.startTime.toISOString(), + to: tree.endTime.toISOString(), + }, + indexPatterns: ['logs-*'], + }) + .expect(200); + expect(body).to.be.empty(); + verifyTree({ + expectations: [ + { + origin: tree.origin.id, + nodeExpectations: { descendants: 0, descendantLevels: 0, ancestors: 0 }, + }, + ], + response: body, + schema: schemaWithAncestry, + genTree: tree, }); }); }); diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/tree_entity_id.ts b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/tree_entity_id.ts new file mode 100644 index 0000000000000..39cce77b8cc9d --- /dev/null +++ b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/tree_entity_id.ts @@ -0,0 +1,375 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import expect from '@kbn/expect'; +import { + SafeResolverAncestry, + SafeResolverChildren, + SafeResolverTree, + SafeLegacyEndpointEvent, +} from '../../../../plugins/security_solution/common/endpoint/types'; +import { parentEntityIDSafeVersion } from '../../../../plugins/security_solution/common/endpoint/models/event'; +import { FtrProviderContext } from '../../ftr_provider_context'; +import { + Tree, + RelatedEventCategory, +} from '../../../../plugins/security_solution/common/endpoint/generate_data'; +import { Options, GeneratedTrees } from '../../services/resolver'; +import { + compareArrays, + checkAncestryFromEntityTreeAPI, + retrieveDistantAncestor, + verifyChildrenFromEntityTreeAPI, + verifyLifecycleStats, + verifyEntityTreeStats, +} from './common'; + +export default function ({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const resolver = getService('resolverGenerator'); + + const relatedEventsToGen = [ + { category: RelatedEventCategory.Driver, count: 2 }, + { category: RelatedEventCategory.File, count: 1 }, + { category: RelatedEventCategory.Registry, count: 1 }, + ]; + const relatedAlerts = 4; + let resolverTrees: GeneratedTrees; + let tree: Tree; + const treeOptions: Options = { + ancestors: 5, + relatedEvents: relatedEventsToGen, + relatedAlerts, + children: 3, + generations: 2, + percentTerminated: 100, + percentWithRelated: 100, + numTrees: 1, + alwaysGenMaxChildrenPerNode: true, + ancestryArraySize: 2, + }; + + describe('Resolver entity tree api', () => { + before(async () => { + await esArchiver.load('endpoint/resolver/api_feature'); + resolverTrees = await resolver.createTrees(treeOptions); + // we only requested a single alert so there's only 1 tree + tree = resolverTrees.trees[0]; + }); + after(async () => { + await resolver.deleteData(resolverTrees); + // this unload is for an endgame-* index so it does not use data streams + await esArchiver.unload('endpoint/resolver/api_feature'); + }); + + describe('ancestry events route', () => { + describe('legacy events', () => { + const endpointID = '5a0c957f-b8e7-4538-965e-57e8bb86ad3a'; + const entityID = '94042'; + + it('should return details for the root node', async () => { + const { body }: { body: SafeResolverAncestry } = await supertest + .get( + `/api/endpoint/resolver/${entityID}/ancestry?legacyEndpointID=${endpointID}&ancestors=5` + ) + .expect(200); + expect(body.ancestors[0].lifecycle.length).to.eql(2); + expect(body.ancestors.length).to.eql(2); + expect(body.nextAncestor).to.eql(null); + }); + + it('should have a populated next parameter', async () => { + const { body }: { body: SafeResolverAncestry } = await supertest + .get( + `/api/endpoint/resolver/${entityID}/ancestry?legacyEndpointID=${endpointID}&ancestors=0` + ) + .expect(200); + expect(body.nextAncestor).to.eql('94041'); + }); + + it('should handle an ancestors param request', async () => { + let { body }: { body: SafeResolverAncestry } = await supertest + .get( + `/api/endpoint/resolver/${entityID}/ancestry?legacyEndpointID=${endpointID}&ancestors=0` + ) + .expect(200); + const next = body.nextAncestor; + + ({ body } = await supertest + .get( + `/api/endpoint/resolver/${next}/ancestry?legacyEndpointID=${endpointID}&ancestors=1` + ) + .expect(200)); + expect(body.ancestors[0].lifecycle.length).to.eql(1); + expect(body.nextAncestor).to.eql(null); + }); + }); + + describe('endpoint events', () => { + it('should return the origin node at the front of the array', async () => { + const { body }: { body: SafeResolverAncestry } = await supertest + .get(`/api/endpoint/resolver/${tree.origin.id}/ancestry?ancestors=9`) + .expect(200); + expect(body.ancestors[0].entityID).to.eql(tree.origin.id); + }); + + it('should return details for the root node', async () => { + const { body }: { body: SafeResolverAncestry } = await supertest + .get(`/api/endpoint/resolver/${tree.origin.id}/ancestry?ancestors=9`) + .expect(200); + // the tree we generated had 5 ancestors + 1 origin node + expect(body.ancestors.length).to.eql(6); + expect(body.ancestors[0].entityID).to.eql(tree.origin.id); + checkAncestryFromEntityTreeAPI(body.ancestors, tree, true); + expect(body.nextAncestor).to.eql(null); + }); + + it('should handle an invalid id', async () => { + const { body }: { body: SafeResolverAncestry } = await supertest + .get(`/api/endpoint/resolver/alskdjflasj/ancestry`) + .expect(200); + expect(body.ancestors).to.be.empty(); + expect(body.nextAncestor).to.eql(null); + }); + + it('should have a populated next parameter', async () => { + const { body }: { body: SafeResolverAncestry } = await supertest + .get(`/api/endpoint/resolver/${tree.origin.id}/ancestry?ancestors=2`) + .expect(200); + // it should have 2 ancestors + 1 origin + expect(body.ancestors.length).to.eql(3); + checkAncestryFromEntityTreeAPI(body.ancestors, tree, false); + const distantGrandparent = retrieveDistantAncestor(body.ancestors); + expect(body.nextAncestor).to.eql( + parentEntityIDSafeVersion(distantGrandparent.lifecycle[0]) + ); + }); + + it('should handle multiple ancestor requests', async () => { + let { body }: { body: SafeResolverAncestry } = await supertest + .get(`/api/endpoint/resolver/${tree.origin.id}/ancestry?ancestors=3`) + .expect(200); + expect(body.ancestors.length).to.eql(4); + const next = body.nextAncestor; + + ({ body } = await supertest + .get(`/api/endpoint/resolver/${next}/ancestry?ancestors=1`) + .expect(200)); + expect(body.ancestors.length).to.eql(2); + checkAncestryFromEntityTreeAPI(body.ancestors, tree, true); + // the highest node in the generated tree will not have a parent ID which causes the server to return + // without setting the pagination so nextAncestor will be null + expect(body.nextAncestor).to.eql(null); + }); + }); + }); + + describe('children route', () => { + describe('legacy events', () => { + const endpointID = '5a0c957f-b8e7-4538-965e-57e8bb86ad3a'; + const entityID = '94041'; + + it('returns child process lifecycle events', async () => { + const { body }: { body: SafeResolverChildren } = await supertest + .get(`/api/endpoint/resolver/${entityID}/children?legacyEndpointID=${endpointID}`) + .expect(200); + expect(body.childNodes.length).to.eql(1); + expect(body.childNodes[0].lifecycle.length).to.eql(2); + expect( + // for some reason the ts server doesn't think `endgame` exists even though we're using ResolverEvent + // here, so to avoid it complaining we'll just force it + (body.childNodes[0].lifecycle[0] as SafeLegacyEndpointEvent).endgame.unique_pid + ).to.eql(94042); + }); + + it('returns multiple levels of child process lifecycle events', async () => { + const { body }: { body: SafeResolverChildren } = await supertest + .get(`/api/endpoint/resolver/93802/children?legacyEndpointID=${endpointID}&children=10`) + .expect(200); + expect(body.childNodes.length).to.eql(10); + expect(body.nextChild).to.be(null); + expect(body.childNodes[0].lifecycle.length).to.eql(1); + expect( + // for some reason the ts server doesn't think `endgame` exists even though we're using ResolverEvent + // here, so to avoid it complaining we'll just force it + (body.childNodes[0].lifecycle[0] as SafeLegacyEndpointEvent).endgame.unique_pid + ).to.eql(93932); + }); + + it('returns no values when there is no more data', async () => { + let { body }: { body: SafeResolverChildren } = await supertest + .get( + // there should only be a single child for this node + `/api/endpoint/resolver/94041/children?legacyEndpointID=${endpointID}&children=1` + ) + .expect(200); + expect(body.nextChild).to.not.be(null); + + ({ body } = await supertest + .get( + `/api/endpoint/resolver/94041/children?legacyEndpointID=${endpointID}&afterChild=${body.nextChild}` + ) + .expect(200)); + expect(body.childNodes).be.empty(); + expect(body.nextChild).to.eql(null); + }); + + it('returns the first page of information when the cursor is invalid', async () => { + const { body }: { body: SafeResolverChildren } = await supertest + .get( + `/api/endpoint/resolver/${entityID}/children?legacyEndpointID=${endpointID}&afterChild=blah` + ) + .expect(200); + expect(body.childNodes.length).to.eql(1); + expect(body.nextChild).to.be(null); + }); + + it('errors on invalid pagination values', async () => { + await supertest.get(`/api/endpoint/resolver/${entityID}/children?children=0`).expect(400); + await supertest + .get(`/api/endpoint/resolver/${entityID}/children?children=20000`) + .expect(400); + await supertest + .get(`/api/endpoint/resolver/${entityID}/children?children=-1`) + .expect(400); + }); + + it('returns empty events without a matching entity id', async () => { + const { body }: { body: SafeResolverChildren } = await supertest + .get(`/api/endpoint/resolver/5555/children`) + .expect(200); + expect(body.nextChild).to.eql(null); + expect(body.childNodes).to.be.empty(); + }); + + it('returns empty events with an invalid endpoint id', async () => { + const { body }: { body: SafeResolverChildren } = await supertest + .get(`/api/endpoint/resolver/${entityID}/children?legacyEndpointID=foo`) + .expect(200); + expect(body.nextChild).to.eql(null); + expect(body.childNodes).to.be.empty(); + }); + }); + + describe('endpoint events', () => { + it('returns all children for the origin', async () => { + const { body }: { body: SafeResolverChildren } = await supertest + .get(`/api/endpoint/resolver/${tree.origin.id}/children?children=100`) + .expect(200); + // there are 2 levels in the children part of the tree and 3 nodes for each = + // 3 children for the origin + 3 children for each of the origin's children = 12 + expect(body.childNodes.length).to.eql(12); + // there will be 4 parents, the origin of the tree, and it's 3 children + verifyChildrenFromEntityTreeAPI(body.childNodes, tree, 4, 3); + expect(body.nextChild).to.eql(null); + }); + + it('returns a single generation of children', async () => { + // this gets a node should have 3 children which were created in succession so that the timestamps + // are ordered correctly to be retrieved in a single call + const distantChildEntityID = Array.from(tree.childrenLevels[0].values())[0].id; + const { body }: { body: SafeResolverChildren } = await supertest + .get(`/api/endpoint/resolver/${distantChildEntityID}/children?children=3`) + .expect(200); + expect(body.childNodes.length).to.eql(3); + verifyChildrenFromEntityTreeAPI(body.childNodes, tree, 1, 3); + expect(body.nextChild).to.not.eql(null); + }); + + it('paginates the children', async () => { + // this gets a node should have 3 children which were created in succession so that the timestamps + // are ordered correctly to be retrieved in a single call + const distantChildEntityID = Array.from(tree.childrenLevels[0].values())[0].id; + let { body }: { body: SafeResolverChildren } = await supertest + .get(`/api/endpoint/resolver/${distantChildEntityID}/children?children=1`) + .expect(200); + expect(body.childNodes.length).to.eql(1); + verifyChildrenFromEntityTreeAPI(body.childNodes, tree, 1, 1); + expect(body.nextChild).to.not.be(null); + + ({ body } = await supertest + .get( + `/api/endpoint/resolver/${distantChildEntityID}/children?children=2&afterChild=${body.nextChild}` + ) + .expect(200)); + expect(body.childNodes.length).to.eql(2); + verifyChildrenFromEntityTreeAPI(body.childNodes, tree, 1, 2); + expect(body.nextChild).to.not.be(null); + + ({ body } = await supertest + .get( + `/api/endpoint/resolver/${distantChildEntityID}/children?children=2&afterChild=${body.nextChild}` + ) + .expect(200)); + expect(body.childNodes.length).to.eql(0); + expect(body.nextChild).to.be(null); + }); + + it('gets all children in two queries', async () => { + // should get all the children of the origin + let { body }: { body: SafeResolverChildren } = await supertest + .get(`/api/endpoint/resolver/${tree.origin.id}/children?children=3`) + .expect(200); + expect(body.childNodes.length).to.eql(3); + verifyChildrenFromEntityTreeAPI(body.childNodes, tree); + expect(body.nextChild).to.not.be(null); + const firstNodes = [...body.childNodes]; + + ({ body } = await supertest + .get( + `/api/endpoint/resolver/${tree.origin.id}/children?children=10&afterChild=${body.nextChild}` + ) + .expect(200)); + expect(body.childNodes.length).to.eql(9); + // put all the results together and we should have all the children + verifyChildrenFromEntityTreeAPI([...firstNodes, ...body.childNodes], tree, 4, 3); + expect(body.nextChild).to.be(null); + }); + }); + }); + + describe('tree api', () => { + describe('legacy events', () => { + const endpointID = '5a0c957f-b8e7-4538-965e-57e8bb86ad3a'; + + it('returns ancestors, events, children, and current process lifecycle', async () => { + const { body }: { body: SafeResolverTree } = await supertest + .get(`/api/endpoint/resolver/93933?legacyEndpointID=${endpointID}`) + .expect(200); + expect(body.ancestry.nextAncestor).to.equal(null); + expect(body.children.nextChild).to.equal(null); + expect(body.children.childNodes.length).to.equal(0); + expect(body.lifecycle.length).to.equal(2); + }); + }); + + describe('endpoint events', () => { + it('returns a tree', async () => { + const { body }: { body: SafeResolverTree } = await supertest + .get( + `/api/endpoint/resolver/${tree.origin.id}?children=100&ancestors=5&events=5&alerts=5` + ) + .expect(200); + + expect(body.children.nextChild).to.equal(null); + expect(body.children.childNodes.length).to.equal(12); + verifyChildrenFromEntityTreeAPI(body.children.childNodes, tree, 4, 3); + verifyLifecycleStats(body.children.childNodes, relatedEventsToGen, relatedAlerts); + + expect(body.ancestry.nextAncestor).to.equal(null); + checkAncestryFromEntityTreeAPI(body.ancestry.ancestors, tree, true); + verifyLifecycleStats(body.ancestry.ancestors, relatedEventsToGen, relatedAlerts); + + expect(body.relatedAlerts.nextAlert).to.equal(null); + compareArrays(tree.origin.relatedAlerts, body.relatedAlerts.alerts, true); + + compareArrays(tree.origin.lifecycle, body.lifecycle, true); + verifyEntityTreeStats(body.stats, relatedEventsToGen, relatedAlerts); + }); + }); + }); + }); +} From 5bc4d75256afc543aec79d93f023fc75158a56a0 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Tue, 24 Nov 2020 11:26:46 -0600 Subject: [PATCH 07/49] [deb/rpm] Move systemd service to /usr/lib/systemd/system (#83571) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../systemd/{etc => usr/lib}/systemd/system/kibana.service | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/dev/build/tasks/os_packages/service_templates/systemd/{etc => usr/lib}/systemd/system/kibana.service (100%) diff --git a/src/dev/build/tasks/os_packages/service_templates/systemd/etc/systemd/system/kibana.service b/src/dev/build/tasks/os_packages/service_templates/systemd/usr/lib/systemd/system/kibana.service similarity index 100% rename from src/dev/build/tasks/os_packages/service_templates/systemd/etc/systemd/system/kibana.service rename to src/dev/build/tasks/os_packages/service_templates/systemd/usr/lib/systemd/system/kibana.service From 38a09b99c4314d2a2aa50a9e2780caad94e5e91c Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 24 Nov 2020 18:42:02 +0100 Subject: [PATCH 08/49] Expression: Add render mode and use it for canvas interactivity (#83559) --- ...c.expressionrenderhandler._constructor_.md | 4 +- ...ressions-public.expressionrenderhandler.md | 2 +- ...ressions-public.iexpressionloaderparams.md | 1 + ...blic.iexpressionloaderparams.rendermode.md | 11 ++ ...interpreterrenderhandlers.getrendermode.md | 11 ++ ...sions-public.iinterpreterrenderhandlers.md | 1 + ...interpreterrenderhandlers.getrendermode.md | 11 ++ ...sions-server.iinterpreterrenderhandlers.md | 1 + .../common/expression_renderers/types.ts | 13 ++ src/plugins/expressions/public/loader.test.ts | 28 ++- src/plugins/expressions/public/loader.ts | 1 + src/plugins/expressions/public/public.api.md | 8 +- src/plugins/expressions/public/render.ts | 8 +- src/plugins/expressions/public/types/index.ts | 2 + src/plugins/expressions/server/server.api.md | 4 + .../functions/external/saved_lens.ts | 1 + .../renderers/__stories__/render.tsx | 1 + .../canvas/public/lib/create_handlers.ts | 3 + .../embeddable/embeddable.test.tsx | 39 ++++ .../embeddable/embeddable.tsx | 3 + .../embeddable/expression_wrapper.tsx | 4 + .../public/pie_visualization/expression.tsx | 1 + .../render_function.test.tsx | 9 + .../pie_visualization/render_function.tsx | 17 +- .../xy_visualization/expression.test.tsx | 84 +++++++++ .../public/xy_visualization/expression.tsx | 172 +++++++++--------- 26 files changed, 344 insertions(+), 96 deletions(-) create mode 100644 docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iexpressionloaderparams.rendermode.md create mode 100644 docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iinterpreterrenderhandlers.getrendermode.md create mode 100644 docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.iinterpreterrenderhandlers.getrendermode.md diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionrenderhandler._constructor_.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionrenderhandler._constructor_.md index fb6ba7ee2621c..fcccd3f6b9618 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionrenderhandler._constructor_.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionrenderhandler._constructor_.md @@ -9,7 +9,7 @@ Constructs a new instance of the `ExpressionRenderHandler` class Signature: ```typescript -constructor(element: HTMLElement, { onRenderError }?: Partial); +constructor(element: HTMLElement, { onRenderError, renderMode }?: Partial); ``` ## Parameters @@ -17,5 +17,5 @@ constructor(element: HTMLElement, { onRenderError }?: PartialHTMLElement | | -| { onRenderError } | Partial<ExpressionRenderHandlerParams> | | +| { onRenderError, renderMode } | Partial<ExpressionRenderHandlerParams> | | diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionrenderhandler.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionrenderhandler.md index 7f7d5792ba684..12c663273bd8c 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionrenderhandler.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionrenderhandler.md @@ -14,7 +14,7 @@ export declare class ExpressionRenderHandler | Constructor | Modifiers | Description | | --- | --- | --- | -| [(constructor)(element, { onRenderError })](./kibana-plugin-plugins-expressions-public.expressionrenderhandler._constructor_.md) | | Constructs a new instance of the ExpressionRenderHandler class | +| [(constructor)(element, { onRenderError, renderMode })](./kibana-plugin-plugins-expressions-public.expressionrenderhandler._constructor_.md) | | Constructs a new instance of the ExpressionRenderHandler class | ## Properties diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iexpressionloaderparams.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iexpressionloaderparams.md index 2dfc67d2af5fa..54eecad0deb50 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iexpressionloaderparams.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iexpressionloaderparams.md @@ -21,6 +21,7 @@ export interface IExpressionLoaderParams | [disableCaching](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.disablecaching.md) | boolean | | | [inspectorAdapters](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.inspectoradapters.md) | Adapters | | | [onRenderError](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.onrendererror.md) | RenderErrorHandlerFnType | | +| [renderMode](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.rendermode.md) | RenderMode | | | [searchContext](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.searchcontext.md) | SerializableState | | | [searchSessionId](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.searchsessionid.md) | string | | | [uiState](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.uistate.md) | unknown | | diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iexpressionloaderparams.rendermode.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iexpressionloaderparams.rendermode.md new file mode 100644 index 0000000000000..2986b81fc67c5 --- /dev/null +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iexpressionloaderparams.rendermode.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-expressions-public](./kibana-plugin-plugins-expressions-public.md) > [IExpressionLoaderParams](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.md) > [renderMode](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.rendermode.md) + +## IExpressionLoaderParams.renderMode property + +Signature: + +```typescript +renderMode?: RenderMode; +``` diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iinterpreterrenderhandlers.getrendermode.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iinterpreterrenderhandlers.getrendermode.md new file mode 100644 index 0000000000000..8cddec1a5359c --- /dev/null +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iinterpreterrenderhandlers.getrendermode.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-expressions-public](./kibana-plugin-plugins-expressions-public.md) > [IInterpreterRenderHandlers](./kibana-plugin-plugins-expressions-public.iinterpreterrenderhandlers.md) > [getRenderMode](./kibana-plugin-plugins-expressions-public.iinterpreterrenderhandlers.getrendermode.md) + +## IInterpreterRenderHandlers.getRenderMode property + +Signature: + +```typescript +getRenderMode: () => RenderMode; +``` diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iinterpreterrenderhandlers.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iinterpreterrenderhandlers.md index ab0273be71402..a65e025451636 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iinterpreterrenderhandlers.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iinterpreterrenderhandlers.md @@ -16,6 +16,7 @@ export interface IInterpreterRenderHandlers | --- | --- | --- | | [done](./kibana-plugin-plugins-expressions-public.iinterpreterrenderhandlers.done.md) | () => void | Done increments the number of rendering successes | | [event](./kibana-plugin-plugins-expressions-public.iinterpreterrenderhandlers.event.md) | (event: any) => void | | +| [getRenderMode](./kibana-plugin-plugins-expressions-public.iinterpreterrenderhandlers.getrendermode.md) | () => RenderMode | | | [onDestroy](./kibana-plugin-plugins-expressions-public.iinterpreterrenderhandlers.ondestroy.md) | (fn: () => void) => void | | | [reload](./kibana-plugin-plugins-expressions-public.iinterpreterrenderhandlers.reload.md) | () => void | | | [uiState](./kibana-plugin-plugins-expressions-public.iinterpreterrenderhandlers.uistate.md) | PersistedState | | diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.iinterpreterrenderhandlers.getrendermode.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.iinterpreterrenderhandlers.getrendermode.md new file mode 100644 index 0000000000000..16db25ab244f6 --- /dev/null +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.iinterpreterrenderhandlers.getrendermode.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-expressions-server](./kibana-plugin-plugins-expressions-server.md) > [IInterpreterRenderHandlers](./kibana-plugin-plugins-expressions-server.iinterpreterrenderhandlers.md) > [getRenderMode](./kibana-plugin-plugins-expressions-server.iinterpreterrenderhandlers.getrendermode.md) + +## IInterpreterRenderHandlers.getRenderMode property + +Signature: + +```typescript +getRenderMode: () => RenderMode; +``` diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.iinterpreterrenderhandlers.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.iinterpreterrenderhandlers.md index ccf6271f712b9..b1496386944fa 100644 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.iinterpreterrenderhandlers.md +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.iinterpreterrenderhandlers.md @@ -16,6 +16,7 @@ export interface IInterpreterRenderHandlers | --- | --- | --- | | [done](./kibana-plugin-plugins-expressions-server.iinterpreterrenderhandlers.done.md) | () => void | Done increments the number of rendering successes | | [event](./kibana-plugin-plugins-expressions-server.iinterpreterrenderhandlers.event.md) | (event: any) => void | | +| [getRenderMode](./kibana-plugin-plugins-expressions-server.iinterpreterrenderhandlers.getrendermode.md) | () => RenderMode | | | [onDestroy](./kibana-plugin-plugins-expressions-server.iinterpreterrenderhandlers.ondestroy.md) | (fn: () => void) => void | | | [reload](./kibana-plugin-plugins-expressions-server.iinterpreterrenderhandlers.reload.md) | () => void | | | [uiState](./kibana-plugin-plugins-expressions-server.iinterpreterrenderhandlers.uistate.md) | PersistedState | | diff --git a/src/plugins/expressions/common/expression_renderers/types.ts b/src/plugins/expressions/common/expression_renderers/types.ts index 0ea3d72e75609..dd3124c7d17ee 100644 --- a/src/plugins/expressions/common/expression_renderers/types.ts +++ b/src/plugins/expressions/common/expression_renderers/types.ts @@ -61,6 +61,18 @@ export interface ExpressionRenderDefinition { export type AnyExpressionRenderDefinition = ExpressionRenderDefinition; +/** + * Mode of the expression render environment. + * This value can be set from a consumer embedding an expression renderer and is accessible + * from within the active render function as part of the handlers. + * The following modes are supported: + * * display (default): The chart is rendered in a container with the main purpose of viewing the chart (e.g. in a container like dashboard or canvas) + * * preview: The chart is rendered in very restricted space (below 100px width and height) and should only show a rough outline + * * edit: The chart is rendered within an editor and configuration elements within the chart should be displayed + * * noInteractivity: The chart is rendered in a non-interactive environment and should not provide any affordances for interaction like brushing + */ +export type RenderMode = 'noInteractivity' | 'edit' | 'preview' | 'display'; + export interface IInterpreterRenderHandlers { /** * Done increments the number of rendering successes @@ -70,5 +82,6 @@ export interface IInterpreterRenderHandlers { reload: () => void; update: (params: any) => void; event: (event: any) => void; + getRenderMode: () => RenderMode; uiState?: PersistedState; } diff --git a/src/plugins/expressions/public/loader.test.ts b/src/plugins/expressions/public/loader.test.ts index bf8b442769563..598b614a326a9 100644 --- a/src/plugins/expressions/public/loader.test.ts +++ b/src/plugins/expressions/public/loader.test.ts @@ -20,17 +20,24 @@ import { first, skip, toArray } from 'rxjs/operators'; import { loader, ExpressionLoader } from './loader'; import { Observable } from 'rxjs'; -import { parseExpression, IInterpreterRenderHandlers } from '../common'; +import { + parseExpression, + IInterpreterRenderHandlers, + RenderMode, + AnyExpressionFunctionDefinition, +} from '../common'; // eslint-disable-next-line -const { __getLastExecution } = require('./services'); +const { __getLastExecution, __getLastRenderMode } = require('./services'); const element: HTMLElement = null as any; jest.mock('./services', () => { + let renderMode: RenderMode | undefined; const renderers: Record = { test: { render: (el: HTMLElement, value: unknown, handlers: IInterpreterRenderHandlers) => { + renderMode = handlers.getRenderMode(); handlers.done(); }, }, @@ -39,9 +46,18 @@ jest.mock('./services', () => { // eslint-disable-next-line const service = new (require('../common/service/expressions_services').ExpressionsService as any)(); + const testFn: AnyExpressionFunctionDefinition = { + fn: () => ({ type: 'render', as: 'test' }), + name: 'testrender', + args: {}, + help: '', + }; + service.registerFunction(testFn); + const moduleMock = { __execution: undefined, __getLastExecution: () => moduleMock.__execution, + __getLastRenderMode: () => renderMode, getRenderersRegistry: () => ({ get: (id: string) => renderers[id], }), @@ -130,6 +146,14 @@ describe('ExpressionLoader', () => { expect(response).toBe(2); }); + it('passes mode to the renderer', async () => { + const expressionLoader = new ExpressionLoader(element, 'testrender', { + renderMode: 'edit', + }); + await expressionLoader.render$.pipe(first()).toPromise(); + expect(__getLastRenderMode()).toEqual('edit'); + }); + it('cancels the previous request when the expression is updated', () => { const expressionLoader = new ExpressionLoader(element, 'var foo', {}); const execution = __getLastExecution(); diff --git a/src/plugins/expressions/public/loader.ts b/src/plugins/expressions/public/loader.ts index 91c482621de36..983a344c0e1a1 100644 --- a/src/plugins/expressions/public/loader.ts +++ b/src/plugins/expressions/public/loader.ts @@ -63,6 +63,7 @@ export class ExpressionLoader { this.renderHandler = new ExpressionRenderHandler(element, { onRenderError: params && params.onRenderError, + renderMode: params?.renderMode, }); this.render$ = this.renderHandler.render$; this.update$ = this.renderHandler.update$; diff --git a/src/plugins/expressions/public/public.api.md b/src/plugins/expressions/public/public.api.md index 17f8e6255f6bb..2a73cd6e208d1 100644 --- a/src/plugins/expressions/public/public.api.md +++ b/src/plugins/expressions/public/public.api.md @@ -530,7 +530,7 @@ export interface ExpressionRenderError extends Error { // @public (undocumented) export class ExpressionRenderHandler { // Warning: (ae-forgotten-export) The symbol "ExpressionRenderHandlerParams" needs to be exported by the entry point index.d.ts - constructor(element: HTMLElement, { onRenderError }?: Partial); + constructor(element: HTMLElement, { onRenderError, renderMode }?: Partial); // (undocumented) destroy: () => void; // (undocumented) @@ -891,6 +891,10 @@ export interface IExpressionLoaderParams { // // (undocumented) onRenderError?: RenderErrorHandlerFnType; + // Warning: (ae-forgotten-export) The symbol "RenderMode" needs to be exported by the entry point index.d.ts + // + // (undocumented) + renderMode?: RenderMode; // (undocumented) searchContext?: SerializableState_2; // (undocumented) @@ -909,6 +913,8 @@ export interface IInterpreterRenderHandlers { // (undocumented) event: (event: any) => void; // (undocumented) + getRenderMode: () => RenderMode; + // (undocumented) onDestroy: (fn: () => void) => void; // (undocumented) reload: () => void; diff --git a/src/plugins/expressions/public/render.ts b/src/plugins/expressions/public/render.ts index 924f8d4830f73..4390033b5be60 100644 --- a/src/plugins/expressions/public/render.ts +++ b/src/plugins/expressions/public/render.ts @@ -22,7 +22,7 @@ import { Observable } from 'rxjs'; import { filter } from 'rxjs/operators'; import { ExpressionRenderError, RenderErrorHandlerFnType, IExpressionLoaderParams } from './types'; import { renderErrorHandler as defaultRenderErrorHandler } from './render_error_handler'; -import { IInterpreterRenderHandlers, ExpressionAstExpression } from '../common'; +import { IInterpreterRenderHandlers, ExpressionAstExpression, RenderMode } from '../common'; import { getRenderersRegistry } from './services'; @@ -30,6 +30,7 @@ export type IExpressionRendererExtraHandlers = Record; export interface ExpressionRenderHandlerParams { onRenderError: RenderErrorHandlerFnType; + renderMode: RenderMode; } export interface ExpressionRendererEvent { @@ -58,7 +59,7 @@ export class ExpressionRenderHandler { constructor( element: HTMLElement, - { onRenderError }: Partial = {} + { onRenderError, renderMode }: Partial = {} ) { this.element = element; @@ -92,6 +93,9 @@ export class ExpressionRenderHandler { event: (data) => { this.eventsSubject.next(data); }, + getRenderMode: () => { + return renderMode || 'display'; + }, }; } diff --git a/src/plugins/expressions/public/types/index.ts b/src/plugins/expressions/public/types/index.ts index 4af36fea169a1..5bae985699476 100644 --- a/src/plugins/expressions/public/types/index.ts +++ b/src/plugins/expressions/public/types/index.ts @@ -23,6 +23,7 @@ import { ExpressionValue, ExpressionsService, SerializableState, + RenderMode, } from '../../common'; /** @@ -54,6 +55,7 @@ export interface IExpressionLoaderParams { inspectorAdapters?: Adapters; onRenderError?: RenderErrorHandlerFnType; searchSessionId?: string; + renderMode?: RenderMode; } export interface ExpressionRenderError extends Error { diff --git a/src/plugins/expressions/server/server.api.md b/src/plugins/expressions/server/server.api.md index e5b499206ebdd..33ff759faa3b1 100644 --- a/src/plugins/expressions/server/server.api.md +++ b/src/plugins/expressions/server/server.api.md @@ -729,6 +729,10 @@ export interface IInterpreterRenderHandlers { done: () => void; // (undocumented) event: (event: any) => void; + // Warning: (ae-forgotten-export) The symbol "RenderMode" needs to be exported by the entry point index.d.ts + // + // (undocumented) + getRenderMode: () => RenderMode; // (undocumented) onDestroy: (fn: () => void) => void; // (undocumented) diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/external/saved_lens.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/external/saved_lens.ts index 765ff50728228..380d07972ca4d 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/external/saved_lens.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/external/saved_lens.ts @@ -83,6 +83,7 @@ export function savedLens(): ExpressionFunctionDefinition< title: args.title === null ? undefined : args.title, disableTriggers: true, palette: args.palette, + renderMode: 'noInteractivity', }, embeddableType: EmbeddableTypes.lens, generatedAt: Date.now(), diff --git a/x-pack/plugins/canvas/canvas_plugin_src/renderers/__stories__/render.tsx b/x-pack/plugins/canvas/canvas_plugin_src/renderers/__stories__/render.tsx index 647c63c2c1042..54702f2654839 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/renderers/__stories__/render.tsx +++ b/x-pack/plugins/canvas/canvas_plugin_src/renderers/__stories__/render.tsx @@ -11,6 +11,7 @@ export const defaultHandlers: RendererHandlers = { destroy: () => action('destroy'), getElementId: () => 'element-id', getFilter: () => 'filter', + getRenderMode: () => 'display', onComplete: (fn) => undefined, onEmbeddableDestroyed: action('onEmbeddableDestroyed'), onEmbeddableInputChange: action('onEmbeddableInputChange'), diff --git a/x-pack/plugins/canvas/public/lib/create_handlers.ts b/x-pack/plugins/canvas/public/lib/create_handlers.ts index ae0956ee21283..9bc4bd5e78fd0 100644 --- a/x-pack/plugins/canvas/public/lib/create_handlers.ts +++ b/x-pack/plugins/canvas/public/lib/create_handlers.ts @@ -23,6 +23,9 @@ export const createHandlers = (): RendererHandlers => ({ getFilter() { return ''; }, + getRenderMode() { + return 'display'; + }, onComplete(fn: () => void) { this.done = fn; }, diff --git a/x-pack/plugins/lens/public/editor_frame_service/embeddable/embeddable.test.tsx b/x-pack/plugins/lens/public/editor_frame_service/embeddable/embeddable.test.tsx index 9f9d7fef9c7b4..3a3258a79c59f 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/embeddable/embeddable.test.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/embeddable/embeddable.test.tsx @@ -262,6 +262,45 @@ describe('embeddable', () => { expect(expressionRenderer.mock.calls[0][0].searchSessionId).toBe(input.searchSessionId); }); + it('should pass render mode to expression', async () => { + const timeRange: TimeRange = { from: 'now-15d', to: 'now' }; + const query: Query = { language: 'kquery', query: '' }; + const filters: Filter[] = [{ meta: { alias: 'test', negate: false, disabled: false } }]; + + const input = { + savedObjectId: '123', + timeRange, + query, + filters, + renderMode: 'noInteractivity', + } as LensEmbeddableInput; + + const embeddable = new Embeddable( + { + timefilter: dataPluginMock.createSetupContract().query.timefilter.timefilter, + attributeService, + expressionRenderer, + basePath, + indexPatternService: {} as IndexPatternsContract, + editable: true, + getTrigger, + documentToExpression: () => + Promise.resolve({ + type: 'expression', + chain: [ + { type: 'function', function: 'my', arguments: {} }, + { type: 'function', function: 'expression', arguments: {} }, + ], + }), + }, + input + ); + await embeddable.initializeSavedVis(input); + embeddable.render(mountpoint); + + expect(expressionRenderer.mock.calls[0][0].renderMode).toEqual('noInteractivity'); + }); + it('should merge external context with query and filters of the saved object', async () => { const timeRange: TimeRange = { from: 'now-15d', to: 'now' }; const query: Query = { language: 'kquery', query: 'external filter' }; diff --git a/x-pack/plugins/lens/public/editor_frame_service/embeddable/embeddable.tsx b/x-pack/plugins/lens/public/editor_frame_service/embeddable/embeddable.tsx index 8139631daa971..76276f8b4c828 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/embeddable/embeddable.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/embeddable/embeddable.tsx @@ -20,6 +20,7 @@ import { PaletteOutput } from 'src/plugins/charts/public'; import { Subscription } from 'rxjs'; import { toExpression, Ast } from '@kbn/interpreter/common'; +import { RenderMode } from 'src/plugins/expressions'; import { ExpressionRendererEvent, ReactExpressionRendererType, @@ -53,6 +54,7 @@ export type LensByValueInput = { export type LensByReferenceInput = SavedObjectEmbeddableInput & EmbeddableInput; export type LensEmbeddableInput = (LensByValueInput | LensByReferenceInput) & { palette?: PaletteOutput; + renderMode?: RenderMode; }; export interface LensEmbeddableOutput extends EmbeddableOutput { @@ -192,6 +194,7 @@ export class Embeddable variables={input.palette ? { theme: { palette: input.palette } } : {}} searchSessionId={this.input.searchSessionId} handleEvent={this.handleEvent} + renderMode={input.renderMode} />, domNode ); diff --git a/x-pack/plugins/lens/public/editor_frame_service/embeddable/expression_wrapper.tsx b/x-pack/plugins/lens/public/editor_frame_service/embeddable/expression_wrapper.tsx index 4a3ba971381fb..d18372246b0e6 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/embeddable/expression_wrapper.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/embeddable/expression_wrapper.tsx @@ -13,6 +13,7 @@ import { ReactExpressionRendererType, } from 'src/plugins/expressions/public'; import { ExecutionContextSearch } from 'src/plugins/data/public'; +import { RenderMode } from 'src/plugins/expressions'; import { getOriginalRequestErrorMessage } from '../error_helper'; export interface ExpressionWrapperProps { @@ -22,6 +23,7 @@ export interface ExpressionWrapperProps { searchContext: ExecutionContextSearch; searchSessionId?: string; handleEvent: (event: ExpressionRendererEvent) => void; + renderMode?: RenderMode; } export function ExpressionWrapper({ @@ -31,6 +33,7 @@ export function ExpressionWrapper({ variables, handleEvent, searchSessionId, + renderMode, }: ExpressionWrapperProps) { return ( @@ -57,6 +60,7 @@ export function ExpressionWrapper({ expression={expression} searchContext={searchContext} searchSessionId={searchSessionId} + renderMode={renderMode} renderError={(errorMessage, error) => (
diff --git a/x-pack/plugins/lens/public/pie_visualization/expression.tsx b/x-pack/plugins/lens/public/pie_visualization/expression.tsx index 3b5226eaa8e1f..5f18ef7c7f637 100644 --- a/x-pack/plugins/lens/public/pie_visualization/expression.tsx +++ b/x-pack/plugins/lens/public/pie_visualization/expression.tsx @@ -139,6 +139,7 @@ export const getPieRenderer = (dependencies: { chartsThemeService={dependencies.chartsThemeService} paletteService={dependencies.paletteService} onClickValue={onClickValue} + renderMode={handlers.getRenderMode()} /> , domNode, diff --git a/x-pack/plugins/lens/public/pie_visualization/render_function.test.tsx b/x-pack/plugins/lens/public/pie_visualization/render_function.test.tsx index c44179ccd8dfc..458b1a75c4c17 100644 --- a/x-pack/plugins/lens/public/pie_visualization/render_function.test.tsx +++ b/x-pack/plugins/lens/public/pie_visualization/render_function.test.tsx @@ -70,6 +70,7 @@ describe('PieVisualization component', () => { onClickValue: jest.fn(), chartsThemeService, paletteService: chartPluginMock.createPaletteRegistry(), + renderMode: 'display' as const, }; } @@ -266,6 +267,14 @@ describe('PieVisualization component', () => { `); }); + test('does not set click listener on noInteractivity render mode', () => { + const defaultArgs = getDefaultArgs(); + const component = shallow( + + ); + expect(component.find(Settings).first().prop('onElementClick')).toBeUndefined(); + }); + test('it shows emptyPlaceholder for undefined grouped data', () => { const defaultData = getDefaultArgs().data; const emptyData: LensMultiTable = { diff --git a/x-pack/plugins/lens/public/pie_visualization/render_function.tsx b/x-pack/plugins/lens/public/pie_visualization/render_function.tsx index 39743a355fd78..20d558fefc3d7 100644 --- a/x-pack/plugins/lens/public/pie_visualization/render_function.tsx +++ b/x-pack/plugins/lens/public/pie_visualization/render_function.tsx @@ -20,7 +20,9 @@ import { RecursivePartial, Position, Settings, + ElementClickListener, } from '@elastic/charts'; +import { RenderMode } from 'src/plugins/expressions'; import { FormatFactory, LensFilterEvent } from '../types'; import { VisualizationContainer } from '../visualization_container'; import { CHART_NAMES, DEFAULT_PERCENT_DECIMALS } from './constants'; @@ -44,6 +46,7 @@ export function PieComponent( chartsThemeService: ChartsPluginSetup['theme']; paletteService: PaletteRegistry; onClickValue: (data: LensFilterEvent['data']) => void; + renderMode: RenderMode; } ) { const [firstTable] = Object.values(props.data.tables); @@ -228,6 +231,12 @@ export function PieComponent( ); } + + const onElementClickHandler: ElementClickListener = (args) => { + const context = getFilterContext(args[0][0] as LayerValue[], groups, firstTable); + + onClickValue(desanitizeFilterContext(context)); + }; return ( { - const context = getFilterContext(args[0][0] as LayerValue[], groups, firstTable); - - onClickValue(desanitizeFilterContext(context)); - }} + onElementClick={ + props.renderMode !== 'noInteractivity' ? onElementClickHandler : undefined + } theme={{ ...chartTheme, background: { diff --git a/x-pack/plugins/lens/public/xy_visualization/expression.test.tsx b/x-pack/plugins/lens/public/xy_visualization/expression.test.tsx index a4b5d741c80f1..0e2b47410c3f9 100644 --- a/x-pack/plugins/lens/public/xy_visualization/expression.test.tsx +++ b/x-pack/plugins/lens/public/xy_visualization/expression.test.tsx @@ -427,6 +427,7 @@ describe('xy_expression', () => { args={args} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -451,6 +452,7 @@ describe('xy_expression', () => { args={{ ...args, layers: [{ ...args.layers[0], seriesType: 'line' }] }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -504,6 +506,7 @@ describe('xy_expression', () => { }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={undefined} @@ -541,6 +544,7 @@ describe('xy_expression', () => { args={multiLayerArgs} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -578,6 +582,7 @@ describe('xy_expression', () => { }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -596,6 +601,7 @@ describe('xy_expression', () => { args={{ ...args, layers: [{ ...args.layers[0], seriesType: 'bar' }] }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -617,6 +623,7 @@ describe('xy_expression', () => { args={{ ...args, layers: [{ ...args.layers[0], seriesType: 'area' }] }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -638,6 +645,7 @@ describe('xy_expression', () => { args={{ ...args, layers: [{ ...args.layers[0], seriesType: 'bar_horizontal' }] }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -664,6 +672,7 @@ describe('xy_expression', () => { args={args} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -688,6 +697,7 @@ describe('xy_expression', () => { }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -773,6 +783,7 @@ describe('xy_expression', () => { }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -791,6 +802,27 @@ describe('xy_expression', () => { }); }); + test('onBrushEnd is not set on noInteractivity mode', () => { + const { args, data } = sampleArgs(); + + const wrapper = mountWithIntl( + + ); + + expect(wrapper.find(Settings).first().prop('onBrushEnd')).toBeUndefined(); + }); + test('onElementClick returns correct context data', () => { const geometry: GeometryValue = { x: 5, y: 1, accessor: 'y1', mark: null, datum: {} }; const series = { @@ -825,6 +857,7 @@ describe('xy_expression', () => { }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -855,6 +888,27 @@ describe('xy_expression', () => { }); }); + test('onElementClick is not triggering event on noInteractivity mode', () => { + const { args, data } = sampleArgs(); + + const wrapper = mountWithIntl( + + ); + + expect(wrapper.find(Settings).first().prop('onElementClick')).toBeUndefined(); + }); + test('it renders stacked bar', () => { const { data, args } = sampleArgs(); const component = shallow( @@ -863,6 +917,7 @@ describe('xy_expression', () => { args={{ ...args, layers: [{ ...args.layers[0], seriesType: 'bar_stacked' }] }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -884,6 +939,7 @@ describe('xy_expression', () => { args={{ ...args, layers: [{ ...args.layers[0], seriesType: 'area_stacked' }] }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -908,6 +964,7 @@ describe('xy_expression', () => { }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -941,6 +998,7 @@ describe('xy_expression', () => { }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -961,6 +1019,7 @@ describe('xy_expression', () => { args={args} formatFactory={getFormatSpy} timeZone="CEST" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -987,6 +1046,7 @@ describe('xy_expression', () => { args={{ ...args, layers: [firstLayer] }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -1007,6 +1067,7 @@ describe('xy_expression', () => { args={{ ...args, layers: [firstLayer] }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -1030,6 +1091,7 @@ describe('xy_expression', () => { args={{ ...args, layers: [firstLayer, secondLayer] }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -1058,6 +1120,7 @@ describe('xy_expression', () => { }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -1080,6 +1143,7 @@ describe('xy_expression', () => { }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -1481,6 +1545,7 @@ describe('xy_expression', () => { args={{ ...args, layers: [{ ...args.layers[0], xScaleType: 'ordinal' }] }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -1501,6 +1566,7 @@ describe('xy_expression', () => { args={{ ...args, layers: [{ ...args.layers[0], yScaleType: 'sqrt' }] }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -1521,6 +1587,7 @@ describe('xy_expression', () => { args={{ ...args }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -1544,6 +1611,7 @@ describe('xy_expression', () => { paletteService={paletteService} minInterval={50} timeZone="UTC" + renderMode="display" onClickValue={onClickValue} onSelectRange={onSelectRange} /> @@ -1563,6 +1631,7 @@ describe('xy_expression', () => { args={{ ...args }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -1598,6 +1667,7 @@ describe('xy_expression', () => { args={{ ...args }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -1631,6 +1701,7 @@ describe('xy_expression', () => { args={{ ...args }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -1664,6 +1735,7 @@ describe('xy_expression', () => { args={{ ...args }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -1697,6 +1769,7 @@ describe('xy_expression', () => { args={{ ...args }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -1797,6 +1870,7 @@ describe('xy_expression', () => { args={args} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -1871,6 +1945,7 @@ describe('xy_expression', () => { args={args} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -1943,6 +2018,7 @@ describe('xy_expression', () => { args={args} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -1967,6 +2043,7 @@ describe('xy_expression', () => { }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -1990,6 +2067,7 @@ describe('xy_expression', () => { }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -2013,6 +2091,7 @@ describe('xy_expression', () => { }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -2048,6 +2127,7 @@ describe('xy_expression', () => { args={{ ...args, fittingFunction: 'Carry' }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -2075,6 +2155,7 @@ describe('xy_expression', () => { args={{ ...args }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -2097,6 +2178,7 @@ describe('xy_expression', () => { args={{ ...args }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -2124,6 +2206,7 @@ describe('xy_expression', () => { args={{ ...args }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} @@ -2157,6 +2240,7 @@ describe('xy_expression', () => { args={{ ...args }} formatFactory={getFormatSpy} timeZone="UTC" + renderMode="display" chartsThemeService={chartsThemeService} paletteService={paletteService} minInterval={50} diff --git a/x-pack/plugins/lens/public/xy_visualization/expression.tsx b/x-pack/plugins/lens/public/xy_visualization/expression.tsx index 54ae3bb759d2c..790416a6c920d 100644 --- a/x-pack/plugins/lens/public/xy_visualization/expression.tsx +++ b/x-pack/plugins/lens/public/xy_visualization/expression.tsx @@ -21,6 +21,8 @@ import { StackMode, VerticalAlignment, HorizontalAlignment, + ElementClickListener, + BrushEndListener, } from '@elastic/charts'; import { I18nProvider } from '@kbn/i18n/react'; import { @@ -31,6 +33,7 @@ import { } from 'src/plugins/expressions/public'; import { IconType } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; +import { RenderMode } from 'src/plugins/expressions'; import { LensMultiTable, FormatFactory, @@ -81,6 +84,7 @@ type XYChartRenderProps = XYChartProps & { minInterval: number | undefined; onClickValue: (data: LensFilterEvent['data']) => void; onSelectRange: (data: LensBrushEvent['data']) => void; + renderMode: RenderMode; }; export const xyChart: ExpressionFunctionDefinition< @@ -235,6 +239,7 @@ export const getXyChartRenderer = (dependencies: { minInterval={await calculateMinInterval(config, dependencies.getIntervalByColumn)} onClickValue={onClickValue} onSelectRange={onSelectRange} + renderMode={handlers.getRenderMode()} /> , domNode, @@ -303,6 +308,7 @@ export function XYChart({ minInterval, onClickValue, onSelectRange, + renderMode, }: XYChartRenderProps) { const { legend, layers, fittingFunction, gridlinesVisibilitySettings, valueLabels } = args; const chartTheme = chartsThemeService.useChartsTheme(); @@ -415,6 +421,87 @@ export function XYChart({ const colorAssignments = getColorAssignments(args.layers, data, formatFactory); + const clickHandler: ElementClickListener = ([[geometry, series]]) => { + // for xyChart series is always XYChartSeriesIdentifier and geometry is always type of GeometryValue + const xySeries = series as XYChartSeriesIdentifier; + const xyGeometry = geometry as GeometryValue; + + const layer = filteredLayers.find((l) => + xySeries.seriesKeys.some((key: string | number) => l.accessors.includes(key.toString())) + ); + if (!layer) { + return; + } + + const table = data.tables[layer.layerId]; + + const points = [ + { + row: table.rows.findIndex((row) => { + if (layer.xAccessor) { + if (layersAlreadyFormatted[layer.xAccessor]) { + // stringify the value to compare with the chart value + return xAxisFormatter.convert(row[layer.xAccessor]) === xyGeometry.x; + } + return row[layer.xAccessor] === xyGeometry.x; + } + }), + column: table.columns.findIndex((col) => col.id === layer.xAccessor), + value: xyGeometry.x, + }, + ]; + + if (xySeries.seriesKeys.length > 1) { + const pointValue = xySeries.seriesKeys[0]; + + points.push({ + row: table.rows.findIndex( + (row) => layer.splitAccessor && row[layer.splitAccessor] === pointValue + ), + column: table.columns.findIndex((col) => col.id === layer.splitAccessor), + value: pointValue, + }); + } + + const xAxisFieldName = table.columns.find((el) => el.id === layer.xAccessor)?.meta?.field; + const timeFieldName = xDomain && xAxisFieldName; + + const context: LensFilterEvent['data'] = { + data: points.map((point) => ({ + row: point.row, + column: point.column, + value: point.value, + table, + })), + timeFieldName, + }; + onClickValue(desanitizeFilterContext(context)); + }; + + const brushHandler: BrushEndListener = ({ x }) => { + if (!x) { + return; + } + const [min, max] = x; + if (!xAxisColumn || !isHistogramViz) { + return; + } + + const table = data.tables[filteredLayers[0].layerId]; + + const xAxisColumnIndex = table.columns.findIndex((el) => el.id === filteredLayers[0].xAccessor); + + const timeFieldName = isTimeViz ? table.columns[xAxisColumnIndex]?.meta?.field : undefined; + + const context: LensBrushEvent['data'] = { + range: [min, max], + table, + column: xAxisColumnIndex, + timeFieldName, + }; + onSelectRange(context); + }; + return ( { - if (!x) { - return; - } - const [min, max] = x; - if (!xAxisColumn || !isHistogramViz) { - return; - } - - const table = data.tables[filteredLayers[0].layerId]; - - const xAxisColumnIndex = table.columns.findIndex( - (el) => el.id === filteredLayers[0].xAccessor - ); - - const timeFieldName = isTimeViz - ? table.columns[xAxisColumnIndex]?.meta?.field - : undefined; - - const context: LensBrushEvent['data'] = { - range: [min, max], - table, - column: xAxisColumnIndex, - timeFieldName, - }; - onSelectRange(context); - }} - onElementClick={([[geometry, series]]) => { - // for xyChart series is always XYChartSeriesIdentifier and geometry is always type of GeometryValue - const xySeries = series as XYChartSeriesIdentifier; - const xyGeometry = geometry as GeometryValue; - - const layer = filteredLayers.find((l) => - xySeries.seriesKeys.some((key: string | number) => l.accessors.includes(key.toString())) - ); - if (!layer) { - return; - } - - const table = data.tables[layer.layerId]; - - const points = [ - { - row: table.rows.findIndex((row) => { - if (layer.xAccessor) { - if (layersAlreadyFormatted[layer.xAccessor]) { - // stringify the value to compare with the chart value - return xAxisFormatter.convert(row[layer.xAccessor]) === xyGeometry.x; - } - return row[layer.xAccessor] === xyGeometry.x; - } - }), - column: table.columns.findIndex((col) => col.id === layer.xAccessor), - value: xyGeometry.x, - }, - ]; - - if (xySeries.seriesKeys.length > 1) { - const pointValue = xySeries.seriesKeys[0]; - - points.push({ - row: table.rows.findIndex( - (row) => layer.splitAccessor && row[layer.splitAccessor] === pointValue - ), - column: table.columns.findIndex((col) => col.id === layer.splitAccessor), - value: pointValue, - }); - } - - const xAxisFieldName = table.columns.find((el) => el.id === layer.xAccessor)?.meta?.field; - const timeFieldName = xDomain && xAxisFieldName; - - const context: LensFilterEvent['data'] = { - data: points.map((point) => ({ - row: point.row, - column: point.column, - value: point.value, - table, - })), - timeFieldName, - }; - onClickValue(desanitizeFilterContext(context)); - }} + onBrushEnd={renderMode !== 'noInteractivity' ? brushHandler : undefined} + onElementClick={renderMode !== 'noInteractivity' ? clickHandler : undefined} /> Date: Tue, 24 Nov 2020 12:42:46 -0500 Subject: [PATCH 09/49] Remove expressions.legacy from README (#79681) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- x-pack/plugins/canvas/README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/canvas/README.md b/x-pack/plugins/canvas/README.md index 7bd9a1994ba7e..f77585b5b062c 100644 --- a/x-pack/plugins/canvas/README.md +++ b/x-pack/plugins/canvas/README.md @@ -149,7 +149,7 @@ yarn start #### Adding a server-side function -> Server side functions may be deprecated in a later version of Kibana as they require using an API marked _legacy_ +> Server side functions may be deprecated in a later version of Kibana Now, let's add a function which runs on the server. @@ -206,9 +206,7 @@ And then in our setup method, register it with the Expressions plugin: ```typescript setup(core: CoreSetup, plugins: CanvasExamplePluginsSetup) { - // .register requires serverFunctions and types, so pass an empty array - // if you don't have any custom types to register - plugins.expressions.__LEGACY.register({ serverFunctions, types: [] }); + serverFunctions.forEach((f) => plugins.expressions.registerFunction(f)); } ``` From c2026dfa7aa90a031f34b884a6e4dbf25c96aeb4 Mon Sep 17 00:00:00 2001 From: Anton Dosov Date: Tue, 24 Nov 2020 18:54:49 +0100 Subject: [PATCH 10/49] Unskip "Copy dashboards to space" (#84115) --- .../dashboard/drilldowns/dashboard_to_dashboard_drilldown.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/functional/apps/dashboard/drilldowns/dashboard_to_dashboard_drilldown.ts b/x-pack/test/functional/apps/dashboard/drilldowns/dashboard_to_dashboard_drilldown.ts index 03765f5aa6033..9326f7e240e3e 100644 --- a/x-pack/test/functional/apps/dashboard/drilldowns/dashboard_to_dashboard_drilldown.ts +++ b/x-pack/test/functional/apps/dashboard/drilldowns/dashboard_to_dashboard_drilldown.ts @@ -166,7 +166,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await spaces.delete(destinationSpaceId); }); - it.skip('Dashboards linked by a drilldown are both copied to a space', async () => { + it('Dashboards linked by a drilldown are both copied to a space', async () => { await PageObjects.copySavedObjectsToSpace.openCopyToSpaceFlyoutForObject( dashboardDrilldownsManage.DASHBOARD_WITH_AREA_CHART_NAME ); From d47460d08d15c71ffcb61f2be642d44ec807d581 Mon Sep 17 00:00:00 2001 From: Mikhail Shustov Date: Tue, 24 Nov 2020 20:56:35 +0300 Subject: [PATCH 11/49] Attempt to fix incremental build error (#84152) * make fetch compatible with CollectorFetchMethod * use Alejandros suggestion --- src/plugins/usage_collection/server/collector/collector_set.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/usage_collection/server/collector/collector_set.ts b/src/plugins/usage_collection/server/collector/collector_set.ts index fe4f3536ffed6..cda4ce36d4e23 100644 --- a/src/plugins/usage_collection/server/collector/collector_set.ts +++ b/src/plugins/usage_collection/server/collector/collector_set.ts @@ -29,7 +29,7 @@ import { import { Collector, CollectorOptions } from './collector'; import { UsageCollector, UsageCollectorOptions } from './usage_collector'; -type AnyCollector = Collector; +type AnyCollector = Collector; type AnyUsageCollector = UsageCollector; interface CollectorSetConfig { From 31a5b15250a59cd5cafcc320c88f5020bf184c62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Tue, 24 Nov 2020 19:27:24 +0100 Subject: [PATCH 12/49] [APM] Use `asTransactionRate` consistently everywhere (#84213) --- .../apm/common/utils/formatters/formatters.ts | 11 ---- .../ServiceMap/Popover/ServiceStatsList.tsx | 8 +-- .../app/TraceOverview/TraceList.tsx | 13 ++--- .../TransactionDetails/Distribution/index.tsx | 53 +++++-------------- .../service_inventory/ServiceList/index.tsx | 21 +------- .../TransactionList/index.tsx | 10 +--- .../charts/transaction_charts/index.tsx | 15 +----- .../public/selectors/chart_selectors.test.ts | 2 +- .../apm/public/selectors/chart_selectors.ts | 4 +- .../translations/translations/ja-JP.json | 7 --- .../translations/translations/zh-CN.json | 7 --- 11 files changed, 29 insertions(+), 122 deletions(-) diff --git a/x-pack/plugins/apm/common/utils/formatters/formatters.ts b/x-pack/plugins/apm/common/utils/formatters/formatters.ts index 2314e915e3161..50ce9db096610 100644 --- a/x-pack/plugins/apm/common/utils/formatters/formatters.ts +++ b/x-pack/plugins/apm/common/utils/formatters/formatters.ts @@ -4,7 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ import numeral from '@elastic/numeral'; -import { i18n } from '@kbn/i18n'; import { Maybe } from '../../../typings/common'; import { NOT_AVAILABLE_LABEL } from '../../i18n'; import { isFiniteNumber } from '../is_finite_number'; @@ -17,16 +16,6 @@ export function asInteger(value: number) { return numeral(value).format('0,0'); } -export function tpmUnit(type?: string) { - return type === 'request' - ? i18n.translate('xpack.apm.formatters.requestsPerMinLabel', { - defaultMessage: 'rpm', - }) - : i18n.translate('xpack.apm.formatters.transactionsPerMinLabel', { - defaultMessage: 'tpm', - }); -} - export function asPercent( numerator: Maybe, denominator: number | undefined, diff --git a/x-pack/plugins/apm/public/components/app/ServiceMap/Popover/ServiceStatsList.tsx b/x-pack/plugins/apm/public/components/app/ServiceMap/Popover/ServiceStatsList.tsx index 8463da0824bde..adbcf897669ae 100644 --- a/x-pack/plugins/apm/public/components/app/ServiceMap/Popover/ServiceStatsList.tsx +++ b/x-pack/plugins/apm/public/components/app/ServiceMap/Popover/ServiceStatsList.tsx @@ -11,7 +11,7 @@ import styled from 'styled-components'; import { asDuration, asPercent, - tpmUnit, + asTransactionRate, } from '../../../../../common/utils/formatters'; import { ServiceNodeStats } from '../../../../../common/service_map'; @@ -55,11 +55,7 @@ export function ServiceStatsList({ defaultMessage: 'Req. per minute (avg.)', } ), - description: isNumber(transactionStats.avgRequestsPerMinute) - ? `${transactionStats.avgRequestsPerMinute.toFixed(2)} ${tpmUnit( - 'request' - )}` - : null, + description: asTransactionRate(transactionStats.avgRequestsPerMinute), }, { title: i18n.translate('xpack.apm.serviceMap.errorRatePopoverStat', { diff --git a/x-pack/plugins/apm/public/components/app/TraceOverview/TraceList.tsx b/x-pack/plugins/apm/public/components/app/TraceOverview/TraceList.tsx index 4704230d7c68c..e68f8a9809bf5 100644 --- a/x-pack/plugins/apm/public/components/app/TraceOverview/TraceList.tsx +++ b/x-pack/plugins/apm/public/components/app/TraceOverview/TraceList.tsx @@ -8,7 +8,10 @@ import { EuiIcon, EuiToolTip } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import React from 'react'; import styled from 'styled-components'; -import { asMillisecondDuration } from '../../../../common/utils/formatters'; +import { + asMillisecondDuration, + asTransactionRate, +} from '../../../../common/utils/formatters'; import { fontSizes, truncate } from '../../../style/variables'; import { EmptyMessage } from '../../shared/EmptyMessage'; import { ImpactBar } from '../../shared/ImpactBar'; @@ -78,13 +81,7 @@ const traceListColumns: Array> = [ }), sortable: true, dataType: 'number', - render: (value: number) => - `${value.toLocaleString()} ${i18n.translate( - 'xpack.apm.tracesTable.tracesPerMinuteUnitLabel', - { - defaultMessage: 'tpm', - } - )}`, + render: (value: number) => asTransactionRate(value), }, { field: 'impact', diff --git a/x-pack/plugins/apm/public/components/app/TransactionDetails/Distribution/index.tsx b/x-pack/plugins/apm/public/components/app/TransactionDetails/Distribution/index.tsx index e92a6c7db8445..bbc99fb122fc6 100644 --- a/x-pack/plugins/apm/public/components/app/TransactionDetails/Distribution/index.tsx +++ b/x-pack/plugins/apm/public/components/app/TransactionDetails/Distribution/index.tsx @@ -22,7 +22,7 @@ import { EuiIconTip, EuiTitle } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import d3 from 'd3'; import { isEmpty } from 'lodash'; -import React, { useCallback } from 'react'; +import React from 'react'; import { ValuesType } from 'utility-types'; import { APIReturnType } from '../../../../services/rest/createCallApmApi'; import { useTheme } from '../../../../../../observability/public'; @@ -70,46 +70,29 @@ export function getFormattedBuckets( ); } -const getFormatYShort = (transactionType: string | undefined) => ( - t: number -) => { +const formatYShort = (t: number) => { return i18n.translate( 'xpack.apm.transactionDetails.transactionsDurationDistributionChart.unitShortLabel', + { + defaultMessage: '{transCount} trans.', + values: { transCount: t }, + } + ); +}; + +const formatYLong = (t: number) => { + return i18n.translate( + 'xpack.apm.transactionDetails.transactionsDurationDistributionChart.transactionTypeUnitLongLabel', { defaultMessage: - '{transCount} {transType, select, request {req.} other {trans.}}', + '{transCount, plural, =0 {transactions} one {transaction} other {transactions}}', values: { transCount: t, - transType: transactionType, }, } ); }; -const getFormatYLong = (transactionType: string | undefined) => (t: number) => { - return transactionType === 'request' - ? i18n.translate( - 'xpack.apm.transactionDetails.transactionsDurationDistributionChart.requestTypeUnitLongLabel', - { - defaultMessage: - '{transCount, plural, =0 {request} one {request} other {requests}}', - values: { - transCount: t, - }, - } - ) - : i18n.translate( - 'xpack.apm.transactionDetails.transactionsDurationDistributionChart.transactionTypeUnitLongLabel', - { - defaultMessage: - '{transCount, plural, =0 {transaction} one {transaction} other {transactions}}', - values: { - transCount: t, - }, - } - ); -}; - interface Props { distribution?: TransactionDistributionAPIResponse; urlParams: IUrlParams; @@ -129,16 +112,6 @@ export function TransactionDistribution({ }: Props) { const theme = useTheme(); - /* eslint-disable-next-line react-hooks/exhaustive-deps */ - const formatYShort = useCallback(getFormatYShort(transactionType), [ - transactionType, - ]); - - /* eslint-disable-next-line react-hooks/exhaustive-deps */ - const formatYLong = useCallback(getFormatYLong(transactionType), [ - transactionType, - ]); - // no data in response if ( (!distribution || distribution.noHits) && diff --git a/x-pack/plugins/apm/public/components/app/service_inventory/ServiceList/index.tsx b/x-pack/plugins/apm/public/components/app/service_inventory/ServiceList/index.tsx index 547a0938bc24d..a4c93f95dc53d 100644 --- a/x-pack/plugins/apm/public/components/app/service_inventory/ServiceList/index.tsx +++ b/x-pack/plugins/apm/public/components/app/service_inventory/ServiceList/index.tsx @@ -14,8 +14,8 @@ import { APIReturnType } from '../../../../services/rest/createCallApmApi'; import { ServiceHealthStatus } from '../../../../../common/service_health_status'; import { asPercent, - asDecimal, asMillisecondDuration, + asTransactionRate, } from '../../../../../common/utils/formatters'; import { NOT_AVAILABLE_LABEL } from '../../../../../common/i18n'; import { fontSizes, px, truncate, unit } from '../../../../style/variables'; @@ -35,16 +35,6 @@ interface Props { } type ServiceListItem = ValuesType; -function formatNumber(value: number) { - if (value === 0) { - return '0'; - } else if (value <= 0.1) { - return '< 0.1'; - } else { - return asDecimal(value); - } -} - function formatString(value?: string | null) { return value || NOT_AVAILABLE_LABEL; } @@ -154,14 +144,7 @@ export const SERVICE_COLUMNS: Array> = [ ), align: 'left', diff --git a/x-pack/plugins/apm/public/components/app/transaction_overview/TransactionList/index.tsx b/x-pack/plugins/apm/public/components/app/transaction_overview/TransactionList/index.tsx index ece923631a2f7..9774538b2a7a7 100644 --- a/x-pack/plugins/apm/public/components/app/transaction_overview/TransactionList/index.tsx +++ b/x-pack/plugins/apm/public/components/app/transaction_overview/TransactionList/index.tsx @@ -10,8 +10,8 @@ import React, { useMemo } from 'react'; import styled from 'styled-components'; import { APIReturnType } from '../../../../services/rest/createCallApmApi'; import { - asDecimal, asMillisecondDuration, + asTransactionRate, } from '../../../../../common/utils/formatters'; import { fontFamilyCode, truncate } from '../../../../style/variables'; import { ImpactBar } from '../../../shared/ImpactBar'; @@ -103,13 +103,7 @@ export function TransactionList({ items, isLoading }: Props) { ), sortable: true, dataType: 'number', - render: (value: number) => - `${asDecimal(value)} ${i18n.translate( - 'xpack.apm.transactionsTable.transactionsPerMinuteUnitLabel', - { - defaultMessage: 'tpm', - } - )}`, + render: (value: number) => asTransactionRate(value), }, { field: 'impact', diff --git a/x-pack/plugins/apm/public/components/shared/charts/transaction_charts/index.tsx b/x-pack/plugins/apm/public/components/shared/charts/transaction_charts/index.tsx index 61d834abda793..3af081c11c9b1 100644 --- a/x-pack/plugins/apm/public/components/shared/charts/transaction_charts/index.tsx +++ b/x-pack/plugins/apm/public/components/shared/charts/transaction_charts/index.tsx @@ -14,20 +14,17 @@ import { } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import React from 'react'; -import { NOT_AVAILABLE_LABEL } from '../../../../../common/i18n'; import { TRANSACTION_PAGE_LOAD, TRANSACTION_REQUEST, TRANSACTION_ROUTE_CHANGE, } from '../../../../../common/transaction_types'; -import { asDecimal, tpmUnit } from '../../../../../common/utils/formatters'; -import { Coordinate } from '../../../../../typings/timeseries'; +import { asTransactionRate } from '../../../../../common/utils/formatters'; import { ChartsSyncContextProvider } from '../../../../context/charts_sync_context'; import { LicenseContext } from '../../../../context/LicenseContext'; import { IUrlParams } from '../../../../context/UrlParamsContext/types'; import { FETCH_STATUS } from '../../../../hooks/useFetcher'; import { ITransactionChartData } from '../../../../selectors/chart_selectors'; -import { isValidCoordinateValue } from '../../../../utils/isValidCoordinateValue'; import { TransactionBreakdownChart } from '../transaction_breakdown_chart'; import { TimeseriesChart } from '../timeseries_chart'; import { TransactionErrorRateChart } from '../transaction_error_rate_chart/'; @@ -46,14 +43,6 @@ export function TransactionCharts({ urlParams, fetchStatus, }: TransactionChartProps) { - const getTPMFormatter = (t: number) => { - return `${asDecimal(t)} ${tpmUnit(urlParams.transactionType)}`; - }; - - const getTPMTooltipFormatter = (y: Coordinate['y']) => { - return isValidCoordinateValue(y) ? getTPMFormatter(y) : NOT_AVAILABLE_LABEL; - }; - const { transactionType } = urlParams; const { responseTimeSeries, tpmSeries } = charts; @@ -104,7 +93,7 @@ export function TransactionCharts({ fetchStatus={fetchStatus} id="requestPerMinutes" timeseries={tpmSeries || []} - yLabelFormat={getTPMTooltipFormatter} + yLabelFormat={asTransactionRate} /> diff --git a/x-pack/plugins/apm/public/selectors/chart_selectors.test.ts b/x-pack/plugins/apm/public/selectors/chart_selectors.test.ts index 4269ec0e6c0f3..a17faebc9aefa 100644 --- a/x-pack/plugins/apm/public/selectors/chart_selectors.test.ts +++ b/x-pack/plugins/apm/public/selectors/chart_selectors.test.ts @@ -144,7 +144,7 @@ describe('chart selectors', () => { { color: errorColor, data: [{ x: 0, y: 0 }], - legendValue: '0.0 tpm', + legendValue: '0 tpm', title: 'HTTP 5xx', type: 'linemark', }, diff --git a/x-pack/plugins/apm/public/selectors/chart_selectors.ts b/x-pack/plugins/apm/public/selectors/chart_selectors.ts index 8330df07c21eb..663fbc9028108 100644 --- a/x-pack/plugins/apm/public/selectors/chart_selectors.ts +++ b/x-pack/plugins/apm/public/selectors/chart_selectors.ts @@ -20,7 +20,7 @@ import { import { IUrlParams } from '../context/UrlParamsContext/types'; import { getEmptySeries } from '../components/shared/charts/helper/get_empty_series'; import { httpStatusCodeToColor } from '../utils/httpStatusCodeToColor'; -import { asDecimal, asDuration, tpmUnit } from '../../common/utils/formatters'; +import { asDuration, asTransactionRate } from '../../common/utils/formatters'; export interface ITpmBucket { title: string; @@ -171,7 +171,7 @@ export function getTpmSeries( return { title: bucket.key, data: bucket.dataPoints, - legendValue: `${asDecimal(bucket.avg)} ${tpmUnit(transactionType || '')}`, + legendValue: asTransactionRate(bucket.avg), type: 'linemark', color: getColor(bucket.key), }; diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 9a28e0e53bef5..ed514eda000aa 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -4874,9 +4874,7 @@ "xpack.apm.formatters.microsTimeUnitLabel": "マイクロ秒", "xpack.apm.formatters.millisTimeUnitLabel": "ミリ秒", "xpack.apm.formatters.minutesTimeUnitLabel": "最低", - "xpack.apm.formatters.requestsPerMinLabel": "1分あたりリクエスト数", "xpack.apm.formatters.secondsTimeUnitLabel": "秒", - "xpack.apm.formatters.transactionsPerMinLabel": "1分あたりトランザクション数", "xpack.apm.header.badge.readOnly.text": "読み込み専用", "xpack.apm.header.badge.readOnly.tooltip": "を保存できませんでした", "xpack.apm.helpMenu.upgradeAssistantLink": "アップグレードアシスタント", @@ -5052,7 +5050,6 @@ "xpack.apm.servicesTable.notFoundLabel": "サービスが見つかりません", "xpack.apm.servicesTable.transactionErrorRate": "エラー率%", "xpack.apm.servicesTable.transactionsPerMinuteColumnLabel": "1 分あたりのトランザクション", - "xpack.apm.servicesTable.transactionsPerMinuteUnitLabel": "1分あたりトランザクション数", "xpack.apm.servicesTable.UpgradeAssistantLink": "Kibana アップグレードアシスタントで詳細をご覧ください", "xpack.apm.settings.agentConfig": "エージェントの編集", "xpack.apm.settings.anomaly_detection.legacy_jobs.body": "以前の統合のレガシー機械学習ジョブが見つかりました。これは、APMアプリでは使用されていません。", @@ -5155,7 +5152,6 @@ "xpack.apm.tracesTable.notFoundLabel": "このクエリのトレースが見つかりません", "xpack.apm.tracesTable.originatingServiceColumnLabel": "発生元サービス", "xpack.apm.tracesTable.tracesPerMinuteColumnLabel": "1 分あたりのトレース", - "xpack.apm.tracesTable.tracesPerMinuteUnitLabel": "1分あたりトランザクション数", "xpack.apm.transactionActionMenu.actionsButtonLabel": "アクション", "xpack.apm.transactionActionMenu.container.subtitle": "このコンテナーのログとインデックスを表示し、さらに詳細を確認できます。", "xpack.apm.transactionActionMenu.container.title": "コンテナーの詳細", @@ -5206,9 +5202,7 @@ "xpack.apm.transactionDetails.traceNotFound": "選択されたトレースが見つかりません", "xpack.apm.transactionDetails.traceSampleTitle": "トレースのサンプル", "xpack.apm.transactionDetails.transactionLabel": "トランザクション", - "xpack.apm.transactionDetails.transactionsDurationDistributionChart.requestTypeUnitLongLabel": "{transCount, plural, =0 {# request} 1 {# 件のリクエスト} other {# 件のリクエスト}}", "xpack.apm.transactionDetails.transactionsDurationDistributionChart.transactionTypeUnitLongLabel": "{transCount, plural, =0 {# transaction} 1 {# 件のトランザクション} other {# 件のトランザクション}}", - "xpack.apm.transactionDetails.transactionsDurationDistributionChart.unitShortLabel": "{transCount} {transType, select, request {件のリクエスト} other {件のトランザクション}}", "xpack.apm.transactionDetails.transactionsDurationDistributionChartTitle": "トラザクション時間の分布", "xpack.apm.transactionDetails.transactionsDurationDistributionChartTooltip.samplingDescription": "各バケットはサンプルトランザクションを示します。利用可能なサンプルがない場合、恐らくエージェントの構成で設定されたサンプリング制限が原因です。", "xpack.apm.transactionDetails.transactionsDurationDistributionChartTooltip.samplingLabel": "サンプリング", @@ -5241,7 +5235,6 @@ "xpack.apm.transactionsTable.nameColumnLabel": "名前", "xpack.apm.transactionsTable.notFoundLabel": "トランザクションが見つかりませんでした。", "xpack.apm.transactionsTable.transactionsPerMinuteColumnLabel": "1 分あたりのトランザクション", - "xpack.apm.transactionsTable.transactionsPerMinuteUnitLabel": "1分あたりトランザクション数", "xpack.apm.tutorial.apmServer.title": "APM Server", "xpack.apm.tutorial.elasticCloud.textPre": "APM Server を有効にするには、[the Elastic Cloud console](https://cloud.elastic.co/deployments?q={cloudId}) に移動し、展開設定で APM を有効にします。有効になったら、このページを更新してください。", "xpack.apm.tutorial.elasticCloudInstructions.title": "APM エージェント", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 66a00c30bd3b9..a500b63fbf863 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -4876,9 +4876,7 @@ "xpack.apm.formatters.microsTimeUnitLabel": "μs", "xpack.apm.formatters.millisTimeUnitLabel": "ms", "xpack.apm.formatters.minutesTimeUnitLabel": "分钟", - "xpack.apm.formatters.requestsPerMinLabel": "rpm", "xpack.apm.formatters.secondsTimeUnitLabel": "s", - "xpack.apm.formatters.transactionsPerMinLabel": "tpm", "xpack.apm.header.badge.readOnly.text": "只读", "xpack.apm.header.badge.readOnly.tooltip": "无法保存", "xpack.apm.helpMenu.upgradeAssistantLink": "升级助手", @@ -5056,7 +5054,6 @@ "xpack.apm.servicesTable.notFoundLabel": "未找到任何服务", "xpack.apm.servicesTable.transactionErrorRate": "错误率 %", "xpack.apm.servicesTable.transactionsPerMinuteColumnLabel": "每分钟事务数", - "xpack.apm.servicesTable.transactionsPerMinuteUnitLabel": "tpm", "xpack.apm.servicesTable.UpgradeAssistantLink": "通过访问 Kibana 升级助手来了解详情", "xpack.apm.settings.agentConfig": "代理配置", "xpack.apm.settings.anomaly_detection.legacy_jobs.body": "我们在以前的集成中发现 APM 应用中不再使用的旧版 Machine Learning 作业", @@ -5159,7 +5156,6 @@ "xpack.apm.tracesTable.notFoundLabel": "未找到与此查询的任何追溯信息", "xpack.apm.tracesTable.originatingServiceColumnLabel": "发起服务", "xpack.apm.tracesTable.tracesPerMinuteColumnLabel": "每分钟追溯次数", - "xpack.apm.tracesTable.tracesPerMinuteUnitLabel": "tpm", "xpack.apm.transactionActionMenu.actionsButtonLabel": "操作", "xpack.apm.transactionActionMenu.container.subtitle": "查看此容器的日志和指标以获取进一步详情。", "xpack.apm.transactionActionMenu.container.title": "容器详情", @@ -5210,9 +5206,7 @@ "xpack.apm.transactionDetails.traceNotFound": "找不到所选跟踪", "xpack.apm.transactionDetails.traceSampleTitle": "跟踪样例", "xpack.apm.transactionDetails.transactionLabel": "事务", - "xpack.apm.transactionDetails.transactionsDurationDistributionChart.requestTypeUnitLongLabel": "{transCount, plural, =0 {# 个请求} one {# 个请求} other {# 个请求}}", "xpack.apm.transactionDetails.transactionsDurationDistributionChart.transactionTypeUnitLongLabel": "{transCount, plural, =0 {# 个事务} one {# 个事务} other {# 个事务}}", - "xpack.apm.transactionDetails.transactionsDurationDistributionChart.unitShortLabel": "{transCount} 个{transType, select, request {请求} other {事务}}", "xpack.apm.transactionDetails.transactionsDurationDistributionChartTitle": "事务持续时间分布", "xpack.apm.transactionDetails.transactionsDurationDistributionChartTooltip.samplingDescription": "每个存储桶将显示一个样例事务。如果没有可用的样例，很可能是在代理配置设置了采样限制。", "xpack.apm.transactionDetails.transactionsDurationDistributionChartTooltip.samplingLabel": "采样", @@ -5245,7 +5239,6 @@ "xpack.apm.transactionsTable.nameColumnLabel": "名称", "xpack.apm.transactionsTable.notFoundLabel": "未找到任何事务。", "xpack.apm.transactionsTable.transactionsPerMinuteColumnLabel": "每分钟事务数", - "xpack.apm.transactionsTable.transactionsPerMinuteUnitLabel": "tpm", "xpack.apm.tutorial.apmServer.title": "APM Server", "xpack.apm.tutorial.elasticCloud.textPre": "要启用 APM Server，请前往 [Elastic Cloud 控制台](https://cloud.elastic.co/deployments?q={cloudId}) 并在部署设置中启用 APM。启用后，请刷新此页面。", "xpack.apm.tutorial.elasticCloudInstructions.title": "APM 代理", From 2634009a5b33ca1d6800f66fce0b6cb727f92ec0 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Tue, 24 Nov 2020 20:32:35 +0100 Subject: [PATCH 13/49] [code coverage] collect for oss integration tests (#83907) * [code coverage] collect for oss integration tests * do not run snapshot test modified with coverage * skip failures * remove debug msg * update file names * Update packages/kbn-optimizer/src/integration_tests/basic_optimization.test.ts Co-authored-by: Spencer Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Spencer --- .../src/integration_tests/basic_optimization.test.ts | 4 ++++ .../shell_scripts/fix_html_reports_parallel.sh | 4 ++-- test/scripts/jenkins_unit.sh | 12 ++++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/packages/kbn-optimizer/src/integration_tests/basic_optimization.test.ts b/packages/kbn-optimizer/src/integration_tests/basic_optimization.test.ts index 46660f0dd958b..16baaddcb84b2 100644 --- a/packages/kbn-optimizer/src/integration_tests/basic_optimization.test.ts +++ b/packages/kbn-optimizer/src/integration_tests/basic_optimization.test.ts @@ -233,6 +233,10 @@ it('uses cache on second run and exist cleanly', async () => { }); it('prepares assets for distribution', async () => { + if (process.env.CODE_COVERAGE) { + // test fails when testing coverage because source includes instrumentation, so skip it + return; + } const config = OptimizerConfig.create({ repoRoot: MOCK_REPO_DIR, pluginScanDirs: [Path.resolve(MOCK_REPO_DIR, 'plugins'), Path.resolve(MOCK_REPO_DIR, 'x-pack')], diff --git a/src/dev/code_coverage/shell_scripts/fix_html_reports_parallel.sh b/src/dev/code_coverage/shell_scripts/fix_html_reports_parallel.sh index 098737eb2f800..01003b6dc880c 100644 --- a/src/dev/code_coverage/shell_scripts/fix_html_reports_parallel.sh +++ b/src/dev/code_coverage/shell_scripts/fix_html_reports_parallel.sh @@ -8,8 +8,8 @@ PWD=$(pwd) du -sh $COMBINED_EXRACT_DIR echo "### Jest: replacing path in json files" -for i in coverage-final xpack-coverage-final; do - sed -i "s|/dev/shm/workspace/kibana|${PWD}|g" $COMBINED_EXRACT_DIR/jest/${i}.json & +for i in oss oss-integration xpack; do + sed -i "s|/dev/shm/workspace/kibana|${PWD}|g" $COMBINED_EXRACT_DIR/jest/${i}-coverage-final.json & done wait diff --git a/test/scripts/jenkins_unit.sh b/test/scripts/jenkins_unit.sh index a9751003e8425..1f6a3d440734b 100755 --- a/test/scripts/jenkins_unit.sh +++ b/test/scripts/jenkins_unit.sh @@ -2,11 +2,23 @@ source test/scripts/jenkins_test_setup.sh +rename_coverage_file() { + test -f target/kibana-coverage/jest/coverage-final.json \ + && mv target/kibana-coverage/jest/coverage-final.json \ + target/kibana-coverage/jest/$1-coverage-final.json +} + if [[ -z "$CODE_COVERAGE" ]] ; then "$(FORCE_COLOR=0 yarn bin)/grunt" jenkins:unit --dev; else echo " -> Running jest tests with coverage" node scripts/jest --ci --verbose --coverage + rename_coverage_file "oss" + echo "" + echo "" + echo " -> Running jest integration tests with coverage" + node --max-old-space-size=8192 scripts/jest_integration --ci --verbose --coverage || true; + rename_coverage_file "oss-integration" echo "" echo "" echo " -> Running mocha tests with coverage" From e8a4b7e7dd773123618c2bd0d4dde2ace7f67121 Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Tue, 24 Nov 2020 11:33:34 -0800 Subject: [PATCH 14/49] [@kbn/utils] Clean target before build (#84253) Signed-off-by: Tyler Smalley --- packages/kbn-utils/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-utils/package.json b/packages/kbn-utils/package.json index a07be96f0d4d8..0859faa7ed0ad 100644 --- a/packages/kbn-utils/package.json +++ b/packages/kbn-utils/package.json @@ -5,7 +5,7 @@ "license": "Apache-2.0", "private": true, "scripts": { - "build": "../../node_modules/.bin/tsc", + "build": "rm -rf target && ../../node_modules/.bin/tsc", "kbn:bootstrap": "yarn build", "kbn:watch": "yarn build --watch" }, From 3612e3f98d579c6f5075eb552151890640d9f154 Mon Sep 17 00:00:00 2001 From: Thomas Neirynck Date: Tue, 24 Nov 2020 14:36:57 -0500 Subject: [PATCH 15/49] [Maps] fix code-owners (#84265) --- .github/CODEOWNERS | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 834662044988d..a536d1b54551c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -142,9 +142,8 @@ #CC# /src/plugins/maps_oss/ @elastic/kibana-gis #CC# /x-pack/plugins/file_upload @elastic/kibana-gis #CC# /x-pack/plugins/maps_legacy_licensing @elastic/kibana-gis -#CC# /src/plugins/home/server/tutorials @elastic/kibana-gis -#CC# /src/plugins/tile_map/ @elastic/kibana-gis -#CC# /src/plugins/region_map/ @elastic/kibana-gis +/src/plugins/tile_map/ @elastic/kibana-gis +/src/plugins/region_map/ @elastic/kibana-gis # Operations /src/dev/ @elastic/kibana-operations From cc35065f5ab027684444f3768f92d5a335e18cfd Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Tue, 24 Nov 2020 21:22:58 +0100 Subject: [PATCH 16/49] Update example docs with correct version of Boom (#84271) --- .../developer/plugin/migrating-legacy-plugins-examples.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/developer/plugin/migrating-legacy-plugins-examples.asciidoc b/docs/developer/plugin/migrating-legacy-plugins-examples.asciidoc index abf51bb3378b7..469f7a4f3adb1 100644 --- a/docs/developer/plugin/migrating-legacy-plugins-examples.asciidoc +++ b/docs/developer/plugin/migrating-legacy-plugins-examples.asciidoc @@ -242,7 +242,7 @@ migration is complete: ---- import { schema } from '@kbn/config-schema'; import { CoreSetup } from 'kibana/server'; -import Boom from 'boom'; +import Boom from '@hapi/boom'; export class DemoPlugin { public setup(core: CoreSetup) { From 13808e019e6ebcf62b392824885b743371120bb1 Mon Sep 17 00:00:00 2001 From: Brandon Kobel Date: Tue, 24 Nov 2020 12:28:15 -0800 Subject: [PATCH 17/49] Deprecate `kibana.index` setting (#83988) * Deprecating `kibana.index` setting * Using ela.st service so this can be changed to the blog in the future * Adding unit tests * Revising deprecation log message * Changing the deprecation log message to be more consistent with others * Updating kibana.index docs also * Using rename deprecation as the "standard" for the deprecation messages * /s/'/` --- docs/setup/settings.asciidoc | 6 ++- src/core/server/kibana_config.test.ts | 66 +++++++++++++++++++++++++++ src/core/server/kibana_config.ts | 14 ++++++ 3 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 src/core/server/kibana_config.test.ts diff --git a/docs/setup/settings.asciidoc b/docs/setup/settings.asciidoc index efc7a1b930932..c22d4466ee09e 100644 --- a/docs/setup/settings.asciidoc +++ b/docs/setup/settings.asciidoc @@ -214,10 +214,12 @@ Please use the `defaultRoute` advanced setting instead. The default application to load. *Default: `"home"`* |[[kibana-index]] `kibana.index:` - | {kib} uses an index in {es} to store saved searches, visualizations, and + | *deprecated* This setting is deprecated and will be removed in 8.0. Multitenancy by changing +`kibana.index` will not be supported starting in 8.0. See https://ela.st/kbn-remove-legacy-multitenancy[8.0 Breaking Changes] +for more details. {kib} uses an index in {es} to store saved searches, visualizations, and dashboards. {kib} creates a new index if the index doesn’t already exist. If you configure a custom index, the name must be lowercase, and conform to the -{es} {ref}/indices-create-index.html[index name limitations]. +{es} {ref}/indices-create-index.html[index name limitations]. *Default: `".kibana"`* | `kibana.autocompleteTimeout:` {ess-icon} diff --git a/src/core/server/kibana_config.test.ts b/src/core/server/kibana_config.test.ts new file mode 100644 index 0000000000000..804c02ae99e4b --- /dev/null +++ b/src/core/server/kibana_config.test.ts @@ -0,0 +1,66 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { config } from './kibana_config'; +import { applyDeprecations, configDeprecationFactory } from '@kbn/config'; + +const CONFIG_PATH = 'kibana'; + +const applyKibanaDeprecations = (settings: Record = {}) => { + const deprecations = config.deprecations!(configDeprecationFactory); + const deprecationMessages: string[] = []; + const _config: any = {}; + _config[CONFIG_PATH] = settings; + const migrated = applyDeprecations( + _config, + deprecations.map((deprecation) => ({ + deprecation, + path: CONFIG_PATH, + })), + (msg) => deprecationMessages.push(msg) + ); + return { + messages: deprecationMessages, + migrated, + }; +}; + +it('set correct defaults ', () => { + const configValue = config.schema.validate({}); + expect(configValue).toMatchInlineSnapshot(` + Object { + "autocompleteTerminateAfter": "PT1M40S", + "autocompleteTimeout": "PT1S", + "enabled": true, + "index": ".kibana", + } + `); +}); + +describe('deprecations', () => { + ['.foo', '.kibana'].forEach((index) => { + it('logs a warning if index is set', () => { + const { messages } = applyKibanaDeprecations({ index }); + expect(messages).toMatchInlineSnapshot(` + Array [ + "\\"kibana.index\\" is deprecated. Multitenancy by changing \\"kibana.index\\" will not be supported starting in 8.0. See https://ela.st/kbn-remove-legacy-multitenancy for more details", + ] + `); + }); + }); +}); diff --git a/src/core/server/kibana_config.ts b/src/core/server/kibana_config.ts index 17f77a6e9328f..ae6897b6a6ad3 100644 --- a/src/core/server/kibana_config.ts +++ b/src/core/server/kibana_config.ts @@ -18,9 +18,22 @@ */ import { schema, TypeOf } from '@kbn/config-schema'; +import { ConfigDeprecationProvider } from '@kbn/config'; export type KibanaConfigType = TypeOf; +const deprecations: ConfigDeprecationProvider = () => [ + (settings, fromPath, log) => { + const kibana = settings[fromPath]; + if (kibana?.index) { + log( + `"kibana.index" is deprecated. Multitenancy by changing "kibana.index" will not be supported starting in 8.0. See https://ela.st/kbn-remove-legacy-multitenancy for more details` + ); + } + return settings; + }, +]; + export const config = { path: 'kibana', schema: schema.object({ @@ -29,4 +42,5 @@ export const config = { autocompleteTerminateAfter: schema.duration({ defaultValue: 100000 }), autocompleteTimeout: schema.duration({ defaultValue: 1000 }), }), + deprecations, }; From 115916956d875bb56e9b57961b0bbb4cf090e3e9 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Tue, 24 Nov 2020 22:15:35 +0100 Subject: [PATCH 18/49] Use correct version of Podium (#84270) --- package.json | 1 - .../src/legacy_logging_server.ts | 2 +- yarn.lock | 17 ----------------- 3 files changed, 1 insertion(+), 19 deletions(-) diff --git a/package.json b/package.json index af80102641dbb..571dc7302f924 100644 --- a/package.json +++ b/package.json @@ -261,7 +261,6 @@ "pdfmake": "^0.1.65", "pegjs": "0.10.0", "pngjs": "^3.4.0", - "podium": "^3.1.2", "prop-types": "^15.7.2", "proper-lockfile": "^3.2.0", "proxy-from-env": "1.0.0", diff --git a/packages/kbn-legacy-logging/src/legacy_logging_server.ts b/packages/kbn-legacy-logging/src/legacy_logging_server.ts index 45e4bda0b007c..1b13eda44fff2 100644 --- a/packages/kbn-legacy-logging/src/legacy_logging_server.ts +++ b/packages/kbn-legacy-logging/src/legacy_logging_server.ts @@ -18,7 +18,7 @@ */ import { ServerExtType, Server } from '@hapi/hapi'; -import Podium from 'podium'; +import Podium from '@hapi/podium'; import { setupLogging } from './setup_logging'; import { attachMetaData } from './metadata'; import { legacyLoggingConfigSchema } from './schema'; diff --git a/yarn.lock b/yarn.lock index 8d47d3e843788..dc171a44dca1f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -17975,15 +17975,6 @@ joi@13.x.x, joi@^13.5.2: isemail "3.x.x" topo "3.x.x" -joi@14.x.x: - version "14.3.1" - resolved "https://registry.yarnpkg.com/joi/-/joi-14.3.1.tgz#164a262ec0b855466e0c35eea2a885ae8b6c703c" - integrity sha512-LQDdM+pkOrpAn4Lp+neNIFV3axv1Vna3j38bisbQhETPMANYRbFJFUyOZcOClYvM/hppMhGWuKSFEK9vjrB+bQ== - dependencies: - hoek "6.x.x" - isemail "3.x.x" - topo "3.x.x" - joi@^17.1.1: version "17.2.1" resolved "https://registry.yarnpkg.com/joi/-/joi-17.2.1.tgz#e5140fdf07e8fecf9bc977c2832d1bdb1e3f2a0a" @@ -22237,14 +22228,6 @@ pnp-webpack-plugin@1.6.4: dependencies: ts-pnp "^1.1.6" -podium@^3.1.2: - version "3.2.0" - resolved "https://registry.yarnpkg.com/podium/-/podium-3.2.0.tgz#2a7c579ddd5408f412d014c9ffac080c41d83477" - integrity sha512-rbwvxwVkI6gRRlxZQ1zUeafrpGxZ7QPHIheinehAvGATvGIPfWRkaTeWedc5P4YjXJXEV8ZbBxPtglNylF9hjw== - dependencies: - hoek "6.x.x" - joi "14.x.x" - polished@^1.9.2: version "1.9.2" resolved "https://registry.yarnpkg.com/polished/-/polished-1.9.2.tgz#d705cac66f3a3ed1bd38aad863e2c1e269baf6b6" From 9ee1ec7f30b87c6c836fd378c97e0b265f400259 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Tue, 24 Nov 2020 21:33:47 +0000 Subject: [PATCH 19/49] chore(NA): rebalance x-pack cigroups (#84099) * chore(NA): rebalance cigroup1 into cigroup5 * chore(NA): get list api integration into cigropup1 again * chore(NA): get apm integration basic into cigropup1 again * chore(NA): move back apm_api_integration trial tests into ciGroup1 * chore(NA): move exception operators data types into ciGroup1 again * chore(NA): move detection engine api security and spaces back into ciGroup1 * chore(NA): add a new xpack cigroup11 * chore(NA): correctly create 11 xpack ci groups * chore(NA): try to balance ciGroup2 and 8 * chore(NA): reset number of xpack parallel worker builds to 10 Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .ci/es-snapshots/Jenkinsfile_verify_es | 1 + .ci/jobs.yml | 1 + test/scripts/jenkins_xpack_build_kibana.sh | 3 ++- vars/kibanaCoverage.groovy | 1 + vars/tasks.groovy | 2 +- x-pack/test/case_api_integration/basic/tests/index.ts | 2 +- .../tests/exception_operators_data_types/index.ts | 2 +- .../security_and_spaces/tests/index.ts | 2 +- x-pack/test/functional/apps/discover/index.ts | 2 +- 9 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.ci/es-snapshots/Jenkinsfile_verify_es b/.ci/es-snapshots/Jenkinsfile_verify_es index a6fe980242afe..3c38d6279a038 100644 --- a/.ci/es-snapshots/Jenkinsfile_verify_es +++ b/.ci/es-snapshots/Jenkinsfile_verify_es @@ -55,6 +55,7 @@ kibanaPipeline(timeoutMinutes: 150) { 'xpack-ciGroup8': kibanaPipeline.xpackCiGroupProcess(8), 'xpack-ciGroup9': kibanaPipeline.xpackCiGroupProcess(9), 'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10), + 'xpack-ciGroup11': kibanaPipeline.xpackCiGroupProcess(11), ]), ]) } diff --git a/.ci/jobs.yml b/.ci/jobs.yml index 3add92aadd256..d4ec8a3d5a699 100644 --- a/.ci/jobs.yml +++ b/.ci/jobs.yml @@ -31,6 +31,7 @@ JOB: - x-pack-ciGroup8 - x-pack-ciGroup9 - x-pack-ciGroup10 + - x-pack-ciGroup11 - x-pack-accessibility - x-pack-visualRegression diff --git a/test/scripts/jenkins_xpack_build_kibana.sh b/test/scripts/jenkins_xpack_build_kibana.sh index 2452e2f5b8c58..8bb6effbec89c 100755 --- a/test/scripts/jenkins_xpack_build_kibana.sh +++ b/test/scripts/jenkins_xpack_build_kibana.sh @@ -22,7 +22,8 @@ node scripts/functional_tests --assert-none-excluded \ --include-tag ciGroup7 \ --include-tag ciGroup8 \ --include-tag ciGroup9 \ - --include-tag ciGroup10 + --include-tag ciGroup10 \ + --include-tag ciGroup11 # Do not build kibana for code coverage run if [[ -z "$CODE_COVERAGE" ]] ; then diff --git a/vars/kibanaCoverage.groovy b/vars/kibanaCoverage.groovy index e75ed8fef9875..c43be9fb17ee0 100644 --- a/vars/kibanaCoverage.groovy +++ b/vars/kibanaCoverage.groovy @@ -249,6 +249,7 @@ def xpackProks() { 'xpack-ciGroup8' : kibanaPipeline.xpackCiGroupProcess(8), 'xpack-ciGroup9' : kibanaPipeline.xpackCiGroupProcess(9), 'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10), + 'xpack-ciGroup11': kibanaPipeline.xpackCiGroupProcess(11), ] } diff --git a/vars/tasks.groovy b/vars/tasks.groovy index 5a8161ebd3608..b6bcc0d93f9c0 100644 --- a/vars/tasks.groovy +++ b/vars/tasks.groovy @@ -94,7 +94,7 @@ def functionalXpack(Map params = [:]) { kibanaPipeline.buildXpack(10) if (config.ciGroups) { - def ciGroups = 1..10 + def ciGroups = 1..11 tasks(ciGroups.collect { kibanaPipeline.xpackCiGroupProcess(it) }) } diff --git a/x-pack/test/case_api_integration/basic/tests/index.ts b/x-pack/test/case_api_integration/basic/tests/index.ts index 2f7af95e264f8..56b473af61e63 100644 --- a/x-pack/test/case_api_integration/basic/tests/index.ts +++ b/x-pack/test/case_api_integration/basic/tests/index.ts @@ -10,7 +10,7 @@ import { FtrProviderContext } from '../../common/ftr_provider_context'; export default ({ loadTestFile }: FtrProviderContext): void => { describe('case api basic', function () { // Fastest ciGroup for the moment. - this.tags('ciGroup2'); + this.tags('ciGroup5'); loadTestFile(require.resolve('./cases/comments/delete_comment')); loadTestFile(require.resolve('./cases/comments/find_comments')); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/index.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/index.ts index d2aca34e27399..0fbb97d284429 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/index.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/index.ts @@ -9,7 +9,7 @@ import { FtrProviderContext } from '../../../common/ftr_provider_context'; // eslint-disable-next-line import/no-default-export export default ({ loadTestFile }: FtrProviderContext): void => { describe('Detection exceptions data types and operators', function () { - this.tags('ciGroup1'); + this.tags('ciGroup11'); loadTestFile(require.resolve('./date')); loadTestFile(require.resolve('./double')); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts index 97d5b079fd206..a2422b9e3bf40 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts @@ -9,7 +9,7 @@ import { FtrProviderContext } from '../../common/ftr_provider_context'; // eslint-disable-next-line import/no-default-export export default ({ loadTestFile }: FtrProviderContext): void => { describe('detection engine api security and spaces enabled', function () { - this.tags('ciGroup1'); + this.tags('ciGroup11'); loadTestFile(require.resolve('./add_actions')); loadTestFile(require.resolve('./add_prepackaged_rules')); diff --git a/x-pack/test/functional/apps/discover/index.ts b/x-pack/test/functional/apps/discover/index.ts index fc91a72c3950f..13426da504bd9 100644 --- a/x-pack/test/functional/apps/discover/index.ts +++ b/x-pack/test/functional/apps/discover/index.ts @@ -7,7 +7,7 @@ import { FtrProviderContext } from '../../ftr_provider_context'; export default function ({ loadTestFile }: FtrProviderContext) { describe('discover', function () { - this.tags('ciGroup8'); + this.tags('ciGroup1'); loadTestFile(require.resolve('./feature_controls')); loadTestFile(require.resolve('./preserve_url')); From dc15aa8ea21a9bf88dbfa3afc77afc81711788b7 Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Tue, 24 Nov 2020 23:26:08 +0100 Subject: [PATCH 20/49] [ML] Fix swim lane for top influencers (#84258) * [ML] fix swim lane with page size for top influencers, fix swim lane sorting * [ML] fix typo --- .../application/explorer/swimlane_container.tsx | 11 ++++++++++- .../application/routing/routes/explorer.tsx | 17 +++++++++++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/ml/public/application/explorer/swimlane_container.tsx b/x-pack/plugins/ml/public/application/explorer/swimlane_container.tsx index 9c7d0f6fe78e2..b166d90f040a6 100644 --- a/x-pack/plugins/ml/public/application/explorer/swimlane_container.tsx +++ b/x-pack/plugins/ml/public/application/explorer/swimlane_container.tsx @@ -184,6 +184,8 @@ export const SwimlaneContainer: FC = ({ return []; } + const sortedLaneValues = swimlaneData.laneLabels; + return swimlaneData.points .map((v) => { const formatted = { ...v, time: v.time * 1000 }; @@ -195,8 +197,15 @@ export const SwimlaneContainer: FC = ({ } return formatted; }) + .sort((a, b) => { + let aIndex = sortedLaneValues.indexOf(a.laneLabel); + let bIndex = sortedLaneValues.indexOf(b.laneLabel); + aIndex = aIndex > -1 ? aIndex : sortedLaneValues.length; + bIndex = bIndex > -1 ? bIndex : sortedLaneValues.length; + return aIndex - bIndex; + }) .filter((v) => v.value > 0); - }, [swimlaneData?.points, filterActive, swimlaneType]); + }, [swimlaneData?.points, filterActive, swimlaneType, swimlaneData?.laneLabels]); const showSwimlane = swimlaneData?.laneLabels?.length > 0 && swimLanePoints.length > 0; diff --git a/x-pack/plugins/ml/public/application/routing/routes/explorer.tsx b/x-pack/plugins/ml/public/application/routing/routes/explorer.tsx index 83f876bcf7b56..f8b4de6903ad2 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/explorer.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/explorer.tsx @@ -231,15 +231,24 @@ const ExplorerUrlStateManager: FC = ({ jobsWithTim : undefined; useEffect(() => { + /** + * For the "View by" swim lane the limit is the cardinality of the influencer values, + * which is known after the initial fetch. + * When looking up for top influencers for selected range in Overall swim lane + * the result is filtered by top influencers values, hence there is no need to set the limit. + */ + const swimlaneLimit = + isViewBySwimLaneData(explorerState?.viewBySwimlaneData) && !selectedCells?.showTopFieldValues + ? explorerState?.viewBySwimlaneData.cardinality + : undefined; + if (explorerState && explorerState.swimlaneContainerWidth > 0) { loadExplorerData({ ...loadExplorerDataConfig, - swimlaneLimit: isViewBySwimLaneData(explorerState?.viewBySwimlaneData) - ? explorerState?.viewBySwimlaneData.cardinality - : undefined, + swimlaneLimit, }); } - }, [JSON.stringify(loadExplorerDataConfig)]); + }, [JSON.stringify(loadExplorerDataConfig), selectedCells?.showTopFieldValues]); if (explorerState === undefined || refresh === undefined || showCharts === undefined) { return null; From f80da6cc39ca833bf1b7cdf05de1ee3a03679e32 Mon Sep 17 00:00:00 2001 From: Lukas Olson Date: Tue, 24 Nov 2020 15:38:12 -0700 Subject: [PATCH 21/49] [data.search] Simplify poll logic and improve types (#82545) * [Search] Add request context and asScoped pattern * Update docs * Unify interface for getting search client * Update examples/search_examples/server/my_strategy.ts Co-authored-by: Anton Dosov * Review feedback * Fix checks * Fix CI * Fix security search * Fix test * Fix test for reals * Fix types * [data.search] Refactor search polling and improve types * Fix & update tests & types * eql totals * doc * Revert "eql totals" This reverts commit 01e8a0684792c7255b4ccaad0fa09fe5e3c9c57c. * lint * response type * shim inside strategies * shim for security * fix eql params Co-authored-by: Anton Dosov Co-authored-by: Liza K Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- ...data-public.painlesserror._constructor_.md | 3 +- ...lugin-plugins-data-public.painlesserror.md | 2 +- ...lic.searchinterceptor.handlesearcherror.md | 3 +- ...n-plugins-data-public.searchinterceptor.md | 2 +- ...gins-data-server.getdefaultsearchparams.md | 12 +- ...gin-plugins-data-server.getshardtimeout.md | 12 +- ...ins-data-server.iesrawsearchresponse.id.md | 11 -- ...-server.iesrawsearchresponse.is_partial.md | 11 -- ...-server.iesrawsearchresponse.is_running.md | 11 -- ...lugins-data-server.iesrawsearchresponse.md | 20 --- .../kibana-plugin-plugins-data-server.md | 2 +- ...ibana-plugin-plugins-data-server.search.md | 18 --- ...plugins-data-server.searchusageobserver.md | 31 ++++ .../search/es_search/es_search_rxjs_utils.ts | 55 ------- .../search/es_search/get_total_loaded.test.ts | 36 ----- .../data/common/search/es_search/index.ts | 5 - .../es_search/shim_abort_signal.test.ts | 64 -------- .../search/es_search/shim_abort_signal.ts | 48 ------ .../common/search/es_search/to_snake_case.ts | 24 --- .../data/common/search/es_search/types.ts | 6 - src/plugins/data/common/search/index.ts | 1 + src/plugins/data/common/search/utils.test.ts | 106 +++++++++++++ .../common/search/{es_search => }/utils.ts | 2 +- src/plugins/data/public/public.api.md | 4 +- .../public/search/errors/painless_error.tsx | 3 +- .../public/search/search_interceptor.test.ts | 21 ++- .../data/public/search/search_interceptor.ts | 11 +- src/plugins/data/server/index.ts | 26 +-- .../data/server/search/collectors/index.ts | 3 +- .../data/server/search/collectors/usage.ts | 21 ++- .../search/es_search/es_search_rxjs_utils.ts | 53 ------- .../search/es_search/es_search_strategy.ts | 38 ++--- .../es_search/get_default_search_params.ts | 41 ----- .../data/server/search/es_search/index.ts | 5 +- .../search/es_search/request_utils.test.ts | 148 ++++++++++++++++++ .../server/search/es_search/request_utils.ts | 66 ++++++++ .../search/es_search/response_utils.test.ts | 69 ++++++++ .../search/es_search/response_utils.ts} | 18 ++- src/plugins/data/server/search/index.ts | 2 +- .../data/server/search/routes/call_msearch.ts | 7 +- src/plugins/data/server/server.api.md | 104 ++++++------ x-pack/plugins/data_enhanced/common/index.ts | 5 +- .../search/es_search/es_search_rxjs_utils.ts | 51 ------ .../common/search/es_search/index.ts | 7 - .../data_enhanced/common/search/index.ts | 2 +- .../common/search/poll_search.ts | 31 ++++ .../data_enhanced/common/search/types.ts | 15 -- .../public/search/search_interceptor.test.ts | 10 +- .../public/search/search_interceptor.ts | 54 ++----- .../server/search/eql_search_strategy.test.ts | 2 +- .../server/search/eql_search_strategy.ts | 81 ++++------ .../server/search/es_search_strategy.ts | 115 ++++++-------- .../search/get_default_search_params.ts | 33 ---- .../server/search/request_utils.ts | 64 ++++++++ .../server/search/response_utils.ts | 38 +++++ .../data_enhanced/server/search/types.ts | 20 +++ .../security_solution/index.ts | 22 ++- .../server/search_strategy/timeline/index.ts | 22 ++- 58 files changed, 842 insertions(+), 855 deletions(-) delete mode 100644 docs/development/plugins/data/server/kibana-plugin-plugins-data-server.iesrawsearchresponse.id.md delete mode 100644 docs/development/plugins/data/server/kibana-plugin-plugins-data-server.iesrawsearchresponse.is_partial.md delete mode 100644 docs/development/plugins/data/server/kibana-plugin-plugins-data-server.iesrawsearchresponse.is_running.md delete mode 100644 docs/development/plugins/data/server/kibana-plugin-plugins-data-server.iesrawsearchresponse.md create mode 100644 docs/development/plugins/data/server/kibana-plugin-plugins-data-server.searchusageobserver.md delete mode 100644 src/plugins/data/common/search/es_search/es_search_rxjs_utils.ts delete mode 100644 src/plugins/data/common/search/es_search/get_total_loaded.test.ts delete mode 100644 src/plugins/data/common/search/es_search/shim_abort_signal.test.ts delete mode 100644 src/plugins/data/common/search/es_search/shim_abort_signal.ts delete mode 100644 src/plugins/data/common/search/es_search/to_snake_case.ts create mode 100644 src/plugins/data/common/search/utils.test.ts rename src/plugins/data/common/search/{es_search => }/utils.ts (96%) delete mode 100644 src/plugins/data/server/search/es_search/es_search_rxjs_utils.ts delete mode 100644 src/plugins/data/server/search/es_search/get_default_search_params.ts create mode 100644 src/plugins/data/server/search/es_search/request_utils.test.ts create mode 100644 src/plugins/data/server/search/es_search/request_utils.ts create mode 100644 src/plugins/data/server/search/es_search/response_utils.test.ts rename src/plugins/data/{common/search/es_search/get_total_loaded.ts => server/search/es_search/response_utils.ts} (69%) delete mode 100644 x-pack/plugins/data_enhanced/common/search/es_search/es_search_rxjs_utils.ts delete mode 100644 x-pack/plugins/data_enhanced/common/search/es_search/index.ts create mode 100644 x-pack/plugins/data_enhanced/common/search/poll_search.ts delete mode 100644 x-pack/plugins/data_enhanced/server/search/get_default_search_params.ts create mode 100644 x-pack/plugins/data_enhanced/server/search/request_utils.ts create mode 100644 x-pack/plugins/data_enhanced/server/search/response_utils.ts create mode 100644 x-pack/plugins/data_enhanced/server/search/types.ts diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.painlesserror._constructor_.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.painlesserror._constructor_.md index 051414eac7585..5f43f8477cb9f 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.painlesserror._constructor_.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.painlesserror._constructor_.md @@ -9,7 +9,7 @@ Constructs a new instance of the `PainlessError` class Signature: ```typescript -constructor(err: IEsError, request: IKibanaSearchRequest); +constructor(err: IEsError); ``` ## Parameters @@ -17,5 +17,4 @@ constructor(err: IEsError, request: IKibanaSearchRequest); | Parameter | Type | Description | | --- | --- | --- | | err | IEsError | | -| request | IKibanaSearchRequest | | diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.painlesserror.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.painlesserror.md index 6ab32f3fb1dfa..c77b8b259136b 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.painlesserror.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.painlesserror.md @@ -14,7 +14,7 @@ export declare class PainlessError extends EsError | Constructor | Modifiers | Description | | --- | --- | --- | -| [(constructor)(err, request)](./kibana-plugin-plugins-data-public.painlesserror._constructor_.md) | | Constructs a new instance of the PainlessError class | +| [(constructor)(err)](./kibana-plugin-plugins-data-public.painlesserror._constructor_.md) | | Constructs a new instance of the PainlessError class | ## Properties diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptor.handlesearcherror.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptor.handlesearcherror.md index 1c8b6eb41a72e..b5ac4a4e53887 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptor.handlesearcherror.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptor.handlesearcherror.md @@ -7,7 +7,7 @@ Signature: ```typescript -protected handleSearchError(e: any, request: IKibanaSearchRequest, timeoutSignal: AbortSignal, options?: ISearchOptions): Error; +protected handleSearchError(e: any, timeoutSignal: AbortSignal, options?: ISearchOptions): Error; ``` ## Parameters @@ -15,7 +15,6 @@ protected handleSearchError(e: any, request: IKibanaSearchRequest, timeoutSignal | Parameter | Type | Description | | --- | --- | --- | | e | any | | -| request | IKibanaSearchRequest | | | timeoutSignal | AbortSignal | | | options | ISearchOptions | | diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptor.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptor.md index 40c7055e4c059..5f266e7d8bd8c 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptor.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptor.md @@ -27,7 +27,7 @@ export declare class SearchInterceptor | Method | Modifiers | Description | | --- | --- | --- | | [getTimeoutMode()](./kibana-plugin-plugins-data-public.searchinterceptor.gettimeoutmode.md) | | | -| [handleSearchError(e, request, timeoutSignal, options)](./kibana-plugin-plugins-data-public.searchinterceptor.handlesearcherror.md) | | | +| [handleSearchError(e, timeoutSignal, options)](./kibana-plugin-plugins-data-public.searchinterceptor.handlesearcherror.md) | | | | [search(request, options)](./kibana-plugin-plugins-data-public.searchinterceptor.search.md) | | Searches using the given search method. Overrides the AbortSignal with one that will abort either when cancelPending is called, when the request times out, or when the original AbortSignal is aborted. Updates pendingCount$ when the request is started/finalized. | | [showError(e)](./kibana-plugin-plugins-data-public.searchinterceptor.showerror.md) | | | diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.getdefaultsearchparams.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.getdefaultsearchparams.md index 3d9191196aaf0..19a4bbbbef86c 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.getdefaultsearchparams.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.getdefaultsearchparams.md @@ -7,11 +7,7 @@ Signature: ```typescript -export declare function getDefaultSearchParams(uiSettingsClient: IUiSettingsClient): Promise<{ - maxConcurrentShardRequests: number | undefined; - ignoreUnavailable: boolean; - trackTotalHits: boolean; -}>; +export declare function getDefaultSearchParams(uiSettingsClient: IUiSettingsClient): Promise>; ``` ## Parameters @@ -22,9 +18,5 @@ export declare function getDefaultSearchParams(uiSettingsClient: IUiSettingsClie Returns: -`Promise<{ - maxConcurrentShardRequests: number | undefined; - ignoreUnavailable: boolean; - trackTotalHits: boolean; -}>` +`Promise>` diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.getshardtimeout.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.getshardtimeout.md index d7e2a597ff33d..87aa32608eb14 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.getshardtimeout.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.getshardtimeout.md @@ -7,11 +7,7 @@ Signature: ```typescript -export declare function getShardTimeout(config: SharedGlobalConfig): { - timeout: string; -} | { - timeout?: undefined; -}; +export declare function getShardTimeout(config: SharedGlobalConfig): Pick; ``` ## Parameters @@ -22,9 +18,5 @@ export declare function getShardTimeout(config: SharedGlobalConfig): { Returns: -`{ - timeout: string; -} | { - timeout?: undefined; -}` +`Pick` diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.iesrawsearchresponse.id.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.iesrawsearchresponse.id.md deleted file mode 100644 index 8e1d5d01bb664..0000000000000 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.iesrawsearchresponse.id.md +++ /dev/null @@ -1,11 +0,0 @@ - - -[Home](./index.md) > [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) > [IEsRawSearchResponse](./kibana-plugin-plugins-data-server.iesrawsearchresponse.md) > [id](./kibana-plugin-plugins-data-server.iesrawsearchresponse.id.md) - -## IEsRawSearchResponse.id property - -Signature: - -```typescript -id?: string; -``` diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.iesrawsearchresponse.is_partial.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.iesrawsearchresponse.is_partial.md deleted file mode 100644 index da2a57a84ab2f..0000000000000 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.iesrawsearchresponse.is_partial.md +++ /dev/null @@ -1,11 +0,0 @@ - - -[Home](./index.md) > [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) > [IEsRawSearchResponse](./kibana-plugin-plugins-data-server.iesrawsearchresponse.md) > [is\_partial](./kibana-plugin-plugins-data-server.iesrawsearchresponse.is_partial.md) - -## IEsRawSearchResponse.is\_partial property - -Signature: - -```typescript -is_partial?: boolean; -``` diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.iesrawsearchresponse.is_running.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.iesrawsearchresponse.is_running.md deleted file mode 100644 index 78b9e07b77890..0000000000000 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.iesrawsearchresponse.is_running.md +++ /dev/null @@ -1,11 +0,0 @@ - - -[Home](./index.md) > [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) > [IEsRawSearchResponse](./kibana-plugin-plugins-data-server.iesrawsearchresponse.md) > [is\_running](./kibana-plugin-plugins-data-server.iesrawsearchresponse.is_running.md) - -## IEsRawSearchResponse.is\_running property - -Signature: - -```typescript -is_running?: boolean; -``` diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.iesrawsearchresponse.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.iesrawsearchresponse.md deleted file mode 100644 index 306c18dea9b0d..0000000000000 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.iesrawsearchresponse.md +++ /dev/null @@ -1,20 +0,0 @@ - - -[Home](./index.md) > [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) > [IEsRawSearchResponse](./kibana-plugin-plugins-data-server.iesrawsearchresponse.md) - -## IEsRawSearchResponse interface - -Signature: - -```typescript -export interface IEsRawSearchResponse extends SearchResponse -``` - -## Properties - -| Property | Type | Description | -| --- | --- | --- | -| [id](./kibana-plugin-plugins-data-server.iesrawsearchresponse.id.md) | string | | -| [is\_partial](./kibana-plugin-plugins-data-server.iesrawsearchresponse.is_partial.md) | boolean | | -| [is\_running](./kibana-plugin-plugins-data-server.iesrawsearchresponse.is_running.md) | boolean | | - diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.md index d9f14950be0e8..c85f294d162bc 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.md @@ -34,6 +34,7 @@ | [getTime(indexPattern, timeRange, options)](./kibana-plugin-plugins-data-server.gettime.md) | | | [parseInterval(interval)](./kibana-plugin-plugins-data-server.parseinterval.md) | | | [plugin(initializerContext)](./kibana-plugin-plugins-data-server.plugin.md) | Static code to be shared externally | +| [searchUsageObserver(logger, usage)](./kibana-plugin-plugins-data-server.searchusageobserver.md) | Rxjs observer for easily doing tap(searchUsageObserver(logger, usage)) in an rxjs chain. | | [shouldReadFieldFromDocValues(aggregatable, esType)](./kibana-plugin-plugins-data-server.shouldreadfieldfromdocvalues.md) | | | [usageProvider(core)](./kibana-plugin-plugins-data-server.usageprovider.md) | | @@ -45,7 +46,6 @@ | [EsQueryConfig](./kibana-plugin-plugins-data-server.esqueryconfig.md) | | | [FieldDescriptor](./kibana-plugin-plugins-data-server.fielddescriptor.md) | | | [FieldFormatConfig](./kibana-plugin-plugins-data-server.fieldformatconfig.md) | | -| [IEsRawSearchResponse](./kibana-plugin-plugins-data-server.iesrawsearchresponse.md) | | | [IEsSearchRequest](./kibana-plugin-plugins-data-server.iessearchrequest.md) | | | [IFieldSubType](./kibana-plugin-plugins-data-server.ifieldsubtype.md) | | | [IFieldType](./kibana-plugin-plugins-data-server.ifieldtype.md) | | diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.search.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.search.md index 77abcacd7704a..4f8a0beefa421 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.search.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.search.md @@ -8,24 +8,6 @@ ```typescript search: { - esSearch: { - utils: { - doSearch: (searchMethod: () => Promise, abortSignal?: AbortSignal | undefined) => import("rxjs").Observable; - shimAbortSignal: >(promise: T, signal: AbortSignal | undefined) => T; - trackSearchStatus: = import("./search").IEsSearchResponse>>(logger: import("src/core/server").Logger, usage?: import("./search").SearchUsage | undefined) => import("rxjs").UnaryFunction, import("rxjs").Observable>; - includeTotalLoaded: () => import("rxjs").OperatorFunction>, { - total: number; - loaded: number; - id?: string | undefined; - isRunning?: boolean | undefined; - isPartial?: boolean | undefined; - rawResponse: import("elasticsearch").SearchResponse; - }>; - toKibanaSearchResponse: = import("../common").IEsRawSearchResponse, KibanaResponse_1 extends import("../common").IKibanaSearchResponse = import("../common").IKibanaSearchResponse>() => import("rxjs").OperatorFunction, KibanaResponse_1>; - getTotalLoaded: typeof getTotalLoaded; - toSnakeCase: typeof toSnakeCase; - }; - }; aggs: { CidrMask: typeof CidrMask; dateHistogramInterval: typeof dateHistogramInterval; diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.searchusageobserver.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.searchusageobserver.md new file mode 100644 index 0000000000000..5e03bb381527e --- /dev/null +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.searchusageobserver.md @@ -0,0 +1,31 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) > [searchUsageObserver](./kibana-plugin-plugins-data-server.searchusageobserver.md) + +## searchUsageObserver() function + +Rxjs observer for easily doing `tap(searchUsageObserver(logger, usage))` in an rxjs chain. + +Signature: + +```typescript +export declare function searchUsageObserver(logger: Logger, usage?: SearchUsage): { + next(response: IEsSearchResponse): void; + error(): void; +}; +``` + +## Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| logger | Logger | | +| usage | SearchUsage | | + +Returns: + +`{ + next(response: IEsSearchResponse): void; + error(): void; +}` + diff --git a/src/plugins/data/common/search/es_search/es_search_rxjs_utils.ts b/src/plugins/data/common/search/es_search/es_search_rxjs_utils.ts deleted file mode 100644 index e3238ea62db57..0000000000000 --- a/src/plugins/data/common/search/es_search/es_search_rxjs_utils.ts +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { from } from 'rxjs'; -import { map } from 'rxjs/operators'; - -import type { SearchResponse } from 'elasticsearch'; -import type { ApiResponse } from '@elastic/elasticsearch'; - -import { shimAbortSignal } from './shim_abort_signal'; -import { getTotalLoaded } from './get_total_loaded'; - -import type { IEsRawSearchResponse } from './types'; -import type { IKibanaSearchResponse } from '../types'; - -export const doSearch = ( - searchMethod: () => Promise, - abortSignal?: AbortSignal -) => from(shimAbortSignal(searchMethod(), abortSignal)); - -export const toKibanaSearchResponse = < - SearchResponse extends IEsRawSearchResponse = IEsRawSearchResponse, - KibanaResponse extends IKibanaSearchResponse = IKibanaSearchResponse ->() => - map, KibanaResponse>( - (response) => - ({ - id: response.body.id, - isPartial: response.body.is_partial || false, - isRunning: response.body.is_running || false, - rawResponse: response.body, - } as KibanaResponse) - ); - -export const includeTotalLoaded = () => - map((response: IKibanaSearchResponse>) => ({ - ...response, - ...getTotalLoaded(response.rawResponse._shards), - })); diff --git a/src/plugins/data/common/search/es_search/get_total_loaded.test.ts b/src/plugins/data/common/search/es_search/get_total_loaded.test.ts deleted file mode 100644 index 74e2873ede762..0000000000000 --- a/src/plugins/data/common/search/es_search/get_total_loaded.test.ts +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { getTotalLoaded } from './get_total_loaded'; - -describe('getTotalLoaded', () => { - it('returns the total/loaded, not including skipped', () => { - const result = getTotalLoaded({ - successful: 10, - failed: 5, - skipped: 5, - total: 100, - }); - - expect(result).toEqual({ - total: 100, - loaded: 15, - }); - }); -}); diff --git a/src/plugins/data/common/search/es_search/index.ts b/src/plugins/data/common/search/es_search/index.ts index 555667a9f5300..d8f7b5091eb8f 100644 --- a/src/plugins/data/common/search/es_search/index.ts +++ b/src/plugins/data/common/search/es_search/index.ts @@ -18,8 +18,3 @@ */ export * from './types'; -export * from './utils'; -export * from './es_search_rxjs_utils'; -export * from './shim_abort_signal'; -export * from './to_snake_case'; -export * from './get_total_loaded'; diff --git a/src/plugins/data/common/search/es_search/shim_abort_signal.test.ts b/src/plugins/data/common/search/es_search/shim_abort_signal.test.ts deleted file mode 100644 index 61af8b4c782ae..0000000000000 --- a/src/plugins/data/common/search/es_search/shim_abort_signal.test.ts +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { shimAbortSignal } from './shim_abort_signal'; - -const createSuccessTransportRequestPromise = ( - body: any, - { statusCode = 200 }: { statusCode?: number } = {} -) => { - const promise = Promise.resolve({ body, statusCode }) as any; - promise.abort = jest.fn(); - - return promise; -}; - -describe('shimAbortSignal', () => { - test('aborts the promise if the signal is aborted', () => { - const promise = createSuccessTransportRequestPromise({ - success: true, - }); - const controller = new AbortController(); - shimAbortSignal(promise, controller.signal); - controller.abort(); - - expect(promise.abort).toHaveBeenCalled(); - }); - - test('returns the original promise', async () => { - const promise = createSuccessTransportRequestPromise({ - success: true, - }); - const controller = new AbortController(); - const response = await shimAbortSignal(promise, controller.signal); - - expect(response).toEqual(expect.objectContaining({ body: { success: true } })); - }); - - test('allows the promise to be aborted manually', () => { - const promise = createSuccessTransportRequestPromise({ - success: true, - }); - const controller = new AbortController(); - const enhancedPromise = shimAbortSignal(promise, controller.signal); - - enhancedPromise.abort(); - expect(promise.abort).toHaveBeenCalled(); - }); -}); diff --git a/src/plugins/data/common/search/es_search/shim_abort_signal.ts b/src/plugins/data/common/search/es_search/shim_abort_signal.ts deleted file mode 100644 index 554a24e268815..0000000000000 --- a/src/plugins/data/common/search/es_search/shim_abort_signal.ts +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/** - * @internal - * TransportRequestPromise extends base Promise with an "abort" method - */ -export interface TransportRequestPromise extends Promise { - abort?: () => void; -} - -/** - * - * @internal - * NOTE: Temporary workaround until https://github.com/elastic/elasticsearch-js/issues/1297 - * is resolved - * - * @param promise a TransportRequestPromise - * @param signal optional AbortSignal - * - * @returns a TransportRequestPromise that will be aborted if the signal is aborted - */ - -export const shimAbortSignal = >( - promise: T, - signal: AbortSignal | undefined -): T => { - if (signal) { - signal.addEventListener('abort', () => promise.abort && promise.abort()); - } - return promise; -}; diff --git a/src/plugins/data/common/search/es_search/to_snake_case.ts b/src/plugins/data/common/search/es_search/to_snake_case.ts deleted file mode 100644 index b222a56fbf602..0000000000000 --- a/src/plugins/data/common/search/es_search/to_snake_case.ts +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { mapKeys, snakeCase } from 'lodash'; - -export function toSnakeCase(obj: Record): Record { - return mapKeys(obj, (value, key) => snakeCase(key)); -} diff --git a/src/plugins/data/common/search/es_search/types.ts b/src/plugins/data/common/search/es_search/types.ts index 7d81cf42e1866..7dbbd01d2cdad 100644 --- a/src/plugins/data/common/search/es_search/types.ts +++ b/src/plugins/data/common/search/es_search/types.ts @@ -30,10 +30,4 @@ export interface IEsSearchRequest extends IKibanaSearchRequest extends SearchResponse { - id?: string; - is_partial?: boolean; - is_running?: boolean; -} - export type IEsSearchResponse = IKibanaSearchResponse>; diff --git a/src/plugins/data/common/search/index.ts b/src/plugins/data/common/search/index.ts index e650cf10db87c..01944d6e37aaf 100644 --- a/src/plugins/data/common/search/index.ts +++ b/src/plugins/data/common/search/index.ts @@ -24,3 +24,4 @@ export * from './search_source'; export * from './tabify'; export * from './types'; export * from './session'; +export * from './utils'; diff --git a/src/plugins/data/common/search/utils.test.ts b/src/plugins/data/common/search/utils.test.ts new file mode 100644 index 0000000000000..94f7b14de4bc3 --- /dev/null +++ b/src/plugins/data/common/search/utils.test.ts @@ -0,0 +1,106 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { isErrorResponse, isCompleteResponse, isPartialResponse } from './utils'; + +describe('utils', () => { + describe('isErrorResponse', () => { + it('returns `true` if the response is undefined', () => { + const isError = isErrorResponse(); + expect(isError).toBe(true); + }); + + it('returns `true` if the response is not running and partial', () => { + const isError = isErrorResponse({ + isPartial: true, + isRunning: false, + rawResponse: {}, + }); + expect(isError).toBe(true); + }); + + it('returns `false` if the response is running and partial', () => { + const isError = isErrorResponse({ + isPartial: true, + isRunning: true, + rawResponse: {}, + }); + expect(isError).toBe(false); + }); + + it('returns `false` if the response is complete', () => { + const isError = isErrorResponse({ + isPartial: false, + isRunning: false, + rawResponse: {}, + }); + expect(isError).toBe(false); + }); + }); + + describe('isCompleteResponse', () => { + it('returns `false` if the response is undefined', () => { + const isError = isCompleteResponse(); + expect(isError).toBe(false); + }); + + it('returns `false` if the response is running and partial', () => { + const isError = isCompleteResponse({ + isPartial: true, + isRunning: true, + rawResponse: {}, + }); + expect(isError).toBe(false); + }); + + it('returns `true` if the response is complete', () => { + const isError = isCompleteResponse({ + isPartial: false, + isRunning: false, + rawResponse: {}, + }); + expect(isError).toBe(true); + }); + }); + + describe('isPartialResponse', () => { + it('returns `false` if the response is undefined', () => { + const isError = isPartialResponse(); + expect(isError).toBe(false); + }); + + it('returns `true` if the response is running and partial', () => { + const isError = isPartialResponse({ + isPartial: true, + isRunning: true, + rawResponse: {}, + }); + expect(isError).toBe(true); + }); + + it('returns `false` if the response is complete', () => { + const isError = isPartialResponse({ + isPartial: false, + isRunning: false, + rawResponse: {}, + }); + expect(isError).toBe(false); + }); + }); +}); diff --git a/src/plugins/data/common/search/es_search/utils.ts b/src/plugins/data/common/search/utils.ts similarity index 96% rename from src/plugins/data/common/search/es_search/utils.ts rename to src/plugins/data/common/search/utils.ts index 6ed222ab0830c..0d544a51c2d45 100644 --- a/src/plugins/data/common/search/es_search/utils.ts +++ b/src/plugins/data/common/search/utils.ts @@ -17,7 +17,7 @@ * under the License. */ -import type { IKibanaSearchResponse } from '../types'; +import type { IKibanaSearchResponse } from './types'; /** * @returns true if response had an error while executing in ES diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index 2c47ecb27184d..12d6fc5ad32c7 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -1686,7 +1686,7 @@ export interface OptionedValueProp { // @public (undocumented) export class PainlessError extends EsError { // Warning: (ae-forgotten-export) The symbol "IEsError" needs to be exported by the entry point index.d.ts - constructor(err: IEsError, request: IKibanaSearchRequest); + constructor(err: IEsError); // (undocumented) getErrorMessage(application: ApplicationStart): JSX.Element; // (undocumented) @@ -2090,7 +2090,7 @@ export class SearchInterceptor { // (undocumented) protected getTimeoutMode(): TimeoutErrorMode; // (undocumented) - protected handleSearchError(e: any, request: IKibanaSearchRequest, timeoutSignal: AbortSignal, options?: ISearchOptions): Error; + protected handleSearchError(e: any, timeoutSignal: AbortSignal, options?: ISearchOptions): Error; // @internal protected pendingCount$: BehaviorSubject; // @internal (undocumented) diff --git a/src/plugins/data/public/search/errors/painless_error.tsx b/src/plugins/data/public/search/errors/painless_error.tsx index 282a602d358c7..3cfe9f4278ba0 100644 --- a/src/plugins/data/public/search/errors/painless_error.tsx +++ b/src/plugins/data/public/search/errors/painless_error.tsx @@ -25,11 +25,10 @@ import { ApplicationStart } from 'kibana/public'; import { IEsError, isEsError } from './types'; import { EsError } from './es_error'; import { getRootCause } from './utils'; -import { IKibanaSearchRequest } from '..'; export class PainlessError extends EsError { painlessStack?: string; - constructor(err: IEsError, request: IKibanaSearchRequest) { + constructor(err: IEsError) { super(err); } diff --git a/src/plugins/data/public/search/search_interceptor.test.ts b/src/plugins/data/public/search/search_interceptor.test.ts index 60274261da25f..6e75f6e5eef9e 100644 --- a/src/plugins/data/public/search/search_interceptor.test.ts +++ b/src/plugins/data/public/search/search_interceptor.test.ts @@ -65,20 +65,17 @@ describe('SearchInterceptor', () => { test('Renders a PainlessError', async () => { searchInterceptor.showError( - new PainlessError( - { - body: { - attributes: { - error: { - failed_shards: { - reason: 'bananas', - }, + new PainlessError({ + body: { + attributes: { + error: { + failed_shards: { + reason: 'bananas', }, }, - } as any, - }, - {} as any - ) + }, + } as any, + }) ); expect(mockCoreSetup.notifications.toasts.addDanger).toBeCalledTimes(1); expect(mockCoreSetup.notifications.toasts.addError).not.toBeCalled(); diff --git a/src/plugins/data/public/search/search_interceptor.ts b/src/plugins/data/public/search/search_interceptor.ts index 3fadb723b27cd..e5abac0d48fef 100644 --- a/src/plugins/data/public/search/search_interceptor.ts +++ b/src/plugins/data/public/search/search_interceptor.ts @@ -93,12 +93,7 @@ export class SearchInterceptor { * @returns `Error` a search service specific error or the original error, if a specific error can't be recognized. * @internal */ - protected handleSearchError( - e: any, - request: IKibanaSearchRequest, - timeoutSignal: AbortSignal, - options?: ISearchOptions - ): Error { + protected handleSearchError(e: any, timeoutSignal: AbortSignal, options?: ISearchOptions): Error { if (timeoutSignal.aborted || get(e, 'body.message') === 'Request timed out') { // Handle a client or a server side timeout const err = new SearchTimeoutError(e, this.getTimeoutMode()); @@ -112,7 +107,7 @@ export class SearchInterceptor { return e; } else if (isEsError(e)) { if (isPainlessError(e)) { - return new PainlessError(e, request); + return new PainlessError(e); } else { return new EsError(e); } @@ -244,7 +239,7 @@ export class SearchInterceptor { this.pendingCount$.next(this.pendingCount$.getValue() + 1); return from(this.runSearch(request, { ...options, abortSignal: combinedSignal })).pipe( catchError((e: Error) => { - return throwError(this.handleSearchError(e, request, timeoutSignal, options)); + return throwError(this.handleSearchError(e, timeoutSignal, options)); }), finalize(() => { this.pendingCount$.next(this.pendingCount$.getValue() - 1); diff --git a/src/plugins/data/server/index.ts b/src/plugins/data/server/index.ts index b3fe412152c9d..a233447cdf438 100644 --- a/src/plugins/data/server/index.ts +++ b/src/plugins/data/server/index.ts @@ -156,7 +156,6 @@ export { IndexPatternAttributes, UI_SETTINGS, IndexPattern, - IEsRawSearchResponse, } from '../common'; /** @@ -189,13 +188,6 @@ import { // tabify tabifyAggResponse, tabifyGetColumns, - // search - toSnakeCase, - shimAbortSignal, - doSearch, - includeTotalLoaded, - toKibanaSearchResponse, - getTotalLoaded, calcAutoIntervalLessThan, } from '../common'; @@ -243,27 +235,17 @@ export { SearchStrategyDependencies, getDefaultSearchParams, getShardTimeout, + getTotalLoaded, + toKibanaSearchResponse, shimHitsTotal, usageProvider, + searchUsageObserver, + shimAbortSignal, SearchUsage, } from './search'; -import { trackSearchStatus } from './search'; - // Search namespace export const search = { - esSearch: { - utils: { - doSearch, - shimAbortSignal, - trackSearchStatus, - includeTotalLoaded, - toKibanaSearchResponse, - // utils: - getTotalLoaded, - toSnakeCase, - }, - }, aggs: { CidrMask, dateHistogramInterval, diff --git a/src/plugins/data/server/search/collectors/index.ts b/src/plugins/data/server/search/collectors/index.ts index 417dc1c2012d3..8ad6501d505eb 100644 --- a/src/plugins/data/server/search/collectors/index.ts +++ b/src/plugins/data/server/search/collectors/index.ts @@ -17,4 +17,5 @@ * under the License. */ -export { usageProvider, SearchUsage } from './usage'; +export type { SearchUsage } from './usage'; +export { usageProvider, searchUsageObserver } from './usage'; diff --git a/src/plugins/data/server/search/collectors/usage.ts b/src/plugins/data/server/search/collectors/usage.ts index e1be92aa13c37..948175a41cb6b 100644 --- a/src/plugins/data/server/search/collectors/usage.ts +++ b/src/plugins/data/server/search/collectors/usage.ts @@ -17,8 +17,9 @@ * under the License. */ -import { CoreSetup } from 'kibana/server'; -import { Usage } from './register'; +import type { CoreSetup, Logger } from 'kibana/server'; +import type { IEsSearchResponse } from '../../../common'; +import type { Usage } from './register'; const SAVED_OBJECT_ID = 'search-telemetry'; @@ -74,3 +75,19 @@ export function usageProvider(core: CoreSetup): SearchUsage { trackSuccess: getTracker('successCount'), }; } + +/** + * Rxjs observer for easily doing `tap(searchUsageObserver(logger, usage))` in an rxjs chain. + */ +export function searchUsageObserver(logger: Logger, usage?: SearchUsage) { + return { + next(response: IEsSearchResponse) { + logger.debug(`trackSearchStatus:next ${response.rawResponse.took}`); + usage?.trackSuccess(response.rawResponse.took); + }, + error() { + logger.debug(`trackSearchStatus:error`); + usage?.trackError(); + }, + }; +} diff --git a/src/plugins/data/server/search/es_search/es_search_rxjs_utils.ts b/src/plugins/data/server/search/es_search/es_search_rxjs_utils.ts deleted file mode 100644 index 3ba2f9c4b2698..0000000000000 --- a/src/plugins/data/server/search/es_search/es_search_rxjs_utils.ts +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -import { pipe } from 'rxjs'; -import { tap } from 'rxjs/operators'; - -import type { Logger, SearchResponse } from 'kibana/server'; -import type { SearchUsage } from '../collectors'; -import type { IEsSearchResponse, IKibanaSearchResponse } from '../../../common/search'; - -/** - * trackSearchStatus is a custom rxjs operator that can be used to track the progress of a search. - * @param Logger - * @param SearchUsage - */ -export const trackSearchStatus = < - KibanaResponse extends IKibanaSearchResponse = IEsSearchResponse> ->( - logger: Logger, - usage?: SearchUsage -) => { - return pipe( - tap( - (response: KibanaResponse) => { - const trackSuccessData = response.rawResponse.took; - - if (trackSuccessData !== undefined) { - logger.debug(`trackSearchStatus:next ${trackSuccessData}`); - usage?.trackSuccess(trackSuccessData); - } - }, - (err: any) => { - logger.debug(`trackSearchStatus:error ${err}`); - usage?.trackError(); - } - ) - ); -}; diff --git a/src/plugins/data/server/search/es_search/es_search_strategy.ts b/src/plugins/data/server/search/es_search/es_search_strategy.ts index 3e2d415eac16f..620df9c8edcb0 100644 --- a/src/plugins/data/server/search/es_search/es_search_strategy.ts +++ b/src/plugins/data/server/search/es_search/es_search_strategy.ts @@ -16,20 +16,15 @@ * specific language governing permissions and limitations * under the License. */ -import { Observable } from 'rxjs'; -import { first } from 'rxjs/operators'; - -import type { Logger } from 'kibana/server'; -import type { ApiResponse } from '@elastic/elasticsearch'; -import type { SharedGlobalConfig } from 'kibana/server'; - -import { doSearch, includeTotalLoaded, toKibanaSearchResponse, toSnakeCase } from '../../../common'; -import { trackSearchStatus } from './es_search_rxjs_utils'; -import { getDefaultSearchParams, getShardTimeout } from '../es_search'; - +import { from, Observable } from 'rxjs'; +import { first, tap } from 'rxjs/operators'; +import type { SearchResponse } from 'elasticsearch'; +import type { Logger, SharedGlobalConfig } from 'kibana/server'; import type { ISearchStrategy } from '../types'; -import type { SearchUsage } from '../collectors/usage'; -import type { IEsRawSearchResponse } from '../../../common'; +import type { SearchUsage } from '../collectors'; +import { getDefaultSearchParams, getShardTimeout, shimAbortSignal } from './request_utils'; +import { toKibanaSearchResponse } from './response_utils'; +import { searchUsageObserver } from '../collectors/usage'; export const esSearchStrategyProvider = ( config$: Observable, @@ -43,19 +38,18 @@ export const esSearchStrategyProvider = ( throw new Error(`Unsupported index pattern type ${request.indexType}`); } - return doSearch>(async () => { + const search = async () => { const config = await config$.pipe(first()).toPromise(); - const params = toSnakeCase({ + const params = { ...(await getDefaultSearchParams(uiSettingsClient)), ...getShardTimeout(config), ...request.params, - }); + }; + const promise = esClient.asCurrentUser.search>(params); + const { body } = await shimAbortSignal(promise, abortSignal); + return toKibanaSearchResponse(body); + }; - return esClient.asCurrentUser.search(params); - }, abortSignal).pipe( - toKibanaSearchResponse(), - trackSearchStatus(logger, usage), - includeTotalLoaded() - ); + return from(search()).pipe(tap(searchUsageObserver(logger, usage))); }, }); diff --git a/src/plugins/data/server/search/es_search/get_default_search_params.ts b/src/plugins/data/server/search/es_search/get_default_search_params.ts deleted file mode 100644 index a01b0885abf3b..0000000000000 --- a/src/plugins/data/server/search/es_search/get_default_search_params.ts +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -import { UI_SETTINGS } from '../../../common/constants'; -import type { SharedGlobalConfig, IUiSettingsClient } from '../../../../../core/server'; - -export function getShardTimeout(config: SharedGlobalConfig) { - const timeout = config.elasticsearch.shardTimeout.asMilliseconds(); - return timeout - ? { - timeout: `${timeout}ms`, - } - : {}; -} - -export async function getDefaultSearchParams(uiSettingsClient: IUiSettingsClient) { - const maxConcurrentShardRequests = await uiSettingsClient.get( - UI_SETTINGS.COURIER_MAX_CONCURRENT_SHARD_REQUESTS - ); - return { - maxConcurrentShardRequests: - maxConcurrentShardRequests > 0 ? maxConcurrentShardRequests : undefined, - ignoreUnavailable: true, // Don't fail if the index/indices don't exist - trackTotalHits: true, - }; -} diff --git a/src/plugins/data/server/search/es_search/index.ts b/src/plugins/data/server/search/es_search/index.ts index 14e8a4e1b0245..f6487e3ef84f5 100644 --- a/src/plugins/data/server/search/es_search/index.ts +++ b/src/plugins/data/server/search/es_search/index.ts @@ -18,7 +18,6 @@ */ export { esSearchStrategyProvider } from './es_search_strategy'; -export * from './get_default_search_params'; -export * from './es_search_rxjs_utils'; - +export * from './request_utils'; +export * from './response_utils'; export { ES_SEARCH_STRATEGY, IEsSearchRequest, IEsSearchResponse } from '../../../common'; diff --git a/src/plugins/data/server/search/es_search/request_utils.test.ts b/src/plugins/data/server/search/es_search/request_utils.test.ts new file mode 100644 index 0000000000000..b63a6b3ae7e9b --- /dev/null +++ b/src/plugins/data/server/search/es_search/request_utils.test.ts @@ -0,0 +1,148 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { getShardTimeout, getDefaultSearchParams, shimAbortSignal } from './request_utils'; +import { IUiSettingsClient, SharedGlobalConfig } from 'kibana/server'; + +const createSuccessTransportRequestPromise = ( + body: any, + { statusCode = 200 }: { statusCode?: number } = {} +) => { + const promise = Promise.resolve({ body, statusCode }) as any; + promise.abort = jest.fn(); + + return promise; +}; + +describe('request utils', () => { + describe('getShardTimeout', () => { + test('returns an empty object if the config does not contain a value', () => { + const result = getShardTimeout(({ + elasticsearch: { + shardTimeout: { + asMilliseconds: jest.fn(), + }, + }, + } as unknown) as SharedGlobalConfig); + expect(result).toEqual({}); + }); + + test('returns an empty object if the config contains 0', () => { + const result = getShardTimeout(({ + elasticsearch: { + shardTimeout: { + asMilliseconds: jest.fn().mockReturnValue(0), + }, + }, + } as unknown) as SharedGlobalConfig); + expect(result).toEqual({}); + }); + + test('returns a duration if the config >= 0', () => { + const result = getShardTimeout(({ + elasticsearch: { + shardTimeout: { + asMilliseconds: jest.fn().mockReturnValue(10), + }, + }, + } as unknown) as SharedGlobalConfig); + expect(result).toEqual({ timeout: '10ms' }); + }); + }); + + describe('getDefaultSearchParams', () => { + describe('max_concurrent_shard_requests', () => { + test('returns value if > 0', async () => { + const result = await getDefaultSearchParams(({ + get: jest.fn().mockResolvedValue(1), + } as unknown) as IUiSettingsClient); + expect(result).toHaveProperty('max_concurrent_shard_requests', 1); + }); + + test('returns undefined if === 0', async () => { + const result = await getDefaultSearchParams(({ + get: jest.fn().mockResolvedValue(0), + } as unknown) as IUiSettingsClient); + expect(result.max_concurrent_shard_requests).toBe(undefined); + }); + + test('returns undefined if undefined', async () => { + const result = await getDefaultSearchParams(({ + get: jest.fn(), + } as unknown) as IUiSettingsClient); + expect(result.max_concurrent_shard_requests).toBe(undefined); + }); + }); + + describe('other defaults', () => { + test('returns ignore_unavailable and track_total_hits', async () => { + const result = await getDefaultSearchParams(({ + get: jest.fn(), + } as unknown) as IUiSettingsClient); + expect(result).toHaveProperty('ignore_unavailable', true); + expect(result).toHaveProperty('track_total_hits', true); + }); + }); + }); + + describe('shimAbortSignal', () => { + test('aborts the promise if the signal is already aborted', async () => { + const promise = createSuccessTransportRequestPromise({ + success: true, + }); + const controller = new AbortController(); + controller.abort(); + shimAbortSignal(promise, controller.signal); + + expect(promise.abort).toHaveBeenCalled(); + }); + + test('aborts the promise if the signal is aborted', () => { + const promise = createSuccessTransportRequestPromise({ + success: true, + }); + const controller = new AbortController(); + shimAbortSignal(promise, controller.signal); + controller.abort(); + + expect(promise.abort).toHaveBeenCalled(); + }); + + test('returns the original promise', async () => { + const promise = createSuccessTransportRequestPromise({ + success: true, + }); + const controller = new AbortController(); + const response = await shimAbortSignal(promise, controller.signal); + + expect(response).toEqual(expect.objectContaining({ body: { success: true } })); + }); + + test('allows the promise to be aborted manually', () => { + const promise = createSuccessTransportRequestPromise({ + success: true, + }); + const controller = new AbortController(); + const enhancedPromise = shimAbortSignal(promise, controller.signal); + + enhancedPromise.abort(); + expect(promise.abort).toHaveBeenCalled(); + }); + }); +}); diff --git a/src/plugins/data/server/search/es_search/request_utils.ts b/src/plugins/data/server/search/es_search/request_utils.ts new file mode 100644 index 0000000000000..03b7db7da8ffe --- /dev/null +++ b/src/plugins/data/server/search/es_search/request_utils.ts @@ -0,0 +1,66 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import type { TransportRequestPromise } from '@elastic/elasticsearch/lib/Transport'; +import type { Search } from '@elastic/elasticsearch/api/requestParams'; +import type { IUiSettingsClient, SharedGlobalConfig } from 'kibana/server'; +import { UI_SETTINGS } from '../../../common'; + +export function getShardTimeout(config: SharedGlobalConfig): Pick { + const timeout = config.elasticsearch.shardTimeout.asMilliseconds(); + return timeout ? { timeout: `${timeout}ms` } : {}; +} + +export async function getDefaultSearchParams( + uiSettingsClient: IUiSettingsClient +): Promise< + Pick +> { + const maxConcurrentShardRequests = await uiSettingsClient.get( + UI_SETTINGS.COURIER_MAX_CONCURRENT_SHARD_REQUESTS + ); + return { + max_concurrent_shard_requests: + maxConcurrentShardRequests > 0 ? maxConcurrentShardRequests : undefined, + ignore_unavailable: true, // Don't fail if the index/indices don't exist + track_total_hits: true, + }; +} + +/** + * Temporary workaround until https://github.com/elastic/elasticsearch-js/issues/1297 is resolved. + * Shims the `AbortSignal` behavior so that, if the given `signal` aborts, the `abort` method on the + * `TransportRequestPromise` is called, actually performing the cancellation. + * @internal + */ +export const shimAbortSignal = (promise: TransportRequestPromise, signal?: AbortSignal) => { + if (!signal) return promise; + const abortHandler = () => { + promise.abort(); + cleanup(); + }; + const cleanup = () => signal.removeEventListener('abort', abortHandler); + if (signal.aborted) { + promise.abort(); + } else { + signal.addEventListener('abort', abortHandler); + promise.then(cleanup, cleanup); + } + return promise; +}; diff --git a/src/plugins/data/server/search/es_search/response_utils.test.ts b/src/plugins/data/server/search/es_search/response_utils.test.ts new file mode 100644 index 0000000000000..f93625980a69c --- /dev/null +++ b/src/plugins/data/server/search/es_search/response_utils.test.ts @@ -0,0 +1,69 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { getTotalLoaded, toKibanaSearchResponse } from './response_utils'; +import { SearchResponse } from 'elasticsearch'; + +describe('response utils', () => { + describe('getTotalLoaded', () => { + it('returns the total/loaded, not including skipped', () => { + const result = getTotalLoaded(({ + _shards: { + successful: 10, + failed: 5, + skipped: 5, + total: 100, + }, + } as unknown) as SearchResponse); + + expect(result).toEqual({ + total: 100, + loaded: 15, + }); + }); + }); + + describe('toKibanaSearchResponse', () => { + it('returns rawResponse, isPartial, isRunning, total, and loaded', () => { + const result = toKibanaSearchResponse(({ + _shards: { + successful: 10, + failed: 5, + skipped: 5, + total: 100, + }, + } as unknown) as SearchResponse); + + expect(result).toEqual({ + rawResponse: { + _shards: { + successful: 10, + failed: 5, + skipped: 5, + total: 100, + }, + }, + isRunning: false, + isPartial: false, + total: 100, + loaded: 15, + }); + }); + }); +}); diff --git a/src/plugins/data/common/search/es_search/get_total_loaded.ts b/src/plugins/data/server/search/es_search/response_utils.ts similarity index 69% rename from src/plugins/data/common/search/es_search/get_total_loaded.ts rename to src/plugins/data/server/search/es_search/response_utils.ts index 233bcf8186666..2f502f55057b8 100644 --- a/src/plugins/data/common/search/es_search/get_total_loaded.ts +++ b/src/plugins/data/server/search/es_search/response_utils.ts @@ -17,14 +17,28 @@ * under the License. */ -import type { ShardsResponse } from 'elasticsearch'; +import { SearchResponse } from 'elasticsearch'; /** * Get the `total`/`loaded` for this response (see `IKibanaSearchResponse`). Note that `skipped` is * not included as it is already included in `successful`. * @internal */ -export function getTotalLoaded({ total, failed, successful }: ShardsResponse) { +export function getTotalLoaded(response: SearchResponse) { + const { total, failed, successful } = response._shards; const loaded = failed + successful; return { total, loaded }; } + +/** + * Get the Kibana representation of this response (see `IKibanaSearchResponse`). + * @internal + */ +export function toKibanaSearchResponse(rawResponse: SearchResponse) { + return { + rawResponse, + isPartial: false, + isRunning: false, + ...getTotalLoaded(rawResponse), + }; +} diff --git a/src/plugins/data/server/search/index.ts b/src/plugins/data/server/search/index.ts index 1be641401b29c..3001bbe3c2f38 100644 --- a/src/plugins/data/server/search/index.ts +++ b/src/plugins/data/server/search/index.ts @@ -19,6 +19,6 @@ export * from './types'; export * from './es_search'; -export { usageProvider, SearchUsage } from './collectors'; +export { usageProvider, SearchUsage, searchUsageObserver } from './collectors'; export * from './aggs'; export { shimHitsTotal } from './routes'; diff --git a/src/plugins/data/server/search/routes/call_msearch.ts b/src/plugins/data/server/search/routes/call_msearch.ts index 603b3ed867b23..923369297889b 100644 --- a/src/plugins/data/server/search/routes/call_msearch.ts +++ b/src/plugins/data/server/search/routes/call_msearch.ts @@ -24,9 +24,8 @@ import { SearchResponse } from 'elasticsearch'; import { IUiSettingsClient, IScopedClusterClient, SharedGlobalConfig } from 'src/core/server'; import type { MsearchRequestBody, MsearchResponse } from '../../../common/search/search_source'; -import { toSnakeCase, shimAbortSignal } from '../../../common/search/es_search'; import { shimHitsTotal } from './shim_hits_total'; -import { getShardTimeout, getDefaultSearchParams } from '..'; +import { getShardTimeout, getDefaultSearchParams, shimAbortSignal } from '..'; /** @internal */ export function convertRequestBody( @@ -71,7 +70,7 @@ export function getCallMsearch(dependencies: CallMsearchDependencies) { const timeout = getShardTimeout(config); // trackTotalHits is not supported by msearch - const { trackTotalHits, ...defaultParams } = await getDefaultSearchParams(uiSettings); + const { track_total_hits: _, ...defaultParams } = await getDefaultSearchParams(uiSettings); const body = convertRequestBody(params.body, timeout); @@ -81,7 +80,7 @@ export function getCallMsearch(dependencies: CallMsearchDependencies) { body, }, { - querystring: toSnakeCase(defaultParams), + querystring: defaultParams, } ), params.signal diff --git a/src/plugins/data/server/server.api.md b/src/plugins/data/server/server.api.md index 6870ad5e2402f..73e2a68cb9667 100644 --- a/src/plugins/data/server/server.api.md +++ b/src/plugins/data/server/server.api.md @@ -34,6 +34,7 @@ import { IScopedClusterClient } from 'src/core/server'; import { ISearchOptions as ISearchOptions_2 } from 'src/plugins/data/public'; import { ISearchSource } from 'src/plugins/data/public'; import { IUiSettingsClient } from 'src/core/server'; +import { IUiSettingsClient as IUiSettingsClient_3 } from 'kibana/server'; import { KibanaRequest } from 'src/core/server'; import { LegacyAPICaller } from 'src/core/server'; import { Logger } from 'src/core/server'; @@ -58,8 +59,9 @@ import { SavedObjectsClientContract as SavedObjectsClientContract_2 } from 'kiba import { Search } from '@elastic/elasticsearch/api/requestParams'; import { SearchResponse } from 'elasticsearch'; import { SerializedFieldFormat as SerializedFieldFormat_2 } from 'src/plugins/expressions/common'; -import { ShardsResponse } from 'elasticsearch'; +import { SharedGlobalConfig as SharedGlobalConfig_2 } from 'kibana/server'; import { ToastInputFields } from 'src/core/public/notifications'; +import { TransportRequestPromise } from '@elastic/elasticsearch/lib/Transport'; import { Type } from '@kbn/config-schema'; import { TypeOf } from '@kbn/config-schema'; import { UiStatsMetricType } from '@kbn/analytics'; @@ -410,25 +412,15 @@ export function getCapabilitiesForRollupIndices(indices: { [key: string]: any; }; -// Warning: (ae-forgotten-export) The symbol "IUiSettingsClient" needs to be exported by the entry point index.d.ts // Warning: (ae-missing-release-tag) "getDefaultSearchParams" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export function getDefaultSearchParams(uiSettingsClient: IUiSettingsClient_2): Promise<{ - maxConcurrentShardRequests: number | undefined; - ignoreUnavailable: boolean; - trackTotalHits: boolean; -}>; +export function getDefaultSearchParams(uiSettingsClient: IUiSettingsClient_3): Promise>; -// Warning: (ae-forgotten-export) The symbol "SharedGlobalConfig" needs to be exported by the entry point index.d.ts // Warning: (ae-missing-release-tag) "getShardTimeout" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export function getShardTimeout(config: SharedGlobalConfig): { - timeout: string; -} | { - timeout?: undefined; -}; +export function getShardTimeout(config: SharedGlobalConfig_2): Pick; // Warning: (ae-forgotten-export) The symbol "IIndexPattern" needs to be exported by the entry point index.d.ts // Warning: (ae-missing-release-tag) "getTime" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -439,6 +431,12 @@ export function getTime(indexPattern: IIndexPattern | undefined, timeRange: Time fieldName?: string; }): import("../..").RangeFilter | undefined; +// @internal +export function getTotalLoaded(response: SearchResponse): { + total: number; + loaded: number; +}; + // Warning: (ae-missing-release-tag) "IAggConfig" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public @@ -455,18 +453,6 @@ export type IAggConfigs = AggConfigs; // @public (undocumented) export type IAggType = AggType; -// Warning: (ae-missing-release-tag) "IEsRawSearchResponse" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) -export interface IEsRawSearchResponse extends SearchResponse { - // (undocumented) - id?: string; - // (undocumented) - is_partial?: boolean; - // (undocumented) - is_running?: boolean; -} - // Warning: (ae-forgotten-export) The symbol "IKibanaSearchRequest" needs to be exported by the entry point index.d.ts // Warning: (ae-forgotten-export) The symbol "ISearchRequestParams" needs to be exported by the entry point index.d.ts // Warning: (ae-missing-release-tag) "IEsSearchRequest" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -1040,24 +1026,6 @@ export interface RefreshInterval { // // @public (undocumented) export const search: { - esSearch: { - utils: { - doSearch: (searchMethod: () => Promise, abortSignal?: AbortSignal | undefined) => import("rxjs").Observable; - shimAbortSignal: >(promise: T, signal: AbortSignal | undefined) => T; - trackSearchStatus: = import("./search").IEsSearchResponse>>(logger: import("src/core/server").Logger, usage?: import("./search").SearchUsage | undefined) => import("rxjs").UnaryFunction, import("rxjs").Observable>; - includeTotalLoaded: () => import("rxjs").OperatorFunction>, { - total: number; - loaded: number; - id?: string | undefined; - isRunning?: boolean | undefined; - isPartial?: boolean | undefined; - rawResponse: import("elasticsearch").SearchResponse; - }>; - toKibanaSearchResponse: = import("../common").IEsRawSearchResponse, KibanaResponse_1 extends import("../common").IKibanaSearchResponse = import("../common").IKibanaSearchResponse>() => import("rxjs").OperatorFunction, KibanaResponse_1>; - getTotalLoaded: typeof getTotalLoaded; - toSnakeCase: typeof toSnakeCase; - }; - }; aggs: { CidrMask: typeof CidrMask; dateHistogramInterval: typeof dateHistogramInterval; @@ -1114,6 +1082,17 @@ export interface SearchUsage { trackSuccess(duration: number): Promise; } +// Warning: (ae-missing-release-tag) "searchUsageObserver" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export function searchUsageObserver(logger: Logger_2, usage?: SearchUsage): { + next(response: IEsSearchResponse): void; + error(): void; +}; + +// @internal +export const shimAbortSignal: (promise: TransportRequestPromise, signal?: AbortSignal | undefined) => TransportRequestPromise; + // @internal export function shimHitsTotal(response: SearchResponse): { hits: { @@ -1176,6 +1155,15 @@ export type TimeRange = { mode?: 'absolute' | 'relative'; }; +// @internal +export function toKibanaSearchResponse(rawResponse: SearchResponse): { + total: number; + loaded: number; + rawResponse: SearchResponse; + isPartial: boolean; + isRunning: boolean; +}; + // Warning: (ae-missing-release-tag) "UI_SETTINGS" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -1247,22 +1235,20 @@ export function usageProvider(core: CoreSetup_2): SearchUsage; // src/plugins/data/server/index.ts:111:26 - (ae-forgotten-export) The symbol "TruncateFormat" needs to be exported by the entry point index.d.ts // src/plugins/data/server/index.ts:137:27 - (ae-forgotten-export) The symbol "isFilterable" needs to be exported by the entry point index.d.ts // src/plugins/data/server/index.ts:137:27 - (ae-forgotten-export) The symbol "isNestedField" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:254:20 - (ae-forgotten-export) The symbol "getRequestInspectorStats" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:254:20 - (ae-forgotten-export) The symbol "getResponseInspectorStats" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:254:20 - (ae-forgotten-export) The symbol "tabifyAggResponse" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:254:20 - (ae-forgotten-export) The symbol "tabifyGetColumns" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:269:5 - (ae-forgotten-export) The symbol "getTotalLoaded" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:270:5 - (ae-forgotten-export) The symbol "toSnakeCase" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:274:1 - (ae-forgotten-export) The symbol "CidrMask" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:275:1 - (ae-forgotten-export) The symbol "dateHistogramInterval" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:284:1 - (ae-forgotten-export) The symbol "InvalidEsCalendarIntervalError" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:285:1 - (ae-forgotten-export) The symbol "InvalidEsIntervalFormatError" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:286:1 - (ae-forgotten-export) The symbol "Ipv4Address" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:290:1 - (ae-forgotten-export) The symbol "isValidEsInterval" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:291:1 - (ae-forgotten-export) The symbol "isValidInterval" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:295:1 - (ae-forgotten-export) The symbol "propFilter" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:298:1 - (ae-forgotten-export) The symbol "toAbsoluteDates" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:299:1 - (ae-forgotten-export) The symbol "calcAutoIntervalLessThan" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:248:20 - (ae-forgotten-export) The symbol "getRequestInspectorStats" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:248:20 - (ae-forgotten-export) The symbol "getResponseInspectorStats" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:248:20 - (ae-forgotten-export) The symbol "tabifyAggResponse" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:248:20 - (ae-forgotten-export) The symbol "tabifyGetColumns" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:250:1 - (ae-forgotten-export) The symbol "CidrMask" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:251:1 - (ae-forgotten-export) The symbol "dateHistogramInterval" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:260:1 - (ae-forgotten-export) The symbol "InvalidEsCalendarIntervalError" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:261:1 - (ae-forgotten-export) The symbol "InvalidEsIntervalFormatError" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:262:1 - (ae-forgotten-export) The symbol "Ipv4Address" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:266:1 - (ae-forgotten-export) The symbol "isValidEsInterval" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:267:1 - (ae-forgotten-export) The symbol "isValidInterval" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:271:1 - (ae-forgotten-export) The symbol "propFilter" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:274:1 - (ae-forgotten-export) The symbol "toAbsoluteDates" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:275:1 - (ae-forgotten-export) The symbol "calcAutoIntervalLessThan" needs to be exported by the entry point index.d.ts // src/plugins/data/server/index_patterns/index_patterns_service.ts:58:14 - (ae-forgotten-export) The symbol "IndexPatternsService" needs to be exported by the entry point index.d.ts // src/plugins/data/server/plugin.ts:88:66 - (ae-forgotten-export) The symbol "DataEnhancements" needs to be exported by the entry point index.d.ts // src/plugins/data/server/search/types.ts:104:5 - (ae-forgotten-export) The symbol "ISearchStartSearchSource" needs to be exported by the entry point index.d.ts diff --git a/x-pack/plugins/data_enhanced/common/index.ts b/x-pack/plugins/data_enhanced/common/index.ts index 61767af030803..dd1a2d39ab5d1 100644 --- a/x-pack/plugins/data_enhanced/common/index.ts +++ b/x-pack/plugins/data_enhanced/common/index.ts @@ -10,9 +10,6 @@ export { EqlRequestParams, EqlSearchStrategyRequest, EqlSearchStrategyResponse, - IAsyncSearchRequest, - IEnhancedEsSearchRequest, IAsyncSearchOptions, - doPartialSearch, - throwOnEsError, + pollSearch, } from './search'; diff --git a/x-pack/plugins/data_enhanced/common/search/es_search/es_search_rxjs_utils.ts b/x-pack/plugins/data_enhanced/common/search/es_search/es_search_rxjs_utils.ts deleted file mode 100644 index 8b25a59ed857a..0000000000000 --- a/x-pack/plugins/data_enhanced/common/search/es_search/es_search_rxjs_utils.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { of, merge, timer, throwError } from 'rxjs'; -import { map, takeWhile, switchMap, expand, mergeMap, tap } from 'rxjs/operators'; -import { ApiResponse } from '@elastic/elasticsearch'; - -import { - doSearch, - IKibanaSearchResponse, - isErrorResponse, -} from '../../../../../../src/plugins/data/common'; -import { AbortError } from '../../../../../../src/plugins/kibana_utils/common'; -import type { IKibanaSearchRequest } from '../../../../../../src/plugins/data/common'; -import type { IAsyncSearchOptions } from '../../../common/search/types'; - -const DEFAULT_POLLING_INTERVAL = 1000; - -export const doPartialSearch = ( - searchMethod: () => Promise, - partialSearchMethod: (id: IKibanaSearchRequest['id']) => Promise, - isCompleteResponse: (response: SearchResponse) => boolean, - getId: (response: SearchResponse) => IKibanaSearchRequest['id'], - requestId: IKibanaSearchRequest['id'], - { abortSignal, pollInterval = DEFAULT_POLLING_INTERVAL }: IAsyncSearchOptions -) => - doSearch( - requestId ? () => partialSearchMethod(requestId) : searchMethod, - abortSignal - ).pipe( - tap((response) => (requestId = getId(response))), - expand(() => timer(pollInterval).pipe(switchMap(() => partialSearchMethod(requestId)))), - takeWhile((response) => !isCompleteResponse(response), true) - ); - -export const normalizeEqlResponse = () => - map((eqlResponse) => ({ - ...eqlResponse, - body: { - ...eqlResponse.body, - ...eqlResponse, - }, - })); - -export const throwOnEsError = () => - mergeMap((r: IKibanaSearchResponse) => - isErrorResponse(r) ? merge(of(r), throwError(new AbortError())) : of(r) - ); diff --git a/x-pack/plugins/data_enhanced/common/search/es_search/index.ts b/x-pack/plugins/data_enhanced/common/search/es_search/index.ts deleted file mode 100644 index bbf9f14ba63c2..0000000000000 --- a/x-pack/plugins/data_enhanced/common/search/es_search/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -export * from './es_search_rxjs_utils'; diff --git a/x-pack/plugins/data_enhanced/common/search/index.ts b/x-pack/plugins/data_enhanced/common/search/index.ts index 44f82386e35c3..34bb21cb91af1 100644 --- a/x-pack/plugins/data_enhanced/common/search/index.ts +++ b/x-pack/plugins/data_enhanced/common/search/index.ts @@ -5,4 +5,4 @@ */ export * from './types'; -export * from './es_search'; +export * from './poll_search'; diff --git a/x-pack/plugins/data_enhanced/common/search/poll_search.ts b/x-pack/plugins/data_enhanced/common/search/poll_search.ts new file mode 100644 index 0000000000000..c0e289c691cfd --- /dev/null +++ b/x-pack/plugins/data_enhanced/common/search/poll_search.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { from, NEVER, Observable, timer } from 'rxjs'; +import { expand, finalize, switchMap, takeUntil, takeWhile, tap } from 'rxjs/operators'; +import type { IKibanaSearchResponse } from '../../../../../src/plugins/data/common'; +import { isErrorResponse, isPartialResponse } from '../../../../../src/plugins/data/common'; +import { AbortError, abortSignalToPromise } from '../../../../../src/plugins/kibana_utils/common'; +import type { IAsyncSearchOptions } from './types'; + +export const pollSearch = ( + search: () => Promise, + { pollInterval = 1000, ...options }: IAsyncSearchOptions = {} +): Observable => { + const aborted = options?.abortSignal + ? abortSignalToPromise(options?.abortSignal) + : { promise: NEVER, cleanup: () => {} }; + + return from(search()).pipe( + expand(() => timer(pollInterval).pipe(switchMap(search))), + tap((response) => { + if (isErrorResponse(response)) throw new AbortError(); + }), + takeWhile(isPartialResponse, true), + takeUntil(from(aborted.promise)), + finalize(aborted.cleanup) + ); +}; diff --git a/x-pack/plugins/data_enhanced/common/search/types.ts b/x-pack/plugins/data_enhanced/common/search/types.ts index 4abf8351114f8..f017462d4050b 100644 --- a/x-pack/plugins/data_enhanced/common/search/types.ts +++ b/x-pack/plugins/data_enhanced/common/search/types.ts @@ -9,27 +9,12 @@ import { ApiResponse, TransportRequestOptions } from '@elastic/elasticsearch/lib import { ISearchOptions, - IEsSearchRequest, IKibanaSearchRequest, IKibanaSearchResponse, } from '../../../../../src/plugins/data/common'; export const ENHANCED_ES_SEARCH_STRATEGY = 'ese'; -export interface IAsyncSearchRequest extends IEsSearchRequest { - /** - * The ID received from the response from the initial request - */ - id?: string; -} - -export interface IEnhancedEsSearchRequest extends IEsSearchRequest { - /** - * Used to determine whether to use the _rollups_search or a regular search endpoint. - */ - isRollup?: boolean; -} - export const EQL_SEARCH_STRATEGY = 'eql'; export type EqlRequestParams = EqlSearch>; diff --git a/x-pack/plugins/data_enhanced/public/search/search_interceptor.test.ts b/x-pack/plugins/data_enhanced/public/search/search_interceptor.test.ts index 044489d58eb0e..3f1cfc7a010c7 100644 --- a/x-pack/plugins/data_enhanced/public/search/search_interceptor.test.ts +++ b/x-pack/plugins/data_enhanced/public/search/search_interceptor.test.ts @@ -117,7 +117,7 @@ describe('EnhancedSearchInterceptor', () => { { time: 10, value: { - isPartial: false, + isPartial: true, isRunning: true, id: 1, rawResponse: { @@ -175,8 +175,6 @@ describe('EnhancedSearchInterceptor', () => { await timeTravel(10); - expect(next).toHaveBeenCalled(); - expect(next.mock.calls[0][0]).toStrictEqual(responses[0].value); expect(error).toHaveBeenCalled(); expect(error.mock.calls[0][0]).toBeInstanceOf(AbortError); }); @@ -212,7 +210,7 @@ describe('EnhancedSearchInterceptor', () => { { time: 10, value: { - isPartial: false, + isPartial: true, isRunning: true, id: 1, }, @@ -280,7 +278,7 @@ describe('EnhancedSearchInterceptor', () => { { time: 10, value: { - isPartial: false, + isPartial: true, isRunning: true, id: 1, }, @@ -320,7 +318,7 @@ describe('EnhancedSearchInterceptor', () => { { time: 10, value: { - isPartial: false, + isPartial: true, isRunning: true, id: 1, }, diff --git a/x-pack/plugins/data_enhanced/public/search/search_interceptor.ts b/x-pack/plugins/data_enhanced/public/search/search_interceptor.ts index e1bd71caddb4d..9aa35b460b1e8 100644 --- a/x-pack/plugins/data_enhanced/public/search/search_interceptor.ts +++ b/x-pack/plugins/data_enhanced/public/search/search_interceptor.ts @@ -4,24 +4,17 @@ * you may not use this file except in compliance with the Elastic License. */ -import { throwError, from, Subscription } from 'rxjs'; -import { tap, takeUntil, finalize, catchError } from 'rxjs/operators'; +import { throwError, Subscription } from 'rxjs'; +import { tap, finalize, catchError } from 'rxjs/operators'; import { TimeoutErrorMode, - IEsSearchResponse, SearchInterceptor, SearchInterceptorDeps, UI_SETTINGS, + IKibanaSearchRequest, } from '../../../../../src/plugins/data/public'; -import { AbortError, abortSignalToPromise } from '../../../../../src/plugins/kibana_utils/public'; - -import { - IAsyncSearchRequest, - ENHANCED_ES_SEARCH_STRATEGY, - IAsyncSearchOptions, - doPartialSearch, - throwOnEsError, -} from '../../common'; +import { AbortError } from '../../../../../src/plugins/kibana_utils/common'; +import { ENHANCED_ES_SEARCH_STRATEGY, IAsyncSearchOptions, pollSearch } from '../../common'; export class EnhancedSearchInterceptor extends SearchInterceptor { private uiSettingsSub: Subscription; @@ -60,49 +53,26 @@ export class EnhancedSearchInterceptor extends SearchInterceptor { if (this.deps.usageCollector) this.deps.usageCollector.trackQueriesCancelled(); }; - public search( - request: IAsyncSearchRequest, - { pollInterval = 1000, ...options }: IAsyncSearchOptions = {} - ) { - let { id } = request; - + public search({ id, ...request }: IKibanaSearchRequest, options: IAsyncSearchOptions = {}) { const { combinedSignal, timeoutSignal, cleanup } = this.setupAbortSignal({ abortSignal: options.abortSignal, timeout: this.searchTimeout, }); - const abortedPromise = abortSignalToPromise(combinedSignal); const strategy = options?.strategy ?? ENHANCED_ES_SEARCH_STRATEGY; + const searchOptions = { ...options, strategy, abortSignal: combinedSignal }; + const search = () => this.runSearch({ id, ...request }, searchOptions); this.pendingCount$.next(this.pendingCount$.getValue() + 1); - return doPartialSearch( - () => this.runSearch(request, { ...options, strategy, abortSignal: combinedSignal }), - (requestId) => - this.runSearch( - { ...request, id: requestId }, - { ...options, strategy, abortSignal: combinedSignal } - ), - (r) => !r.isRunning, - (response) => response.id, - id, - { pollInterval } - ).pipe( - tap((r) => { - id = r.id ?? id; - }), - throwOnEsError(), - takeUntil(from(abortedPromise.promise)), + return pollSearch(search, { ...options, abortSignal: combinedSignal }).pipe( + tap((response) => (id = response.id)), catchError((e: AbortError) => { - if (id) { - this.deps.http.delete(`/internal/search/${strategy}/${id}`); - } - - return throwError(this.handleSearchError(e, request, timeoutSignal, options)); + if (id) this.deps.http.delete(`/internal/search/${strategy}/${id}`); + return throwError(this.handleSearchError(e, timeoutSignal, options)); }), finalize(() => { this.pendingCount$.next(this.pendingCount$.getValue() - 1); cleanup(); - abortedPromise.cleanup(); }) ); } diff --git a/x-pack/plugins/data_enhanced/server/search/eql_search_strategy.test.ts b/x-pack/plugins/data_enhanced/server/search/eql_search_strategy.test.ts index cd94d91db8c5e..f2d7725954a26 100644 --- a/x-pack/plugins/data_enhanced/server/search/eql_search_strategy.test.ts +++ b/x-pack/plugins/data_enhanced/server/search/eql_search_strategy.test.ts @@ -178,7 +178,7 @@ describe('EQL search strategy', () => { expect(requestOptions).toEqual( expect.objectContaining({ - max_retries: 2, + maxRetries: 2, ignore: [300], }) ); diff --git a/x-pack/plugins/data_enhanced/server/search/eql_search_strategy.ts b/x-pack/plugins/data_enhanced/server/search/eql_search_strategy.ts index 7b3d0db450b04..26325afc378f7 100644 --- a/x-pack/plugins/data_enhanced/server/search/eql_search_strategy.ts +++ b/x-pack/plugins/data_enhanced/server/search/eql_search_strategy.ts @@ -4,21 +4,19 @@ * you may not use this file except in compliance with the Elastic License. */ +import { tap } from 'rxjs/operators'; import type { Logger } from 'kibana/server'; -import type { ApiResponse } from '@elastic/elasticsearch'; - -import { search } from '../../../../../src/plugins/data/server'; -import { - doPartialSearch, - normalizeEqlResponse, -} from '../../common/search/es_search/es_search_rxjs_utils'; -import { getAsyncOptions, getDefaultSearchParams } from './get_default_search_params'; - -import type { ISearchStrategy, IEsRawSearchResponse } from '../../../../../src/plugins/data/server'; +import type { ISearchStrategy } from '../../../../../src/plugins/data/server'; import type { EqlSearchStrategyRequest, EqlSearchStrategyResponse, -} from '../../common/search/types'; + IAsyncSearchOptions, +} from '../../common'; +import { getDefaultSearchParams, shimAbortSignal } from '../../../../../src/plugins/data/server'; +import { pollSearch } from '../../common'; +import { getDefaultAsyncGetParams, getIgnoreThrottled } from './request_utils'; +import { toEqlKibanaSearchResponse } from './response_utils'; +import { EqlSearchResponse } from './types'; export const eqlSearchStrategyProvider = ( logger: Logger @@ -26,48 +24,37 @@ export const eqlSearchStrategyProvider = ( return { cancel: async (id, options, { esClient }) => { logger.debug(`_eql/delete ${id}`); - await esClient.asCurrentUser.eql.delete({ - id, - }); + await esClient.asCurrentUser.eql.delete({ id }); }, - search: (request, options, { esClient, uiSettingsClient }) => { - logger.debug(`_eql/search ${JSON.stringify(request.params) || request.id}`); + search: ({ id, ...request }, options: IAsyncSearchOptions, { esClient, uiSettingsClient }) => { + logger.debug(`_eql/search ${JSON.stringify(request.params) || id}`); - const { utils } = search.esSearch; - const asyncOptions = getAsyncOptions(); - const requestOptions = utils.toSnakeCase({ ...request.options }); const client = esClient.asCurrentUser.eql; - return doPartialSearch>( - async () => { - const { ignoreThrottled, ignoreUnavailable } = await getDefaultSearchParams( - uiSettingsClient - ); - - return client.search( - utils.toSnakeCase({ - ignoreThrottled, - ignoreUnavailable, - ...asyncOptions, + const search = async () => { + const { track_total_hits: _, ...defaultParams } = await getDefaultSearchParams( + uiSettingsClient + ); + const params = id + ? getDefaultAsyncGetParams() + : { + ...(await getIgnoreThrottled(uiSettingsClient)), + ...defaultParams, + ...getDefaultAsyncGetParams(), ...request.params, - }) as EqlSearchStrategyRequest['params'], - requestOptions - ); - }, - (id) => - client.get( - { - id: id!, - ...utils.toSnakeCase(asyncOptions), - }, - requestOptions - ), - (response) => !response.body.is_running, - (response) => response.body.id, - request.id, - options - ).pipe(normalizeEqlResponse(), utils.toKibanaSearchResponse()); + }; + const promise = id + ? client.get({ ...params, id }, request.options) + : client.search( + params as EqlSearchStrategyRequest['params'], + request.options + ); + const response = await shimAbortSignal(promise, options.abortSignal); + return toEqlKibanaSearchResponse(response); + }; + + return pollSearch(search, options).pipe(tap((response) => (id = response.id))); }, }; }; diff --git a/x-pack/plugins/data_enhanced/server/search/es_search_strategy.ts b/x-pack/plugins/data_enhanced/server/search/es_search_strategy.ts index 2070610ceb20e..e1c7d7b5fc22e 100644 --- a/x-pack/plugins/data_enhanced/server/search/es_search_strategy.ts +++ b/x-pack/plugins/data_enhanced/server/search/es_search_strategy.ts @@ -4,86 +4,67 @@ * you may not use this file except in compliance with the Elastic License. */ +import type { Observable } from 'rxjs'; +import type { Logger, SharedGlobalConfig } from 'kibana/server'; +import { first, tap } from 'rxjs/operators'; +import { SearchResponse } from 'elasticsearch'; import { from } from 'rxjs'; -import { first, map } from 'rxjs/operators'; -import { Observable } from 'rxjs'; - -import type { SearchResponse } from 'elasticsearch'; -import type { ApiResponse } from '@elastic/elasticsearch'; - -import { - getShardTimeout, - shimHitsTotal, - search, - SearchStrategyDependencies, -} from '../../../../../src/plugins/data/server'; -import { doPartialSearch } from '../../common/search/es_search/es_search_rxjs_utils'; -import { getDefaultSearchParams, getAsyncOptions } from './get_default_search_params'; - -import type { SharedGlobalConfig, Logger } from '../../../../../src/core/server'; - import type { + IEsSearchRequest, + IEsSearchResponse, + ISearchOptions, ISearchStrategy, + SearchStrategyDependencies, SearchUsage, - IEsRawSearchResponse, - ISearchOptions, - IEsSearchResponse, } from '../../../../../src/plugins/data/server'; - -import type { IEnhancedEsSearchRequest } from '../../common'; - -const { utils } = search.esSearch; - -interface IEsRawAsyncSearchResponse extends IEsRawSearchResponse { - response: SearchResponse; -} +import { + getDefaultSearchParams, + getShardTimeout, + getTotalLoaded, + searchUsageObserver, + shimAbortSignal, +} from '../../../../../src/plugins/data/server'; +import type { IAsyncSearchOptions } from '../../common'; +import { pollSearch } from '../../common'; +import { + getDefaultAsyncGetParams, + getDefaultAsyncSubmitParams, + getIgnoreThrottled, +} from './request_utils'; +import { toAsyncKibanaSearchResponse } from './response_utils'; +import { AsyncSearchResponse } from './types'; export const enhancedEsSearchStrategyProvider = ( config$: Observable, logger: Logger, usage?: SearchUsage -): ISearchStrategy => { +): ISearchStrategy => { function asyncSearch( - request: IEnhancedEsSearchRequest, - options: ISearchOptions, + { id, ...request }: IEsSearchRequest, + options: IAsyncSearchOptions, { esClient, uiSettingsClient }: SearchStrategyDependencies ) { - const asyncOptions = getAsyncOptions(); const client = esClient.asCurrentUser.asyncSearch; - return doPartialSearch>( - async () => - client.submit( - utils.toSnakeCase({ - ...(await getDefaultSearchParams(uiSettingsClient)), - batchedReduceSize: 64, - keepOnCompletion: !!options.sessionId, // Always return an ID, even if the request completes quickly - ...asyncOptions, - ...request.params, - }) - ), - (id) => - client.get({ - id: id!, - ...utils.toSnakeCase({ ...asyncOptions }), - }), - (response) => !response.body.is_running, - (response) => response.body.id, - request.id, - options - ).pipe( - utils.toKibanaSearchResponse(), - map((response) => ({ - ...response, - rawResponse: shimHitsTotal(response.rawResponse.response!), - })), - utils.trackSearchStatus(logger, usage), - utils.includeTotalLoaded() + const search = async () => { + const params = id + ? getDefaultAsyncGetParams() + : { ...(await getDefaultAsyncSubmitParams(uiSettingsClient, options)), ...request.params }; + const promise = id + ? client.get({ ...params, id }) + : client.submit(params); + const { body } = await shimAbortSignal(promise, options.abortSignal); + return toAsyncKibanaSearchResponse(body); + }; + + return pollSearch(search, options).pipe( + tap((response) => (id = response.id)), + tap(searchUsageObserver(logger, usage)) ); } async function rollupSearch( - request: IEnhancedEsSearchRequest, + request: IEsSearchRequest, options: ISearchOptions, { esClient, uiSettingsClient }: SearchStrategyDependencies ): Promise { @@ -91,11 +72,12 @@ export const enhancedEsSearchStrategyProvider = ( const { body, index, ...params } = request.params!; const method = 'POST'; const path = encodeURI(`/${index}/_rollup_search`); - const querystring = utils.toSnakeCase({ + const querystring = { ...getShardTimeout(config), + ...(await getIgnoreThrottled(uiSettingsClient)), ...(await getDefaultSearchParams(uiSettingsClient)), ...params, - }); + }; const promise = esClient.asCurrentUser.transport.request({ method, @@ -104,17 +86,16 @@ export const enhancedEsSearchStrategyProvider = ( querystring, }); - const esResponse = await utils.shimAbortSignal(promise, options?.abortSignal); - + const esResponse = await shimAbortSignal(promise, options?.abortSignal); const response = esResponse.body as SearchResponse; return { rawResponse: response, - ...utils.getTotalLoaded(response._shards), + ...getTotalLoaded(response), }; } return { - search: (request, options, deps) => { + search: (request, options: IAsyncSearchOptions, deps) => { logger.debug(`search ${JSON.stringify(request.params) || request.id}`); return request.indexType !== 'rollup' diff --git a/x-pack/plugins/data_enhanced/server/search/get_default_search_params.ts b/x-pack/plugins/data_enhanced/server/search/get_default_search_params.ts deleted file mode 100644 index fdda78798808f..0000000000000 --- a/x-pack/plugins/data_enhanced/server/search/get_default_search_params.ts +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { IUiSettingsClient } from 'src/core/server'; -import { UI_SETTINGS } from '../../../../../src/plugins/data/common'; - -import { getDefaultSearchParams as getBaseSearchParams } from '../../../../../src/plugins/data/server'; - -/** - @internal - */ -export async function getDefaultSearchParams(uiSettingsClient: IUiSettingsClient) { - const ignoreThrottled = !(await uiSettingsClient.get(UI_SETTINGS.SEARCH_INCLUDE_FROZEN)); - - return { - ignoreThrottled, - ...(await getBaseSearchParams(uiSettingsClient)), - }; -} - -/** - @internal - */ -export const getAsyncOptions = (): { - waitForCompletionTimeout: string; - keepAlive: string; -} => ({ - waitForCompletionTimeout: '100ms', // Wait up to 100ms for the response to return - keepAlive: '1m', // Extend the TTL for this search request by one minute, -}); diff --git a/x-pack/plugins/data_enhanced/server/search/request_utils.ts b/x-pack/plugins/data_enhanced/server/search/request_utils.ts new file mode 100644 index 0000000000000..f54ab2199c905 --- /dev/null +++ b/x-pack/plugins/data_enhanced/server/search/request_utils.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { IUiSettingsClient } from 'kibana/server'; +import { + AsyncSearchGet, + AsyncSearchSubmit, + Search, +} from '@elastic/elasticsearch/api/requestParams'; +import { ISearchOptions, UI_SETTINGS } from '../../../../../src/plugins/data/common'; +import { getDefaultSearchParams } from '../../../../../src/plugins/data/server'; + +/** + * @internal + */ +export async function getIgnoreThrottled( + uiSettingsClient: IUiSettingsClient +): Promise> { + const includeFrozen = await uiSettingsClient.get(UI_SETTINGS.SEARCH_INCLUDE_FROZEN); + return { ignore_throttled: !includeFrozen }; +} + +/** + @internal + */ +export async function getDefaultAsyncSubmitParams( + uiSettingsClient: IUiSettingsClient, + options: ISearchOptions +): Promise< + Pick< + AsyncSearchSubmit, + | 'batched_reduce_size' + | 'keep_alive' + | 'wait_for_completion_timeout' + | 'ignore_throttled' + | 'max_concurrent_shard_requests' + | 'ignore_unavailable' + | 'track_total_hits' + | 'keep_on_completion' + > +> { + return { + batched_reduce_size: 64, + keep_on_completion: !!options.sessionId, // Always return an ID, even if the request completes quickly + ...getDefaultAsyncGetParams(), + ...(await getIgnoreThrottled(uiSettingsClient)), + ...(await getDefaultSearchParams(uiSettingsClient)), + }; +} + +/** + @internal + */ +export function getDefaultAsyncGetParams(): Pick< + AsyncSearchGet, + 'keep_alive' | 'wait_for_completion_timeout' +> { + return { + keep_alive: '1m', // Extend the TTL for this search request by one minute + wait_for_completion_timeout: '100ms', // Wait up to 100ms for the response to return + }; +} diff --git a/x-pack/plugins/data_enhanced/server/search/response_utils.ts b/x-pack/plugins/data_enhanced/server/search/response_utils.ts new file mode 100644 index 0000000000000..716e7d72d80e7 --- /dev/null +++ b/x-pack/plugins/data_enhanced/server/search/response_utils.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ApiResponse } from '@elastic/elasticsearch'; +import { getTotalLoaded } from '../../../../../src/plugins/data/server'; +import { AsyncSearchResponse, EqlSearchResponse } from './types'; +import { EqlSearchStrategyResponse } from '../../common/search'; + +/** + * Get the Kibana representation of an async search response (see `IKibanaSearchResponse`). + */ +export function toAsyncKibanaSearchResponse(response: AsyncSearchResponse) { + return { + id: response.id, + rawResponse: response.response, + isPartial: response.is_partial, + isRunning: response.is_running, + ...getTotalLoaded(response.response), + }; +} + +/** + * Get the Kibana representation of an EQL search response (see `IKibanaSearchResponse`). + * (EQL does not provide _shard info, so total/loaded cannot be calculated.) + */ +export function toEqlKibanaSearchResponse( + response: ApiResponse +): EqlSearchStrategyResponse { + return { + id: response.body.id, + rawResponse: response, + isPartial: response.body.is_partial, + isRunning: response.body.is_running, + }; +} diff --git a/x-pack/plugins/data_enhanced/server/search/types.ts b/x-pack/plugins/data_enhanced/server/search/types.ts new file mode 100644 index 0000000000000..f01ac51a1516e --- /dev/null +++ b/x-pack/plugins/data_enhanced/server/search/types.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchResponse } from 'elasticsearch'; + +export interface AsyncSearchResponse { + id?: string; + response: SearchResponse; + is_partial: boolean; + is_running: boolean; +} + +export interface EqlSearchResponse extends SearchResponse { + id?: string; + is_partial: boolean; + is_running: boolean; +} diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/index.ts index baacad65e140f..8b2cce01cf07a 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/index.ts @@ -4,8 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -import { mergeMap } from 'rxjs/operators'; -import { ISearchStrategy, PluginStart } from '../../../../../../src/plugins/data/server'; +import { map, mergeMap } from 'rxjs/operators'; +import { + ISearchStrategy, + PluginStart, + shimHitsTotal, +} from '../../../../../../src/plugins/data/server'; import { ENHANCED_ES_SEARCH_STRATEGY } from '../../../../data_enhanced/common'; import { FactoryQueryTypes, @@ -28,9 +32,17 @@ export const securitySolutionSearchStrategyProvider = = securitySolutionFactory[request.factoryQueryType]; const dsl = queryFactory.buildDsl(request); - return es - .search({ ...request, params: dsl }, options, deps) - .pipe(mergeMap((esSearchRes) => queryFactory.parse(request, esSearchRes))); + return es.search({ ...request, params: dsl }, options, deps).pipe( + map((response) => { + return { + ...response, + ...{ + rawResponse: shimHitsTotal(response.rawResponse), + }, + }; + }), + mergeMap((esSearchRes) => queryFactory.parse(request, esSearchRes)) + ); }, cancel: async (id, options, deps) => { if (es.cancel) { diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/index.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/index.ts index 29ad37e76264f..5ad00a727c3b6 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/timeline/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/index.ts @@ -4,8 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -import { mergeMap } from 'rxjs/operators'; -import { ISearchStrategy, PluginStart } from '../../../../../../src/plugins/data/server'; +import { map, mergeMap } from 'rxjs/operators'; +import { + ISearchStrategy, + PluginStart, + shimHitsTotal, +} from '../../../../../../src/plugins/data/server'; import { ENHANCED_ES_SEARCH_STRATEGY } from '../../../../data_enhanced/common'; import { TimelineFactoryQueryTypes, @@ -29,9 +33,17 @@ export const securitySolutionTimelineSearchStrategyProvider = queryFactory.parse(request, esSearchRes))); + return es.search({ ...request, params: dsl }, options, deps).pipe( + map((response) => { + return { + ...response, + ...{ + rawResponse: shimHitsTotal(response.rawResponse), + }, + }; + }), + mergeMap((esSearchRes) => queryFactory.parse(request, esSearchRes)) + ); }, cancel: async (id, options, deps) => { if (es.cancel) { From 5f53d856c093a50f68932b81647e6332dbe0ace4 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Tue, 24 Nov 2020 17:54:26 -0500 Subject: [PATCH 22/49] [Fleet] Only display log level in Filter that are present in the logs (#84277) --- .../components/agent_logs/constants.tsx | 2 ++ .../components/agent_logs/filter_log_level.tsx | 17 +++++++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/constants.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/constants.tsx index 41069e7107862..ea98de3560246 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/constants.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/constants.tsx @@ -32,4 +32,6 @@ export const AGENT_LOG_LEVELS = { DEBUG: 'debug', }; +export const ORDERED_FILTER_LOG_LEVELS = ['error', 'warning', 'warn', 'notice', 'info', 'debug']; + export const DEFAULT_LOG_LEVEL = AGENT_LOG_LEVELS.INFO; diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/filter_log_level.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/filter_log_level.tsx index a45831b2bbd2a..6aee9e065a96d 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/filter_log_level.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/filter_log_level.tsx @@ -6,10 +6,19 @@ import React, { memo, useState, useEffect } from 'react'; import { EuiPopover, EuiFilterButton, EuiFilterSelectItem } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import { AGENT_LOG_LEVELS, AGENT_LOG_INDEX_PATTERN, LOG_LEVEL_FIELD } from './constants'; +import { ORDERED_FILTER_LOG_LEVELS, AGENT_LOG_INDEX_PATTERN, LOG_LEVEL_FIELD } from './constants'; import { useStartServices } from '../../../../../hooks'; -const LEVEL_VALUES = Object.values(AGENT_LOG_LEVELS); +function sortLogLevels(levels: string[]): string[] { + return [ + ...new Set([ + // order by severity for known level + ...ORDERED_FILTER_LOG_LEVELS.filter((level) => levels.includes(level)), + // Add unknown log level + ...levels.sort(), + ]), + ]; +} export const LogLevelFilter: React.FunctionComponent<{ selectedLevels: string[]; @@ -18,7 +27,7 @@ export const LogLevelFilter: React.FunctionComponent<{ const { data } = useStartServices(); const [isOpen, setIsOpen] = useState(false); const [isLoading, setIsLoading] = useState(false); - const [levelValues, setLevelValues] = useState(LEVEL_VALUES); + const [levelValues, setLevelValues] = useState([]); useEffect(() => { const fetchValues = async () => { @@ -32,7 +41,7 @@ export const LogLevelFilter: React.FunctionComponent<{ field: LOG_LEVEL_FIELD, query: '', }); - setLevelValues([...new Set([...LEVEL_VALUES, ...values.sort()])]); + setLevelValues(sortLogLevels(values)); } catch (e) { setLevelValues([]); } From 2ce1e09aad408d352a8802cec46f868453574f34 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 24 Nov 2020 17:03:56 -0600 Subject: [PATCH 23/49] [Security Solution][Detections] Rule Form Fixes (#84169) * Prevent error from being displayed when choosing action throttle Addresses #83230. This field was previously refactored to not require a callback prop; simply updating the value via `field.setValue` is sufficient for our use case. This fix removes the errant code that assumed a callback prop, since such a prop does not exist on the underlying component. * Fix UI bug on ML Jobs popover EUI links now add an SVG if they're an external link; our use of a div was causing that to wrap. Since the div was only needed to change the text size, a refactor makes this all work. * Exercise editing of tags in E2E tests These tests were recently skipped due to their improper teardown. Since that's a broader issue across most of these tests, I'm reopening these so we can get the coverage provided here for the time being. * useFetchIndex defaults to isLoading: false In the case where no index pattern is provided, the hook exits without doing any work but does not update the loading state; this had the downstream effect of disabling a form field that was waiting for this hook to stop loading. * Move situational action into ... the situation We only need to clear existing tags in the case where we're editing the rule (and it has tags); in all other cases, this method fails. This fixes things by moving that conditional logic (clear the tags field) into the test that needs it (editing custom rules). --- .../integration/alerts_detection_rules_custom.spec.ts | 5 +++-- x-pack/plugins/security_solution/cypress/objects/rule.ts | 1 + .../security_solution/cypress/screens/create_new_rule.ts | 3 +++ .../components/ml_popover/jobs_table/jobs_table.tsx | 8 +++++--- .../public/common/containers/source/index.tsx | 2 +- .../components/rules/throttle_select_field/index.tsx | 7 +++---- 6 files changed, 16 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_custom.spec.ts b/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_custom.spec.ts index 596b92d064050..dfe984cba3816 100644 --- a/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_custom.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_custom.spec.ts @@ -38,6 +38,7 @@ import { SCHEDULE_INTERVAL_AMOUNT_INPUT, SCHEDULE_INTERVAL_UNITS_INPUT, SEVERITY_DROPDOWN, + TAGS_CLEAR_BUTTON, TAGS_FIELD, } from '../screens/create_new_rule'; import { @@ -215,8 +216,7 @@ describe('Custom detection rules creation', () => { }); }); -// FLAKY: https://github.com/elastic/kibana/issues/83772 -describe.skip('Custom detection rules deletion and edition', () => { +describe('Custom detection rules deletion and edition', () => { beforeEach(() => { esArchiverLoad('custom_rules'); loginAndWaitForPageWithoutDateRange(DETECTIONS_URL); @@ -328,6 +328,7 @@ describe.skip('Custom detection rules deletion and edition', () => { cy.get(ACTIONS_THROTTLE_INPUT).invoke('val').should('eql', 'no_actions'); goToAboutStepTab(); + cy.get(TAGS_CLEAR_BUTTON).click({ force: true }); fillAboutRule(editedRule); saveEditedRule(); diff --git a/x-pack/plugins/security_solution/cypress/objects/rule.ts b/x-pack/plugins/security_solution/cypress/objects/rule.ts index 0bb4c8e356091..8ba545e242b47 100644 --- a/x-pack/plugins/security_solution/cypress/objects/rule.ts +++ b/x-pack/plugins/security_solution/cypress/objects/rule.ts @@ -265,4 +265,5 @@ export const editedRule = { ...existingRule, severity: 'Medium', description: 'Edited Rule description', + tags: [...existingRule.tags, 'edited'], }; diff --git a/x-pack/plugins/security_solution/cypress/screens/create_new_rule.ts b/x-pack/plugins/security_solution/cypress/screens/create_new_rule.ts index 618ddbad9f44a..d802e97363a68 100644 --- a/x-pack/plugins/security_solution/cypress/screens/create_new_rule.ts +++ b/x-pack/plugins/security_solution/cypress/screens/create_new_rule.ts @@ -146,6 +146,9 @@ export const TAGS_FIELD = export const TAGS_INPUT = '[data-test-subj="detectionEngineStepAboutRuleTags"] [data-test-subj="comboBoxSearchInput"]'; +export const TAGS_CLEAR_BUTTON = + '[data-test-subj="detectionEngineStepAboutRuleTags"] [data-test-subj="comboBoxClearButton"]'; + export const THRESHOLD_FIELD_SELECTION = '.euiFilterSelectItem'; export const THRESHOLD_INPUT_AREA = '[data-test-subj="thresholdInput"]'; diff --git a/x-pack/plugins/security_solution/public/common/components/ml_popover/jobs_table/jobs_table.tsx b/x-pack/plugins/security_solution/public/common/components/ml_popover/jobs_table/jobs_table.tsx index 5e3045efe1f4d..54a2381ecf587 100644 --- a/x-pack/plugins/security_solution/public/common/components/ml_popover/jobs_table/jobs_table.tsx +++ b/x-pack/plugins/security_solution/public/common/components/ml_popover/jobs_table/jobs_table.tsx @@ -57,9 +57,11 @@ const JobName = ({ id, description, basePath }: JobNameProps) => { return ( - - {id} - + + + {id} + + {description.length > truncateThreshold ? `${description.substring(0, truncateThreshold)}...` diff --git a/x-pack/plugins/security_solution/public/common/containers/source/index.tsx b/x-pack/plugins/security_solution/public/common/containers/source/index.tsx index 7e73a40f2f748..f245857f3d0db 100644 --- a/x-pack/plugins/security_solution/public/common/containers/source/index.tsx +++ b/x-pack/plugins/security_solution/public/common/containers/source/index.tsx @@ -126,7 +126,7 @@ export const useFetchIndex = ( const { data, notifications } = useKibana().services; const abortCtrl = useRef(new AbortController()); const previousIndexesName = useRef([]); - const [isLoading, setLoading] = useState(true); + const [isLoading, setLoading] = useState(false); const [state, setState] = useState({ browserFields: DEFAULT_BROWSER_FIELDS, diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/throttle_select_field/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/throttle_select_field/index.tsx index bf3498b28cd45..f0326913909b5 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/throttle_select_field/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/throttle_select_field/index.tsx @@ -25,14 +25,13 @@ export const DEFAULT_THROTTLE_OPTION = THROTTLE_OPTIONS[0]; type ThrottleSelectField = typeof SelectField; export const ThrottleSelectField: ThrottleSelectField = (props) => { + const { setValue } = props.field; const onChange = useCallback( (e) => { const throttle = e.target.value; - props.field.setValue(throttle); - props.handleChange(throttle); + setValue(throttle); }, - // eslint-disable-next-line react-hooks/exhaustive-deps - [props.field.setValue, props.handleChange] + [setValue] ); const newEuiFieldProps = { ...props.euiFieldProps, onChange }; return ; From a1b568b604c1db9db7f319f0247048d67c649f16 Mon Sep 17 00:00:00 2001 From: "Christiane (Tina) Heiligers" Date: Tue, 24 Nov 2020 16:32:46 -0700 Subject: [PATCH 24/49] Changes code ownership from kibana-telemetry to kibana-core (#84281) --- .github/CODEOWNERS | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a536d1b54551c..bb4c500283020 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -204,6 +204,28 @@ #CC# /x-pack/plugins/features/ @elastic/kibana-core #CC# /x-pack/plugins/global_search/ @elastic/kibana-core +# Kibana Telemetry +/packages/kbn-analytics/ @elastic/kibana-core +/packages/kbn-telemetry-tools/ @elastic/kibana-core +/src/plugins/kibana_usage_collection/ @elastic/kibana-core +/src/plugins/newsfeed/ @elastic/kibana-core +/src/plugins/telemetry/ @elastic/kibana-core +/src/plugins/telemetry_collection_manager/ @elastic/kibana-core +/src/plugins/telemetry_management_section/ @elastic/kibana-core +/src/plugins/usage_collection/ @elastic/kibana-core +/x-pack/plugins/telemetry_collection_xpack/ @elastic/kibana-core +/.telemetryrc.json @elastic/kibana-core +/x-pack/.telemetryrc.json @elastic/kibana-core +src/plugins/telemetry/schema/legacy_oss_plugins.json @elastic/kibana-core +src/plugins/telemetry/schema/oss_plugins.json @elastic/kibana-core +x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @elastic/kibana-core + +# Kibana Localization +/src/dev/i18n/ @elastic/kibana-localization @elastic/kibana-core +/src/core/public/i18n/ @elastic/kibana-localization @elastic/kibana-core +/packages/kbn-i18n/ @elastic/kibana-localization @elastic/kibana-core +#CC# /x-pack/plugins/translations/ @elastic/kibana-localization @elastic/kibana-core + # Security /src/core/server/csp/ @elastic/kibana-security @elastic/kibana-core /src/plugins/security_oss/ @elastic/kibana-security @@ -221,28 +243,6 @@ #CC# /x-pack/plugins/security_solution/ @elastic/kibana-security #CC# /x-pack/plugins/security/ @elastic/kibana-security -# Kibana Localization -/src/dev/i18n/ @elastic/kibana-localization -/src/core/public/i18n/ @elastic/kibana-localization -/packages/kbn-i18n/ @elastic/kibana-localization -#CC# /x-pack/plugins/translations/ @elastic/kibana-localization - -# Kibana Telemetry -/packages/kbn-analytics/ @elastic/kibana-telemetry -/packages/kbn-telemetry-tools/ @elastic/kibana-telemetry -/src/plugins/kibana_usage_collection/ @elastic/kibana-telemetry -/src/plugins/newsfeed/ @elastic/kibana-telemetry -/src/plugins/telemetry/ @elastic/kibana-telemetry -/src/plugins/telemetry_collection_manager/ @elastic/kibana-telemetry -/src/plugins/telemetry_management_section/ @elastic/kibana-telemetry -/src/plugins/usage_collection/ @elastic/kibana-telemetry -/x-pack/plugins/telemetry_collection_xpack/ @elastic/kibana-telemetry -/.telemetryrc.json @elastic/kibana-telemetry -/x-pack/.telemetryrc.json @elastic/kibana-telemetry -src/plugins/telemetry/schema/legacy_oss_plugins.json @elastic/kibana-telemetry -src/plugins/telemetry/schema/oss_plugins.json @elastic/kibana-telemetry -x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @elastic/kibana-telemetry - # Kibana Alerting Services /x-pack/plugins/alerts/ @elastic/kibana-alerting-services /x-pack/plugins/actions/ @elastic/kibana-alerting-services From aa8ec78060765cb614eb46346a099d0560ce22b6 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 25 Nov 2020 00:18:18 +0000 Subject: [PATCH 25/49] chore(NA): enable yarn prefer offline and local mirror for development (#84124) * chore(NA): enable --prefer-offline by default * chore(NA): use prefer offline in the yarnrc * chore(NA): update kbn pm * chore(NA): add yarn offline mirror integration * chore(NA): remove non wanted prune feature due to switching between branches * chore(NA): re-introduce babel require hook --- .ci/packer_cache_for_branch.sh | 3 ++- .gitignore | 3 +++ .yarnrc | 5 +++++ package.json | 6 +++--- src/dev/ci_setup/setup.sh | 2 +- vars/kibanaCoverage.groovy | 4 ++-- vars/kibanaTeamAssign.groovy | 2 +- yarn.lock | 6 +++--- 8 files changed, 20 insertions(+), 11 deletions(-) create mode 100644 .yarnrc diff --git a/.ci/packer_cache_for_branch.sh b/.ci/packer_cache_for_branch.sh index 0d9b22b04dbd0..bc427bf927f11 100755 --- a/.ci/packer_cache_for_branch.sh +++ b/.ci/packer_cache_for_branch.sh @@ -49,7 +49,8 @@ tar -cf "$HOME/.kibana/bootstrap_cache/$branch.tar" \ .chromium \ .es \ .chromedriver \ - .geckodriver; + .geckodriver \ + .yarn-local-mirror; echo "created $HOME/.kibana/bootstrap_cache/$branch.tar" diff --git a/.gitignore b/.gitignore index 45034583cffbb..b786a419383b9 100644 --- a/.gitignore +++ b/.gitignore @@ -75,3 +75,6 @@ report.asciidoc # TS incremental build cache *.tsbuildinfo + +# Yarn local mirror content +.yarn-local-mirror diff --git a/.yarnrc b/.yarnrc new file mode 100644 index 0000000000000..eceec9ca34a22 --- /dev/null +++ b/.yarnrc @@ -0,0 +1,5 @@ +# Configure an offline yarn mirror in the data folder +yarn-offline-mirror ".yarn-local-mirror" + +# Always look into the cache first before fetching online +--install.prefer-offline true diff --git a/package.json b/package.json index 571dc7302f924..ab1cb90c900ac 100644 --- a/package.json +++ b/package.json @@ -243,7 +243,7 @@ "moment": "^2.24.0", "moment-duration-format": "^2.3.2", "moment-timezone": "^0.5.27", - "monaco-editor": "~0.17.0", + "monaco-editor": "^0.17.0", "mustache": "^2.3.2", "ngreact": "^0.5.1", "nock": "12.0.3", @@ -589,7 +589,7 @@ "babel-loader": "^8.0.6", "babel-plugin-add-module-exports": "^1.0.2", "babel-plugin-istanbul": "^6.0.0", - "babel-plugin-require-context-hook": "npm:babel-plugin-require-context-hook-babel7@1.0.0", + "babel-plugin-require-context-hook": "^1.0.0", "babel-plugin-styled-components": "^1.10.7", "babel-plugin-transform-react-remove-prop-types": "^0.4.24", "backport": "5.6.0", @@ -775,7 +775,7 @@ "react-fast-compare": "^2.0.4", "react-grid-layout": "^0.16.2", "react-markdown": "^4.3.1", - "react-monaco-editor": "~0.27.0", + "react-monaco-editor": "^0.27.0", "react-popper-tooltip": "^2.10.1", "react-resize-detector": "^4.2.0", "react-reverse-portal": "^1.0.4", diff --git a/src/dev/ci_setup/setup.sh b/src/dev/ci_setup/setup.sh index aabc1e75b9025..61f578ba33971 100755 --- a/src/dev/ci_setup/setup.sh +++ b/src/dev/ci_setup/setup.sh @@ -14,7 +14,7 @@ echo " -- TEST_ES_SNAPSHOT_VERSION='$TEST_ES_SNAPSHOT_VERSION'" ### install dependencies ### echo " -- installing node.js dependencies" -yarn kbn bootstrap --prefer-offline +yarn kbn bootstrap ### ### Download es snapshots diff --git a/vars/kibanaCoverage.groovy b/vars/kibanaCoverage.groovy index c43be9fb17ee0..521672e4bf48c 100644 --- a/vars/kibanaCoverage.groovy +++ b/vars/kibanaCoverage.groovy @@ -145,7 +145,7 @@ def generateReports(title) { source src/dev/ci_setup/setup_env.sh true # bootstrap from x-pack folder cd x-pack - yarn kbn bootstrap --prefer-offline + yarn kbn bootstrap # Return to project root cd .. . src/dev/code_coverage/shell_scripts/extract_archives.sh @@ -172,7 +172,7 @@ def uploadCombinedReports() { def ingestData(jobName, buildNum, buildUrl, previousSha, teamAssignmentsPath, title) { kibanaPipeline.bash(""" source src/dev/ci_setup/setup_env.sh - yarn kbn bootstrap --prefer-offline + yarn kbn bootstrap # Using existing target/kibana-coverage folder . src/dev/code_coverage/shell_scripts/generate_team_assignments_and_ingest_coverage.sh '${jobName}' ${buildNum} '${buildUrl}' '${previousSha}' '${teamAssignmentsPath}' """, title) diff --git a/vars/kibanaTeamAssign.groovy b/vars/kibanaTeamAssign.groovy index caf1ee36e25a8..590d3af4b7bf9 100644 --- a/vars/kibanaTeamAssign.groovy +++ b/vars/kibanaTeamAssign.groovy @@ -1,7 +1,7 @@ def generateTeamAssignments(teamAssignmentsPath, title) { kibanaPipeline.bash(""" source src/dev/ci_setup/setup_env.sh - yarn kbn bootstrap --prefer-offline + yarn kbn bootstrap # Build team assignments dat file node scripts/generate_team_assignments.js --verbose --dest '${teamAssignmentsPath}' diff --git a/yarn.lock b/yarn.lock index dc171a44dca1f..1cde1266ca38f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7849,7 +7849,7 @@ babel-plugin-react-docgen@^4.1.0: react-docgen "^5.0.0" recast "^0.14.7" -babel-plugin-require-context-hook@^1.0.0, "babel-plugin-require-context-hook@npm:babel-plugin-require-context-hook-babel7@1.0.0": +babel-plugin-require-context-hook@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/babel-plugin-require-context-hook-babel7/-/babel-plugin-require-context-hook-babel7-1.0.0.tgz#1273d4cee7e343d0860966653759a45d727e815d" integrity sha512-kez0BAN/cQoyO1Yu1nre1bQSYZEF93Fg7VQiBHFfMWuaZTy7vJSTT4FY68FwHTYG53Nyt0A7vpSObSVxwweQeQ== @@ -20267,7 +20267,7 @@ moment-timezone@^0.5.27: resolved "https://registry.yarnpkg.com/moment/-/moment-2.28.0.tgz#cdfe73ce01327cee6537b0fafac2e0f21a237d75" integrity sha512-Z5KOjYmnHyd/ukynmFd/WwyXHd7L4J9vTI/nn5Ap9AVUgaAE15VvQ9MOGmJJygEUklupqIrFnor/tjTwRU+tQw== -monaco-editor@~0.17.0: +monaco-editor@^0.17.0: version "0.17.1" resolved "https://registry.yarnpkg.com/monaco-editor/-/monaco-editor-0.17.1.tgz#8fbe96ca54bfa75262706e044f8f780e904aa45c" integrity sha512-JAc0mtW7NeO+0SwPRcdkfDbWLgkqL9WfP1NbpP9wNASsW6oWqgZqNIWt4teymGjZIXTElx3dnQmUYHmVrJ7HxA== @@ -23323,7 +23323,7 @@ react-moment-proptypes@^1.7.0: dependencies: moment ">=1.6.0" -react-monaco-editor@~0.27.0: +react-monaco-editor@^0.27.0: version "0.27.0" resolved "https://registry.yarnpkg.com/react-monaco-editor/-/react-monaco-editor-0.27.0.tgz#2dbf47b8fd4d8e4763934051f07291d9b128bb89" integrity sha512-Im40xO4DuFlQ6kVcSBHC+p70fD/5aErUy1uyLT9RZ4nlehn6BOPpwmcw/2IN/LfMvy8X4WmLuuvrNftBZLH+vA== From 80dfd91abf0840da99dc71f6534d6a997367ca19 Mon Sep 17 00:00:00 2001 From: Uladzislau Lasitsa Date: Wed, 25 Nov 2020 11:09:59 +0300 Subject: [PATCH 26/49] [TSVB] Remove jQuery dependency from plugins (#83809) * Use rxjs instead of jquery for eventBus. * Fix comments * Removed one check because property is private. * Resolve comments Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../lib/{active_cursor.js => active_cursor.ts} | 8 +++----- .../visualizations/views/timeseries/index.js | 10 +++++----- 2 files changed, 8 insertions(+), 10 deletions(-) rename src/plugins/vis_type_timeseries/public/application/visualizations/lib/{active_cursor.js => active_cursor.ts} (78%) diff --git a/src/plugins/vis_type_timeseries/public/application/visualizations/lib/active_cursor.js b/src/plugins/vis_type_timeseries/public/application/visualizations/lib/active_cursor.ts similarity index 78% rename from src/plugins/vis_type_timeseries/public/application/visualizations/lib/active_cursor.js rename to src/plugins/vis_type_timeseries/public/application/visualizations/lib/active_cursor.ts index 427ced4dc3f2a..59a846aa66a07 100644 --- a/src/plugins/vis_type_timeseries/public/application/visualizations/lib/active_cursor.js +++ b/src/plugins/vis_type_timeseries/public/application/visualizations/lib/active_cursor.ts @@ -17,9 +17,7 @@ * under the License. */ -// TODO: Remove bus when action/triggers are available with LegacyPluginApi or metric is converted to Embeddable -import $ from 'jquery'; +import { Subject } from 'rxjs'; +import { PointerEvent } from '@elastic/charts'; -export const ACTIVE_CURSOR = 'ACTIVE_CURSOR'; - -export const eventBus = $({}); +export const activeCursor$ = new Subject(); diff --git a/src/plugins/vis_type_timeseries/public/application/visualizations/views/timeseries/index.js b/src/plugins/vis_type_timeseries/public/application/visualizations/views/timeseries/index.js index 36624cfeea0c2..b13d82387a707 100644 --- a/src/plugins/vis_type_timeseries/public/application/visualizations/views/timeseries/index.js +++ b/src/plugins/vis_type_timeseries/public/application/visualizations/views/timeseries/index.js @@ -34,7 +34,7 @@ import { } from '@elastic/charts'; import { EuiIcon } from '@elastic/eui'; import { getTimezone } from '../../../lib/get_timezone'; -import { eventBus, ACTIVE_CURSOR } from '../../lib/active_cursor'; +import { activeCursor$ } from '../../lib/active_cursor'; import { getUISettings, getChartsSetup } from '../../../../services'; import { GRID_LINE_CONFIG, ICON_TYPES_MAP, STACKED_OPTIONS } from '../../constants'; import { AreaSeriesDecorator } from './decorators/area_decorator'; @@ -54,7 +54,7 @@ const generateAnnotationData = (values, formatter) => const decorateFormatter = (formatter) => ({ value }) => formatter(value); const handleCursorUpdate = (cursor) => { - eventBus.trigger(ACTIVE_CURSOR, cursor); + activeCursor$.next(cursor); }; export const TimeSeries = ({ @@ -73,16 +73,16 @@ export const TimeSeries = ({ const chartRef = useRef(); useEffect(() => { - const updateCursor = (_, cursor) => { + const updateCursor = (cursor) => { if (chartRef.current) { chartRef.current.dispatchExternalPointerEvent(cursor); } }; - eventBus.on(ACTIVE_CURSOR, updateCursor); + const subscription = activeCursor$.subscribe(updateCursor); return () => { - eventBus.off(ACTIVE_CURSOR, undefined, updateCursor); + subscription.unsubscribe(); }; }, []); From 8dfa489da26100a11408d3770a808842e4daf8b0 Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Wed, 25 Nov 2020 09:28:20 +0100 Subject: [PATCH 27/49] [ILM] Relax POST policy route validation (#84203) * relax policy post route validation * update comment --- .../api/policies/register_create_route.ts | 116 ++---------------- 1 file changed, 11 insertions(+), 105 deletions(-) diff --git a/x-pack/plugins/index_lifecycle_management/server/routes/api/policies/register_create_route.ts b/x-pack/plugins/index_lifecycle_management/server/routes/api/policies/register_create_route.ts index d8e40e3b30410..8d21b1f164541 100644 --- a/x-pack/plugins/index_lifecycle_management/server/routes/api/policies/register_create_route.ts +++ b/x-pack/plugins/index_lifecycle_management/server/routes/api/policies/register_create_route.ts @@ -23,114 +23,20 @@ async function createPolicy(client: ElasticsearchClient, name: string, phases: a return client.ilm.putLifecycle({ policy: name, body }, options); } -const minAgeSchema = schema.maybe(schema.string()); - -const setPrioritySchema = schema.maybe( - schema.object({ - priority: schema.nullable(schema.number()), - }) -); - -const unfollowSchema = schema.maybe(schema.object({})); // Unfollow has no options - -const migrateSchema = schema.maybe(schema.object({ enabled: schema.literal(false) })); - -const allocateNodeSchema = schema.maybe(schema.recordOf(schema.string(), schema.string())); -const allocateSchema = schema.maybe( - schema.object({ - number_of_replicas: schema.maybe(schema.number()), - include: allocateNodeSchema, - exclude: allocateNodeSchema, - require: allocateNodeSchema, - }) -); - -const forcemergeSchema = schema.maybe( - schema.object({ - max_num_segments: schema.number(), - index_codec: schema.maybe(schema.literal('best_compression')), - }) -); - -const hotPhaseSchema = schema.object({ - min_age: minAgeSchema, - actions: schema.object({ - set_priority: setPrioritySchema, - unfollow: unfollowSchema, - rollover: schema.maybe( - schema.object({ - max_age: schema.maybe(schema.string()), - max_size: schema.maybe(schema.string()), - max_docs: schema.maybe(schema.number()), - }) - ), - forcemerge: forcemergeSchema, - }), -}); - -const warmPhaseSchema = schema.maybe( - schema.object({ - min_age: minAgeSchema, - actions: schema.object({ - migrate: migrateSchema, - set_priority: setPrioritySchema, - unfollow: unfollowSchema, - readonly: schema.maybe(schema.object({})), // Readonly has no options - allocate: allocateSchema, - shrink: schema.maybe( - schema.object({ - number_of_shards: schema.number(), - }) - ), - forcemerge: forcemergeSchema, - }), - }) -); - -const coldPhaseSchema = schema.maybe( - schema.object({ - min_age: minAgeSchema, - actions: schema.object({ - migrate: migrateSchema, - set_priority: setPrioritySchema, - unfollow: unfollowSchema, - allocate: allocateSchema, - freeze: schema.maybe(schema.object({})), // Freeze has no options - searchable_snapshot: schema.maybe( - schema.object({ - snapshot_repository: schema.string(), - }) - ), - }), - }) -); - -const deletePhaseSchema = schema.maybe( - schema.object({ - min_age: minAgeSchema, - actions: schema.object({ - wait_for_snapshot: schema.maybe( - schema.object({ - policy: schema.string(), - }) - ), - delete: schema.maybe( - schema.object({ - delete_searchable_snapshot: schema.maybe(schema.boolean()), - }) - ), - }), - }) -); - -// Per https://www.elastic.co/guide/en/elasticsearch/reference/current/_actions.html +/** + * We intentionally do not deeply validate the posted policy object to avoid erroring on valid ES + * policy configuration Kibana UI does not know or should not know about. For instance, the + * `force_merge_index` setting of the `searchable_snapshot` action. + * + * We only specify a rough structure based on https://www.elastic.co/guide/en/elasticsearch/reference/current/_actions.html. + */ const bodySchema = schema.object({ name: schema.string(), phases: schema.object({ - hot: hotPhaseSchema, - warm: warmPhaseSchema, - cold: coldPhaseSchema, - delete: deletePhaseSchema, + hot: schema.any(), + warm: schema.maybe(schema.any()), + cold: schema.maybe(schema.any()), + delete: schema.maybe(schema.any()), }), }); From a763d3302f8c52f681d4b4f81e244c3dc83242eb Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 25 Nov 2020 09:43:23 +0100 Subject: [PATCH 28/49] [Lens] Calculation operations (#83789) --- .../workspace_panel/workspace_panel.tsx | 2 +- .../public/indexpattern_datasource/index.ts | 2 + .../indexpattern.test.ts | 19 ++- .../indexpattern_datasource/indexpattern.tsx | 1 + .../definitions/calculations/counter_rate.tsx | 91 +++++++++++ .../calculations/cumulative_sum.tsx | 91 +++++++++++ .../definitions/calculations/derivative.tsx | 90 +++++++++++ .../definitions/calculations/index.ts | 10 ++ .../calculations/moving_average.tsx | 147 ++++++++++++++++++ .../definitions/calculations/utils.ts | 67 ++++++++ .../operations/definitions/column_types.ts | 2 +- .../operations/definitions/index.ts | 20 ++- .../operations/layer_helpers.ts | 11 +- .../operations/operations.test.ts | 16 ++ 14 files changed, 552 insertions(+), 17 deletions(-) create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/derivative.tsx create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/index.ts create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.ts diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx index 95aeedbd857ca..6c2c01d944cd9 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx @@ -161,7 +161,7 @@ export function WorkspacePanel({ const expression = useMemo( () => { - if (!configurationValidationError) { + if (!configurationValidationError || configurationValidationError.length === 0) { try { return buildExpression({ visualization: activeVisualization, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/index.ts index 793f3387e707d..5f7eddd807c93 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/index.ts @@ -37,12 +37,14 @@ export class IndexPatternDatasource { getIndexPatternDatasource, renameColumns, formatColumn, + counterRate, getTimeScaleFunction, getSuffixFormatter, } = await import('../async_services'); return core.getStartServices().then(([coreStart, { data }]) => { data.fieldFormats.register([getSuffixFormatter(data.fieldFormats.deserialize)]); expressions.registerFunction(getTimeScaleFunction(data)); + expressions.registerFunction(counterRate); expressions.registerFunction(renameColumns); expressions.registerFunction(formatColumn); return getIndexPatternDatasource({ diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts index 3cf9bdc3a92f1..c3247b251d88a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts @@ -661,19 +661,30 @@ describe('IndexPattern Data Source', () => { it('should skip columns that are being referenced', () => { publicAPI = indexPatternDatasource.getPublicAPI({ state: { + ...enrichBaseState(baseState), layers: { first: { indexPatternId: '1', columnOrder: ['col1', 'col2'], columns: { - // @ts-ignore this is too little information for a real column col1: { + label: 'Sum', dataType: 'number', - }, + isBucketed: false, + + operationType: 'sum', + sourceField: 'test', + params: {}, + } as IndexPatternColumn, col2: { - // @ts-expect-error update once we have a reference operation outside tests + label: 'Cumulative sum', + dataType: 'number', + isBucketed: false, + + operationType: 'cumulative_sum', references: ['col1'], - }, + params: {}, + } as IndexPatternColumn, }, }, }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx index 2c64431867df0..289b6bbe3f25b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx @@ -78,6 +78,7 @@ export function columnToOperation(column: IndexPatternColumn, uniqueLabel?: stri export * from './rename_columns'; export * from './format_column'; export * from './time_scale'; +export * from './counter_rate'; export * from './suffix_formatter'; export function getIndexPatternDatasource({ diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx new file mode 100644 index 0000000000000..d256b74696a4c --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import { FormattedIndexPatternColumn, ReferenceBasedIndexPatternColumn } from '../column_types'; +import { IndexPatternLayer } from '../../../types'; +import { checkForDateHistogram, dateBasedOperationToExpression, hasDateField } from './utils'; +import { OperationDefinition } from '..'; + +const ofName = (name?: string) => { + return i18n.translate('xpack.lens.indexPattern.CounterRateOf', { + defaultMessage: 'Counter rate of {name}', + values: { + name: + name ?? + i18n.translate('xpack.lens.indexPattern.incompleteOperation', { + defaultMessage: '(incomplete)', + }), + }, + }); +}; + +export type CounterRateIndexPatternColumn = FormattedIndexPatternColumn & + ReferenceBasedIndexPatternColumn & { + operationType: 'counter_rate'; + }; + +export const counterRateOperation: OperationDefinition< + CounterRateIndexPatternColumn, + 'fullReference' +> = { + type: 'counter_rate', + priority: 1, + displayName: i18n.translate('xpack.lens.indexPattern.counterRate', { + defaultMessage: 'Counter rate', + }), + input: 'fullReference', + selectionStyle: 'field', + requiredReferences: [ + { + input: ['field'], + specificOperations: ['max'], + validateMetadata: (meta) => meta.dataType === 'number' && !meta.isBucketed, + }, + ], + getPossibleOperation: () => { + return { + dataType: 'number', + isBucketed: false, + scale: 'ratio', + }; + }, + getDefaultLabel: (column, indexPattern, columns) => { + return ofName(columns[column.references[0]]?.label); + }, + toExpression: (layer, columnId) => { + return dateBasedOperationToExpression(layer, columnId, 'lens_counter_rate'); + }, + buildColumn: ({ referenceIds, previousColumn, layer }) => { + const metric = layer.columns[referenceIds[0]]; + return { + label: ofName(metric?.label), + dataType: 'number', + operationType: 'counter_rate', + isBucketed: false, + scale: 'ratio', + references: referenceIds, + params: + previousColumn?.dataType === 'number' && + previousColumn.params && + 'format' in previousColumn.params && + previousColumn.params.format + ? { format: previousColumn.params.format } + : undefined, + }; + }, + isTransferable: (column, newIndexPattern) => { + return hasDateField(newIndexPattern); + }, + getErrorMessage: (layer: IndexPatternLayer) => { + return checkForDateHistogram( + layer, + i18n.translate('xpack.lens.indexPattern.counterRate', { + defaultMessage: 'Counter rate', + }) + ); + }, +}; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx new file mode 100644 index 0000000000000..9244aaaf90ab7 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import { FormattedIndexPatternColumn, ReferenceBasedIndexPatternColumn } from '../column_types'; +import { IndexPatternLayer } from '../../../types'; +import { checkForDateHistogram, dateBasedOperationToExpression } from './utils'; +import { OperationDefinition } from '..'; + +const ofName = (name?: string) => { + return i18n.translate('xpack.lens.indexPattern.cumulativeSumOf', { + defaultMessage: 'Cumulative sum rate of {name}', + values: { + name: + name ?? + i18n.translate('xpack.lens.indexPattern.incompleteOperation', { + defaultMessage: '(incomplete)', + }), + }, + }); +}; + +export type CumulativeSumIndexPatternColumn = FormattedIndexPatternColumn & + ReferenceBasedIndexPatternColumn & { + operationType: 'cumulative_sum'; + }; + +export const cumulativeSumOperation: OperationDefinition< + CumulativeSumIndexPatternColumn, + 'fullReference' +> = { + type: 'cumulative_sum', + priority: 1, + displayName: i18n.translate('xpack.lens.indexPattern.cumulativeSum', { + defaultMessage: 'Cumulative sum', + }), + input: 'fullReference', + selectionStyle: 'field', + requiredReferences: [ + { + input: ['field'], + specificOperations: ['count', 'sum'], + validateMetadata: (meta) => meta.dataType === 'number' && !meta.isBucketed, + }, + ], + getPossibleOperation: () => { + return { + dataType: 'number', + isBucketed: false, + scale: 'ratio', + }; + }, + getDefaultLabel: (column, indexPattern, columns) => { + return ofName(columns[column.references[0]]?.label); + }, + toExpression: (layer, columnId) => { + return dateBasedOperationToExpression(layer, columnId, 'cumulative_sum'); + }, + buildColumn: ({ referenceIds, previousColumn, layer }) => { + const metric = layer.columns[referenceIds[0]]; + return { + label: ofName(metric?.label), + dataType: 'number', + operationType: 'cumulative_sum', + isBucketed: false, + scale: 'ratio', + references: referenceIds, + params: + previousColumn?.dataType === 'number' && + previousColumn.params && + 'format' in previousColumn.params && + previousColumn.params.format + ? { format: previousColumn.params.format } + : undefined, + }; + }, + isTransferable: () => { + return true; + }, + getErrorMessage: (layer: IndexPatternLayer) => { + return checkForDateHistogram( + layer, + i18n.translate('xpack.lens.indexPattern.cumulativeSum', { + defaultMessage: 'Cumulative sum', + }) + ); + }, +}; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/derivative.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/derivative.tsx new file mode 100644 index 0000000000000..7398f7e07ea4e --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/derivative.tsx @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import { FormattedIndexPatternColumn, ReferenceBasedIndexPatternColumn } from '../column_types'; +import { IndexPatternLayer } from '../../../types'; +import { checkForDateHistogram, dateBasedOperationToExpression, hasDateField } from './utils'; +import { OperationDefinition } from '..'; + +const ofName = (name?: string) => { + return i18n.translate('xpack.lens.indexPattern.derivativeOf', { + defaultMessage: 'Differences of {name}', + values: { + name: + name ?? + i18n.translate('xpack.lens.indexPattern.incompleteOperation', { + defaultMessage: '(incomplete)', + }), + }, + }); +}; + +export type DerivativeIndexPatternColumn = FormattedIndexPatternColumn & + ReferenceBasedIndexPatternColumn & { + operationType: 'derivative'; + }; + +export const derivativeOperation: OperationDefinition< + DerivativeIndexPatternColumn, + 'fullReference' +> = { + type: 'derivative', + priority: 1, + displayName: i18n.translate('xpack.lens.indexPattern.derivative', { + defaultMessage: 'Differences', + }), + input: 'fullReference', + selectionStyle: 'full', + requiredReferences: [ + { + input: ['field'], + validateMetadata: (meta) => meta.dataType === 'number' && !meta.isBucketed, + }, + ], + getPossibleOperation: () => { + return { + dataType: 'number', + isBucketed: false, + scale: 'ratio', + }; + }, + getDefaultLabel: (column, indexPattern, columns) => { + return ofName(columns[column.references[0]]?.label); + }, + toExpression: (layer, columnId) => { + return dateBasedOperationToExpression(layer, columnId, 'derivative'); + }, + buildColumn: ({ referenceIds, previousColumn, layer }) => { + const metric = layer.columns[referenceIds[0]]; + return { + label: ofName(metric?.label), + dataType: 'number', + operationType: 'derivative', + isBucketed: false, + scale: 'ratio', + references: referenceIds, + params: + previousColumn?.dataType === 'number' && + previousColumn.params && + 'format' in previousColumn.params && + previousColumn.params.format + ? { format: previousColumn.params.format } + : undefined, + }; + }, + isTransferable: (column, newIndexPattern) => { + return hasDateField(newIndexPattern); + }, + getErrorMessage: (layer: IndexPatternLayer) => { + return checkForDateHistogram( + layer, + i18n.translate('xpack.lens.indexPattern.derivative', { + defaultMessage: 'Differences', + }) + ); + }, +}; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/index.ts new file mode 100644 index 0000000000000..30e87aef46a0d --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { counterRateOperation, CounterRateIndexPatternColumn } from './counter_rate'; +export { cumulativeSumOperation, CumulativeSumIndexPatternColumn } from './cumulative_sum'; +export { derivativeOperation, DerivativeIndexPatternColumn } from './derivative'; +export { movingAverageOperation, MovingAverageIndexPatternColumn } from './moving_average'; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx new file mode 100644 index 0000000000000..795281d0fd994 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx @@ -0,0 +1,147 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import { useState } from 'react'; +import React from 'react'; +import { EuiFormRow } from '@elastic/eui'; +import { EuiFieldNumber } from '@elastic/eui'; +import { FormattedIndexPatternColumn, ReferenceBasedIndexPatternColumn } from '../column_types'; +import { IndexPatternLayer } from '../../../types'; +import { checkForDateHistogram, dateBasedOperationToExpression, hasDateField } from './utils'; +import { updateColumnParam } from '../../layer_helpers'; +import { useDebounceWithOptions } from '../helpers'; +import type { OperationDefinition, ParamEditorProps } from '..'; + +const ofName = (name?: string) => { + return i18n.translate('xpack.lens.indexPattern.movingAverageOf', { + defaultMessage: 'Moving average of {name}', + values: { + name: + name ?? + i18n.translate('xpack.lens.indexPattern.incompleteOperation', { + defaultMessage: '(incomplete)', + }), + }, + }); +}; + +export type MovingAverageIndexPatternColumn = FormattedIndexPatternColumn & + ReferenceBasedIndexPatternColumn & { + operationType: 'moving_average'; + params: { + window: number; + }; + }; + +export const movingAverageOperation: OperationDefinition< + MovingAverageIndexPatternColumn, + 'fullReference' +> = { + type: 'moving_average', + priority: 1, + displayName: i18n.translate('xpack.lens.indexPattern.movingAverage', { + defaultMessage: 'Moving Average', + }), + input: 'fullReference', + selectionStyle: 'full', + requiredReferences: [ + { + input: ['field'], + validateMetadata: (meta) => meta.dataType === 'number' && !meta.isBucketed, + }, + ], + getPossibleOperation: () => { + return { + dataType: 'number', + isBucketed: false, + scale: 'ratio', + }; + }, + getDefaultLabel: (column, indexPattern, columns) => { + return ofName(columns[column.references[0]]?.label); + }, + toExpression: (layer, columnId) => { + return dateBasedOperationToExpression(layer, columnId, 'moving_average', { + window: [(layer.columns[columnId] as MovingAverageIndexPatternColumn).params.window], + }); + }, + buildColumn: ({ referenceIds, previousColumn, layer }) => { + const metric = layer.columns[referenceIds[0]]; + return { + label: ofName(metric?.label), + dataType: 'number', + operationType: 'moving_average', + isBucketed: false, + scale: 'ratio', + references: referenceIds, + params: + previousColumn?.dataType === 'number' && + previousColumn.params && + 'format' in previousColumn.params && + previousColumn.params.format + ? { format: previousColumn.params.format, window: 5 } + : { window: 5 }, + }; + }, + paramEditor: MovingAverageParamEditor, + isTransferable: (column, newIndexPattern) => { + return hasDateField(newIndexPattern); + }, + getErrorMessage: (layer: IndexPatternLayer) => { + return checkForDateHistogram( + layer, + i18n.translate('xpack.lens.indexPattern.movingAverage', { + defaultMessage: 'Moving Average', + }) + ); + }, +}; + +function MovingAverageParamEditor({ + state, + setState, + currentColumn, + layerId, +}: ParamEditorProps) { + const [inputValue, setInputValue] = useState(String(currentColumn.params.window)); + + useDebounceWithOptions( + () => { + if (inputValue === '') { + return; + } + const inputNumber = Number(inputValue); + setState( + updateColumnParam({ + state, + layerId, + currentColumn, + paramName: 'window', + value: inputNumber, + }) + ); + }, + { skipFirstRender: true }, + 256, + [inputValue] + ); + return ( + + ) => setInputValue(e.target.value)} + /> + + ); +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.ts new file mode 100644 index 0000000000000..c64a292280603 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import { ExpressionFunctionAST } from '@kbn/interpreter/common'; +import { IndexPattern, IndexPatternLayer } from '../../../types'; +import { ReferenceBasedIndexPatternColumn } from '../column_types'; + +/** + * Checks whether the current layer includes a date histogram and returns an error otherwise + */ +export function checkForDateHistogram(layer: IndexPatternLayer, name: string) { + const buckets = layer.columnOrder.filter((colId) => layer.columns[colId].isBucketed); + const hasDateHistogram = buckets.some( + (colId) => layer.columns[colId].operationType === 'date_histogram' + ); + if (hasDateHistogram) { + return undefined; + } + return [ + i18n.translate('xpack.lens.indexPattern.calculations.dateHistogramErrorMessage', { + defaultMessage: + '{name} requires a date histogram to work. Choose a different function or add a date histogram.', + values: { + name, + }, + }), + ]; +} + +export function hasDateField(indexPattern: IndexPattern) { + return indexPattern.fields.some((field) => field.type === 'date'); +} + +/** + * Creates an expression ast for a date based operation (cumulative sum, derivative, moving average, counter rate) + */ +export function dateBasedOperationToExpression( + layer: IndexPatternLayer, + columnId: string, + functionName: string, + additionalArgs: Record = {} +): ExpressionFunctionAST[] { + const currentColumn = (layer.columns[columnId] as unknown) as ReferenceBasedIndexPatternColumn; + const buckets = layer.columnOrder.filter((colId) => layer.columns[colId].isBucketed); + const dateColumnIndex = buckets.findIndex( + (colId) => layer.columns[colId].operationType === 'date_histogram' + )!; + buckets.splice(dateColumnIndex, 1); + + return [ + { + type: 'function', + function: functionName, + arguments: { + by: buckets, + inputColumnId: [currentColumn.references[0]], + outputColumnId: [columnId], + outputColumnName: [currentColumn.label], + ...additionalArgs, + }, + }, + ]; +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/column_types.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/column_types.ts index 13bddc0c2ec26..aef9bb7731d4c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/column_types.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/column_types.ts @@ -16,7 +16,7 @@ export interface BaseIndexPatternColumn extends Operation { // export interface FormattedIndexPatternColumn extends BaseIndexPatternColumn { export type FormattedIndexPatternColumn = BaseIndexPatternColumn & { params?: { - format: { + format?: { id: string; params?: { decimals: number; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index 0e7e125944e71..392377234d76d 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -23,6 +23,16 @@ import { MedianIndexPatternColumn, } from './metrics'; import { dateHistogramOperation, DateHistogramIndexPatternColumn } from './date_histogram'; +import { + cumulativeSumOperation, + CumulativeSumIndexPatternColumn, + counterRateOperation, + CounterRateIndexPatternColumn, + derivativeOperation, + DerivativeIndexPatternColumn, + movingAverageOperation, + MovingAverageIndexPatternColumn, +} from './calculations'; import { countOperation, CountIndexPatternColumn } from './count'; import { StateSetter, OperationMetadata } from '../../../types'; import type { BaseIndexPatternColumn, ReferenceBasedIndexPatternColumn } from './column_types'; @@ -52,7 +62,11 @@ export type IndexPatternColumn = | CardinalityIndexPatternColumn | SumIndexPatternColumn | MedianIndexPatternColumn - | CountIndexPatternColumn; + | CountIndexPatternColumn + | CumulativeSumIndexPatternColumn + | CounterRateIndexPatternColumn + | DerivativeIndexPatternColumn + | MovingAverageIndexPatternColumn; export type FieldBasedIndexPatternColumn = Extract; @@ -73,6 +87,10 @@ const internalOperationDefinitions = [ medianOperation, countOperation, rangeOperation, + cumulativeSumOperation, + counterRateOperation, + derivativeOperation, + movingAverageOperation, ]; export { termsOperation } from './terms'; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index 1495a876a2c8e..58a066c81a1a7 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -424,7 +424,6 @@ export function deleteColumn({ }; } - // @ts-expect-error this fails statically because there are no references added const extraDeletions: string[] = 'references' in column ? column.references : []; const hypotheticalColumns = { ...layer.columns }; @@ -452,11 +451,9 @@ export function getColumnOrder(layer: IndexPatternLayer): string[] { ); // If a reference has another reference as input, put it last in sort order referenceBased.sort(([idA, a], [idB, b]) => { - // @ts-expect-error not statically analyzed if ('references' in a && a.references.includes(idB)) { return 1; } - // @ts-expect-error not statically analyzed if ('references' in b && b.references.includes(idA)) { return -1; } @@ -517,14 +514,12 @@ export function getErrorMessages(layer: IndexPatternLayer): string[] | undefined } if ('references' in column) { - // @ts-expect-error references are not statically analyzed yet column.references.forEach((referenceId, index) => { if (!layer.columns[referenceId]) { errors.push( i18n.translate('xpack.lens.indexPattern.missingReferenceError', { defaultMessage: 'Dimension {dimensionLabel} is incomplete', values: { - // @ts-expect-error references are not statically analyzed yet dimensionLabel: column.label, }, }) @@ -544,7 +539,6 @@ export function getErrorMessages(layer: IndexPatternLayer): string[] | undefined i18n.translate('xpack.lens.indexPattern.invalidReferenceConfiguration', { defaultMessage: 'Dimension {dimensionLabel} does not have a valid configuration', values: { - // @ts-expect-error references are not statically analyzed yet dimensionLabel: column.label, }, }) @@ -560,10 +554,7 @@ export function getErrorMessages(layer: IndexPatternLayer): string[] | undefined export function isReferenced(layer: IndexPatternLayer, columnId: string): boolean { const allReferences = Object.values(layer.columns).flatMap((col) => - 'references' in col - ? // @ts-expect-error not statically analyzed - col.references - : [] + 'references' in col ? col.references : [] ); return allReferences.includes(columnId); } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts index d6f5b10cf64e1..63d0fd3d4e5c5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts @@ -247,6 +247,22 @@ describe('getOperationTypesForField', () => { "operationType": "sum", "type": "field", }, + Object { + "operationType": "cumulative_sum", + "type": "fullReference", + }, + Object { + "operationType": "counter_rate", + "type": "fullReference", + }, + Object { + "operationType": "derivative", + "type": "fullReference", + }, + Object { + "operationType": "moving_average", + "type": "fullReference", + }, Object { "field": "bytes", "operationType": "min", From 6cd4d8465703807aac16db38e0783eb46037da78 Mon Sep 17 00:00:00 2001 From: Marco Liberati Date: Wed, 25 Nov 2020 10:29:08 +0100 Subject: [PATCH 29/49] [Lens] Fix Treemap outer labels with transparent background (#84245) --- .../lens/public/pie_visualization/render_function.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/lens/public/pie_visualization/render_function.tsx b/x-pack/plugins/lens/public/pie_visualization/render_function.tsx index 20d558fefc3d7..56ecf57f2dff7 100644 --- a/x-pack/plugins/lens/public/pie_visualization/render_function.tsx +++ b/x-pack/plugins/lens/public/pie_visualization/render_function.tsx @@ -68,6 +68,7 @@ export function PieComponent( } = props.args; const chartTheme = chartsThemeService.useChartsTheme(); const chartBaseTheme = chartsThemeService.useChartsBaseTheme(); + const isDarkMode = chartsThemeService.useDarkMode(); if (!hideLabels) { firstTable.columns.forEach((column) => { @@ -128,7 +129,9 @@ export function PieComponent( if (shape === 'treemap') { // Only highlight the innermost color of the treemap, as it accurately represents area if (layerIndex < bucketColumns.length - 1) { - return 'rgba(0,0,0,0)'; + // Mind the difference here: the contrast computation for the text ignores the alpha/opacity + // therefore change it for dask mode + return isDarkMode ? 'rgba(0,0,0,0)' : 'rgba(255,255,255,0)'; } // only use the top level series layer for coloring if (seriesLayers.length > 1) { @@ -263,6 +266,7 @@ export function PieComponent( theme={{ ...chartTheme, background: { + ...chartTheme.background, color: undefined, // removes background for embeddables }, }} From 4aa1683b3b95a8f4842a82e38a3ac7306fdd7a1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Kopyci=C5=84ski?= Date: Wed, 25 Nov 2020 10:45:32 +0100 Subject: [PATCH 30/49] [Security Solution] Cleanup graphiql (#82595) --- package.json | 1 - .../server/lib/compose/kibana.ts | 3 +- .../lib/framework/kibana_framework_adapter.ts | 32 +---------- .../security_solution/server/plugin.ts | 2 +- .../security_solution/feature_controls.ts | 53 ------------------- 5 files changed, 3 insertions(+), 88 deletions(-) diff --git a/package.json b/package.json index ab1cb90c900ac..1febfc2380b7a 100644 --- a/package.json +++ b/package.json @@ -160,7 +160,6 @@ "apollo-server-core": "^1.3.6", "apollo-server-errors": "^2.0.2", "apollo-server-hapi": "^1.3.6", - "apollo-server-module-graphiql": "^1.3.4", "archiver": "^3.1.1", "axios": "^0.19.2", "bluebird": "3.5.5", diff --git a/x-pack/plugins/security_solution/server/lib/compose/kibana.ts b/x-pack/plugins/security_solution/server/lib/compose/kibana.ts index 433ee4a5f99fa..b680b19f31813 100644 --- a/x-pack/plugins/security_solution/server/lib/compose/kibana.ts +++ b/x-pack/plugins/security_solution/server/lib/compose/kibana.ts @@ -23,10 +23,9 @@ import { EndpointAppContext } from '../../endpoint/types'; export function compose( core: CoreSetup, plugins: SetupPlugins, - isProductionMode: boolean, endpointContext: EndpointAppContext ): AppBackendLibs { - const framework = new KibanaBackendFrameworkAdapter(core, plugins, isProductionMode); + const framework = new KibanaBackendFrameworkAdapter(core, plugins); const sources = new Sources(new ConfigurationSourcesAdapter()); const sourceStatus = new SourceStatus(new ElasticsearchSourceStatusAdapter(framework)); diff --git a/x-pack/plugins/security_solution/server/lib/framework/kibana_framework_adapter.ts b/x-pack/plugins/security_solution/server/lib/framework/kibana_framework_adapter.ts index e36fb1144e93f..8327af846d1ac 100644 --- a/x-pack/plugins/security_solution/server/lib/framework/kibana_framework_adapter.ts +++ b/x-pack/plugins/security_solution/server/lib/framework/kibana_framework_adapter.ts @@ -4,7 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -import * as GraphiQL from 'apollo-server-module-graphiql'; import { GraphQLSchema } from 'graphql'; import { runHttpQuery } from 'apollo-server-core'; import { schema as configSchema } from '@kbn/config-schema'; @@ -31,7 +30,7 @@ export class KibanaBackendFrameworkAdapter implements FrameworkAdapter { private router: IRouter; private security: SetupPlugins['security']; - constructor(core: CoreSetup, plugins: SetupPlugins, private isProductionMode: boolean) { + constructor(core: CoreSetup, plugins: SetupPlugins) { this.router = core.http.createRouter(); this.security = plugins.security; } @@ -90,35 +89,6 @@ export class KibanaBackendFrameworkAdapter implements FrameworkAdapter { } } ); - - if (!this.isProductionMode) { - this.router.get( - { - path: `${routePath}/graphiql`, - validate: false, - options: { - tags: ['access:securitySolution'], - }, - }, - async (context, request, response) => { - const graphiqlString = await GraphiQL.resolveGraphiQLString( - request.query, - { - endpointURL: routePath, - passHeader: "'kbn-xsrf': 'graphiql'", - }, - request - ); - - return response.ok({ - body: graphiqlString, - headers: { - 'content-type': 'text/html', - }, - }); - } - ); - } } private async getCurrentUserInfo(request: KibanaRequest): Promise { diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index d963b3b093d81..088af40a84ae0 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -290,7 +290,7 @@ export class Plugin implements IPlugin { diff --git a/x-pack/test/api_integration/apis/security_solution/feature_controls.ts b/x-pack/test/api_integration/apis/security_solution/feature_controls.ts index 0137a90ce9817..9377c255f2d19 100644 --- a/x-pack/test/api_integration/apis/security_solution/feature_controls.ts +++ b/x-pack/test/api_integration/apis/security_solution/feature_controls.ts @@ -19,8 +19,6 @@ const introspectionQuery = gql` `; export default function ({ getService }: FtrProviderContext) { - const config = getService('config'); - const supertest = getService('supertestWithoutAuth'); const security = getService('security'); const spaces = getService('spaces'); const clientFactory = getService('securitySolutionGraphQLClientFactory'); @@ -38,18 +36,6 @@ export default function ({ getService }: FtrProviderContext) { expect(result.response.data).to.be.an('object'); }; - const expectGraphIQL404 = (result: any) => { - expect(result.error).to.be(undefined); - expect(result.response).not.to.be(undefined); - expect(result.response).to.have.property('statusCode', 404); - }; - - const expectGraphIQLResponse = (result: any) => { - expect(result.error).to.be(undefined); - expect(result.response).not.to.be(undefined); - expect(result.response).to.have.property('statusCode', 200); - }; - const executeGraphQLQuery = async (username: string, password: string, spaceId?: string) => { const queryOptions = { query: introspectionQuery, @@ -71,23 +57,7 @@ export default function ({ getService }: FtrProviderContext) { }; }; - const executeGraphIQLRequest = async (username: string, password: string, spaceId?: string) => { - const basePath = spaceId ? `/s/${spaceId}` : ''; - - return supertest - .get(`${basePath}/api/security_solution/graphql/graphiql`) - .auth(username, password) - .then((response: any) => ({ error: undefined, response })) - .catch((error: any) => ({ error, response: undefined })); - }; - describe('feature controls', () => { - let isProdOrCi = false; - before(() => { - const kbnConfig = config.get('servers.kibana'); - isProdOrCi = - !!process.env.CI || !(kbnConfig.hostname === 'localhost' && kbnConfig.port === 5620); - }); it(`APIs can't be accessed by user with no privileges`, async () => { const username = 'logstash_read'; const roleName = 'logstash_read'; @@ -103,9 +73,6 @@ export default function ({ getService }: FtrProviderContext) { const graphQLResult = await executeGraphQLQuery(username, password); expectGraphQL403(graphQLResult); - - const graphQLIResult = await executeGraphIQLRequest(username, password); - expectGraphIQL404(graphQLIResult); } finally { await security.role.delete(roleName); await security.user.delete(username); @@ -134,13 +101,6 @@ export default function ({ getService }: FtrProviderContext) { const graphQLResult = await executeGraphQLQuery(username, password); expectGraphQLResponse(graphQLResult); - - const graphQLIResult = await executeGraphIQLRequest(username, password); - if (!isProdOrCi) { - expectGraphIQLResponse(graphQLIResult); - } else { - expectGraphIQL404(graphQLIResult); - } } finally { await security.role.delete(roleName); await security.user.delete(username); @@ -172,9 +132,6 @@ export default function ({ getService }: FtrProviderContext) { const graphQLResult = await executeGraphQLQuery(username, password); expectGraphQL403(graphQLResult); - - const graphQLIResult = await executeGraphIQLRequest(username, password); - expectGraphIQL404(graphQLIResult); } finally { await security.role.delete(roleName); await security.user.delete(username); @@ -233,21 +190,11 @@ export default function ({ getService }: FtrProviderContext) { it('user_1 can access APIs in space_1', async () => { const graphQLResult = await executeGraphQLQuery(username, password, space1Id); expectGraphQLResponse(graphQLResult); - - const graphQLIResult = await executeGraphIQLRequest(username, password, space1Id); - if (!isProdOrCi) { - expectGraphIQLResponse(graphQLIResult); - } else { - expectGraphIQL404(graphQLIResult); - } }); it(`user_1 can't access APIs in space_2`, async () => { const graphQLResult = await executeGraphQLQuery(username, password, space2Id); expectGraphQL403(graphQLResult); - - const graphQLIResult = await executeGraphIQLRequest(username, password, space2Id); - expectGraphIQL404(graphQLIResult); }); }); }); From f294a9e2abb896fcb37e8fdf815c46fc69b3965e Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Wed, 25 Nov 2020 11:07:08 +0100 Subject: [PATCH 31/49] [Discover] Refactor discover.js controller topnav code (#79062) * Move discover.js functions to helper functions in separate files * Convert to TypeScript * Add unit tests * Add removeField function to SearchSource --- ...plugin-plugins-data-public.searchsource.md | 1 + ...ns-data-public.searchsource.removefield.md | 24 + .../data/common/search/search_source/mocks.ts | 1 + .../search_source/search_source.test.ts | 9 + .../search/search_source/search_source.ts | 14 +- src/plugins/data/public/public.api.md | 1 + .../discover/public/__mocks__/config.ts | 30 ++ .../public/__mocks__/index_pattern.ts | 74 +++ .../public/__mocks__/index_patterns.ts | 32 ++ .../discover/public/__mocks__/saved_search.ts | 41 ++ .../public/application/angular/discover.js | 510 +++--------------- .../angular/discover_state.test.ts | 4 +- .../application/angular/discover_state.ts | 15 +- ...s.snap => open_search_panel.test.tsx.snap} | 4 +- .../top_nav/get_top_nav_links.test.ts | 85 +++ .../components/top_nav/get_top_nav_links.ts | 148 +++++ .../top_nav/on_save_search.test.tsx | 47 ++ .../components/top_nav/on_save_search.tsx | 158 ++++++ ...nel.test.js => open_search_panel.test.tsx} | 4 +- ..._search_panel.js => open_search_panel.tsx} | 14 +- ...ch_panel.js => show_open_search_panel.tsx} | 9 +- .../public/application/helpers/breadcrumbs.ts | 28 + .../helpers/calc_field_counts.test.ts | 58 ++ .../application/helpers/calc_field_counts.ts | 38 ++ .../helpers/get_index_pattern_id.ts | 60 --- .../helpers/get_sharing_data.test.ts | 70 +++ .../application/helpers/get_sharing_data.ts | 88 +++ .../helpers/persist_saved_search.ts | 65 +++ .../helpers/resolve_index_pattern.test.ts | 56 ++ .../helpers/resolve_index_pattern.ts | 158 ++++++ .../helpers/update_search_source.test.ts | 51 ++ .../helpers/update_search_source.ts | 54 ++ .../discover/public/saved_searches/types.ts | 7 +- 33 files changed, 1449 insertions(+), 509 deletions(-) create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.removefield.md create mode 100644 src/plugins/discover/public/__mocks__/config.ts create mode 100644 src/plugins/discover/public/__mocks__/index_pattern.ts create mode 100644 src/plugins/discover/public/__mocks__/index_patterns.ts create mode 100644 src/plugins/discover/public/__mocks__/saved_search.ts rename src/plugins/discover/public/application/components/top_nav/__snapshots__/{open_search_panel.test.js.snap => open_search_panel.test.tsx.snap} (96%) create mode 100644 src/plugins/discover/public/application/components/top_nav/get_top_nav_links.test.ts create mode 100644 src/plugins/discover/public/application/components/top_nav/get_top_nav_links.ts create mode 100644 src/plugins/discover/public/application/components/top_nav/on_save_search.test.tsx create mode 100644 src/plugins/discover/public/application/components/top_nav/on_save_search.tsx rename src/plugins/discover/public/application/components/top_nav/{open_search_panel.test.js => open_search_panel.test.tsx} (89%) rename src/plugins/discover/public/application/components/top_nav/{open_search_panel.js => open_search_panel.tsx} (94%) rename src/plugins/discover/public/application/components/top_nav/{show_open_search_panel.js => show_open_search_panel.tsx} (87%) create mode 100644 src/plugins/discover/public/application/helpers/calc_field_counts.test.ts create mode 100644 src/plugins/discover/public/application/helpers/calc_field_counts.ts delete mode 100644 src/plugins/discover/public/application/helpers/get_index_pattern_id.ts create mode 100644 src/plugins/discover/public/application/helpers/get_sharing_data.test.ts create mode 100644 src/plugins/discover/public/application/helpers/get_sharing_data.ts create mode 100644 src/plugins/discover/public/application/helpers/persist_saved_search.ts create mode 100644 src/plugins/discover/public/application/helpers/resolve_index_pattern.test.ts create mode 100644 src/plugins/discover/public/application/helpers/resolve_index_pattern.ts create mode 100644 src/plugins/discover/public/application/helpers/update_search_source.test.ts create mode 100644 src/plugins/discover/public/application/helpers/update_search_source.ts diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.md index 548fa66e6e518..df302e9f3b0d3 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.md @@ -41,6 +41,7 @@ export declare class SearchSource | [getSearchRequestBody()](./kibana-plugin-plugins-data-public.searchsource.getsearchrequestbody.md) | | Returns body contents of the search request, often referred as query DSL. | | [getSerializedFields()](./kibana-plugin-plugins-data-public.searchsource.getserializedfields.md) | | serializes search source fields (which can later be passed to [ISearchStartSearchSource](./kibana-plugin-plugins-data-public.isearchstartsearchsource.md)) | | [onRequestStart(handler)](./kibana-plugin-plugins-data-public.searchsource.onrequeststart.md) | | Add a handler that will be notified whenever requests start | +| [removeField(field)](./kibana-plugin-plugins-data-public.searchsource.removefield.md) | | remove field | | [serialize()](./kibana-plugin-plugins-data-public.searchsource.serialize.md) | | Serializes the instance to a JSON string and a set of referenced objects. Use this method to get a representation of the search source which can be stored in a saved object.The references returned by this function can be mixed with other references in the same object, however make sure there are no name-collisions. The references will be named kibanaSavedObjectMeta.searchSourceJSON.index and kibanaSavedObjectMeta.searchSourceJSON.filter[<number>].meta.index.Using createSearchSource, the instance can be re-created. | | [setField(field, value)](./kibana-plugin-plugins-data-public.searchsource.setfield.md) | | sets value to a single search source field | | [setFields(newFields)](./kibana-plugin-plugins-data-public.searchsource.setfields.md) | | Internal, do not use. Overrides all search source fields with the new field array. | diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.removefield.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.removefield.md new file mode 100644 index 0000000000000..1e6b63be997ff --- /dev/null +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.removefield.md @@ -0,0 +1,24 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [SearchSource](./kibana-plugin-plugins-data-public.searchsource.md) > [removeField](./kibana-plugin-plugins-data-public.searchsource.removefield.md) + +## SearchSource.removeField() method + +remove field + +Signature: + +```typescript +removeField(field: K): this; +``` + +## Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| field | K | | + +Returns: + +`this` + diff --git a/src/plugins/data/common/search/search_source/mocks.ts b/src/plugins/data/common/search/search_source/mocks.ts index ea7d6b4441ccf..dd2b0eaccc86e 100644 --- a/src/plugins/data/common/search/search_source/mocks.ts +++ b/src/plugins/data/common/search/search_source/mocks.ts @@ -28,6 +28,7 @@ export const searchSourceInstanceMock: MockedKeys = { setPreferredSearchStrategyId: jest.fn(), setFields: jest.fn().mockReturnThis(), setField: jest.fn().mockReturnThis(), + removeField: jest.fn().mockReturnThis(), getId: jest.fn(), getFields: jest.fn(), getField: jest.fn(), diff --git a/src/plugins/data/common/search/search_source/search_source.test.ts b/src/plugins/data/common/search/search_source/search_source.test.ts index 98d66310c040e..e7bdcb159f3cb 100644 --- a/src/plugins/data/common/search/search_source/search_source.test.ts +++ b/src/plugins/data/common/search/search_source/search_source.test.ts @@ -82,6 +82,15 @@ describe('SearchSource', () => { }); }); + describe('#removeField()', () => { + test('remove property', () => { + const searchSource = new SearchSource({}, searchSourceDependencies); + searchSource.setField('aggs', 5); + searchSource.removeField('aggs'); + expect(searchSource.getField('aggs')).toBeFalsy(); + }); + }); + describe(`#setField('index')`, () => { describe('auto-sourceFiltering', () => { describe('new index pattern assigned', () => { diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index 9bc65ca341980..79ef3a3f11ca5 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -142,10 +142,18 @@ export class SearchSource { */ setField(field: K, value: SearchSourceFields[K]) { if (value == null) { - delete this.fields[field]; - } else { - this.fields[field] = value; + return this.removeField(field); } + this.fields[field] = value; + return this; + } + + /** + * remove field + * @param field: field name + */ + removeField(field: K) { + delete this.fields[field]; return this; } diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index 12d6fc5ad32c7..7c4d3ee27cf4a 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -2169,6 +2169,7 @@ export class SearchSource { // (undocumented) history: SearchRequest[]; onRequestStart(handler: (searchSource: SearchSource, options?: ISearchOptions) => Promise): void; + removeField(field: K): this; serialize(): { searchSourceJSON: string; references: import("src/core/server").SavedObjectReference[]; diff --git a/src/plugins/discover/public/__mocks__/config.ts b/src/plugins/discover/public/__mocks__/config.ts new file mode 100644 index 0000000000000..a6cdfedd795b5 --- /dev/null +++ b/src/plugins/discover/public/__mocks__/config.ts @@ -0,0 +1,30 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { IUiSettingsClient } from '../../../../core/public'; + +export const configMock = ({ + get: (key: string) => { + if (key === 'defaultIndex') { + return 'the-index-pattern-id'; + } + + return ''; + }, +} as unknown) as IUiSettingsClient; diff --git a/src/plugins/discover/public/__mocks__/index_pattern.ts b/src/plugins/discover/public/__mocks__/index_pattern.ts new file mode 100644 index 0000000000000..696079ec72a73 --- /dev/null +++ b/src/plugins/discover/public/__mocks__/index_pattern.ts @@ -0,0 +1,74 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { IndexPattern, indexPatterns } from '../kibana_services'; +import { IIndexPatternFieldList } from '../../../data/common/index_patterns/fields'; + +const fields = [ + { + name: '_index', + type: 'string', + scripted: false, + filterable: true, + }, + { + name: 'message', + type: 'string', + scripted: false, + filterable: false, + }, + { + name: 'extension', + type: 'string', + scripted: false, + filterable: true, + }, + { + name: 'bytes', + type: 'number', + scripted: false, + filterable: true, + }, + { + name: 'scripted', + type: 'number', + scripted: true, + filterable: false, + }, +] as IIndexPatternFieldList; + +fields.getByName = (name: string) => { + return fields.find((field) => field.name === name); +}; + +const indexPattern = ({ + id: 'the-index-pattern-id', + title: 'the-index-pattern-title', + metaFields: ['_index', '_score'], + flattenHit: undefined, + formatHit: jest.fn((hit) => hit._source), + fields, + getComputedFields: () => ({}), + getSourceFiltering: () => ({}), + getFieldByName: () => ({}), +} as unknown) as IndexPattern; + +indexPattern.flattenHit = indexPatterns.flattenHitWrapper(indexPattern, indexPattern.metaFields); + +export const indexPatternMock = indexPattern; diff --git a/src/plugins/discover/public/__mocks__/index_patterns.ts b/src/plugins/discover/public/__mocks__/index_patterns.ts new file mode 100644 index 0000000000000..f413a111a1d79 --- /dev/null +++ b/src/plugins/discover/public/__mocks__/index_patterns.ts @@ -0,0 +1,32 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { IndexPatternsService } from '../../../data/common'; +import { indexPatternMock } from './index_pattern'; + +export const indexPatternsMock = ({ + getCache: () => { + return [indexPatternMock]; + }, + get: (id: string) => { + if (id === 'the-index-pattern-id') { + return indexPatternMock; + } + }, +} as unknown) as IndexPatternsService; diff --git a/src/plugins/discover/public/__mocks__/saved_search.ts b/src/plugins/discover/public/__mocks__/saved_search.ts new file mode 100644 index 0000000000000..11f36fdfde67c --- /dev/null +++ b/src/plugins/discover/public/__mocks__/saved_search.ts @@ -0,0 +1,41 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { SavedSearch } from '../saved_searches'; + +export const savedSearchMock = ({ + id: 'the-saved-search-id', + type: 'search', + attributes: { + title: 'the-saved-search-title', + kibanaSavedObjectMeta: { + searchSourceJSON: + '{"highlightAll":true,"version":true,"query":{"query":"foo : \\"bar\\" ","language":"kuery"},"filter":[],"indexRefName":"kibanaSavedObjectMeta.searchSourceJSON.index"}', + }, + }, + references: [ + { + name: 'kibanaSavedObjectMeta.searchSourceJSON.index', + type: 'index-pattern', + id: 'the-index-pattern-id', + }, + ], + migrationVersion: { search: '7.5.0' }, + error: undefined, +} as unknown) as SavedSearch; diff --git a/src/plugins/discover/public/application/angular/discover.js b/src/plugins/discover/public/application/angular/discover.js index 9319c58db3e33..272c2f2ca6187 100644 --- a/src/plugins/discover/public/application/angular/discover.js +++ b/src/plugins/discover/public/application/angular/discover.js @@ -18,8 +18,7 @@ */ import _ from 'lodash'; -import React from 'react'; -import { Subscription, Subject, merge } from 'rxjs'; +import { merge, Subject, Subscription } from 'rxjs'; import { debounceTime } from 'rxjs/operators'; import moment from 'moment'; import dateMath from '@elastic/datemath'; @@ -28,31 +27,52 @@ import { getState, splitState } from './discover_state'; import { RequestAdapter } from '../../../../inspector/public'; import { + connectToQueryState, esFilters, indexPatterns as indexPatternsUtils, - connectToQueryState, syncQueryStateWithUrl, } from '../../../../data/public'; -import { SavedObjectSaveModal, showSaveModal } from '../../../../saved_objects/public'; -import { getSortArray, getSortForSearchSource } from './doc_table'; +import { getSortArray } from './doc_table'; import { createFixedScroll } from './directives/fixed_scroll'; import * as columnActions from './doc_table/actions/columns'; import indexTemplateLegacy from './discover_legacy.html'; -import { showOpenSearchPanel } from '../components/top_nav/show_open_search_panel'; import { addHelpMenuToAppChrome } from '../components/help_menu/help_menu_util'; import { discoverResponseHandler } from './response_handler'; import { + getAngularModule, + getHeaderActionMenuMounter, getRequestInspectorStats, getResponseInspectorStats, getServices, - getHeaderActionMenuMounter, getUrlTracker, - unhashUrl, + redirectWhenMissing, subscribeWithScope, tabifyAggResponse, - getAngularModule, - redirectWhenMissing, } from '../../kibana_services'; +import { + getRootBreadcrumbs, + getSavedSearchBreadcrumbs, + setBreadcrumbsTitle, +} from '../helpers/breadcrumbs'; +import { validateTimeRange } from '../helpers/validate_time_range'; +import { popularizeField } from '../helpers/popularize_field'; +import { getSwitchIndexPatternAppState } from '../helpers/get_switch_index_pattern_app_state'; +import { addFatalError } from '../../../../kibana_legacy/public'; +import { METRIC_TYPE } from '@kbn/analytics'; +import { SEARCH_SESSION_ID_QUERY_PARAM } from '../../url_generator'; +import { removeQueryParam, getQueryParams } from '../../../../kibana_utils/public'; +import { + DEFAULT_COLUMNS_SETTING, + MODIFY_COLUMNS_ON_SWITCH, + SAMPLE_SIZE_SETTING, + SEARCH_ON_PAGE_LOAD_SETTING, +} from '../../../common'; +import { resolveIndexPattern, loadIndexPattern } from '../helpers/resolve_index_pattern'; +import { getTopNavLinks } from '../components/top_nav/get_top_nav_links'; +import { updateSearchSource } from '../helpers/update_search_source'; +import { calcFieldCounts } from '../helpers/calc_field_counts'; + +const services = getServices(); const { core, @@ -61,30 +81,11 @@ const { history: getHistory, indexPatterns, filterManager, - share, timefilter, toastNotifications, uiSettings: config, trackUiMetric, -} = getServices(); - -import { getRootBreadcrumbs, getSavedSearchBreadcrumbs } from '../helpers/breadcrumbs'; -import { validateTimeRange } from '../helpers/validate_time_range'; -import { popularizeField } from '../helpers/popularize_field'; -import { getSwitchIndexPatternAppState } from '../helpers/get_switch_index_pattern_app_state'; -import { getIndexPatternId } from '../helpers/get_index_pattern_id'; -import { addFatalError } from '../../../../kibana_legacy/public'; -import { - DEFAULT_COLUMNS_SETTING, - SAMPLE_SIZE_SETTING, - SORT_DEFAULT_ORDER_SETTING, - SEARCH_ON_PAGE_LOAD_SETTING, - DOC_HIDE_TIME_COLUMN_SETTING, - MODIFY_COLUMNS_ON_SWITCH, -} from '../../../common'; -import { METRIC_TYPE } from '@kbn/analytics'; -import { SEARCH_SESSION_ID_QUERY_PARAM } from '../../url_generator'; -import { removeQueryParam, getQueryParams } from '../../../../kibana_utils/public'; +} = services; const fetchStatuses = { UNINITIALIZED: 'uninitialized', @@ -132,24 +133,7 @@ app.config(($routeProvider) => { const { appStateContainer } = getState({ history }); const { index } = appStateContainer.getState(); return Promise.props({ - ip: indexPatterns.getCache().then((indexPatternList) => { - /** - * In making the indexPattern modifiable it was placed in appState. Unfortunately, - * the load order of AppState conflicts with the load order of many other things - * so in order to get the name of the index we should use, and to switch to the - * default if necessary, we parse the appState with a temporary State object and - * then destroy it immediatly after we're done - * - * @type {State} - */ - const id = getIndexPatternId(index, indexPatternList, config.get('defaultIndex')); - return Promise.props({ - list: indexPatternList, - loaded: indexPatterns.get(id), - stateVal: index, - stateValFound: !!index && id === index, - }); - }), + ip: loadIndexPattern(index, data.indexPatterns, config), savedSearch: getServices() .getSavedSearchById(savedSearchId) .then((savedSearch) => { @@ -204,7 +188,11 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise let inspectorRequest; const savedSearch = $route.current.locals.savedObjects.savedSearch; $scope.searchSource = savedSearch.searchSource; - $scope.indexPattern = resolveIndexPatternLoading(); + $scope.indexPattern = resolveIndexPattern( + $route.current.locals.savedObjects.ip, + $scope.searchSource, + toastNotifications + ); //used for functional testing $scope.fetchCounter = 0; @@ -216,22 +204,22 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise // used for restoring background session let isInitialSearch = true; + const state = getState({ + getStateDefaults, + storeInSessionStorage: config.get('state:storeInSessionStorage'), + history, + toasts: core.notifications.toasts, + }); const { appStateContainer, startSync: startStateSync, stopSync: stopStateSync, setAppState, replaceUrlAppState, - isAppStateDirty, kbnUrlStateStorage, getPreviousAppState, - resetInitialAppState, - } = getState({ - defaultAppState: getStateDefaults(), - storeInSessionStorage: config.get('state:storeInSessionStorage'), - history, - toasts: core.notifications.toasts, - }); + } = state; + if (appStateContainer.getState().index !== $scope.indexPattern.id) { //used index pattern is different than the given by url/state which is invalid setAppState({ index: $scope.indexPattern.id }); @@ -349,145 +337,36 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise unlistenHistoryBasePath(); }); - const getTopNavLinks = () => { - const newSearch = { - id: 'new', - label: i18n.translate('discover.localMenu.localMenu.newSearchTitle', { - defaultMessage: 'New', - }), - description: i18n.translate('discover.localMenu.newSearchDescription', { - defaultMessage: 'New Search', - }), - run: function () { - $scope.$evalAsync(() => { - history.push('/'); - }); - }, - testId: 'discoverNewButton', - }; - - const saveSearch = { - id: 'save', - label: i18n.translate('discover.localMenu.saveTitle', { - defaultMessage: 'Save', - }), - description: i18n.translate('discover.localMenu.saveSearchDescription', { - defaultMessage: 'Save Search', - }), - testId: 'discoverSaveButton', - run: async () => { - const onSave = ({ - newTitle, - newCopyOnSave, - isTitleDuplicateConfirmed, - onTitleDuplicate, - }) => { - const currentTitle = savedSearch.title; - savedSearch.title = newTitle; - savedSearch.copyOnSave = newCopyOnSave; - const saveOptions = { - confirmOverwrite: false, - isTitleDuplicateConfirmed, - onTitleDuplicate, - }; - return saveDataSource(saveOptions).then((response) => { - // If the save wasn't successful, put the original values back. - if (!response.id || response.error) { - savedSearch.title = currentTitle; - } else { - resetInitialAppState(); - } - return response; - }); - }; - - const saveModal = ( - {}} - title={savedSearch.title} - showCopyOnSave={!!savedSearch.id} - objectType="search" - description={i18n.translate('discover.localMenu.saveSaveSearchDescription', { - defaultMessage: - 'Save your Discover search so you can use it in visualizations and dashboards', - })} - showDescription={false} - /> - ); - showSaveModal(saveModal, core.i18n.Context); - }, - }; - - const openSearch = { - id: 'open', - label: i18n.translate('discover.localMenu.openTitle', { - defaultMessage: 'Open', - }), - description: i18n.translate('discover.localMenu.openSavedSearchDescription', { - defaultMessage: 'Open Saved Search', - }), - testId: 'discoverOpenButton', - run: () => { - showOpenSearchPanel({ - makeUrl: (searchId) => `#/view/${encodeURIComponent(searchId)}`, - I18nContext: core.i18n.Context, - }); - }, - }; - - const shareSearch = { - id: 'share', - label: i18n.translate('discover.localMenu.shareTitle', { - defaultMessage: 'Share', - }), - description: i18n.translate('discover.localMenu.shareSearchDescription', { - defaultMessage: 'Share Search', - }), - testId: 'shareTopNavButton', - run: async (anchorElement) => { - const sharingData = await this.getSharingData(); - share.toggleShareContextMenu({ - anchorElement, - allowEmbed: false, - allowShortUrl: uiCapabilities.discover.createShortUrl, - shareableUrl: unhashUrl(window.location.href), - objectId: savedSearch.id, - objectType: 'search', - sharingData: { - ...sharingData, - title: savedSearch.title, - }, - isDirty: !savedSearch.id || isAppStateDirty(), - }); - }, - }; - - const inspectSearch = { - id: 'inspect', - label: i18n.translate('discover.localMenu.inspectTitle', { - defaultMessage: 'Inspect', - }), - description: i18n.translate('discover.localMenu.openInspectorForSearchDescription', { - defaultMessage: 'Open Inspector for search', - }), - testId: 'openInspectorButton', - run() { - getServices().inspector.open(inspectorAdapters, { - title: savedSearch.title, - }); - }, - }; + const getFieldCounts = async () => { + // the field counts aren't set until we have the data back, + // so we wait for the fetch to be done before proceeding + if ($scope.fetchStatus === fetchStatuses.COMPLETE) { + return $scope.fieldCounts; + } - return [ - newSearch, - ...(uiCapabilities.discover.save ? [saveSearch] : []), - openSearch, - shareSearch, - inspectSearch, - ]; + return await new Promise((resolve) => { + const unwatch = $scope.$watch('fetchStatus', (newValue) => { + if (newValue === fetchStatuses.COMPLETE) { + unwatch(); + resolve($scope.fieldCounts); + } + }); + }); }; - $scope.topNavMenu = getTopNavLinks(); + + $scope.topNavMenu = getTopNavLinks({ + getFieldCounts, + indexPattern: $scope.indexPattern, + inspectorAdapters, + navigateTo: (path) => { + $scope.$evalAsync(() => { + history.push(path); + }); + }, + savedSearch, + services, + state, + }); $scope.searchSource .setField('index', $scope.indexPattern) @@ -511,96 +390,8 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise const pageTitleSuffix = savedSearch.id && savedSearch.title ? `: ${savedSearch.title}` : ''; chrome.docTitle.change(`Discover${pageTitleSuffix}`); - const discoverBreadcrumbsTitle = i18n.translate('discover.discoverBreadcrumbTitle', { - defaultMessage: 'Discover', - }); - - if (savedSearch.id && savedSearch.title) { - chrome.setBreadcrumbs([ - { - text: discoverBreadcrumbsTitle, - href: '#/', - }, - { text: savedSearch.title }, - ]); - } else { - chrome.setBreadcrumbs([ - { - text: discoverBreadcrumbsTitle, - }, - ]); - } - const getFieldCounts = async () => { - // the field counts aren't set until we have the data back, - // so we wait for the fetch to be done before proceeding - if ($scope.fetchStatus === fetchStatuses.COMPLETE) { - return $scope.fieldCounts; - } - - return await new Promise((resolve) => { - const unwatch = $scope.$watch('fetchStatus', (newValue) => { - if (newValue === fetchStatuses.COMPLETE) { - unwatch(); - resolve($scope.fieldCounts); - } - }); - }); - }; - - const getSharingDataFields = async (selectedFields, timeFieldName, hideTimeColumn) => { - if (selectedFields.length === 1 && selectedFields[0] === '_source') { - const fieldCounts = await getFieldCounts(); - return { - searchFields: null, - selectFields: _.keys(fieldCounts).sort(), - }; - } - - const fields = - timeFieldName && !hideTimeColumn ? [timeFieldName, ...selectedFields] : selectedFields; - return { - searchFields: fields, - selectFields: fields, - }; - }; - - this.getSharingData = async () => { - const searchSource = $scope.searchSource.createCopy(); - - const { searchFields, selectFields } = await getSharingDataFields( - $scope.state.columns, - $scope.indexPattern.timeFieldName, - config.get(DOC_HIDE_TIME_COLUMN_SETTING) - ); - searchSource.setField('fields', searchFields); - searchSource.setField( - 'sort', - getSortForSearchSource( - $scope.state.sort, - $scope.indexPattern, - config.get(SORT_DEFAULT_ORDER_SETTING) - ) - ); - searchSource.setField('highlight', null); - searchSource.setField('highlightAll', null); - searchSource.setField('aggs', null); - searchSource.setField('size', null); - - const body = await searchSource.getSearchRequestBody(); - return { - searchRequest: { - index: searchSource.getField('index').title, - body, - }, - fields: selectFields, - metaFields: $scope.indexPattern.metaFields, - conflictedTypesFields: $scope.indexPattern.fields - .filter((f) => f.type === 'conflict') - .map((f) => f.name), - indexPatternId: searchSource.getField('index').id, - }; - }; + setBreadcrumbsTitle(savedSearch, chrome); function getStateDefaults() { const query = $scope.searchSource.getField('query') || data.query.queryString.getDefaultQuery(); @@ -739,57 +530,6 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise }); }); - async function saveDataSource(saveOptions) { - await $scope.updateDataSource(); - - savedSearch.columns = $scope.state.columns; - savedSearch.sort = $scope.state.sort; - - try { - const id = await savedSearch.save(saveOptions); - $scope.$evalAsync(() => { - if (id) { - toastNotifications.addSuccess({ - title: i18n.translate('discover.notifications.savedSearchTitle', { - defaultMessage: `Search '{savedSearchTitle}' was saved`, - values: { - savedSearchTitle: savedSearch.title, - }, - }), - 'data-test-subj': 'saveSearchSuccess', - }); - - if (savedSearch.id !== $route.current.params.id) { - history.push(`/view/${encodeURIComponent(savedSearch.id)}`); - } else { - // Update defaults so that "reload saved query" functions correctly - setAppState(getStateDefaults()); - chrome.docTitle.change(savedSearch.lastSavedTitle); - chrome.setBreadcrumbs([ - { - text: discoverBreadcrumbsTitle, - href: '#/', - }, - { text: savedSearch.title }, - ]); - } - } - }); - return { id }; - } catch (saveError) { - toastNotifications.addDanger({ - title: i18n.translate('discover.notifications.notSavedSearchTitle', { - defaultMessage: `Search '{savedSearchTitle}' was not saved.`, - values: { - savedSearchTitle: savedSearch.title, - }, - }), - text: saveError.message, - }); - return { error: saveError }; - } - } - $scope.opts.fetch = $scope.fetch = function () { // ignore requests to fetch before the app inits if (!init.complete) return; @@ -907,16 +647,11 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise $scope.hits = resp.hits.total; $scope.rows = resp.hits.hits; - // if we haven't counted yet, reset the counts - const counts = ($scope.fieldCounts = $scope.fieldCounts || {}); - - $scope.rows.forEach((hit) => { - const fields = Object.keys($scope.indexPattern.flattenHit(hit)); - fields.forEach((fieldName) => { - counts[fieldName] = (counts[fieldName] || 0) + 1; - }); - }); - + $scope.fieldCounts = calcFieldCounts( + $scope.fieldCounts || {}, + resp.hits.hits, + $scope.indexPattern + ); $scope.fetchStatus = fetchStatuses.COMPLETE; } @@ -944,13 +679,6 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise }; }; - $scope.toMoment = function (datetime) { - if (!datetime) { - return; - } - return moment(datetime).format(config.get('dateFormat')); - }; - $scope.resetQuery = function () { history.push( $route.current.params.id ? `/view/${encodeURIComponent($route.current.params.id)}` : '/' @@ -979,20 +707,11 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise }; $scope.updateDataSource = () => { - const { indexPattern, searchSource } = $scope; - searchSource - .setField('index', $scope.indexPattern) - .setField('size', $scope.opts.sampleSize) - .setField( - 'sort', - getSortForSearchSource( - $scope.state.sort, - indexPattern, - config.get(SORT_DEFAULT_ORDER_SETTING) - ) - ) - .setField('query', data.query.queryString.getQuery() || null) - .setField('filter', filterManager.getFilters()); + updateSearchSource($scope.searchSource, { + indexPattern: $scope.indexPattern, + services, + sort: $scope.state.sort, + }); return Promise.resolve(); }; @@ -1044,11 +763,6 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise const columns = columnActions.moveColumn($scope.state.columns, columnName, newIndex); setAppState({ columns }); }; - - $scope.scrollToTop = function () { - $window.scrollTo(0, 0); - }; - async function setupVisualization() { // If no timefield has been specified we don't create a histogram of messages if (!getTimeField()) return; @@ -1085,62 +799,6 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise }); } - function getIndexPatternWarning(index) { - return i18n.translate('discover.valueIsNotConfiguredIndexPatternIDWarningTitle', { - defaultMessage: '{stateVal} is not a configured index pattern ID', - values: { - stateVal: `"${index}"`, - }, - }); - } - - function resolveIndexPatternLoading() { - const { - loaded: loadedIndexPattern, - stateVal, - stateValFound, - } = $route.current.locals.savedObjects.ip; - - const ownIndexPattern = $scope.searchSource.getOwnField('index'); - - if (ownIndexPattern && !stateVal) { - return ownIndexPattern; - } - - if (stateVal && !stateValFound) { - const warningTitle = getIndexPatternWarning(); - - if (ownIndexPattern) { - toastNotifications.addWarning({ - title: warningTitle, - text: i18n.translate('discover.showingSavedIndexPatternWarningDescription', { - defaultMessage: - 'Showing the saved index pattern: "{ownIndexPatternTitle}" ({ownIndexPatternId})', - values: { - ownIndexPatternTitle: ownIndexPattern.title, - ownIndexPatternId: ownIndexPattern.id, - }, - }), - }); - return ownIndexPattern; - } - - toastNotifications.addWarning({ - title: warningTitle, - text: i18n.translate('discover.showingDefaultIndexPatternWarningDescription', { - defaultMessage: - 'Showing the default index pattern: "{loadedIndexPatternTitle}" ({loadedIndexPatternId})', - values: { - loadedIndexPatternTitle: loadedIndexPattern.title, - loadedIndexPatternId: loadedIndexPattern.id, - }, - }), - }); - } - - return loadedIndexPattern; - } - addHelpMenuToAppChrome(chrome); init(); diff --git a/src/plugins/discover/public/application/angular/discover_state.test.ts b/src/plugins/discover/public/application/angular/discover_state.test.ts index b7b36ca960167..2914ce8f17a09 100644 --- a/src/plugins/discover/public/application/angular/discover_state.test.ts +++ b/src/plugins/discover/public/application/angular/discover_state.test.ts @@ -29,7 +29,7 @@ describe('Test discover state', () => { history = createBrowserHistory(); history.push('/'); state = getState({ - defaultAppState: { index: 'test' }, + getStateDefaults: () => ({ index: 'test' }), history, }); await state.replaceUrlAppState({}); @@ -84,7 +84,7 @@ describe('Test discover state with legacy migration', () => { "/#?_a=(query:(query_string:(analyze_wildcard:!t,query:'type:nice%20name:%22yeah%22')))" ); state = getState({ - defaultAppState: { index: 'test' }, + getStateDefaults: () => ({ index: 'test' }), history, }); expect(state.appStateContainer.getState()).toMatchInlineSnapshot(` diff --git a/src/plugins/discover/public/application/angular/discover_state.ts b/src/plugins/discover/public/application/angular/discover_state.ts index 5ddb6a92b5fd4..3c6ef1d3e4334 100644 --- a/src/plugins/discover/public/application/angular/discover_state.ts +++ b/src/plugins/discover/public/application/angular/discover_state.ts @@ -65,7 +65,7 @@ interface GetStateParams { /** * Default state used for merging with with URL state to get the initial state */ - defaultAppState?: AppState; + getStateDefaults?: () => AppState; /** * Determins the use of long vs. short/hashed urls */ @@ -123,7 +123,11 @@ export interface GetStateReturn { /** * Returns whether the current app state is different to the initial state */ - isAppStateDirty: () => void; + isAppStateDirty: () => boolean; + /** + * Reset AppState to default, discarding all changes + */ + resetAppState: () => void; } const APP_STATE_URL_KEY = '_a'; @@ -132,11 +136,12 @@ const APP_STATE_URL_KEY = '_a'; * Used to sync URL with UI state */ export function getState({ - defaultAppState = {}, + getStateDefaults, storeInSessionStorage = false, history, toasts, }: GetStateParams): GetStateReturn { + const defaultAppState = getStateDefaults ? getStateDefaults() : {}; const stateStorage = createKbnUrlStateStorage({ useHash: storeInSessionStorage, history, @@ -185,6 +190,10 @@ export function getState({ resetInitialAppState: () => { initialAppState = appStateContainer.getState(); }, + resetAppState: () => { + const defaultState = getStateDefaults ? getStateDefaults() : {}; + setState(appStateContainerModified, defaultState); + }, getPreviousAppState: () => previousAppState, flushToUrl: () => stateStorage.flush(), isAppStateDirty: () => !isEqualState(initialAppState, appStateContainer.getState()), diff --git a/src/plugins/discover/public/application/components/top_nav/__snapshots__/open_search_panel.test.js.snap b/src/plugins/discover/public/application/components/top_nav/__snapshots__/open_search_panel.test.tsx.snap similarity index 96% rename from src/plugins/discover/public/application/components/top_nav/__snapshots__/open_search_panel.test.js.snap rename to src/plugins/discover/public/application/components/top_nav/__snapshots__/open_search_panel.test.tsx.snap index 42cd8613b1de0..2c2674b158bfc 100644 --- a/src/plugins/discover/public/application/components/top_nav/__snapshots__/open_search_panel.test.js.snap +++ b/src/plugins/discover/public/application/components/top_nav/__snapshots__/open_search_panel.test.tsx.snap @@ -3,7 +3,7 @@ exports[`render 1`] = ` { + const topNavLinks = getTopNavLinks({ + getFieldCounts: jest.fn(), + indexPattern: indexPatternMock, + inspectorAdapters: inspectorPluginMock, + navigateTo: jest.fn(), + savedSearch: savedSearchMock, + services, + state, + }); + expect(topNavLinks).toMatchInlineSnapshot(` + Array [ + Object { + "description": "New Search", + "id": "new", + "label": "New", + "run": [Function], + "testId": "discoverNewButton", + }, + Object { + "description": "Save Search", + "id": "save", + "label": "Save", + "run": [Function], + "testId": "discoverSaveButton", + }, + Object { + "description": "Open Saved Search", + "id": "open", + "label": "Open", + "run": [Function], + "testId": "discoverOpenButton", + }, + Object { + "description": "Share Search", + "id": "share", + "label": "Share", + "run": [Function], + "testId": "shareTopNavButton", + }, + Object { + "description": "Open Inspector for search", + "id": "inspect", + "label": "Inspect", + "run": [Function], + "testId": "openInspectorButton", + }, + ] + `); +}); diff --git a/src/plugins/discover/public/application/components/top_nav/get_top_nav_links.ts b/src/plugins/discover/public/application/components/top_nav/get_top_nav_links.ts new file mode 100644 index 0000000000000..62542e9ace4dd --- /dev/null +++ b/src/plugins/discover/public/application/components/top_nav/get_top_nav_links.ts @@ -0,0 +1,148 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { i18n } from '@kbn/i18n'; +import { showOpenSearchPanel } from './show_open_search_panel'; +import { getSharingData } from '../../helpers/get_sharing_data'; +import { unhashUrl } from '../../../../../kibana_utils/public'; +import { DiscoverServices } from '../../../build_services'; +import { Adapters } from '../../../../../inspector/common/adapters'; +import { SavedSearch } from '../../../saved_searches'; +import { onSaveSearch } from './on_save_search'; +import { GetStateReturn } from '../../angular/discover_state'; +import { IndexPattern } from '../../../kibana_services'; + +/** + * Helper function to build the top nav links + */ +export const getTopNavLinks = ({ + getFieldCounts, + indexPattern, + inspectorAdapters, + navigateTo, + savedSearch, + services, + state, +}: { + getFieldCounts: () => Promise>; + indexPattern: IndexPattern; + inspectorAdapters: Adapters; + navigateTo: (url: string) => void; + savedSearch: SavedSearch; + services: DiscoverServices; + state: GetStateReturn; +}) => { + const newSearch = { + id: 'new', + label: i18n.translate('discover.localMenu.localMenu.newSearchTitle', { + defaultMessage: 'New', + }), + description: i18n.translate('discover.localMenu.newSearchDescription', { + defaultMessage: 'New Search', + }), + run: () => navigateTo('/'), + testId: 'discoverNewButton', + }; + + const saveSearch = { + id: 'save', + label: i18n.translate('discover.localMenu.saveTitle', { + defaultMessage: 'Save', + }), + description: i18n.translate('discover.localMenu.saveSearchDescription', { + defaultMessage: 'Save Search', + }), + testId: 'discoverSaveButton', + run: () => onSaveSearch({ savedSearch, services, indexPattern, navigateTo, state }), + }; + + const openSearch = { + id: 'open', + label: i18n.translate('discover.localMenu.openTitle', { + defaultMessage: 'Open', + }), + description: i18n.translate('discover.localMenu.openSavedSearchDescription', { + defaultMessage: 'Open Saved Search', + }), + testId: 'discoverOpenButton', + run: () => + showOpenSearchPanel({ + makeUrl: (searchId) => `#/view/${encodeURIComponent(searchId)}`, + I18nContext: services.core.i18n.Context, + }), + }; + + const shareSearch = { + id: 'share', + label: i18n.translate('discover.localMenu.shareTitle', { + defaultMessage: 'Share', + }), + description: i18n.translate('discover.localMenu.shareSearchDescription', { + defaultMessage: 'Share Search', + }), + testId: 'shareTopNavButton', + run: async (anchorElement: HTMLElement) => { + if (!services.share) { + return; + } + const sharingData = await getSharingData( + savedSearch.searchSource, + state.appStateContainer.getState(), + services.uiSettings, + getFieldCounts + ); + services.share.toggleShareContextMenu({ + anchorElement, + allowEmbed: false, + allowShortUrl: !!services.capabilities.discover.createShortUrl, + shareableUrl: unhashUrl(window.location.href), + objectId: savedSearch.id, + objectType: 'search', + sharingData: { + ...sharingData, + title: savedSearch.title, + }, + isDirty: !savedSearch.id || state.isAppStateDirty(), + }); + }, + }; + + const inspectSearch = { + id: 'inspect', + label: i18n.translate('discover.localMenu.inspectTitle', { + defaultMessage: 'Inspect', + }), + description: i18n.translate('discover.localMenu.openInspectorForSearchDescription', { + defaultMessage: 'Open Inspector for search', + }), + testId: 'openInspectorButton', + run: () => { + services.inspector.open(inspectorAdapters, { + title: savedSearch.title, + }); + }, + }; + + return [ + newSearch, + ...(services.capabilities.discover.save ? [saveSearch] : []), + openSearch, + shareSearch, + inspectSearch, + ]; +}; diff --git a/src/plugins/discover/public/application/components/top_nav/on_save_search.test.tsx b/src/plugins/discover/public/application/components/top_nav/on_save_search.test.tsx new file mode 100644 index 0000000000000..b96af355fafd0 --- /dev/null +++ b/src/plugins/discover/public/application/components/top_nav/on_save_search.test.tsx @@ -0,0 +1,47 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { showSaveModal } from '../../../../../saved_objects/public'; +jest.mock('../../../../../saved_objects/public'); + +import { onSaveSearch } from './on_save_search'; +import { indexPatternMock } from '../../../__mocks__/index_pattern'; +import { savedSearchMock } from '../../../__mocks__/saved_search'; +import { DiscoverServices } from '../../../build_services'; +import { GetStateReturn } from '../../angular/discover_state'; +import { i18nServiceMock } from '../../../../../../core/public/mocks'; + +test('onSaveSearch', async () => { + const serviceMock = ({ + core: { + i18n: i18nServiceMock.create(), + }, + } as unknown) as DiscoverServices; + const stateMock = ({} as unknown) as GetStateReturn; + + await onSaveSearch({ + indexPattern: indexPatternMock, + navigateTo: jest.fn(), + savedSearch: savedSearchMock, + services: serviceMock, + state: stateMock, + }); + + expect(showSaveModal).toHaveBeenCalled(); +}); diff --git a/src/plugins/discover/public/application/components/top_nav/on_save_search.tsx b/src/plugins/discover/public/application/components/top_nav/on_save_search.tsx new file mode 100644 index 0000000000000..c3343968a4685 --- /dev/null +++ b/src/plugins/discover/public/application/components/top_nav/on_save_search.tsx @@ -0,0 +1,158 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import React from 'react'; +import { i18n } from '@kbn/i18n'; +import { SavedObjectSaveModal, showSaveModal } from '../../../../../saved_objects/public'; +import { SavedSearch } from '../../../saved_searches'; +import { IndexPattern } from '../../../../../data/common/index_patterns/index_patterns'; +import { DiscoverServices } from '../../../build_services'; +import { GetStateReturn } from '../../angular/discover_state'; +import { setBreadcrumbsTitle } from '../../helpers/breadcrumbs'; +import { persistSavedSearch } from '../../helpers/persist_saved_search'; + +async function saveDataSource({ + indexPattern, + navigateTo, + savedSearch, + saveOptions, + services, + state, +}: { + indexPattern: IndexPattern; + navigateTo: (url: string) => void; + savedSearch: SavedSearch; + saveOptions: { + confirmOverwrite: boolean; + isTitleDuplicateConfirmed: boolean; + onTitleDuplicate: () => void; + }; + services: DiscoverServices; + state: GetStateReturn; +}) { + const prevSavedSearchId = savedSearch.id; + function onSuccess(id: string) { + if (id) { + services.toastNotifications.addSuccess({ + title: i18n.translate('discover.notifications.savedSearchTitle', { + defaultMessage: `Search '{savedSearchTitle}' was saved`, + values: { + savedSearchTitle: savedSearch.title, + }, + }), + 'data-test-subj': 'saveSearchSuccess', + }); + + if (savedSearch.id !== prevSavedSearchId) { + navigateTo(`/view/${encodeURIComponent(savedSearch.id)}`); + } else { + // Update defaults so that "reload saved query" functions correctly + state.resetAppState(); + services.chrome.docTitle.change(savedSearch.lastSavedTitle!); + setBreadcrumbsTitle(savedSearch, services.chrome); + } + } + } + + function onError(error: Error) { + services.toastNotifications.addDanger({ + title: i18n.translate('discover.notifications.notSavedSearchTitle', { + defaultMessage: `Search '{savedSearchTitle}' was not saved.`, + values: { + savedSearchTitle: savedSearch.title, + }, + }), + text: error.message, + }); + } + return persistSavedSearch(savedSearch, { + indexPattern, + onError, + onSuccess, + saveOptions, + services, + state: state.appStateContainer.getState(), + }); +} + +export async function onSaveSearch({ + indexPattern, + navigateTo, + savedSearch, + services, + state, +}: { + indexPattern: IndexPattern; + navigateTo: (path: string) => void; + savedSearch: SavedSearch; + services: DiscoverServices; + state: GetStateReturn; +}) { + const onSave = async ({ + newTitle, + newCopyOnSave, + isTitleDuplicateConfirmed, + onTitleDuplicate, + }: { + newTitle: string; + newCopyOnSave: boolean; + isTitleDuplicateConfirmed: boolean; + onTitleDuplicate: () => void; + }) => { + const currentTitle = savedSearch.title; + savedSearch.title = newTitle; + savedSearch.copyOnSave = newCopyOnSave; + const saveOptions = { + confirmOverwrite: false, + isTitleDuplicateConfirmed, + onTitleDuplicate, + }; + const response = await saveDataSource({ + indexPattern, + saveOptions, + services, + navigateTo, + savedSearch, + state, + }); + // If the save wasn't successful, put the original values back. + if (!response.id || response.error) { + savedSearch.title = currentTitle; + } else { + state.resetInitialAppState(); + } + return response; + }; + + const saveModal = ( + {}} + title={savedSearch.title} + showCopyOnSave={!!savedSearch.id} + objectType="search" + description={i18n.translate('discover.localMenu.saveSaveSearchDescription', { + defaultMessage: + 'Save your Discover search so you can use it in visualizations and dashboards', + })} + showDescription={false} + /> + ); + showSaveModal(saveModal, services.core.i18n.Context); +} diff --git a/src/plugins/discover/public/application/components/top_nav/open_search_panel.test.js b/src/plugins/discover/public/application/components/top_nav/open_search_panel.test.tsx similarity index 89% rename from src/plugins/discover/public/application/components/top_nav/open_search_panel.test.js rename to src/plugins/discover/public/application/components/top_nav/open_search_panel.test.tsx index 50ab02c8e273d..4b06964c7bc39 100644 --- a/src/plugins/discover/public/application/components/top_nav/open_search_panel.test.js +++ b/src/plugins/discover/public/application/components/top_nav/open_search_panel.test.tsx @@ -24,7 +24,7 @@ jest.mock('../../../kibana_services', () => { return { getServices: () => ({ core: { uiSettings: {}, savedObjects: {} }, - addBasePath: (path) => path, + addBasePath: (path: string) => path, }), }; }); @@ -32,6 +32,6 @@ jest.mock('../../../kibana_services', () => { import { OpenSearchPanel } from './open_search_panel'; test('render', () => { - const component = shallow( {}} makeUrl={() => {}} />); + const component = shallow(); expect(component).toMatchSnapshot(); }); diff --git a/src/plugins/discover/public/application/components/top_nav/open_search_panel.js b/src/plugins/discover/public/application/components/top_nav/open_search_panel.tsx similarity index 94% rename from src/plugins/discover/public/application/components/top_nav/open_search_panel.js rename to src/plugins/discover/public/application/components/top_nav/open_search_panel.tsx index 9a6840c29bf1c..62441f7d827d9 100644 --- a/src/plugins/discover/public/application/components/top_nav/open_search_panel.js +++ b/src/plugins/discover/public/application/components/top_nav/open_search_panel.tsx @@ -16,9 +16,7 @@ * specific language governing permissions and limitations * under the License. */ - import React from 'react'; -import PropTypes from 'prop-types'; import rison from 'rison-node'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; @@ -37,7 +35,12 @@ import { getServices } from '../../../kibana_services'; const SEARCH_OBJECT_TYPE = 'search'; -export function OpenSearchPanel(props) { +interface OpenSearchPanelProps { + onClose: () => void; + makeUrl: (id: string) => string; +} + +export function OpenSearchPanel(props: OpenSearchPanelProps) { const { core: { uiSettings, savedObjects }, addBasePath, @@ -102,8 +105,3 @@ export function OpenSearchPanel(props) { ); } - -OpenSearchPanel.propTypes = { - onClose: PropTypes.func.isRequired, - makeUrl: PropTypes.func.isRequired, -}; diff --git a/src/plugins/discover/public/application/components/top_nav/show_open_search_panel.js b/src/plugins/discover/public/application/components/top_nav/show_open_search_panel.tsx similarity index 87% rename from src/plugins/discover/public/application/components/top_nav/show_open_search_panel.js rename to src/plugins/discover/public/application/components/top_nav/show_open_search_panel.tsx index e40d700b48885..d9a5cdcb063d3 100644 --- a/src/plugins/discover/public/application/components/top_nav/show_open_search_panel.js +++ b/src/plugins/discover/public/application/components/top_nav/show_open_search_panel.tsx @@ -19,11 +19,18 @@ import React from 'react'; import ReactDOM from 'react-dom'; +import { I18nStart } from 'kibana/public'; import { OpenSearchPanel } from './open_search_panel'; let isOpen = false; -export function showOpenSearchPanel({ makeUrl, I18nContext }) { +export function showOpenSearchPanel({ + makeUrl, + I18nContext, +}: { + makeUrl: (path: string) => string; + I18nContext: I18nStart['Context']; +}) { if (isOpen) { return; } diff --git a/src/plugins/discover/public/application/helpers/breadcrumbs.ts b/src/plugins/discover/public/application/helpers/breadcrumbs.ts index 17492b02f7eab..96a9f546a0636 100644 --- a/src/plugins/discover/public/application/helpers/breadcrumbs.ts +++ b/src/plugins/discover/public/application/helpers/breadcrumbs.ts @@ -17,7 +17,9 @@ * under the License. */ +import { ChromeStart } from 'kibana/public'; import { i18n } from '@kbn/i18n'; +import { SavedSearch } from '../../saved_searches'; export function getRootBreadcrumbs() { return [ @@ -38,3 +40,29 @@ export function getSavedSearchBreadcrumbs($route: any) { }, ]; } + +/** + * Helper function to set the Discover's breadcrumb + * if there's an active savedSearch, its title is appended + */ +export function setBreadcrumbsTitle(savedSearch: SavedSearch, chrome: ChromeStart) { + const discoverBreadcrumbsTitle = i18n.translate('discover.discoverBreadcrumbTitle', { + defaultMessage: 'Discover', + }); + + if (savedSearch.id && savedSearch.title) { + chrome.setBreadcrumbs([ + { + text: discoverBreadcrumbsTitle, + href: '#/', + }, + { text: savedSearch.title }, + ]); + } else { + chrome.setBreadcrumbs([ + { + text: discoverBreadcrumbsTitle, + }, + ]); + } +} diff --git a/src/plugins/discover/public/application/helpers/calc_field_counts.test.ts b/src/plugins/discover/public/application/helpers/calc_field_counts.test.ts new file mode 100644 index 0000000000000..ce3319bf8a667 --- /dev/null +++ b/src/plugins/discover/public/application/helpers/calc_field_counts.test.ts @@ -0,0 +1,58 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { calcFieldCounts } from './calc_field_counts'; +import { indexPatternMock } from '../../__mocks__/index_pattern'; + +describe('calcFieldCounts', () => { + test('returns valid field count data', async () => { + const rows = [ + { _id: 1, _source: { message: 'test1', bytes: 20 } }, + { _id: 2, _source: { name: 'test2', extension: 'jpg' } }, + ]; + const result = calcFieldCounts({}, rows, indexPatternMock); + expect(result).toMatchInlineSnapshot(` + Object { + "_index": 2, + "_score": 2, + "bytes": 1, + "extension": 1, + "message": 1, + "name": 1, + } + `); + }); + test('updates field count data', async () => { + const rows = [ + { _id: 1, _source: { message: 'test1', bytes: 20 } }, + { _id: 2, _source: { name: 'test2', extension: 'jpg' } }, + ]; + const result = calcFieldCounts({ message: 2 }, rows, indexPatternMock); + expect(result).toMatchInlineSnapshot(` + Object { + "_index": 2, + "_score": 2, + "bytes": 1, + "extension": 1, + "message": 3, + "name": 1, + } + `); + }); +}); diff --git a/src/plugins/discover/public/application/helpers/calc_field_counts.ts b/src/plugins/discover/public/application/helpers/calc_field_counts.ts new file mode 100644 index 0000000000000..02c0299995e19 --- /dev/null +++ b/src/plugins/discover/public/application/helpers/calc_field_counts.ts @@ -0,0 +1,38 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { IndexPattern } from '../../kibana_services'; + +/** + * This function is recording stats of the available fields, for usage in sidebar and sharing + * Note that this values aren't displayed, but used for internal calculations + */ +export function calcFieldCounts( + counts = {} as Record, + rows: Array>, + indexPattern: IndexPattern +) { + for (const hit of rows) { + const fields = Object.keys(indexPattern.flattenHit(hit)); + for (const fieldName of fields) { + counts[fieldName] = (counts[fieldName] || 0) + 1; + } + } + + return counts; +} diff --git a/src/plugins/discover/public/application/helpers/get_index_pattern_id.ts b/src/plugins/discover/public/application/helpers/get_index_pattern_id.ts deleted file mode 100644 index 601f892e3c56a..0000000000000 --- a/src/plugins/discover/public/application/helpers/get_index_pattern_id.ts +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { IIndexPattern } from '../../../../data/common/index_patterns'; - -export function findIndexPatternById( - indexPatterns: IIndexPattern[], - id: string -): IIndexPattern | undefined { - if (!Array.isArray(indexPatterns) || !id) { - return; - } - return indexPatterns.find((o) => o.id === id); -} - -/** - * Checks if the given defaultIndex exists and returns - * the first available index pattern id if not - */ -export function getFallbackIndexPatternId( - indexPatterns: IIndexPattern[], - defaultIndex: string = '' -): string { - if (defaultIndex && findIndexPatternById(indexPatterns, defaultIndex)) { - return defaultIndex; - } - return !indexPatterns || !indexPatterns.length || !indexPatterns[0].id ? '' : indexPatterns[0].id; -} - -/** - * A given index pattern id is checked for existence and a fallback is provided if it doesn't exist - * The provided defaultIndex is usually configured in Advanced Settings, if it's also invalid - * the first entry of the given list of Indexpatterns is used - */ -export function getIndexPatternId( - id: string = '', - indexPatterns: IIndexPattern[], - defaultIndex: string = '' -): string { - if (!id || !findIndexPatternById(indexPatterns, id)) { - return getFallbackIndexPatternId(indexPatterns, defaultIndex); - } - return id; -} diff --git a/src/plugins/discover/public/application/helpers/get_sharing_data.test.ts b/src/plugins/discover/public/application/helpers/get_sharing_data.test.ts new file mode 100644 index 0000000000000..8ce9789d1dc84 --- /dev/null +++ b/src/plugins/discover/public/application/helpers/get_sharing_data.test.ts @@ -0,0 +1,70 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { getSharingData } from './get_sharing_data'; +import { IUiSettingsClient } from 'kibana/public'; +import { createSearchSourceMock } from '../../../../data/common/search/search_source/mocks'; +import { indexPatternMock } from '../../__mocks__/index_pattern'; + +describe('getSharingData', () => { + test('returns valid data for sharing', async () => { + const searchSourceMock = createSearchSourceMock({ index: indexPatternMock }); + const result = await getSharingData( + searchSourceMock, + { columns: [] }, + ({ + get: () => { + return false; + }, + } as unknown) as IUiSettingsClient, + () => Promise.resolve({}) + ); + expect(result).toMatchInlineSnapshot(` + Object { + "conflictedTypesFields": Array [], + "fields": Array [], + "indexPatternId": "the-index-pattern-id", + "metaFields": Array [ + "_index", + "_score", + ], + "searchRequest": Object { + "body": Object { + "_source": Object { + "includes": Array [], + }, + "docvalue_fields": Array [], + "query": Object { + "bool": Object { + "filter": Array [], + "must": Array [], + "must_not": Array [], + "should": Array [], + }, + }, + "script_fields": Object {}, + "sort": Array [], + "stored_fields": Array [], + }, + "index": "the-index-pattern-title", + }, + } + `); + }); +}); diff --git a/src/plugins/discover/public/application/helpers/get_sharing_data.ts b/src/plugins/discover/public/application/helpers/get_sharing_data.ts new file mode 100644 index 0000000000000..0edaa356cba7d --- /dev/null +++ b/src/plugins/discover/public/application/helpers/get_sharing_data.ts @@ -0,0 +1,88 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { IUiSettingsClient } from 'kibana/public'; +import { DOC_HIDE_TIME_COLUMN_SETTING, SORT_DEFAULT_ORDER_SETTING } from '../../../common'; +import { getSortForSearchSource } from '../angular/doc_table'; +import { SearchSource } from '../../../../data/common'; +import { AppState } from '../angular/discover_state'; +import { SortOrder } from '../../saved_searches/types'; + +const getSharingDataFields = async ( + getFieldCounts: () => Promise>, + selectedFields: string[], + timeFieldName: string, + hideTimeColumn: boolean +) => { + if (selectedFields.length === 1 && selectedFields[0] === '_source') { + const fieldCounts = await getFieldCounts(); + return { + searchFields: undefined, + selectFields: Object.keys(fieldCounts).sort(), + }; + } + + const fields = + timeFieldName && !hideTimeColumn ? [timeFieldName, ...selectedFields] : selectedFields; + return { + searchFields: fields, + selectFields: fields, + }; +}; + +/** + * Preparing data to share the current state as link or CSV/Report + */ +export async function getSharingData( + currentSearchSource: SearchSource, + state: AppState, + config: IUiSettingsClient, + getFieldCounts: () => Promise> +) { + const searchSource = currentSearchSource.createCopy(); + const index = searchSource.getField('index')!; + + const { searchFields, selectFields } = await getSharingDataFields( + getFieldCounts, + state.columns || [], + index.timeFieldName || '', + config.get(DOC_HIDE_TIME_COLUMN_SETTING) + ); + searchSource.setField('fields', searchFields); + searchSource.setField( + 'sort', + getSortForSearchSource(state.sort as SortOrder[], index, config.get(SORT_DEFAULT_ORDER_SETTING)) + ); + searchSource.removeField('highlight'); + searchSource.removeField('highlightAll'); + searchSource.removeField('aggs'); + searchSource.removeField('size'); + + const body = await searchSource.getSearchRequestBody(); + + return { + searchRequest: { + index: index.title, + body, + }, + fields: selectFields, + metaFields: index.metaFields, + conflictedTypesFields: index.fields.filter((f) => f.type === 'conflict').map((f) => f.name), + indexPatternId: index.id, + }; +} diff --git a/src/plugins/discover/public/application/helpers/persist_saved_search.ts b/src/plugins/discover/public/application/helpers/persist_saved_search.ts new file mode 100644 index 0000000000000..8e956eff598f3 --- /dev/null +++ b/src/plugins/discover/public/application/helpers/persist_saved_search.ts @@ -0,0 +1,65 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { updateSearchSource } from './update_search_source'; +import { IndexPattern } from '../../../../data/public'; +import { SavedSearch } from '../../saved_searches'; +import { AppState } from '../angular/discover_state'; +import { SortOrder } from '../../saved_searches/types'; +import { SavedObjectSaveOpts } from '../../../../saved_objects/public'; +import { DiscoverServices } from '../../build_services'; + +/** + * Helper function to update and persist the given savedSearch + */ +export async function persistSavedSearch( + savedSearch: SavedSearch, + { + indexPattern, + onError, + onSuccess, + services, + saveOptions, + state, + }: { + indexPattern: IndexPattern; + onError: (error: Error, savedSearch: SavedSearch) => void; + onSuccess: (id: string) => void; + saveOptions: SavedObjectSaveOpts; + services: DiscoverServices; + state: AppState; + } +) { + updateSearchSource(savedSearch.searchSource, { + indexPattern, + services, + sort: state.sort as SortOrder[], + }); + + savedSearch.columns = state.columns || []; + savedSearch.sort = (state.sort as SortOrder[]) || []; + + try { + const id = await savedSearch.save(saveOptions); + onSuccess(id); + return { id }; + } catch (saveError) { + onError(saveError, savedSearch); + return { error: saveError }; + } +} diff --git a/src/plugins/discover/public/application/helpers/resolve_index_pattern.test.ts b/src/plugins/discover/public/application/helpers/resolve_index_pattern.test.ts new file mode 100644 index 0000000000000..826f738c381a4 --- /dev/null +++ b/src/plugins/discover/public/application/helpers/resolve_index_pattern.test.ts @@ -0,0 +1,56 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { + loadIndexPattern, + getFallbackIndexPatternId, + IndexPatternSavedObject, +} from './resolve_index_pattern'; +import { indexPatternsMock } from '../../__mocks__/index_patterns'; +import { indexPatternMock } from '../../__mocks__/index_pattern'; +import { configMock } from '../../__mocks__/config'; + +describe('Resolve index pattern tests', () => { + test('returns valid data for an existing index pattern', async () => { + const indexPatternId = 'the-index-pattern-id'; + const result = await loadIndexPattern(indexPatternId, indexPatternsMock, configMock); + expect(result.loaded).toEqual(indexPatternMock); + expect(result.stateValFound).toEqual(true); + expect(result.stateVal).toEqual(indexPatternId); + }); + test('returns fallback data for an invalid index pattern', async () => { + const indexPatternId = 'invalid-id'; + const result = await loadIndexPattern(indexPatternId, indexPatternsMock, configMock); + expect(result.loaded).toEqual(indexPatternMock); + expect(result.stateValFound).toBe(false); + expect(result.stateVal).toBe(indexPatternId); + }); + test('getFallbackIndexPatternId with an empty indexPatterns array', async () => { + const result = await getFallbackIndexPatternId([], ''); + expect(result).toBe(''); + }); + test('getFallbackIndexPatternId with an indexPatterns array', async () => { + const list = await indexPatternsMock.getCache(); + const result = await getFallbackIndexPatternId( + (list as unknown) as IndexPatternSavedObject[], + '' + ); + expect(result).toBe('the-index-pattern-id'); + }); +}); diff --git a/src/plugins/discover/public/application/helpers/resolve_index_pattern.ts b/src/plugins/discover/public/application/helpers/resolve_index_pattern.ts new file mode 100644 index 0000000000000..61f7f087501ba --- /dev/null +++ b/src/plugins/discover/public/application/helpers/resolve_index_pattern.ts @@ -0,0 +1,158 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { i18n } from '@kbn/i18n'; +import { IUiSettingsClient, SavedObject, ToastsStart } from 'kibana/public'; +import { IndexPattern } from '../../kibana_services'; +import { IndexPatternsService, SearchSource } from '../../../../data/common'; + +export type IndexPatternSavedObject = SavedObject & { title: string }; + +interface IndexPatternData { + /** + * List of existing index patterns + */ + list: IndexPatternSavedObject[]; + /** + * Loaded index pattern (might be default index pattern if requested was not found) + */ + loaded: IndexPattern; + /** + * Id of the requested index pattern + */ + stateVal: string; + /** + * Determines if requested index pattern was found + */ + stateValFound: boolean; +} + +export function findIndexPatternById( + indexPatterns: IndexPatternSavedObject[], + id: string +): IndexPatternSavedObject | undefined { + if (!Array.isArray(indexPatterns) || !id) { + return; + } + return indexPatterns.find((o) => o.id === id); +} + +/** + * Checks if the given defaultIndex exists and returns + * the first available index pattern id if not + */ +export function getFallbackIndexPatternId( + indexPatterns: IndexPatternSavedObject[], + defaultIndex: string = '' +): string { + if (defaultIndex && findIndexPatternById(indexPatterns, defaultIndex)) { + return defaultIndex; + } + return indexPatterns && indexPatterns[0]?.id ? indexPatterns[0].id : ''; +} + +/** + * A given index pattern id is checked for existence and a fallback is provided if it doesn't exist + * The provided defaultIndex is usually configured in Advanced Settings, if it's also invalid + * the first entry of the given list of Indexpatterns is used + */ +export function getIndexPatternId( + id: string = '', + indexPatterns: IndexPatternSavedObject[] = [], + defaultIndex: string = '' +): string { + if (!id || !findIndexPatternById(indexPatterns, id)) { + return getFallbackIndexPatternId(indexPatterns, defaultIndex); + } + return id; +} + +/** + * Function to load the given index pattern by id, providing a fallback if it doesn't exist + */ +export async function loadIndexPattern( + id: string, + indexPatterns: IndexPatternsService, + config: IUiSettingsClient +): Promise { + const indexPatternList = ((await indexPatterns.getCache()) as unknown) as IndexPatternSavedObject[]; + + const actualId = getIndexPatternId(id, indexPatternList, config.get('defaultIndex')); + return { + list: indexPatternList || [], + loaded: await indexPatterns.get(actualId), + stateVal: id, + stateValFound: !!id && actualId === id, + }; +} + +/** + * Function used in the discover controller to message the user about the state of the current + * index pattern + */ +export function resolveIndexPattern( + ip: IndexPatternData, + searchSource: SearchSource, + toastNotifications: ToastsStart +) { + const { loaded: loadedIndexPattern, stateVal, stateValFound } = ip; + + const ownIndexPattern = searchSource.getOwnField('index'); + + if (ownIndexPattern && !stateVal) { + return ownIndexPattern; + } + + if (stateVal && !stateValFound) { + const warningTitle = i18n.translate('discover.valueIsNotConfiguredIndexPatternIDWarningTitle', { + defaultMessage: '{stateVal} is not a configured index pattern ID', + values: { + stateVal: `"${stateVal}"`, + }, + }); + + if (ownIndexPattern) { + toastNotifications.addWarning({ + title: warningTitle, + text: i18n.translate('discover.showingSavedIndexPatternWarningDescription', { + defaultMessage: + 'Showing the saved index pattern: "{ownIndexPatternTitle}" ({ownIndexPatternId})', + values: { + ownIndexPatternTitle: ownIndexPattern.title, + ownIndexPatternId: ownIndexPattern.id, + }, + }), + }); + return ownIndexPattern; + } + + toastNotifications.addWarning({ + title: warningTitle, + text: i18n.translate('discover.showingDefaultIndexPatternWarningDescription', { + defaultMessage: + 'Showing the default index pattern: "{loadedIndexPatternTitle}" ({loadedIndexPatternId})', + values: { + loadedIndexPatternTitle: loadedIndexPattern.title, + loadedIndexPatternId: loadedIndexPattern.id, + }, + }), + }); + } + + return loadedIndexPattern; +} diff --git a/src/plugins/discover/public/application/helpers/update_search_source.test.ts b/src/plugins/discover/public/application/helpers/update_search_source.test.ts new file mode 100644 index 0000000000000..91832325432ef --- /dev/null +++ b/src/plugins/discover/public/application/helpers/update_search_source.test.ts @@ -0,0 +1,51 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { updateSearchSource } from './update_search_source'; +import { createSearchSourceMock } from '../../../../data/common/search/search_source/mocks'; +import { indexPatternMock } from '../../__mocks__/index_pattern'; +import { IUiSettingsClient } from 'kibana/public'; +import { DiscoverServices } from '../../build_services'; +import { dataPluginMock } from '../../../../data/public/mocks'; +import { SAMPLE_SIZE_SETTING } from '../../../common'; +import { SortOrder } from '../../saved_searches/types'; + +describe('updateSearchSource', () => { + test('updates a given search source', async () => { + const searchSourceMock = createSearchSourceMock({}); + const sampleSize = 250; + const result = updateSearchSource(searchSourceMock, { + indexPattern: indexPatternMock, + services: ({ + data: dataPluginMock.createStartContract(), + uiSettings: ({ + get: (key: string) => { + if (key === SAMPLE_SIZE_SETTING) { + return sampleSize; + } + return false; + }, + } as unknown) as IUiSettingsClient, + } as unknown) as DiscoverServices, + sort: [] as SortOrder[], + }); + expect(result.getField('index')).toEqual(indexPatternMock); + expect(result.getField('size')).toEqual(sampleSize); + }); +}); diff --git a/src/plugins/discover/public/application/helpers/update_search_source.ts b/src/plugins/discover/public/application/helpers/update_search_source.ts new file mode 100644 index 0000000000000..324dc8a48457a --- /dev/null +++ b/src/plugins/discover/public/application/helpers/update_search_source.ts @@ -0,0 +1,54 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { getSortForSearchSource } from '../angular/doc_table'; +import { SAMPLE_SIZE_SETTING, SORT_DEFAULT_ORDER_SETTING } from '../../../common'; +import { IndexPattern, ISearchSource } from '../../../../data/common/'; +import { SortOrder } from '../../saved_searches/types'; +import { DiscoverServices } from '../../build_services'; + +/** + * Helper function to update the given searchSource before fetching/sharing/persisting + */ +export function updateSearchSource( + searchSource: ISearchSource, + { + indexPattern, + services, + sort, + }: { + indexPattern: IndexPattern; + services: DiscoverServices; + sort: SortOrder[]; + } +) { + const { uiSettings, data } = services; + const usedSort = getSortForSearchSource( + sort, + indexPattern, + uiSettings.get(SORT_DEFAULT_ORDER_SETTING) + ); + + searchSource + .setField('index', indexPattern) + .setField('size', uiSettings.get(SAMPLE_SIZE_SETTING)) + .setField('sort', usedSort) + .setField('query', data.query.queryString.getQuery() || null) + .setField('filter', data.query.filterManager.getFilters()); + return searchSource; +} diff --git a/src/plugins/discover/public/saved_searches/types.ts b/src/plugins/discover/public/saved_searches/types.ts index 13361cb647ddc..d5e5dd765a364 100644 --- a/src/plugins/discover/public/saved_searches/types.ts +++ b/src/plugins/discover/public/saved_searches/types.ts @@ -17,18 +17,21 @@ * under the License. */ -import { ISearchSource } from '../../../data/public'; +import { SearchSource } from '../../../data/public'; +import { SavedObjectSaveOpts } from '../../../saved_objects/public'; export type SortOrder = [string, string]; export interface SavedSearch { readonly id: string; title: string; - searchSource: ISearchSource; + searchSource: SearchSource; description?: string; columns: string[]; sort: SortOrder[]; destroy: () => void; + save: (saveOptions: SavedObjectSaveOpts) => Promise; lastSavedTitle?: string; + copyOnSave?: boolean; } export interface SavedSearchLoader { get: (id: string) => Promise; From 09e326e1368f14647fb79d2fc8077aa87aaaaff9 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 25 Nov 2020 12:12:43 +0100 Subject: [PATCH 32/49] [Lens] Fix label input debouncing (#84121) --- .../dimension_panel/dimension_editor.tsx | 6 +----- .../operations/definitions/filters/filter_popover.tsx | 4 ---- .../definitions/shared_components/label_input.tsx | 6 +----- 3 files changed, 2 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index e5c05a1cf8c7a..0a67c157bd837 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -6,7 +6,7 @@ import './dimension_editor.scss'; import _ from 'lodash'; -import React, { useState, useMemo, useEffect } from 'react'; +import React, { useState, useMemo } from 'react'; import { i18n } from '@kbn/i18n'; import { EuiListGroup, @@ -46,10 +46,6 @@ export interface DimensionEditorProps extends IndexPatternDimensionEditorProps { const LabelInput = ({ value, onChange }: { value: string; onChange: (value: string) => void }) => { const [inputValue, setInputValue] = useState(value); - useEffect(() => { - setInputValue(value); - }, [value, setInputValue]); - const onChangeDebounced = useMemo(() => _.debounce(onChange, 256), [onChange]); const handleInputChange = (e: React.ChangeEvent) => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filter_popover.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filter_popover.tsx index b9d9d6306b9ae..ca84c072be5ce 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filter_popover.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filter_popover.tsx @@ -110,10 +110,6 @@ export const QueryInput = ({ }) => { const [inputValue, setInputValue] = useState(value); - React.useEffect(() => { - setInputValue(value); - }, [value, setInputValue]); - useDebounce(() => onChange(inputValue), 256, [inputValue]); const handleInputChange = (input: Query) => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/shared_components/label_input.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/shared_components/label_input.tsx index f0ee30bb4331b..ddcb5633b376f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/shared_components/label_input.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/shared_components/label_input.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { useState, useEffect } from 'react'; +import React, { useState } from 'react'; import useDebounce from 'react-use/lib/useDebounce'; import { EuiFieldText, keys } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; @@ -28,10 +28,6 @@ export const LabelInput = ({ }) => { const [inputValue, setInputValue] = useState(value); - useEffect(() => { - setInputValue(value); - }, [value, setInputValue]); - useDebounce(() => onChange(inputValue), 256, [inputValue]); const handleInputChange = (e: React.ChangeEvent) => { From c1b807eda3d34bff02cc54b91f2648392da2734c Mon Sep 17 00:00:00 2001 From: Pete Harverson Date: Wed, 25 Nov 2020 11:36:31 +0000 Subject: [PATCH 33/49] [ML] Fix Anomaly Explorer population charts when multiple causes in anomaly (#84254) --- x-pack/plugins/ml/public/application/util/chart_utils.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/ml/public/application/util/chart_utils.js b/x-pack/plugins/ml/public/application/util/chart_utils.js index f2dec5f16df1d..d142d2e246659 100644 --- a/x-pack/plugins/ml/public/application/util/chart_utils.js +++ b/x-pack/plugins/ml/public/application/util/chart_utils.js @@ -77,7 +77,10 @@ export function chartExtendedLimits(data = [], functionDescription) { metricValue = actualValue; } - if (d.anomalyScore !== undefined) { + // Check for both an anomaly and for an actual value as anomalies in detectors with + // by and over fields and more than one cause will not have actual / typical values + // at the top level of the anomaly record. + if (d.anomalyScore !== undefined && actualValue !== undefined) { _min = Math.min(_min, metricValue, actualValue, typicalValue); _max = Math.max(_max, metricValue, actualValue, typicalValue); } else { From 4367a10d1b2f12d5f8f891b83ec4563c53d1f03b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Wed, 25 Nov 2020 12:44:02 +0100 Subject: [PATCH 34/49] [APM] Rename `ChartsSyncContext` to `PointerEventContext` (#84272) --- .../app/TransactionDetails/index.tsx | 6 +++--- .../components/app/service_metrics/index.tsx | 6 +++--- .../app/service_node_metrics/index.tsx | 10 +++++----- .../components/app/service_overview/index.tsx | 6 +++--- .../shared/charts/timeseries_chart.tsx | 14 ++++++-------- .../transaction_breakdown_chart_contents.tsx | 18 ++++++++++-------- .../shared/charts/transaction_charts/index.tsx | 6 +++--- ...ext.tsx => chart_pointer_event_context.tsx} | 16 +++++++++------- ...ts_sync.tsx => use_chart_pointer_event.tsx} | 8 ++++---- 9 files changed, 46 insertions(+), 44 deletions(-) rename x-pack/plugins/apm/public/context/{charts_sync_context.tsx => chart_pointer_event_context.tsx} (52%) rename x-pack/plugins/apm/public/hooks/{use_charts_sync.tsx => use_chart_pointer_event.tsx} (56%) diff --git a/x-pack/plugins/apm/public/components/app/TransactionDetails/index.tsx b/x-pack/plugins/apm/public/components/app/TransactionDetails/index.tsx index 8a99773a97baf..8f335ddc71c72 100644 --- a/x-pack/plugins/apm/public/components/app/TransactionDetails/index.tsx +++ b/x-pack/plugins/apm/public/components/app/TransactionDetails/index.tsx @@ -25,7 +25,7 @@ import { TransactionCharts } from '../../shared/charts/transaction_charts'; import { TransactionDistribution } from './Distribution'; import { WaterfallWithSummmary } from './WaterfallWithSummmary'; import { FETCH_STATUS } from '../../../hooks/useFetcher'; -import { ChartsSyncContextProvider } from '../../../context/charts_sync_context'; +import { ChartPointerEventContextProvider } from '../../../context/chart_pointer_event_context'; import { useTrackPageview } from '../../../../../observability/public'; import { Projection } from '../../../../common/projections'; import { fromQuery, toQuery } from '../../shared/Links/url_helpers'; @@ -125,13 +125,13 @@ export function TransactionDetails({ - + - + diff --git a/x-pack/plugins/apm/public/components/app/service_metrics/index.tsx b/x-pack/plugins/apm/public/components/app/service_metrics/index.tsx index 5dc1645a1760d..d0f8fc1e61332 100644 --- a/x-pack/plugins/apm/public/components/app/service_metrics/index.tsx +++ b/x-pack/plugins/apm/public/components/app/service_metrics/index.tsx @@ -16,7 +16,7 @@ import React, { useMemo } from 'react'; import { useServiceMetricCharts } from '../../../hooks/useServiceMetricCharts'; import { MetricsChart } from '../../shared/charts/metrics_chart'; import { useUrlParams } from '../../../hooks/useUrlParams'; -import { ChartsSyncContextProvider } from '../../../context/charts_sync_context'; +import { ChartPointerEventContextProvider } from '../../../context/chart_pointer_event_context'; import { Projection } from '../../../../common/projections'; import { LocalUIFilters } from '../../shared/LocalUIFilters'; import { SearchBar } from '../../shared/search_bar'; @@ -57,7 +57,7 @@ export function ServiceMetrics({ - + {data.charts.map((chart) => ( @@ -73,7 +73,7 @@ export function ServiceMetrics({ ))} - + diff --git a/x-pack/plugins/apm/public/components/app/service_node_metrics/index.tsx b/x-pack/plugins/apm/public/components/app/service_node_metrics/index.tsx index 59e919199be76..a74ff574bc0c8 100644 --- a/x-pack/plugins/apm/public/components/app/service_node_metrics/index.tsx +++ b/x-pack/plugins/apm/public/components/app/service_node_metrics/index.tsx @@ -22,7 +22,7 @@ import React from 'react'; import { RouteComponentProps } from 'react-router-dom'; import styled from 'styled-components'; import { SERVICE_NODE_NAME_MISSING } from '../../../../common/service_nodes'; -import { ChartsSyncContextProvider } from '../../../context/charts_sync_context'; +import { ChartPointerEventContextProvider } from '../../../context/chart_pointer_event_context'; import { useAgentName } from '../../../hooks/useAgentName'; import { FETCH_STATUS, useFetcher } from '../../../hooks/useFetcher'; import { useServiceMetricCharts } from '../../../hooks/useServiceMetricCharts'; @@ -178,7 +178,7 @@ export function ServiceNodeMetrics({ match }: ServiceNodeMetricsProps) { )} {agentName && ( - + {data.charts.map((chart) => ( @@ -194,12 +194,12 @@ export function ServiceNodeMetrics({ match }: ServiceNodeMetricsProps) { ))} - + )} {agentName && ( - + {data.charts.map((chart) => ( @@ -215,7 +215,7 @@ export function ServiceNodeMetrics({ match }: ServiceNodeMetricsProps) { ))} - + )} diff --git a/x-pack/plugins/apm/public/components/app/service_overview/index.tsx b/x-pack/plugins/apm/public/components/app/service_overview/index.tsx index 33027f3946d1f..ddf3107a8ab1e 100644 --- a/x-pack/plugins/apm/public/components/app/service_overview/index.tsx +++ b/x-pack/plugins/apm/public/components/app/service_overview/index.tsx @@ -15,7 +15,7 @@ import { i18n } from '@kbn/i18n'; import React from 'react'; import { useTrackPageview } from '../../../../../observability/public'; import { isRumAgentName } from '../../../../common/agent_name'; -import { ChartsSyncContextProvider } from '../../../context/charts_sync_context'; +import { ChartPointerEventContextProvider } from '../../../context/chart_pointer_event_context'; import { TransactionBreakdownChart } from '../../shared/charts/transaction_breakdown_chart'; import { TransactionErrorRateChart } from '../../shared/charts/transaction_error_rate_chart'; import { ServiceMapLink } from '../../shared/Links/apm/ServiceMapLink'; @@ -43,7 +43,7 @@ export function ServiceOverview({ useTrackPageview({ app: 'apm', path: 'service_overview', delay: 15000 }); return ( - + @@ -170,6 +170,6 @@ export function ServiceOverview({ - + ); } diff --git a/x-pack/plugins/apm/public/components/shared/charts/timeseries_chart.tsx b/x-pack/plugins/apm/public/components/shared/charts/timeseries_chart.tsx index fe3d9a1edc1fb..c4f5abe104aa9 100644 --- a/x-pack/plugins/apm/public/components/shared/charts/timeseries_chart.tsx +++ b/x-pack/plugins/apm/public/components/shared/charts/timeseries_chart.tsx @@ -24,7 +24,7 @@ import { useChartTheme } from '../../../../../observability/public'; import { TimeSeries } from '../../../../typings/timeseries'; import { FETCH_STATUS } from '../../../hooks/useFetcher'; import { useUrlParams } from '../../../hooks/useUrlParams'; -import { useChartsSync } from '../../../hooks/use_charts_sync'; +import { useChartPointerEvent } from '../../../hooks/use_chart_pointer_event'; import { unit } from '../../../style/variables'; import { Annotations } from './annotations'; import { ChartContainer } from './chart_container'; @@ -60,15 +60,15 @@ export function TimeseriesChart({ const history = useHistory(); const chartRef = React.createRef(); const chartTheme = useChartTheme(); - const { event, setEvent } = useChartsSync(); + const { pointerEvent, setPointerEvent } = useChartPointerEvent(); const { urlParams } = useUrlParams(); const { start, end } = urlParams; useEffect(() => { - if (event.chartId !== id && chartRef.current) { - chartRef.current.dispatchExternalPointerEvent(event); + if (pointerEvent && pointerEvent?.chartId !== id && chartRef.current) { + chartRef.current.dispatchExternalPointerEvent(pointerEvent); } - }, [event, chartRef, id]); + }, [pointerEvent, chartRef, id]); const min = moment.utc(start).valueOf(); const max = moment.utc(end).valueOf(); @@ -89,9 +89,7 @@ export function TimeseriesChart({ onBrushEnd({ x, history })} theme={chartTheme} - onPointerUpdate={(currEvent: any) => { - setEvent(currEvent); - }} + onPointerUpdate={setPointerEvent} externalPointerEvents={{ tooltip: { visible: true, placement: Placement.Bottom }, }} diff --git a/x-pack/plugins/apm/public/components/shared/charts/transaction_breakdown_chart/transaction_breakdown_chart_contents.tsx b/x-pack/plugins/apm/public/components/shared/charts/transaction_breakdown_chart/transaction_breakdown_chart_contents.tsx index 8070868f831b2..04c07c01442a4 100644 --- a/x-pack/plugins/apm/public/components/shared/charts/transaction_breakdown_chart/transaction_breakdown_chart_contents.tsx +++ b/x-pack/plugins/apm/public/components/shared/charts/transaction_breakdown_chart/transaction_breakdown_chart_contents.tsx @@ -23,7 +23,7 @@ import { asPercent } from '../../../../../common/utils/formatters'; import { TimeSeries } from '../../../../../typings/timeseries'; import { FETCH_STATUS } from '../../../../hooks/useFetcher'; import { useUrlParams } from '../../../../hooks/useUrlParams'; -import { useChartsSync as useChartsSync2 } from '../../../../hooks/use_charts_sync'; +import { useChartPointerEvent } from '../../../../hooks/use_chart_pointer_event'; import { unit } from '../../../../style/variables'; import { Annotations } from '../../charts/annotations'; import { ChartContainer } from '../../charts/chart_container'; @@ -45,15 +45,19 @@ export function TransactionBreakdownChartContents({ const history = useHistory(); const chartRef = React.createRef(); const chartTheme = useChartTheme(); - const { event, setEvent } = useChartsSync2(); + const { pointerEvent, setPointerEvent } = useChartPointerEvent(); const { urlParams } = useUrlParams(); const { start, end } = urlParams; useEffect(() => { - if (event.chartId !== 'timeSpentBySpan' && chartRef.current) { - chartRef.current.dispatchExternalPointerEvent(event); + if ( + pointerEvent && + pointerEvent.chartId !== 'timeSpentBySpan' && + chartRef.current + ) { + chartRef.current.dispatchExternalPointerEvent(pointerEvent); } - }, [chartRef, event]); + }, [chartRef, pointerEvent]); const min = moment.utc(start).valueOf(); const max = moment.utc(end).valueOf(); @@ -71,9 +75,7 @@ export function TransactionBreakdownChartContents({ theme={chartTheme} xDomain={{ min, max }} flatLegend - onPointerUpdate={(currEvent: any) => { - setEvent(currEvent); - }} + onPointerUpdate={setPointerEvent} externalPointerEvents={{ tooltip: { visible: true, placement: Placement.Bottom }, }} diff --git a/x-pack/plugins/apm/public/components/shared/charts/transaction_charts/index.tsx b/x-pack/plugins/apm/public/components/shared/charts/transaction_charts/index.tsx index 3af081c11c9b1..221f17bb9e1d5 100644 --- a/x-pack/plugins/apm/public/components/shared/charts/transaction_charts/index.tsx +++ b/x-pack/plugins/apm/public/components/shared/charts/transaction_charts/index.tsx @@ -20,7 +20,7 @@ import { TRANSACTION_ROUTE_CHANGE, } from '../../../../../common/transaction_types'; import { asTransactionRate } from '../../../../../common/utils/formatters'; -import { ChartsSyncContextProvider } from '../../../../context/charts_sync_context'; +import { ChartPointerEventContextProvider } from '../../../../context/chart_pointer_event_context'; import { LicenseContext } from '../../../../context/LicenseContext'; import { IUrlParams } from '../../../../context/UrlParamsContext/types'; import { FETCH_STATUS } from '../../../../hooks/useFetcher'; @@ -51,7 +51,7 @@ export function TransactionCharts({ return ( <> - + @@ -109,7 +109,7 @@ export function TransactionCharts({ - + ); } diff --git a/x-pack/plugins/apm/public/context/charts_sync_context.tsx b/x-pack/plugins/apm/public/context/chart_pointer_event_context.tsx similarity index 52% rename from x-pack/plugins/apm/public/context/charts_sync_context.tsx rename to x-pack/plugins/apm/public/context/chart_pointer_event_context.tsx index d983a857a26ec..ea60206463258 100644 --- a/x-pack/plugins/apm/public/context/charts_sync_context.tsx +++ b/x-pack/plugins/apm/public/context/chart_pointer_event_context.tsx @@ -12,21 +12,23 @@ import React, { useState, } from 'react'; -export const ChartsSyncContext = createContext<{ - event: any; - setEvent: Dispatch>; +import { PointerEvent } from '@elastic/charts'; + +export const ChartPointerEventContext = createContext<{ + pointerEvent: PointerEvent | null; + setPointerEvent: Dispatch>; } | null>(null); -export function ChartsSyncContextProvider({ +export function ChartPointerEventContextProvider({ children, }: { children: ReactNode; }) { - const [event, setEvent] = useState({}); + const [pointerEvent, setPointerEvent] = useState(null); return ( - ); diff --git a/x-pack/plugins/apm/public/hooks/use_charts_sync.tsx b/x-pack/plugins/apm/public/hooks/use_chart_pointer_event.tsx similarity index 56% rename from x-pack/plugins/apm/public/hooks/use_charts_sync.tsx rename to x-pack/plugins/apm/public/hooks/use_chart_pointer_event.tsx index cde5c84a6097b..058ec594e2d22 100644 --- a/x-pack/plugins/apm/public/hooks/use_charts_sync.tsx +++ b/x-pack/plugins/apm/public/hooks/use_chart_pointer_event.tsx @@ -5,13 +5,13 @@ */ import { useContext } from 'react'; -import { ChartsSyncContext } from '../context/charts_sync_context'; +import { ChartPointerEventContext } from '../context/chart_pointer_event_context'; -export function useChartsSync() { - const context = useContext(ChartsSyncContext); +export function useChartPointerEvent() { + const context = useContext(ChartPointerEventContext); if (!context) { - throw new Error('Missing ChartsSync context provider'); + throw new Error('Missing ChartPointerEventContext provider'); } return context; From 0f780229b5c195f41e4182f13c3a2d952356a816 Mon Sep 17 00:00:00 2001 From: Chris Roberson Date: Wed, 25 Nov 2020 08:49:17 -0500 Subject: [PATCH 35/49] Put the cluster_uuid in quotes (#83987) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../public/lib/get_safe_for_external_link.test.ts | 4 ++-- .../public/lib/get_safe_for_external_link.ts | 11 ++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/monitoring/public/lib/get_safe_for_external_link.test.ts b/x-pack/plugins/monitoring/public/lib/get_safe_for_external_link.test.ts index 4b40c7f4a88dc..8c1b9d0c475dd 100644 --- a/x-pack/plugins/monitoring/public/lib/get_safe_for_external_link.test.ts +++ b/x-pack/plugins/monitoring/public/lib/get_safe_for_external_link.test.ts @@ -51,7 +51,7 @@ describe('getSafeForExternalLink', () => { location ) ).toBe( - `#/overview?_g=(cluster_uuid:NDKg6VXAT6-TaGzEK2Zy7g,filters:!(),refreshInterval:(pause:!t,value:10000),time:(from:'2017-09-07T20:12:04.011Z',to:'2017-09-07T20:18:55.733Z'))` + `#/overview?_g=(cluster_uuid:'NDKg6VXAT6-TaGzEK2Zy7g',filters:!(),refreshInterval:(pause:!t,value:10000),time:(from:'2017-09-07T20:12:04.011Z',to:'2017-09-07T20:18:55.733Z'))` ); }); @@ -68,7 +68,7 @@ describe('getSafeForExternalLink', () => { location ) ).toBe( - `#/overview?_g=(filters:!(),refreshInterval:(pause:!t,value:10000),time:(from:'2017-09-07T20:12:04.011Z',to:'2017-09-07T20:18:55.733Z'),cluster_uuid:NDKg6VXAT6-TaGzEK2Zy7g)` + `#/overview?_g=(filters:!(),refreshInterval:(pause:!t,value:10000),time:(from:'2017-09-07T20:12:04.011Z',to:'2017-09-07T20:18:55.733Z'),cluster_uuid:'NDKg6VXAT6-TaGzEK2Zy7g')` ); }); }); diff --git a/x-pack/plugins/monitoring/public/lib/get_safe_for_external_link.ts b/x-pack/plugins/monitoring/public/lib/get_safe_for_external_link.ts index 3730ed6411227..86d571b87bc94 100644 --- a/x-pack/plugins/monitoring/public/lib/get_safe_for_external_link.ts +++ b/x-pack/plugins/monitoring/public/lib/get_safe_for_external_link.ts @@ -24,15 +24,16 @@ export function getSafeForExternalLink( let newGlobalState = globalStateExecResult[1]; Object.keys(globalState).forEach((globalStateKey) => { + let value = globalState[globalStateKey]; + if (globalStateKey === 'cluster_uuid') { + value = `'${value}'`; + } const keyRegExp = new RegExp(`${globalStateKey}:([^,]+)`); const execResult = keyRegExp.exec(newGlobalState); if (execResult && execResult.length) { - newGlobalState = newGlobalState.replace( - execResult[0], - `${globalStateKey}:${globalState[globalStateKey]}` - ); + newGlobalState = newGlobalState.replace(execResult[0], `${globalStateKey}:${value}`); } else { - newGlobalState += `,${globalStateKey}:${globalState[globalStateKey]}`; + newGlobalState += `,${globalStateKey}:${value}`; } }); From ced1dadb8a5c2dcf50f590dbe3c5db38d512e0c1 Mon Sep 17 00:00:00 2001 From: Chris Roberson Date: Wed, 25 Nov 2020 08:50:03 -0500 Subject: [PATCH 36/49] [Monitoring] Only look at ES for the missing data alert for now (#83839) * Only look at ES for the missing data alert for now * PR feedback * Fix tests --- .../missing_monitoring_data_alert.test.ts | 67 +------ .../fetch_missing_monitoring_data.test.ts | 89 --------- .../alerts/fetch_missing_monitoring_data.ts | 178 +++--------------- 3 files changed, 35 insertions(+), 299 deletions(-) diff --git a/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.test.ts b/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.test.ts index 57d01dc6a1100..1332148a61cdd 100644 --- a/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.test.ts +++ b/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.test.ts @@ -65,13 +65,6 @@ describe('MissingMonitoringDataAlert', () => { clusterUuid, gapDuration, }, - { - stackProduct: 'kibana', - stackProductUuid: 'kibanaUuid1', - stackProductName: 'kibanaInstance1', - clusterUuid, - gapDuration: gapDuration + 10, - }, ]; const getUiSettingsService = () => ({ asScopedToClient: jest.fn(), @@ -140,7 +133,7 @@ describe('MissingMonitoringDataAlert', () => { // @ts-ignore params: alert.defaultParams, } as any); - const count = 2; + const count = 1; expect(replaceState).toHaveBeenCalledWith({ alertStates: [ { @@ -187,61 +180,17 @@ describe('MissingMonitoringDataAlert', () => { lastCheckedMS: 0, }, }, - { - ccs: undefined, - cluster: { clusterUuid, clusterName }, - gapDuration: gapDuration + 10, - stackProduct: 'kibana', - stackProductName: 'kibanaInstance1', - stackProductUuid: 'kibanaUuid1', - ui: { - isFiring: true, - message: { - text: - 'For the past an hour, we have not detected any monitoring data from the Kibana instance: kibanaInstance1, starting at #absolute', - nextSteps: [ - { - text: '#start_linkView all Kibana instances#end_link', - tokens: [ - { - startToken: '#start_link', - endToken: '#end_link', - type: 'link', - url: 'kibana/instances', - }, - ], - }, - { - text: 'Verify monitoring settings on the instance', - }, - ], - tokens: [ - { - startToken: '#absolute', - type: 'time', - isAbsolute: true, - isRelative: false, - timestamp: 1, - }, - ], - }, - severity: 'danger', - resolvedMS: 0, - triggeredMS: 1, - lastCheckedMS: 0, - }, - }, ], }); expect(scheduleActions).toHaveBeenCalledWith('default', { - internalFullMessage: `We have not detected any monitoring data for 2 stack product(s) in cluster: testCluster. [View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#/overview?_g=(cluster_uuid:abc123))`, - internalShortMessage: `We have not detected any monitoring data for 2 stack product(s) in cluster: testCluster. Verify these stack products are up and running, then double check the monitoring settings.`, + internalFullMessage: `We have not detected any monitoring data for 1 stack product(s) in cluster: testCluster. [View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#/overview?_g=(cluster_uuid:abc123))`, + internalShortMessage: `We have not detected any monitoring data for 1 stack product(s) in cluster: testCluster. Verify these stack products are up and running, then double check the monitoring settings.`, action: `[View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#/overview?_g=(cluster_uuid:abc123))`, actionPlain: 'Verify these stack products are up and running, then double check the monitoring settings.', clusterName, count, - stackProducts: 'Elasticsearch node: esName1, Kibana instance: kibanaInstance1', + stackProducts: 'Elasticsearch node: esName1', state: 'firing', }); }); @@ -442,16 +391,16 @@ describe('MissingMonitoringDataAlert', () => { // @ts-ignore params: alert.defaultParams, } as any); - const count = 2; + const count = 1; expect(scheduleActions).toHaveBeenCalledWith('default', { - internalFullMessage: `We have not detected any monitoring data for 2 stack product(s) in cluster: testCluster. Verify these stack products are up and running, then double check the monitoring settings.`, - internalShortMessage: `We have not detected any monitoring data for 2 stack product(s) in cluster: testCluster. Verify these stack products are up and running, then double check the monitoring settings.`, + internalFullMessage: `We have not detected any monitoring data for 1 stack product(s) in cluster: testCluster. Verify these stack products are up and running, then double check the monitoring settings.`, + internalShortMessage: `We have not detected any monitoring data for 1 stack product(s) in cluster: testCluster. Verify these stack products are up and running, then double check the monitoring settings.`, action: `[View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#/overview?_g=(cluster_uuid:abc123))`, actionPlain: 'Verify these stack products are up and running, then double check the monitoring settings.', clusterName, count, - stackProducts: 'Elasticsearch node: esName1, Kibana instance: kibanaInstance1', + stackProducts: 'Elasticsearch node: esName1', state: 'firing', }); }); diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.test.ts index b09f5a88dba9c..7edd7496805a0 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.test.ts @@ -75,63 +75,6 @@ describe('fetchMissingMonitoringData', () => { timestamp: 2, }, ]), - kibana_uuids: getResponse('.monitoring-kibana-*', [ - { - uuid: 'kibanaUuid1', - nameSource: { - kibana_stats: { - kibana: { - name: 'kibanaName1', - }, - }, - }, - timestamp: 4, - }, - ]), - logstash_uuids: getResponse('.monitoring-logstash-*', [ - { - uuid: 'logstashUuid1', - nameSource: { - logstash_stats: { - logstash: { - host: 'logstashName1', - }, - }, - }, - timestamp: 2, - }, - ]), - beats: { - beats_uuids: getResponse('.monitoring-beats-*', [ - { - uuid: 'beatUuid1', - nameSource: { - beats_stats: { - beat: { - name: 'beatName1', - }, - }, - }, - timestamp: 0, - }, - ]), - }, - apms: { - apm_uuids: getResponse('.monitoring-beats-*', [ - { - uuid: 'apmUuid1', - nameSource: { - beats_stats: { - beat: { - name: 'apmName1', - type: 'apm-server', - }, - }, - }, - timestamp: 1, - }, - ]), - }, })), }, }, @@ -162,38 +105,6 @@ describe('fetchMissingMonitoringData', () => { gapDuration: 8, ccs: null, }, - { - stackProduct: 'kibana', - stackProductUuid: 'kibanaUuid1', - stackProductName: 'kibanaName1', - clusterUuid: 'clusterUuid1', - gapDuration: 6, - ccs: null, - }, - { - stackProduct: 'logstash', - stackProductUuid: 'logstashUuid1', - stackProductName: 'logstashName1', - clusterUuid: 'clusterUuid1', - gapDuration: 8, - ccs: null, - }, - { - stackProduct: 'beats', - stackProductUuid: 'beatUuid1', - stackProductName: 'beatName1', - clusterUuid: 'clusterUuid1', - gapDuration: 10, - ccs: null, - }, - { - stackProduct: 'apm', - stackProductUuid: 'apmUuid1', - stackProductName: 'apmName1', - clusterUuid: 'clusterUuid1', - gapDuration: 9, - ccs: null, - }, ]); }); diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.ts index 0fa90e1d6fb39..b4e12e5d86139 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.ts @@ -5,25 +5,11 @@ */ import { get } from 'lodash'; import { AlertCluster, AlertMissingData } from '../../../common/types/alerts'; -import { - KIBANA_SYSTEM_ID, - BEATS_SYSTEM_ID, - APM_SYSTEM_ID, - LOGSTASH_SYSTEM_ID, - ELASTICSEARCH_SYSTEM_ID, -} from '../../../common/constants'; +import { ELASTICSEARCH_SYSTEM_ID } from '../../../common/constants'; interface ClusterBucketESResponse { key: string; - kibana_uuids?: UuidResponse; - logstash_uuids?: UuidResponse; - es_uuids?: UuidResponse; - beats?: { - beats_uuids: UuidResponse; - }; - apms?: { - apm_uuids: UuidResponse; - }; + es_uuids: UuidResponse; } interface UuidResponse { @@ -48,44 +34,11 @@ interface TopHitESResponse { source_node?: { name: string; }; - kibana_stats?: { - kibana: { - name: string; - }; - }; - logstash_stats?: { - logstash: { - host: string; - }; - }; - beats_stats?: { - beat: { - name: string; - type: string; - }; - }; }; } -function getStackProductFromIndex(index: string, beatType: string) { - if (index.includes('-kibana-')) { - return KIBANA_SYSTEM_ID; - } - if (index.includes('-beats-')) { - if (beatType === 'apm-server') { - return APM_SYSTEM_ID; - } - return BEATS_SYSTEM_ID; - } - if (index.includes('-logstash-')) { - return LOGSTASH_SYSTEM_ID; - } - if (index.includes('-es-')) { - return ELASTICSEARCH_SYSTEM_ID; - } - return ''; -} - +// TODO: only Elasticsearch until we can figure out how to handle upgrades for the rest of the stack +// https://github.com/elastic/kibana/issues/83309 export async function fetchMissingMonitoringData( callCluster: any, clusters: AlertCluster[], @@ -95,37 +48,6 @@ export async function fetchMissingMonitoringData( startMs: number ): Promise { const endMs = nowInMs; - - const nameFields = [ - 'source_node.name', - 'kibana_stats.kibana.name', - 'logstash_stats.logstash.host', - 'beats_stats.beat.name', - 'beats_stats.beat.type', - ]; - const subAggs = { - most_recent: { - max: { - field: 'timestamp', - }, - }, - document: { - top_hits: { - size: 1, - sort: [ - { - timestamp: { - order: 'desc', - }, - }, - ], - _source: { - includes: ['_index', ...nameFields], - }, - }, - }, - }; - const params = { index, filterPath: ['aggregations.clusters.buckets'], @@ -163,61 +85,28 @@ export async function fetchMissingMonitoringData( field: 'node_stats.node_id', size, }, - aggs: subAggs, - }, - kibana_uuids: { - terms: { - field: 'kibana_stats.kibana.uuid', - size, - }, - aggs: subAggs, - }, - beats: { - filter: { - bool: { - must_not: { - term: { - 'beats_stats.beat.type': 'apm-server', - }, - }, - }, - }, aggs: { - beats_uuids: { - terms: { - field: 'beats_stats.beat.uuid', - size, + most_recent: { + max: { + field: 'timestamp', }, - aggs: subAggs, }, - }, - }, - apms: { - filter: { - bool: { - must: { - term: { - 'beats_stats.beat.type': 'apm-server', + document: { + top_hits: { + size: 1, + sort: [ + { + timestamp: { + order: 'desc', + }, + }, + ], + _source: { + includes: ['_index', 'source_node.name'], }, }, }, }, - aggs: { - apm_uuids: { - terms: { - field: 'beats_stats.beat.uuid', - size, - }, - aggs: subAggs, - }, - }, - }, - logstash_uuids: { - terms: { - field: 'logstash_stats.logstash.uuid', - size, - }, - aggs: subAggs, }, }, }, @@ -234,33 +123,20 @@ export async function fetchMissingMonitoringData( const uniqueList: { [id: string]: AlertMissingData } = {}; for (const clusterBucket of clusterBuckets) { const clusterUuid = clusterBucket.key; - - const uuidBuckets = [ - ...(clusterBucket.es_uuids?.buckets || []), - ...(clusterBucket.kibana_uuids?.buckets || []), - ...(clusterBucket.logstash_uuids?.buckets || []), - ...(clusterBucket.beats?.beats_uuids.buckets || []), - ...(clusterBucket.apms?.apm_uuids.buckets || []), - ]; + const uuidBuckets = clusterBucket.es_uuids.buckets; for (const uuidBucket of uuidBuckets) { const stackProductUuid = uuidBucket.key; const indexName = get(uuidBucket, `document.hits.hits[0]._index`); - const stackProduct = getStackProductFromIndex( - indexName, - get(uuidBucket, `document.hits.hits[0]._source.beats_stats.beat.type`) - ); const differenceInMs = nowInMs - uuidBucket.most_recent.value; - let stackProductName = stackProductUuid; - for (const nameField of nameFields) { - stackProductName = get(uuidBucket, `document.hits.hits[0]._source.${nameField}`); - if (stackProductName) { - break; - } - } + const stackProductName = get( + uuidBucket, + `document.hits.hits[0]._source.source_node.name`, + stackProductUuid + ); - uniqueList[`${clusterUuid}${stackProduct}${stackProductUuid}`] = { - stackProduct, + uniqueList[`${clusterUuid}${stackProductUuid}`] = { + stackProduct: ELASTICSEARCH_SYSTEM_ID, stackProductUuid, stackProductName, clusterUuid, From 5ee0104fe1d400135eda157f535996a22d63dec1 Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Wed, 25 Nov 2020 15:20:56 +0100 Subject: [PATCH 37/49] Use .kibana instead of .kibana_current to mark migration completion (#83373) --- rfcs/text/0013_saved_object_migrations.md | 92 ++++++++++++++++------- 1 file changed, 63 insertions(+), 29 deletions(-) diff --git a/rfcs/text/0013_saved_object_migrations.md b/rfcs/text/0013_saved_object_migrations.md index 1a0967d110d06..6e125c28c04c0 100644 --- a/rfcs/text/0013_saved_object_migrations.md +++ b/rfcs/text/0013_saved_object_migrations.md @@ -13,6 +13,7 @@ - [4.2.1 Idempotent migrations performed without coordination](#421-idempotent-migrations-performed-without-coordination) - [4.2.1.1 Restrictions](#4211-restrictions) - [4.2.1.2 Migration algorithm: Cloned index per version](#4212-migration-algorithm-cloned-index-per-version) + - [Known weaknesses:](#known-weaknesses) - [4.2.1.3 Upgrade and rollback procedure](#4213-upgrade-and-rollback-procedure) - [4.2.1.4 Handling documents that belong to a disabled plugin](#4214-handling-documents-that-belong-to-a-disabled-plugin) - [5. Alternatives](#5-alternatives) @@ -192,26 +193,24 @@ id's deterministically with e.g. UUIDv5. ### 4.2.1.2 Migration algorithm: Cloned index per version Note: - The description below assumes the migration algorithm is released in 7.10.0. - So < 7.10.0 will use `.kibana` and >= 7.10.0 will use `.kibana_current`. + So >= 7.10.0 will use the new algorithm. - We refer to the alias and index that outdated nodes use as the source alias and source index. - Every version performs a migration even if mappings or documents aren't outdated. -1. Locate the source index by fetching aliases (including `.kibana` for - versions prior to v7.10.0) +1. Locate the source index by fetching kibana indices: ``` - GET '/_alias/.kibana_current,.kibana_7.10.0,.kibana' + GET '/_indices/.kibana,.kibana_7.10.0' ``` The source index is: - 1. the index the `.kibana_current` alias points to, or if it doesn’t exist, - 2. the index the `.kibana` alias points to, or if it doesn't exist, - 3. the v6.x `.kibana` index + 1. the index the `.kibana` alias points to, or if it doesn't exist, + 2. the v6.x `.kibana` index If none of the aliases exists, this is a new Elasticsearch cluster and no migrations are necessary. Create the `.kibana_7.10.0_001` index with the - following aliases: `.kibana_current` and `.kibana_7.10.0`. + following aliases: `.kibana` and `.kibana_7.10.0`. 2. If the source is a < v6.5 `.kibana` index or < 7.4 `.kibana_task_manager` index prepare the legacy index for a migration: 1. Mark the legacy index as read-only and wait for all in-flight operations to drain (requires https://github.com/elastic/elasticsearch/pull/58094). This prevents any further writes from outdated nodes. Assuming this API is similar to the existing `//_close` API, we expect to receive `"acknowledged" : true` and `"shards_acknowledged" : true`. If all shards don’t acknowledge within the timeout, retry the operation until it succeeds. @@ -235,13 +234,13 @@ Note: atomically so that other Kibana instances will always see either a `.kibana` index or an alias, but never neither. 6. Use the cloned `.kibana_pre6.5.0_001` as the source for the rest of the migration algorithm. -3. If `.kibana_current` and `.kibana_7.10.0` both exists and are pointing to the same index this version's migration has already been completed. +3. If `.kibana` and `.kibana_7.10.0` both exists and are pointing to the same index this version's migration has already been completed. 1. Because the same version can have plugins enabled at any point in time, perform the mappings update in step (6) and migrate outdated documents with step (7). 2. Skip to step (9) to start serving traffic. 4. Fail the migration if: - 1. `.kibana_current` is pointing to an index that belongs to a later version of Kibana .e.g. `.kibana_7.12.0_001` + 1. `.kibana` is pointing to an index that belongs to a later version of Kibana .e.g. `.kibana_7.12.0_001` 2. (Only in 8.x) The source index contains documents that belong to an unknown Saved Object type (from a disabled plugin). Log an error explaining that the plugin that created these documents needs to be enabled again or that these objects should be deleted. See section (4.2.1.4). 5. Mark the source index as read-only and wait for all in-flight operations to drain (requires https://github.com/elastic/elasticsearch/pull/58094). This prevents any further writes from outdated nodes. Assuming this API is similar to the existing `//_close` API, we expect to receive `"acknowledged" : true` and `"shards_acknowledged" : true`. If all shards don’t acknowledge within the timeout, retry the operation until it succeeds. 6. Clone the source index into a new target index which has writes enabled. All nodes on the same version will use the same fixed index name e.g. `.kibana_7.10.0_001`. The `001` postfix isn't used by Kibana, but allows for re-indexing an index should this be required by an Elasticsearch upgrade. E.g. re-index `.kibana_7.10.0_001` into `.kibana_7.10.0_002` and point the `.kibana_7.10.0` alias to `.kibana_7.10.0_002`. @@ -257,24 +256,62 @@ Note: 8. Transform documents by reading batches of outdated documents from the target index then transforming and updating them with optimistic concurrency control. 1. Ignore any version conflict errors. 2. If a document transform throws an exception, add the document to a failure list and continue trying to transform all other documents. If any failures occured, log the complete list of documents that failed to transform. Fail the migration. -9. Mark the migration as complete by doing a single atomic operation (requires https://github.com/elastic/elasticsearch/pull/58100) that: - 3. Checks that `.kibana_current` alias is still pointing to the source index - 4. Points the `.kibana_7.10.0` and `.kibana_current` aliases to the target index. - 5. If this fails with a "required alias [.kibana_current] does not exist" error fetch `.kibana_current` again: - 1. If `.kibana_current` is _not_ pointing to our target index fail the migration. - 2. If `.kibana_current` is pointing to our target index the migration has succeeded and we can proceed to step (9). -10. Start serving traffic. - -This algorithm shares a weakness with our existing migration algorithm -(since v7.4). When the task manager index gets reindexed a reindex script is -applied. Because we delete the original task manager index there is no way to -rollback a failed task manager migration without a snapshot. +9. Mark the migration as complete. This is done as a single atomic + operation (requires https://github.com/elastic/elasticsearch/pull/58100) + to guarantees when multiple versions of Kibana are performing the + migration in parallel, only one version will win. E.g. if 7.11 and 7.12 + are started in parallel and migrate from a 7.9 index, either 7.11 or 7.12 + should succeed and accept writes, but not both. + 3. Checks that `.kibana` alias is still pointing to the source index + 4. Points the `.kibana_7.10.0` and `.kibana` aliases to the target index. + 5. If this fails with a "required alias [.kibana] does not exist" error fetch `.kibana` again: + 1. If `.kibana` is _not_ pointing to our target index fail the migration. + 2. If `.kibana` is pointing to our target index the migration has succeeded and we can proceed to step (10). +10. Start serving traffic. All saved object reads/writes happen through the + version-specific alias `.kibana_7.10.0`. Together with the limitations, this algorithm ensures that migrations are idempotent. If two nodes are started simultaneously, both of them will start transforming documents in that version's target index, but because migrations are idempotent, it doesn’t matter which node’s writes win. +#### Known weaknesses: +(Also present in our existing migration algorithm since v7.4) +When the task manager index gets reindexed a reindex script is applied. +Because we delete the original task manager index there is no way to rollback +a failed task manager migration without a snapshot. Although losing the task +manager data has a fairly low impact. + +(Also present in our existing migration algorithm since v6.5) +If the outdated instance isn't shutdown before starting the migration, the +following data-loss scenario is possible: +1. Upgrade a 7.9 index without shutting down the 7.9 nodes +2. Kibana v7.10 performs a migration and after completing points `.kibana` + alias to `.kibana_7.11.0_001` +3. Kibana v7.9 writes unmigrated documents into `.kibana`. +4. Kibana v7.10 performs a query based on the updated mappings of documents so + results potentially don't match the acknowledged write from step (3). + +Note: + - Data loss won't occur if both nodes have the updated migration algorithm + proposed in this RFC. It is only when one of the nodes use the existing + algorithm that data loss is possible. + - Once v7.10 is restarted it will transform any outdated documents making + these visible to queries again. + +It is possible to work around this weakness by introducing a new alias such as +`.kibana_current` so that after a migration the `.kibana` alias will continue +to point to the outdated index. However, we decided to keep using the +`.kibana` alias despite this weakness for the following reasons: + - Users might rely on `.kibana` alias for snapshots, so if this alias no + longer points to the latest index their snapshots would no longer backup + kibana's latest data. + - Introducing another alias introduces complexity for users and support. + The steps to diagnose, fix or rollback a failed migration will deviate + depending on the 7.x version of Kibana you are using. + - The existing Kibana documentation clearly states that outdated nodes should + be shutdown, this scenario has never been supported by Kibana. +
In the future, this algorithm could enable (2.6) "read-only functionality during the downtime window" but this is outside of the scope of this RFC. @@ -303,12 +340,9 @@ To rollback to a previous version of Kibana without a snapshot: (Assumes the migration to 7.11.0 failed) 1. Shutdown all Kibana nodes. 2. Remove the index created by the failed Kibana migration by using the version-specific alias e.g. `DELETE /.kibana_7.11.0` -3. Identify the rollback index: - 1. If rolling back to a Kibana version < 7.10.0 use `.kibana` - 2. If rolling back to a Kibana version >= 7.10.0 use the version alias of the Kibana version you wish to rollback to e.g. `.kibana_7.10.0` -4. Point the `.kibana_current` alias to the rollback index. -5. Remove the write block from the rollback index. -6. Start the rollback Kibana nodes. All running Kibana nodes should be on the same rollback version, have the same plugins enabled and use the same configuration. +3. Remove the write block from the rollback index using the `.kibana` alias + `PUT /.kibana/_settings {"index.blocks.write": false}` +4. Start the rollback Kibana nodes. All running Kibana nodes should be on the same rollback version, have the same plugins enabled and use the same configuration. ### 4.2.1.4 Handling documents that belong to a disabled plugin It is possible for a plugin to create documents in one version of Kibana, but then when upgrading Kibana to a newer version, that plugin is disabled. Because the plugin is disabled it cannot register it's Saved Objects type including the mappings or any migration transformation functions. These "orphan" documents could cause future problems: @@ -378,7 +412,7 @@ There are several approaches we could take to dealing with these orphan document deterministically perform the delete and re-clone operation without coordination. -5. Transform outdated documents (step 7) on every startup +5. Transform outdated documents (step 8) on every startup Advantages: - Outdated documents belonging to disabled plugins will be upgraded as soon as the plugin is enabled again. From b3430e3f09deac0ed3e577d4f27ef7f9865f2b01 Mon Sep 17 00:00:00 2001 From: Liza Katz Date: Wed, 25 Nov 2020 16:32:05 +0200 Subject: [PATCH 38/49] [Search] Search batching using bfetch (again) (#84043) Re-merging after cypress fixes --- ...plugin-plugins-data-public.plugin.setup.md | 4 +- ...ata-public.searchinterceptordeps.bfetch.md | 11 ++ ...ugins-data-public.searchinterceptordeps.md | 1 + ...plugin-plugins-data-server.plugin.setup.md | 4 +- .../create_streaming_batched_function.test.ts | 127 +++++++++++++++++- .../create_streaming_batched_function.ts | 87 ++++++++---- src/plugins/bfetch/public/batching/types.ts | 31 +++++ src/plugins/bfetch/public/index.ts | 2 + src/plugins/bfetch/public/plugin.ts | 2 +- .../public/streaming/fetch_streaming.test.ts | 27 ++++ .../public/streaming/fetch_streaming.ts | 4 +- .../streaming/from_streaming_xhr.test.ts | 34 +++++ .../public/streaming/from_streaming_xhr.ts | 22 ++- src/plugins/data/kibana.json | 1 + src/plugins/data/public/plugin.ts | 3 +- src/plugins/data/public/public.api.md | 5 +- .../public/search/search_interceptor.test.ts | 26 ++-- .../data/public/search/search_interceptor.ts | 37 +++-- .../data/public/search/search_service.test.ts | 3 + .../data/public/search/search_service.ts | 5 +- src/plugins/data/public/types.ts | 2 + src/plugins/data/server/plugin.ts | 5 +- .../data/server/search/search_service.test.ts | 18 ++- .../data/server/search/search_service.ts | 56 +++++++- src/plugins/data/server/server.api.md | 5 +- src/plugins/data/server/ui_settings.ts | 7 +- src/plugins/embeddable/public/public.api.md | 1 + x-pack/plugins/data_enhanced/kibana.json | 1 + x-pack/plugins/data_enhanced/public/plugin.ts | 5 +- .../public/search/search_interceptor.test.ts | 25 ++-- .../security_solution/cypress/cypress.json | 1 + .../fixtures/overview_search_strategy.json | 6 +- .../cypress/integration/alerts.spec.ts | 14 +- .../alerts_detection_rules.spec.ts | 3 +- .../alerts_detection_rules_custom.spec.ts | 2 +- .../alerts_detection_rules_export.spec.ts | 3 +- .../cypress/integration/overview.spec.ts | 16 +-- .../cypress/support/commands.js | 70 ++++++++-- .../cypress/support/index.d.ts | 7 +- .../es_archives/timeline_alerts/mappings.json | 5 +- 40 files changed, 562 insertions(+), 126 deletions(-) create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptordeps.bfetch.md create mode 100644 src/plugins/bfetch/public/batching/types.ts diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.plugin.setup.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.plugin.setup.md index a0c9b38792825..1ed6059c23062 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.plugin.setup.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.plugin.setup.md @@ -7,7 +7,7 @@ Signature: ```typescript -setup(core: CoreSetup, { expressions, uiActions, usageCollection }: DataSetupDependencies): DataPublicPluginSetup; +setup(core: CoreSetup, { bfetch, expressions, uiActions, usageCollection }: DataSetupDependencies): DataPublicPluginSetup; ``` ## Parameters @@ -15,7 +15,7 @@ setup(core: CoreSetup, { expressio | Parameter | Type | Description | | --- | --- | --- | | core | CoreSetup<DataStartDependencies, DataPublicPluginStart> | | -| { expressions, uiActions, usageCollection } | DataSetupDependencies | | +| { bfetch, expressions, uiActions, usageCollection } | DataSetupDependencies | | Returns: diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptordeps.bfetch.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptordeps.bfetch.md new file mode 100644 index 0000000000000..5b7c635c71529 --- /dev/null +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptordeps.bfetch.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [SearchInterceptorDeps](./kibana-plugin-plugins-data-public.searchinterceptordeps.md) > [bfetch](./kibana-plugin-plugins-data-public.searchinterceptordeps.bfetch.md) + +## SearchInterceptorDeps.bfetch property + +Signature: + +```typescript +bfetch: BfetchPublicSetup; +``` diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptordeps.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptordeps.md index 3653394d28b92..543566b783c23 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptordeps.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptordeps.md @@ -14,6 +14,7 @@ export interface SearchInterceptorDeps | Property | Type | Description | | --- | --- | --- | +| [bfetch](./kibana-plugin-plugins-data-public.searchinterceptordeps.bfetch.md) | BfetchPublicSetup | | | [http](./kibana-plugin-plugins-data-public.searchinterceptordeps.http.md) | CoreSetup['http'] | | | [session](./kibana-plugin-plugins-data-public.searchinterceptordeps.session.md) | ISessionService | | | [startServices](./kibana-plugin-plugins-data-public.searchinterceptordeps.startservices.md) | Promise<[CoreStart, any, unknown]> | | diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.plugin.setup.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.plugin.setup.md index 43129891c5412..b90018c3d9cdd 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.plugin.setup.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.plugin.setup.md @@ -7,7 +7,7 @@ Signature: ```typescript -setup(core: CoreSetup, { expressions, usageCollection }: DataPluginSetupDependencies): { +setup(core: CoreSetup, { bfetch, expressions, usageCollection }: DataPluginSetupDependencies): { __enhance: (enhancements: DataEnhancements) => void; search: ISearchSetup; fieldFormats: { @@ -21,7 +21,7 @@ setup(core: CoreSetup, { expressio | Parameter | Type | Description | | --- | --- | --- | | core | CoreSetup<DataPluginStartDependencies, DataPluginStart> | | -| { expressions, usageCollection } | DataPluginSetupDependencies | | +| { bfetch, expressions, usageCollection } | DataPluginSetupDependencies | | Returns: diff --git a/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts b/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts index d7dde8f1b93d3..3498f205b3286 100644 --- a/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts +++ b/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts @@ -19,7 +19,7 @@ import { createStreamingBatchedFunction } from './create_streaming_batched_function'; import { fetchStreaming as fetchStreamingReal } from '../streaming/fetch_streaming'; -import { defer, of } from '../../../kibana_utils/public'; +import { AbortError, defer, of } from '../../../kibana_utils/public'; import { Subject } from 'rxjs'; const getPromiseState = (promise: Promise): Promise<'resolved' | 'rejected' | 'pending'> => @@ -168,6 +168,28 @@ describe('createStreamingBatchedFunction()', () => { expect(fetchStreaming).toHaveBeenCalledTimes(1); }); + test('ignores a request with an aborted signal', async () => { + const { fetchStreaming } = setup(); + const fn = createStreamingBatchedFunction({ + url: '/test', + fetchStreaming, + maxItemAge: 5, + flushOnMaxItems: 3, + }); + + const abortController = new AbortController(); + abortController.abort(); + + of(fn({ foo: 'bar' }, abortController.signal)); + fn({ baz: 'quix' }); + + await new Promise((r) => setTimeout(r, 6)); + const { body } = fetchStreaming.mock.calls[0][0]; + expect(JSON.parse(body)).toEqual({ + batch: [{ baz: 'quix' }], + }); + }); + test('sends POST request to correct endpoint with items in array batched sorted in call order', async () => { const { fetchStreaming } = setup(); const fn = createStreamingBatchedFunction({ @@ -423,6 +445,73 @@ describe('createStreamingBatchedFunction()', () => { expect(result3).toEqual({ b: '3' }); }); + describe('when requests are aborted', () => { + test('aborts stream when all are aborted', async () => { + const { fetchStreaming } = setup(); + const fn = createStreamingBatchedFunction({ + url: '/test', + fetchStreaming, + maxItemAge: 5, + flushOnMaxItems: 3, + }); + + const abortController = new AbortController(); + const promise = fn({ a: '1' }, abortController.signal); + const promise2 = fn({ a: '2' }, abortController.signal); + await new Promise((r) => setTimeout(r, 6)); + + expect(await isPending(promise)).toBe(true); + expect(await isPending(promise2)).toBe(true); + + abortController.abort(); + await new Promise((r) => setTimeout(r, 6)); + + expect(await isPending(promise)).toBe(false); + expect(await isPending(promise2)).toBe(false); + const [, error] = await of(promise); + const [, error2] = await of(promise2); + expect(error).toBeInstanceOf(AbortError); + expect(error2).toBeInstanceOf(AbortError); + expect(fetchStreaming.mock.calls[0][0].signal.aborted).toBeTruthy(); + }); + + test('rejects promise on abort and lets others continue', async () => { + const { fetchStreaming, stream } = setup(); + const fn = createStreamingBatchedFunction({ + url: '/test', + fetchStreaming, + maxItemAge: 5, + flushOnMaxItems: 3, + }); + + const abortController = new AbortController(); + const promise = fn({ a: '1' }, abortController.signal); + const promise2 = fn({ a: '2' }); + await new Promise((r) => setTimeout(r, 6)); + + expect(await isPending(promise)).toBe(true); + + abortController.abort(); + await new Promise((r) => setTimeout(r, 6)); + + expect(await isPending(promise)).toBe(false); + const [, error] = await of(promise); + expect(error).toBeInstanceOf(AbortError); + + stream.next( + JSON.stringify({ + id: 1, + result: { b: '2' }, + }) + '\n' + ); + + await new Promise((r) => setTimeout(r, 1)); + + const [result2] = await of(promise2); + expect(result2).toEqual({ b: '2' }); + }); + }); + describe('when stream closes prematurely', () => { test('rejects pending promises with CONNECTION error code', async () => { const { fetchStreaming, stream } = setup(); @@ -558,5 +647,41 @@ describe('createStreamingBatchedFunction()', () => { }); }); }); + + test('rejects with STREAM error on JSON parse error only pending promises', async () => { + const { fetchStreaming, stream } = setup(); + const fn = createStreamingBatchedFunction({ + url: '/test', + fetchStreaming, + maxItemAge: 5, + flushOnMaxItems: 3, + }); + + const promise1 = of(fn({ a: '1' })); + const promise2 = of(fn({ a: '2' })); + + await new Promise((r) => setTimeout(r, 6)); + + stream.next( + JSON.stringify({ + id: 1, + result: { b: '1' }, + }) + '\n' + ); + + stream.next('Not a JSON\n'); + + await new Promise((r) => setTimeout(r, 1)); + + const [, error1] = await promise1; + const [result1] = await promise2; + expect(error1).toMatchObject({ + message: 'Unexpected token N in JSON at position 0', + code: 'STREAM', + }); + expect(result1).toMatchObject({ + b: '1', + }); + }); }); }); diff --git a/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts b/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts index 89793fff6b325..f3971ed04efa7 100644 --- a/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts +++ b/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts @@ -17,7 +17,7 @@ * under the License. */ -import { defer, Defer } from '../../../kibana_utils/public'; +import { AbortError, abortSignalToPromise, defer } from '../../../kibana_utils/public'; import { ItemBufferParams, TimedItemBufferParams, @@ -27,13 +27,7 @@ import { } from '../../common'; import { fetchStreaming, split } from '../streaming'; import { normalizeError } from '../../common'; - -export interface BatchItem { - payload: Payload; - future: Defer; -} - -export type BatchedFunc = (payload: Payload) => Promise; +import { BatchedFunc, BatchItem } from './types'; export interface BatchedFunctionProtocolError extends ErrorLike { code: string; @@ -82,43 +76,84 @@ export const createStreamingBatchedFunction = ( flushOnMaxItems = 25, maxItemAge = 10, } = params; - const [fn] = createBatchedFunction, BatchItem>({ - onCall: (payload: Payload) => { + const [fn] = createBatchedFunction({ + onCall: (payload: Payload, signal?: AbortSignal) => { const future = defer(); const entry: BatchItem = { payload, future, + signal, }; return [future.promise, entry]; }, onBatch: async (items) => { try { - let responsesReceived = 0; - const batch = items.map(({ payload }) => payload); + // Filter out any items whose signal is already aborted + items = items.filter((item) => { + if (item.signal?.aborted) item.future.reject(new AbortError()); + return !item.signal?.aborted; + }); + + const donePromises: Array> = items.map((item) => { + return new Promise((resolve) => { + const { promise: abortPromise, cleanup } = item.signal + ? abortSignalToPromise(item.signal) + : { + promise: undefined, + cleanup: () => {}, + }; + + const onDone = () => { + resolve(); + cleanup(); + }; + if (abortPromise) + abortPromise.catch(() => { + item.future.reject(new AbortError()); + onDone(); + }); + item.future.promise.then(onDone, onDone); + }); + }); + + // abort when all items were either resolved, rejected or aborted + const abortController = new AbortController(); + let isBatchDone = false; + Promise.all(donePromises).then(() => { + isBatchDone = true; + abortController.abort(); + }); + const batch = items.map((item) => item.payload); + const { stream } = fetchStreamingInjected({ url, body: JSON.stringify({ batch }), method: 'POST', + signal: abortController.signal, }); + + const handleStreamError = (error: any) => { + const normalizedError = normalizeError(error); + normalizedError.code = 'STREAM'; + for (const { future } of items) future.reject(normalizedError); + }; + stream.pipe(split('\n')).subscribe({ next: (json: string) => { - const response = JSON.parse(json) as BatchResponseItem; - if (response.error) { - responsesReceived++; - items[response.id].future.reject(response.error); - } else if (response.result !== undefined) { - responsesReceived++; - items[response.id].future.resolve(response.result); + try { + const response = JSON.parse(json) as BatchResponseItem; + if (response.error) { + items[response.id].future.reject(response.error); + } else if (response.result !== undefined) { + items[response.id].future.resolve(response.result); + } + } catch (e) { + handleStreamError(e); } }, - error: (error) => { - const normalizedError = normalizeError(error); - normalizedError.code = 'STREAM'; - for (const { future } of items) future.reject(normalizedError); - }, + error: handleStreamError, complete: () => { - const streamTerminatedPrematurely = responsesReceived !== items.length; - if (streamTerminatedPrematurely) { + if (!isBatchDone) { const error: BatchedFunctionProtocolError = { message: 'Connection terminated prematurely.', code: 'CONNECTION', diff --git a/src/plugins/bfetch/public/batching/types.ts b/src/plugins/bfetch/public/batching/types.ts new file mode 100644 index 0000000000000..68860c5d9eedf --- /dev/null +++ b/src/plugins/bfetch/public/batching/types.ts @@ -0,0 +1,31 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Defer } from '../../../kibana_utils/public'; + +export interface BatchItem { + payload: Payload; + future: Defer; + signal?: AbortSignal; +} + +export type BatchedFunc = ( + payload: Payload, + signal?: AbortSignal +) => Promise; diff --git a/src/plugins/bfetch/public/index.ts b/src/plugins/bfetch/public/index.ts index 8707e5a438159..7ff110105faa0 100644 --- a/src/plugins/bfetch/public/index.ts +++ b/src/plugins/bfetch/public/index.ts @@ -23,6 +23,8 @@ import { BfetchPublicPlugin } from './plugin'; export { BfetchPublicSetup, BfetchPublicStart, BfetchPublicContract } from './plugin'; export { split } from './streaming'; +export { BatchedFunc } from './batching/types'; + export function plugin(initializerContext: PluginInitializerContext) { return new BfetchPublicPlugin(initializerContext); } diff --git a/src/plugins/bfetch/public/plugin.ts b/src/plugins/bfetch/public/plugin.ts index 5f01957c0908e..72aaa862b0ad2 100644 --- a/src/plugins/bfetch/public/plugin.ts +++ b/src/plugins/bfetch/public/plugin.ts @@ -22,9 +22,9 @@ import { fetchStreaming as fetchStreamingStatic, FetchStreamingParams } from './ import { removeLeadingSlash } from '../common'; import { createStreamingBatchedFunction, - BatchedFunc, StreamingBatchedFunctionParams, } from './batching/create_streaming_batched_function'; +import { BatchedFunc } from './batching/types'; // eslint-disable-next-line export interface BfetchPublicSetupDependencies {} diff --git a/src/plugins/bfetch/public/streaming/fetch_streaming.test.ts b/src/plugins/bfetch/public/streaming/fetch_streaming.test.ts index 27adc6dc8b549..7a6827b8fee8e 100644 --- a/src/plugins/bfetch/public/streaming/fetch_streaming.test.ts +++ b/src/plugins/bfetch/public/streaming/fetch_streaming.test.ts @@ -132,6 +132,33 @@ test('completes stream observable when request finishes', async () => { expect(spy).toHaveBeenCalledTimes(1); }); +test('completes stream observable when aborted', async () => { + const env = setup(); + const abort = new AbortController(); + const { stream } = fetchStreaming({ + url: 'http://example.com', + signal: abort.signal, + }); + + const spy = jest.fn(); + stream.subscribe({ + complete: spy, + }); + + expect(spy).toHaveBeenCalledTimes(0); + + (env.xhr as any).responseText = 'foo'; + env.xhr.onprogress!({} as any); + + abort.abort(); + + (env.xhr as any).readyState = 4; + (env.xhr as any).status = 200; + env.xhr.onreadystatechange!({} as any); + + expect(spy).toHaveBeenCalledTimes(1); +}); + test('promise throws when request errors', async () => { const env = setup(); const { stream } = fetchStreaming({ diff --git a/src/plugins/bfetch/public/streaming/fetch_streaming.ts b/src/plugins/bfetch/public/streaming/fetch_streaming.ts index 899e8a1824a41..3deee0cf66add 100644 --- a/src/plugins/bfetch/public/streaming/fetch_streaming.ts +++ b/src/plugins/bfetch/public/streaming/fetch_streaming.ts @@ -24,6 +24,7 @@ export interface FetchStreamingParams { headers?: Record; method?: 'GET' | 'POST'; body?: string; + signal?: AbortSignal; } /** @@ -35,6 +36,7 @@ export function fetchStreaming({ headers = {}, method = 'POST', body = '', + signal, }: FetchStreamingParams) { const xhr = new window.XMLHttpRequest(); @@ -45,7 +47,7 @@ export function fetchStreaming({ // Set the HTTP headers Object.entries(headers).forEach(([k, v]) => xhr.setRequestHeader(k, v)); - const stream = fromStreamingXhr(xhr); + const stream = fromStreamingXhr(xhr, signal); // Send the payload to the server xhr.send(body); diff --git a/src/plugins/bfetch/public/streaming/from_streaming_xhr.test.ts b/src/plugins/bfetch/public/streaming/from_streaming_xhr.test.ts index 40eb3d5e2556b..b15bf9bdfbbb0 100644 --- a/src/plugins/bfetch/public/streaming/from_streaming_xhr.test.ts +++ b/src/plugins/bfetch/public/streaming/from_streaming_xhr.test.ts @@ -21,6 +21,7 @@ import { fromStreamingXhr } from './from_streaming_xhr'; const createXhr = (): XMLHttpRequest => (({ + abort: () => {}, onprogress: () => {}, onreadystatechange: () => {}, readyState: 0, @@ -100,6 +101,39 @@ test('completes observable when request reaches end state', () => { expect(complete).toHaveBeenCalledTimes(1); }); +test('completes observable when aborted', () => { + const xhr = createXhr(); + const abortController = new AbortController(); + const observable = fromStreamingXhr(xhr, abortController.signal); + + const next = jest.fn(); + const complete = jest.fn(); + observable.subscribe({ + next, + complete, + }); + + (xhr as any).responseText = '1'; + xhr.onprogress!({} as any); + + (xhr as any).responseText = '2'; + xhr.onprogress!({} as any); + + expect(complete).toHaveBeenCalledTimes(0); + + (xhr as any).readyState = 2; + abortController.abort(); + + expect(complete).toHaveBeenCalledTimes(1); + + // Shouldn't trigger additional events + (xhr as any).readyState = 4; + (xhr as any).status = 200; + xhr.onreadystatechange!({} as any); + + expect(complete).toHaveBeenCalledTimes(1); +}); + test('errors observable if request returns with error', () => { const xhr = createXhr(); const observable = fromStreamingXhr(xhr); diff --git a/src/plugins/bfetch/public/streaming/from_streaming_xhr.ts b/src/plugins/bfetch/public/streaming/from_streaming_xhr.ts index bba8151958492..5df1f5258cb2d 100644 --- a/src/plugins/bfetch/public/streaming/from_streaming_xhr.ts +++ b/src/plugins/bfetch/public/streaming/from_streaming_xhr.ts @@ -26,13 +26,17 @@ import { Observable, Subject } from 'rxjs'; export const fromStreamingXhr = ( xhr: Pick< XMLHttpRequest, - 'onprogress' | 'onreadystatechange' | 'readyState' | 'status' | 'responseText' - > + 'onprogress' | 'onreadystatechange' | 'readyState' | 'status' | 'responseText' | 'abort' + >, + signal?: AbortSignal ): Observable => { const subject = new Subject(); let index = 0; + let aborted = false; const processBatch = () => { + if (aborted) return; + const { responseText } = xhr; if (index >= responseText.length) return; subject.next(responseText.substr(index)); @@ -41,7 +45,19 @@ export const fromStreamingXhr = ( xhr.onprogress = processBatch; + const onBatchAbort = () => { + if (xhr.readyState !== 4) { + aborted = true; + xhr.abort(); + subject.complete(); + if (signal) signal.removeEventListener('abort', onBatchAbort); + } + }; + + if (signal) signal.addEventListener('abort', onBatchAbort); + xhr.onreadystatechange = () => { + if (aborted) return; // Older browsers don't support onprogress, so we need // to call this here, too. It's safe to call this multiple // times even for the same progress event. @@ -49,6 +65,8 @@ export const fromStreamingXhr = ( // 4 is the magic number that means the request is done if (xhr.readyState === 4) { + if (signal) signal.removeEventListener('abort', onBatchAbort); + // 0 indicates a network failure. 400+ messages are considered server errors if (xhr.status === 0 || xhr.status >= 400) { subject.error(new Error(`Batch request failed with status ${xhr.status}`)); diff --git a/src/plugins/data/kibana.json b/src/plugins/data/kibana.json index d6f2534bd5e3b..3e4d08c8faa1b 100644 --- a/src/plugins/data/kibana.json +++ b/src/plugins/data/kibana.json @@ -4,6 +4,7 @@ "server": true, "ui": true, "requiredPlugins": [ + "bfetch", "expressions", "uiActions" ], diff --git a/src/plugins/data/public/plugin.ts b/src/plugins/data/public/plugin.ts index 7e8283476ffc5..dded52310a99c 100644 --- a/src/plugins/data/public/plugin.ts +++ b/src/plugins/data/public/plugin.ts @@ -105,7 +105,7 @@ export class DataPublicPlugin public setup( core: CoreSetup, - { expressions, uiActions, usageCollection }: DataSetupDependencies + { bfetch, expressions, uiActions, usageCollection }: DataSetupDependencies ): DataPublicPluginSetup { const startServices = createStartServicesGetter(core.getStartServices); @@ -152,6 +152,7 @@ export class DataPublicPlugin ); const searchService = this.searchService.setup(core, { + bfetch, usageCollection, expressions, }); diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index 7c4d3ee27cf4a..a6daaf834a424 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -12,6 +12,7 @@ import { ApiResponse as ApiResponse_2 } from '@elastic/elasticsearch/lib/Transpo import { ApplicationStart } from 'kibana/public'; import { Assign } from '@kbn/utility-types'; import { BehaviorSubject } from 'rxjs'; +import { BfetchPublicSetup } from 'src/plugins/bfetch/public'; import Boom from '@hapi/boom'; import { CoreSetup } from 'src/core/public'; import { CoreSetup as CoreSetup_2 } from 'kibana/public'; @@ -1734,7 +1735,7 @@ export class Plugin implements Plugin_2); // (undocumented) - setup(core: CoreSetup, { expressions, uiActions, usageCollection }: DataSetupDependencies): DataPublicPluginSetup; + setup(core: CoreSetup, { bfetch, expressions, uiActions, usageCollection }: DataSetupDependencies): DataPublicPluginSetup; // (undocumented) start(core: CoreStart_2, { uiActions }: DataStartDependencies): DataPublicPluginStart; // (undocumented) @@ -2113,6 +2114,8 @@ export class SearchInterceptor { // // @public (undocumented) export interface SearchInterceptorDeps { + // (undocumented) + bfetch: BfetchPublicSetup; // (undocumented) http: CoreSetup_2['http']; // (undocumented) diff --git a/src/plugins/data/public/search/search_interceptor.test.ts b/src/plugins/data/public/search/search_interceptor.test.ts index 6e75f6e5eef9e..6dc52d7016797 100644 --- a/src/plugins/data/public/search/search_interceptor.test.ts +++ b/src/plugins/data/public/search/search_interceptor.test.ts @@ -25,9 +25,13 @@ import { AbortError } from '../../../kibana_utils/public'; import { SearchTimeoutError, PainlessError, TimeoutErrorMode } from './errors'; import { searchServiceMock } from './mocks'; import { ISearchStart } from '.'; +import { bfetchPluginMock } from '../../../bfetch/public/mocks'; +import { BfetchPublicSetup } from 'src/plugins/bfetch/public'; let searchInterceptor: SearchInterceptor; let mockCoreSetup: MockedKeys; +let bfetchSetup: jest.Mocked; +let fetchMock: jest.Mock; const flushPromises = () => new Promise((resolve) => setImmediate(resolve)); jest.useFakeTimers(); @@ -39,7 +43,11 @@ describe('SearchInterceptor', () => { mockCoreSetup = coreMock.createSetup(); mockCoreStart = coreMock.createStart(); searchMock = searchServiceMock.createStartContract(); + fetchMock = jest.fn(); + bfetchSetup = bfetchPluginMock.createSetupContract(); + bfetchSetup.batchedFunction.mockReturnValue(fetchMock); searchInterceptor = new SearchInterceptor({ + bfetch: bfetchSetup, toasts: mockCoreSetup.notifications.toasts, startServices: new Promise((resolve) => { resolve([mockCoreStart, {}, {}]); @@ -91,7 +99,7 @@ describe('SearchInterceptor', () => { describe('search', () => { test('Observable should resolve if fetch is successful', async () => { const mockResponse: any = { result: 200 }; - mockCoreSetup.http.fetch.mockResolvedValueOnce(mockResponse); + fetchMock.mockResolvedValueOnce(mockResponse); const mockRequest: IEsSearchRequest = { params: {}, }; @@ -102,7 +110,7 @@ describe('SearchInterceptor', () => { describe('Should throw typed errors', () => { test('Observable should fail if fetch has an internal error', async () => { const mockResponse: any = new Error('Internal Error'); - mockCoreSetup.http.fetch.mockRejectedValue(mockResponse); + fetchMock.mockRejectedValue(mockResponse); const mockRequest: IEsSearchRequest = { params: {}, }; @@ -118,7 +126,7 @@ describe('SearchInterceptor', () => { message: 'Request timed out', }, }; - mockCoreSetup.http.fetch.mockRejectedValueOnce(mockResponse); + fetchMock.mockRejectedValueOnce(mockResponse); const mockRequest: IEsSearchRequest = { params: {}, }; @@ -134,7 +142,7 @@ describe('SearchInterceptor', () => { message: 'Request timed out', }, }; - mockCoreSetup.http.fetch.mockRejectedValue(mockResponse); + fetchMock.mockRejectedValue(mockResponse); const mockRequest: IEsSearchRequest = { params: {}, }; @@ -155,7 +163,7 @@ describe('SearchInterceptor', () => { message: 'Request timed out', }, }; - mockCoreSetup.http.fetch.mockRejectedValue(mockResponse); + fetchMock.mockRejectedValue(mockResponse); const mockRequest: IEsSearchRequest = { params: {}, }; @@ -176,7 +184,7 @@ describe('SearchInterceptor', () => { message: 'Request timed out', }, }; - mockCoreSetup.http.fetch.mockRejectedValue(mockResponse); + fetchMock.mockRejectedValue(mockResponse); const mockRequest: IEsSearchRequest = { params: {}, }; @@ -209,7 +217,7 @@ describe('SearchInterceptor', () => { }, }, }; - mockCoreSetup.http.fetch.mockRejectedValueOnce(mockResponse); + fetchMock.mockRejectedValueOnce(mockResponse); const mockRequest: IEsSearchRequest = { params: {}, }; @@ -219,7 +227,7 @@ describe('SearchInterceptor', () => { test('Observable should fail if user aborts (test merged signal)', async () => { const abortController = new AbortController(); - mockCoreSetup.http.fetch.mockImplementationOnce((options: any) => { + fetchMock.mockImplementationOnce((options: any) => { return new Promise((resolve, reject) => { options.signal.addEventListener('abort', () => { reject(new AbortError()); @@ -257,7 +265,7 @@ describe('SearchInterceptor', () => { const error = (e: any) => { expect(e).toBeInstanceOf(AbortError); - expect(mockCoreSetup.http.fetch).not.toBeCalled(); + expect(fetchMock).not.toBeCalled(); done(); }; response.subscribe({ error }); diff --git a/src/plugins/data/public/search/search_interceptor.ts b/src/plugins/data/public/search/search_interceptor.ts index e5abac0d48fef..055b3a71705bf 100644 --- a/src/plugins/data/public/search/search_interceptor.ts +++ b/src/plugins/data/public/search/search_interceptor.ts @@ -17,17 +17,17 @@ * under the License. */ -import { get, memoize, trimEnd } from 'lodash'; +import { get, memoize } from 'lodash'; import { BehaviorSubject, throwError, timer, defer, from, Observable, NEVER } from 'rxjs'; import { catchError, finalize } from 'rxjs/operators'; import { PublicMethodsOf } from '@kbn/utility-types'; import { CoreStart, CoreSetup, ToastsSetup } from 'kibana/public'; import { i18n } from '@kbn/i18n'; +import { BatchedFunc, BfetchPublicSetup } from 'src/plugins/bfetch/public'; import { IKibanaSearchRequest, IKibanaSearchResponse, ISearchOptions, - ES_SEARCH_STRATEGY, ISessionService, } from '../../common'; import { SearchUsageCollector } from './collectors'; @@ -44,6 +44,7 @@ import { toMountPoint } from '../../../kibana_react/public'; import { AbortError, getCombinedAbortSignal } from '../../../kibana_utils/public'; export interface SearchInterceptorDeps { + bfetch: BfetchPublicSetup; http: CoreSetup['http']; uiSettings: CoreSetup['uiSettings']; startServices: Promise<[CoreStart, any, unknown]>; @@ -69,6 +70,10 @@ export class SearchInterceptor { * @internal */ protected application!: CoreStart['application']; + private batchedFetch!: BatchedFunc< + { request: IKibanaSearchRequest; options: ISearchOptions }, + IKibanaSearchResponse + >; /* * @internal @@ -79,6 +84,10 @@ export class SearchInterceptor { this.deps.startServices.then(([coreStart]) => { this.application = coreStart.application; }); + + this.batchedFetch = deps.bfetch.batchedFunction({ + url: '/internal/bsearch', + }); } /* @@ -123,24 +132,14 @@ export class SearchInterceptor { request: IKibanaSearchRequest, options?: ISearchOptions ): Promise { - const { id, ...searchRequest } = request; - const path = trimEnd( - `/internal/search/${options?.strategy ?? ES_SEARCH_STRATEGY}/${id ?? ''}`, - '/' + const { abortSignal, ...requestOptions } = options || {}; + return this.batchedFetch( + { + request, + options: requestOptions, + }, + abortSignal ); - const body = JSON.stringify({ - sessionId: options?.sessionId, - isStored: options?.isStored, - isRestore: options?.isRestore, - ...searchRequest, - }); - - return this.deps.http.fetch({ - method: 'POST', - path, - body, - signal: options?.abortSignal, - }); } /** diff --git a/src/plugins/data/public/search/search_service.test.ts b/src/plugins/data/public/search/search_service.test.ts index 20041a02067d9..3179da4d03a1a 100644 --- a/src/plugins/data/public/search/search_service.test.ts +++ b/src/plugins/data/public/search/search_service.test.ts @@ -21,6 +21,7 @@ import { coreMock } from '../../../../core/public/mocks'; import { CoreSetup, CoreStart } from '../../../../core/public'; import { SearchService, SearchServiceSetupDependencies } from './search_service'; +import { bfetchPluginMock } from '../../../bfetch/public/mocks'; describe('Search service', () => { let searchService: SearchService; @@ -39,8 +40,10 @@ describe('Search service', () => { describe('setup()', () => { it('exposes proper contract', async () => { + const bfetch = bfetchPluginMock.createSetupContract(); const setup = searchService.setup(mockCoreSetup, ({ packageInfo: { version: '8' }, + bfetch, expressions: { registerFunction: jest.fn(), registerType: jest.fn() }, } as unknown) as SearchServiceSetupDependencies); expect(setup).toHaveProperty('aggs'); diff --git a/src/plugins/data/public/search/search_service.ts b/src/plugins/data/public/search/search_service.ts index 96fb3f91ea85f..b76b5846d3d93 100644 --- a/src/plugins/data/public/search/search_service.ts +++ b/src/plugins/data/public/search/search_service.ts @@ -19,6 +19,7 @@ import { Plugin, CoreSetup, CoreStart, PluginInitializerContext } from 'src/core/public'; import { BehaviorSubject } from 'rxjs'; +import { BfetchPublicSetup } from 'src/plugins/bfetch/public'; import { ISearchSetup, ISearchStart, SearchEnhancements } from './types'; import { handleResponse } from './fetch'; @@ -49,6 +50,7 @@ import { aggShardDelay } from '../../common/search/aggs/buckets/shard_delay_fn'; /** @internal */ export interface SearchServiceSetupDependencies { + bfetch: BfetchPublicSetup; expressions: ExpressionsSetup; usageCollection?: UsageCollectionSetup; } @@ -70,7 +72,7 @@ export class SearchService implements Plugin { public setup( { http, getStartServices, notifications, uiSettings }: CoreSetup, - { expressions, usageCollection }: SearchServiceSetupDependencies + { bfetch, expressions, usageCollection }: SearchServiceSetupDependencies ): ISearchSetup { this.usageCollector = createUsageCollector(getStartServices, usageCollection); @@ -80,6 +82,7 @@ export class SearchService implements Plugin { * all pending search requests, as well as getting the number of pending search requests. */ this.searchInterceptor = new SearchInterceptor({ + bfetch, toasts: notifications.toasts, http, uiSettings, diff --git a/src/plugins/data/public/types.ts b/src/plugins/data/public/types.ts index 21a03a49fe058..4082fbe55094c 100644 --- a/src/plugins/data/public/types.ts +++ b/src/plugins/data/public/types.ts @@ -19,6 +19,7 @@ import React from 'react'; import { CoreStart } from 'src/core/public'; +import { BfetchPublicSetup } from 'src/plugins/bfetch/public'; import { IStorageWrapper } from 'src/plugins/kibana_utils/public'; import { ExpressionsSetup } from 'src/plugins/expressions/public'; import { UiActionsSetup, UiActionsStart } from 'src/plugins/ui_actions/public'; @@ -36,6 +37,7 @@ export interface DataPublicPluginEnhancements { } export interface DataSetupDependencies { + bfetch: BfetchPublicSetup; expressions: ExpressionsSetup; uiActions: UiActionsSetup; usageCollection?: UsageCollectionSetup; diff --git a/src/plugins/data/server/plugin.ts b/src/plugins/data/server/plugin.ts index 3ec4e7e64e382..bba2c368ff7d1 100644 --- a/src/plugins/data/server/plugin.ts +++ b/src/plugins/data/server/plugin.ts @@ -19,6 +19,7 @@ import { PluginInitializerContext, CoreSetup, CoreStart, Plugin, Logger } from 'src/core/server'; import { ExpressionsServerSetup } from 'src/plugins/expressions/server'; +import { BfetchServerSetup } from 'src/plugins/bfetch/server'; import { ConfigSchema } from '../config'; import { IndexPatternsService, IndexPatternsServiceStart } from './index_patterns'; import { ISearchSetup, ISearchStart, SearchEnhancements } from './search'; @@ -51,6 +52,7 @@ export interface DataPluginStart { } export interface DataPluginSetupDependencies { + bfetch: BfetchServerSetup; expressions: ExpressionsServerSetup; usageCollection?: UsageCollectionSetup; } @@ -85,7 +87,7 @@ export class DataServerPlugin public setup( core: CoreSetup, - { expressions, usageCollection }: DataPluginSetupDependencies + { bfetch, expressions, usageCollection }: DataPluginSetupDependencies ) { this.indexPatterns.setup(core); this.scriptsService.setup(core); @@ -96,6 +98,7 @@ export class DataServerPlugin core.uiSettings.register(getUiSettings()); const searchSetup = this.searchService.setup(core, { + bfetch, expressions, usageCollection, }); diff --git a/src/plugins/data/server/search/search_service.test.ts b/src/plugins/data/server/search/search_service.test.ts index 0700afd8d6c83..8a52d1d415f9b 100644 --- a/src/plugins/data/server/search/search_service.test.ts +++ b/src/plugins/data/server/search/search_service.test.ts @@ -25,6 +25,8 @@ import { createFieldFormatsStartMock } from '../field_formats/mocks'; import { createIndexPatternsStartMock } from '../index_patterns/mocks'; import { SearchService, SearchServiceSetupDependencies } from './search_service'; +import { bfetchPluginMock } from '../../../bfetch/server/mocks'; +import { of } from 'rxjs'; describe('Search service', () => { let plugin: SearchService; @@ -35,15 +37,29 @@ describe('Search service', () => { const mockLogger: any = { debug: () => {}, }; - plugin = new SearchService(coreMock.createPluginInitializerContext({}), mockLogger); + const context = coreMock.createPluginInitializerContext({}); + context.config.create = jest.fn().mockImplementation(() => { + return of({ + search: { + aggs: { + shardDelay: { + enabled: true, + }, + }, + }, + }); + }); + plugin = new SearchService(context, mockLogger); mockCoreSetup = coreMock.createSetup(); mockCoreStart = coreMock.createStart(); }); describe('setup()', () => { it('exposes proper contract', async () => { + const bfetch = bfetchPluginMock.createSetupContract(); const setup = plugin.setup(mockCoreSetup, ({ packageInfo: { version: '8' }, + bfetch, expressions: { registerFunction: jest.fn(), registerType: jest.fn(), diff --git a/src/plugins/data/server/search/search_service.ts b/src/plugins/data/server/search/search_service.ts index b44980164d097..a9539a8fd3c15 100644 --- a/src/plugins/data/server/search/search_service.ts +++ b/src/plugins/data/server/search/search_service.ts @@ -29,7 +29,8 @@ import { SharedGlobalConfig, StartServicesAccessor, } from 'src/core/server'; -import { first, switchMap } from 'rxjs/operators'; +import { catchError, first, map, switchMap } from 'rxjs/operators'; +import { BfetchServerSetup } from 'src/plugins/bfetch/server'; import { ExpressionsServerSetup } from 'src/plugins/expressions/server'; import { ISearchSetup, @@ -43,7 +44,7 @@ import { AggsService } from './aggs'; import { FieldFormatsStart } from '../field_formats'; import { IndexPatternsServiceStart } from '../index_patterns'; -import { getCallMsearch, registerMsearchRoute, registerSearchRoute } from './routes'; +import { getCallMsearch, registerMsearchRoute, registerSearchRoute, shimHitsTotal } from './routes'; import { ES_SEARCH_STRATEGY, esSearchStrategyProvider } from './es_search'; import { DataPluginStart } from '../plugin'; import { UsageCollectionSetup } from '../../../usage_collection/server'; @@ -85,6 +86,7 @@ type StrategyMap = Record>; /** @internal */ export interface SearchServiceSetupDependencies { + bfetch: BfetchServerSetup; expressions: ExpressionsServerSetup; usageCollection?: UsageCollectionSetup; } @@ -106,6 +108,7 @@ export class SearchService implements Plugin { private readonly searchSourceService = new SearchSourceService(); private defaultSearchStrategyName: string = ES_SEARCH_STRATEGY; private searchStrategies: StrategyMap = {}; + private coreStart?: CoreStart; private sessionService: BackgroundSessionService = new BackgroundSessionService(); constructor( @@ -115,7 +118,7 @@ export class SearchService implements Plugin { public setup( core: CoreSetup<{}, DataPluginStart>, - { expressions, usageCollection }: SearchServiceSetupDependencies + { bfetch, expressions, usageCollection }: SearchServiceSetupDependencies ): ISearchSetup { const usage = usageCollection ? usageProvider(core) : undefined; @@ -128,10 +131,13 @@ export class SearchService implements Plugin { registerMsearchRoute(router, routeDependencies); registerSessionRoutes(router); + core.getStartServices().then(([coreStart]) => { + this.coreStart = coreStart; + }); + core.http.registerRouteHandlerContext('search', async (context, request) => { - const [coreStart] = await core.getStartServices(); - const search = this.asScopedProvider(coreStart)(request); - const session = this.sessionService.asScopedProvider(coreStart)(request); + const search = this.asScopedProvider(this.coreStart!)(request); + const session = this.sessionService.asScopedProvider(this.coreStart!)(request); return { ...search, session }; }); @@ -146,6 +152,44 @@ export class SearchService implements Plugin { ) ); + bfetch.addBatchProcessingRoute< + { request: IKibanaSearchResponse; options?: ISearchOptions }, + any + >('/internal/bsearch', (request) => { + const search = this.asScopedProvider(this.coreStart!)(request); + + return { + onBatchItem: async ({ request: requestData, options }) => { + return search + .search(requestData, options) + .pipe( + first(), + map((response) => { + return { + ...response, + ...{ + rawResponse: shimHitsTotal(response.rawResponse), + }, + }; + }), + catchError((err) => { + // eslint-disable-next-line no-throw-literal + throw { + statusCode: err.statusCode || 500, + body: { + message: err.message, + attributes: { + error: err.body?.error || err.message, + }, + }, + }; + }) + ) + .toPromise(); + }, + }; + }); + core.savedObjects.registerType(searchTelemetry); if (usageCollection) { registerUsageCollector(usageCollection, this.initializerContext); diff --git a/src/plugins/data/server/server.api.md b/src/plugins/data/server/server.api.md index 73e2a68cb9667..86ec784834ace 100644 --- a/src/plugins/data/server/server.api.md +++ b/src/plugins/data/server/server.api.md @@ -9,6 +9,7 @@ import { Adapters } from 'src/plugins/inspector/common'; import { ApiResponse } from '@elastic/elasticsearch'; import { Assign } from '@kbn/utility-types'; import { BehaviorSubject } from 'rxjs'; +import { BfetchServerSetup } from 'src/plugins/bfetch/server'; import { ConfigDeprecationProvider } from '@kbn/config'; import { CoreSetup } from 'src/core/server'; import { CoreSetup as CoreSetup_2 } from 'kibana/server'; @@ -948,7 +949,7 @@ export function parseInterval(interval: string): moment.Duration | null; export class Plugin implements Plugin_2 { constructor(initializerContext: PluginInitializerContext_2); // (undocumented) - setup(core: CoreSetup, { expressions, usageCollection }: DataPluginSetupDependencies): { + setup(core: CoreSetup, { bfetch, expressions, usageCollection }: DataPluginSetupDependencies): { __enhance: (enhancements: DataEnhancements) => void; search: ISearchSetup; fieldFormats: { @@ -1250,7 +1251,7 @@ export function usageProvider(core: CoreSetup_2): SearchUsage; // src/plugins/data/server/index.ts:274:1 - (ae-forgotten-export) The symbol "toAbsoluteDates" needs to be exported by the entry point index.d.ts // src/plugins/data/server/index.ts:275:1 - (ae-forgotten-export) The symbol "calcAutoIntervalLessThan" needs to be exported by the entry point index.d.ts // src/plugins/data/server/index_patterns/index_patterns_service.ts:58:14 - (ae-forgotten-export) The symbol "IndexPatternsService" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/plugin.ts:88:66 - (ae-forgotten-export) The symbol "DataEnhancements" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/plugin.ts:90:74 - (ae-forgotten-export) The symbol "DataEnhancements" needs to be exported by the entry point index.d.ts // src/plugins/data/server/search/types.ts:104:5 - (ae-forgotten-export) The symbol "ISearchStartSearchSource" needs to be exported by the entry point index.d.ts // (No @packageDocumentation comment for this package) diff --git a/src/plugins/data/server/ui_settings.ts b/src/plugins/data/server/ui_settings.ts index 9393700a0e771..f5360f626ac66 100644 --- a/src/plugins/data/server/ui_settings.ts +++ b/src/plugins/data/server/ui_settings.ts @@ -267,14 +267,13 @@ export function getUiSettings(): Record> { }, [UI_SETTINGS.COURIER_BATCH_SEARCHES]: { name: i18n.translate('data.advancedSettings.courier.batchSearchesTitle', { - defaultMessage: 'Batch concurrent searches', + defaultMessage: 'Use legacy search', }), value: false, type: 'boolean', description: i18n.translate('data.advancedSettings.courier.batchSearchesText', { - defaultMessage: `When disabled, dashboard panels will load individually, and search requests will terminate when users navigate - away or update the query. When enabled, dashboard panels will load together when all of the data is loaded, and - searches will not terminate.`, + defaultMessage: `Kibana uses a new search and batching infrastructure. + Enable this option if you prefer to fallback to the legacy synchronous behavior`, }), deprecation: { message: i18n.translate('data.advancedSettings.courier.batchSearchesTextDeprecation', { diff --git a/src/plugins/embeddable/public/public.api.md b/src/plugins/embeddable/public/public.api.md index f3f3682404e32..023cb3d19b632 100644 --- a/src/plugins/embeddable/public/public.api.md +++ b/src/plugins/embeddable/public/public.api.md @@ -12,6 +12,7 @@ import { ApiResponse as ApiResponse_2 } from '@elastic/elasticsearch'; import { ApplicationStart as ApplicationStart_2 } from 'kibana/public'; import { Assign } from '@kbn/utility-types'; import { BehaviorSubject } from 'rxjs'; +import { BfetchPublicSetup } from 'src/plugins/bfetch/public'; import Boom from '@hapi/boom'; import { CoreSetup as CoreSetup_2 } from 'src/core/public'; import { CoreSetup as CoreSetup_3 } from 'kibana/public'; diff --git a/x-pack/plugins/data_enhanced/kibana.json b/x-pack/plugins/data_enhanced/kibana.json index bc7c8410d3df1..eea0101ec4ed7 100644 --- a/x-pack/plugins/data_enhanced/kibana.json +++ b/x-pack/plugins/data_enhanced/kibana.json @@ -6,6 +6,7 @@ "xpack", "data_enhanced" ], "requiredPlugins": [ + "bfetch", "data", "features" ], diff --git a/x-pack/plugins/data_enhanced/public/plugin.ts b/x-pack/plugins/data_enhanced/public/plugin.ts index 948858a5ed4c1..fa3206446f9fc 100644 --- a/x-pack/plugins/data_enhanced/public/plugin.ts +++ b/x-pack/plugins/data_enhanced/public/plugin.ts @@ -7,6 +7,7 @@ import React from 'react'; import { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from 'src/core/public'; import { DataPublicPluginSetup, DataPublicPluginStart } from '../../../../src/plugins/data/public'; +import { BfetchPublicSetup } from '../../../../src/plugins/bfetch/public'; import { setAutocompleteService } from './services'; import { setupKqlQuerySuggestionProvider, KUERY_LANGUAGE_NAME } from './autocomplete'; @@ -16,6 +17,7 @@ import { createConnectedBackgroundSessionIndicator } from './search'; import { ConfigSchema } from '../config'; export interface DataEnhancedSetupDependencies { + bfetch: BfetchPublicSetup; data: DataPublicPluginSetup; } export interface DataEnhancedStartDependencies { @@ -33,7 +35,7 @@ export class DataEnhancedPlugin public setup( core: CoreSetup, - { data }: DataEnhancedSetupDependencies + { bfetch, data }: DataEnhancedSetupDependencies ) { data.autocomplete.addQuerySuggestionProvider( KUERY_LANGUAGE_NAME, @@ -41,6 +43,7 @@ export class DataEnhancedPlugin ); this.enhancedSearchInterceptor = new EnhancedSearchInterceptor({ + bfetch, toasts: core.notifications.toasts, http: core.http, uiSettings: core.uiSettings, diff --git a/x-pack/plugins/data_enhanced/public/search/search_interceptor.test.ts b/x-pack/plugins/data_enhanced/public/search/search_interceptor.test.ts index 3f1cfc7a010c7..f4d7422d1c7e2 100644 --- a/x-pack/plugins/data_enhanced/public/search/search_interceptor.test.ts +++ b/x-pack/plugins/data_enhanced/public/search/search_interceptor.test.ts @@ -11,6 +11,7 @@ import { UI_SETTINGS } from '../../../../../src/plugins/data/common'; import { AbortError } from '../../../../../src/plugins/kibana_utils/public'; import { SearchTimeoutError } from 'src/plugins/data/public'; import { dataPluginMock } from '../../../../../src/plugins/data/public/mocks'; +import { bfetchPluginMock } from '../../../../../src/plugins/bfetch/public/mocks'; const timeTravel = (msToRun = 0) => { jest.advanceTimersByTime(msToRun); @@ -24,12 +25,13 @@ const complete = jest.fn(); let searchInterceptor: EnhancedSearchInterceptor; let mockCoreSetup: MockedKeys; let mockCoreStart: MockedKeys; +let fetchMock: jest.Mock; jest.useFakeTimers(); function mockFetchImplementation(responses: any[]) { let i = 0; - mockCoreSetup.http.fetch.mockImplementation(() => { + fetchMock.mockImplementation(() => { const { time = 0, value = {}, isError = false } = responses[i++]; return new Promise((resolve, reject) => setTimeout(() => { @@ -46,6 +48,7 @@ describe('EnhancedSearchInterceptor', () => { mockCoreSetup = coreMock.createSetup(); mockCoreStart = coreMock.createStart(); const dataPluginMockStart = dataPluginMock.createStartContract(); + fetchMock = jest.fn(); mockCoreSetup.uiSettings.get.mockImplementation((name: string) => { switch (name) { @@ -74,7 +77,11 @@ describe('EnhancedSearchInterceptor', () => { ]); }); + const bfetchMock = bfetchPluginMock.createSetupContract(); + bfetchMock.batchedFunction.mockReturnValue(fetchMock); + searchInterceptor = new EnhancedSearchInterceptor({ + bfetch: bfetchMock, toasts: mockCoreSetup.notifications.toasts, startServices: mockPromise as any, http: mockCoreSetup.http, @@ -245,7 +252,7 @@ describe('EnhancedSearchInterceptor', () => { expect(error).toHaveBeenCalled(); expect(error.mock.calls[0][0]).toBeInstanceOf(AbortError); - expect(mockCoreSetup.http.fetch).toHaveBeenCalledTimes(2); + expect(fetchMock).toHaveBeenCalledTimes(2); expect(mockCoreSetup.http.delete).toHaveBeenCalled(); }); @@ -269,7 +276,7 @@ describe('EnhancedSearchInterceptor', () => { expect(error).toHaveBeenCalled(); expect(error.mock.calls[0][0]).toBeInstanceOf(SearchTimeoutError); - expect(mockCoreSetup.http.fetch).toHaveBeenCalled(); + expect(fetchMock).toHaveBeenCalled(); expect(mockCoreSetup.http.delete).not.toHaveBeenCalled(); }); @@ -301,7 +308,7 @@ describe('EnhancedSearchInterceptor', () => { expect(next).toHaveBeenCalled(); expect(error).not.toHaveBeenCalled(); - expect(mockCoreSetup.http.fetch).toHaveBeenCalled(); + expect(fetchMock).toHaveBeenCalled(); expect(mockCoreSetup.http.delete).not.toHaveBeenCalled(); // Long enough to reach the timeout but not long enough to reach the next response @@ -309,7 +316,7 @@ describe('EnhancedSearchInterceptor', () => { expect(error).toHaveBeenCalled(); expect(error.mock.calls[0][0]).toBeInstanceOf(SearchTimeoutError); - expect(mockCoreSetup.http.fetch).toHaveBeenCalledTimes(2); + expect(fetchMock).toHaveBeenCalledTimes(2); expect(mockCoreSetup.http.delete).toHaveBeenCalled(); }); @@ -343,7 +350,7 @@ describe('EnhancedSearchInterceptor', () => { expect(next).toHaveBeenCalled(); expect(error).not.toHaveBeenCalled(); - expect(mockCoreSetup.http.fetch).toHaveBeenCalled(); + expect(fetchMock).toHaveBeenCalled(); expect(mockCoreSetup.http.delete).not.toHaveBeenCalled(); // Long enough to reach the timeout but not long enough to reach the next response @@ -351,7 +358,7 @@ describe('EnhancedSearchInterceptor', () => { expect(error).toHaveBeenCalled(); expect(error.mock.calls[0][0]).toBe(responses[1].value); - expect(mockCoreSetup.http.fetch).toHaveBeenCalledTimes(2); + expect(fetchMock).toHaveBeenCalledTimes(2); expect(mockCoreSetup.http.delete).toHaveBeenCalled(); }); }); @@ -383,9 +390,7 @@ describe('EnhancedSearchInterceptor', () => { await timeTravel(); - const areAllRequestsAborted = mockCoreSetup.http.fetch.mock.calls.every( - ([{ signal }]) => signal?.aborted - ); + const areAllRequestsAborted = fetchMock.mock.calls.every(([_, signal]) => signal?.aborted); expect(areAllRequestsAborted).toBe(true); expect(mockUsageCollector.trackQueriesCancelled).toBeCalledTimes(1); }); diff --git a/x-pack/plugins/security_solution/cypress/cypress.json b/x-pack/plugins/security_solution/cypress/cypress.json index 173514565c8bb..364db54b4b5d9 100644 --- a/x-pack/plugins/security_solution/cypress/cypress.json +++ b/x-pack/plugins/security_solution/cypress/cypress.json @@ -1,6 +1,7 @@ { "baseUrl": "http://localhost:5601", "defaultCommandTimeout": 120000, + "experimentalNetworkStubbing": true, "retries": { "runMode": 2 }, diff --git a/x-pack/plugins/security_solution/cypress/fixtures/overview_search_strategy.json b/x-pack/plugins/security_solution/cypress/fixtures/overview_search_strategy.json index d0c7517015091..7a6d9d8ae294e 100644 --- a/x-pack/plugins/security_solution/cypress/fixtures/overview_search_strategy.json +++ b/x-pack/plugins/security_solution/cypress/fixtures/overview_search_strategy.json @@ -8,8 +8,7 @@ "filebeatZeek": 71129, "packetbeatDNS": 1090, "packetbeatFlow": 722153, - "packetbeatTLS": 340, - "__typename": "OverviewNetworkData" + "packetbeatTLS": 340 }, "overviewHost": { "auditbeatAuditd": 123, @@ -27,7 +26,6 @@ "endgameSecurity": 397, "filebeatSystemModule": 890, "winlogbeatSecurity": 70, - "winlogbeatMWSysmonOperational": 30, - "__typename": "OverviewHostData" + "winlogbeatMWSysmonOperational": 30 } } diff --git a/x-pack/plugins/security_solution/cypress/integration/alerts.spec.ts b/x-pack/plugins/security_solution/cypress/integration/alerts.spec.ts index db841d2a732c4..8e3b30cddd121 100644 --- a/x-pack/plugins/security_solution/cypress/integration/alerts.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/alerts.spec.ts @@ -25,7 +25,7 @@ import { markInProgressFirstAlert, goToInProgressAlerts, } from '../tasks/alerts'; -import { esArchiverLoad } from '../tasks/es_archiver'; +import { esArchiverLoad, esArchiverUnload } from '../tasks/es_archiver'; import { loginAndWaitForPage } from '../tasks/login'; import { DETECTIONS_URL } from '../urls/navigation'; @@ -37,6 +37,10 @@ describe('Alerts', () => { loginAndWaitForPage(DETECTIONS_URL); }); + afterEach(() => { + esArchiverUnload('alerts'); + }); + it('Closes and opens alerts', () => { waitForAlertsPanelToBeLoaded(); waitForAlertsToBeLoaded(); @@ -165,6 +169,10 @@ describe('Alerts', () => { loginAndWaitForPage(DETECTIONS_URL); }); + afterEach(() => { + esArchiverUnload('closed_alerts'); + }); + it('Open one alert when more than one closed alerts are selected', () => { waitForAlerts(); goToClosedAlerts(); @@ -212,6 +220,10 @@ describe('Alerts', () => { loginAndWaitForPage(DETECTIONS_URL); }); + afterEach(() => { + esArchiverUnload('alerts'); + }); + it('Mark one alert in progress when more than one open alerts are selected', () => { waitForAlerts(); waitForAlertsToBeLoaded(); diff --git a/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules.spec.ts b/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules.spec.ts index 6a62caecfaa67..2d21e3d333c07 100644 --- a/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules.spec.ts @@ -83,7 +83,8 @@ describe('Alerts detection rules', () => { }); }); - it('Auto refreshes rules', () => { + // FIXME: UI hangs on loading + it.skip('Auto refreshes rules', () => { cy.clock(Date.now()); loginAndWaitForPageWithoutDateRange(DETECTIONS_URL); diff --git a/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_custom.spec.ts b/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_custom.spec.ts index dfe984cba3816..5fee3c0bce13c 100644 --- a/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_custom.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_custom.spec.ts @@ -225,7 +225,7 @@ describe('Custom detection rules deletion and edition', () => { goToManageAlertsDetectionRules(); }); - after(() => { + afterEach(() => { esArchiverUnload('custom_rules'); }); diff --git a/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_export.spec.ts b/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_export.spec.ts index c2be6b2883c88..eb8448233c624 100644 --- a/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_export.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_export.spec.ts @@ -17,8 +17,7 @@ import { DETECTIONS_URL } from '../urls/navigation'; const EXPECTED_EXPORTED_RULE_FILE_PATH = 'cypress/test_files/expected_rules_export.ndjson'; -// FLAKY: https://github.com/elastic/kibana/issues/69849 -describe.skip('Export rules', () => { +describe('Export rules', () => { before(() => { esArchiverLoad('export_rule'); cy.server(); diff --git a/x-pack/plugins/security_solution/cypress/integration/overview.spec.ts b/x-pack/plugins/security_solution/cypress/integration/overview.spec.ts index 69094cad7456e..0d12019adbc99 100644 --- a/x-pack/plugins/security_solution/cypress/integration/overview.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/overview.spec.ts @@ -11,9 +11,12 @@ import { loginAndWaitForPage } from '../tasks/login'; import { OVERVIEW_URL } from '../urls/navigation'; +import overviewFixture from '../fixtures/overview_search_strategy.json'; +import emptyInstance from '../fixtures/empty_instance.json'; + describe('Overview Page', () => { it('Host stats render with correct values', () => { - cy.stubSearchStrategyApi('overview_search_strategy'); + cy.stubSearchStrategyApi(overviewFixture, 'overviewHost'); loginAndWaitForPage(OVERVIEW_URL); expandHostStats(); @@ -23,7 +26,7 @@ describe('Overview Page', () => { }); it('Network stats render with correct values', () => { - cy.stubSearchStrategyApi('overview_search_strategy'); + cy.stubSearchStrategyApi(overviewFixture, 'overviewNetwork'); loginAndWaitForPage(OVERVIEW_URL); expandNetworkStats(); @@ -33,14 +36,9 @@ describe('Overview Page', () => { }); describe('with no data', () => { - before(() => { - cy.server(); - cy.fixture('empty_instance').as('emptyInstance'); - loginAndWaitForPage(OVERVIEW_URL); - cy.route('POST', '**/internal/search/securitySolutionIndexFields', '@emptyInstance'); - }); - it('Splash screen should be here', () => { + cy.stubSearchStrategyApi(emptyInstance, undefined, 'securitySolutionIndexFields'); + loginAndWaitForPage(OVERVIEW_URL); cy.get(OVERVIEW_EMPTY_PAGE).should('be.visible'); }); }); diff --git a/x-pack/plugins/security_solution/cypress/support/commands.js b/x-pack/plugins/security_solution/cypress/support/commands.js index c249e0a77690c..95a52794628b1 100644 --- a/x-pack/plugins/security_solution/cypress/support/commands.js +++ b/x-pack/plugins/security_solution/cypress/support/commands.js @@ -30,24 +30,66 @@ // -- This is will overwrite an existing command -- // Cypress.Commands.overwrite("visit", (originalFn, url, options) => { ... }) -Cypress.Commands.add('stubSecurityApi', function (dataFileName) { - cy.on('window:before:load', (win) => { - win.fetch = null; - }); - cy.server(); - cy.fixture(dataFileName).as(`${dataFileName}JSON`); - cy.route('POST', 'api/solutions/security/graphql', `@${dataFileName}JSON`); -}); +import { findIndex } from 'lodash/fp'; + +const getFindRequestConfig = (searchStrategyName, factoryQueryType) => { + if (!factoryQueryType) { + return { + options: { strategy: searchStrategyName }, + }; + } + + return { + options: { strategy: searchStrategyName }, + request: { factoryQueryType }, + }; +}; Cypress.Commands.add( 'stubSearchStrategyApi', - function (dataFileName, searchStrategyName = 'securitySolutionSearchStrategy') { - cy.on('window:before:load', (win) => { - win.fetch = null; + function (stubObject, factoryQueryType, searchStrategyName = 'securitySolutionSearchStrategy') { + cy.route2('POST', '/internal/bsearch', (req) => { + const bodyObj = JSON.parse(req.body); + const findRequestConfig = getFindRequestConfig(searchStrategyName, factoryQueryType); + + const requestIndex = findIndex(findRequestConfig, bodyObj.batch); + + if (requestIndex > -1) { + return req.reply((res) => { + const responseObjectsArray = res.body.split('\n').map((responseString) => { + try { + return JSON.parse(responseString); + } catch { + return responseString; + } + }); + const responseIndex = findIndex({ id: requestIndex }, responseObjectsArray); + + const stubbedResponseObjectsArray = [...responseObjectsArray]; + stubbedResponseObjectsArray[responseIndex] = { + ...stubbedResponseObjectsArray[responseIndex], + result: { + ...stubbedResponseObjectsArray[responseIndex].result, + ...stubObject, + }, + }; + + const stubbedResponse = stubbedResponseObjectsArray + .map((object) => { + try { + return JSON.stringify(object); + } catch { + return object; + } + }) + .join('\n'); + + res.send(stubbedResponse); + }); + } + + req.reply(); }); - cy.server(); - cy.fixture(dataFileName).as(`${dataFileName}JSON`); - cy.route('POST', `internal/search/${searchStrategyName}`, `@${dataFileName}JSON`); } ); diff --git a/x-pack/plugins/security_solution/cypress/support/index.d.ts b/x-pack/plugins/security_solution/cypress/support/index.d.ts index fb55a2890c8b7..06285abba6531 100644 --- a/x-pack/plugins/security_solution/cypress/support/index.d.ts +++ b/x-pack/plugins/security_solution/cypress/support/index.d.ts @@ -7,8 +7,11 @@ declare namespace Cypress { interface Chainable { promisify(): Promise; - stubSecurityApi(dataFileName: string): Chainable; - stubSearchStrategyApi(dataFileName: string, searchStrategyName?: string): Chainable; + stubSearchStrategyApi( + stubObject: Record, + factoryQueryType?: string, + searchStrategyName?: string + ): Chainable; attachFile(fileName: string, fileType?: string): Chainable; waitUntil( fn: (subject: Subject) => boolean | Chainable, diff --git a/x-pack/test/security_solution_cypress/es_archives/timeline_alerts/mappings.json b/x-pack/test/security_solution_cypress/es_archives/timeline_alerts/mappings.json index a1a9e7bfeae7f..abdec252471b7 100644 --- a/x-pack/test/security_solution_cypress/es_archives/timeline_alerts/mappings.json +++ b/x-pack/test/security_solution_cypress/es_archives/timeline_alerts/mappings.json @@ -157,6 +157,9 @@ "throttle": { "type": "keyword" }, + "updatedAt": { + "type": "date" + }, "updatedBy": { "type": "keyword" } @@ -9060,4 +9063,4 @@ } } } -} \ No newline at end of file +} From 58297fa131255158975449915b469ef45bbba86b Mon Sep 17 00:00:00 2001 From: Brandon Kobel Date: Wed, 25 Nov 2020 06:36:24 -0800 Subject: [PATCH 39/49] Deprecate `xpack.task_manager.index` setting (#84155) * Deprecate `xpack.task_manager.index` setting * Updating developer docs about configuring task manager settings --- x-pack/plugins/task_manager/README.md | 2 +- .../plugins/task_manager/server/index.test.ts | 43 +++++++++++++++++++ x-pack/plugins/task_manager/server/index.ts | 18 ++++++-- 3 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 x-pack/plugins/task_manager/server/index.test.ts diff --git a/x-pack/plugins/task_manager/README.md b/x-pack/plugins/task_manager/README.md index 744d657bcd790..d3c8ecb6c4505 100644 --- a/x-pack/plugins/task_manager/README.md +++ b/x-pack/plugins/task_manager/README.md @@ -43,7 +43,7 @@ The task_manager can be configured via `taskManager` config options (e.g. `taskM - `max_attempts` - The maximum number of times a task will be attempted before being abandoned as failed - `poll_interval` - How often the background worker should check the task_manager index for more work - `max_poll_inactivity_cycles` - How many poll intervals is work allowed to block polling for before it's timed out. This does not include task execution, as task execution does not block the polling, but rather includes work needed to manage Task Manager's state. -- `index` - The name of the index that the task_manager +- `index` - **deprecated** The name of the index that the task_manager will use. This is deprecated, and will be removed starting in 8.0 - `max_workers` - The maximum number of tasks a Kibana will run concurrently (defaults to 10) - `credentials` - Encrypted user credentials. All tasks will run in the security context of this user. See [this issue](https://github.com/elastic/dev/issues/1045) for a discussion on task scheduler security. - `override_num_workers`: An object of `taskType: number` that overrides the `num_workers` for tasks diff --git a/x-pack/plugins/task_manager/server/index.test.ts b/x-pack/plugins/task_manager/server/index.test.ts new file mode 100644 index 0000000000000..3f25f4403d358 --- /dev/null +++ b/x-pack/plugins/task_manager/server/index.test.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { config } from './index'; +import { applyDeprecations, configDeprecationFactory } from '@kbn/config'; + +const CONFIG_PATH = 'xpack.task_manager'; + +const applyTaskManagerDeprecations = (settings: Record = {}) => { + const deprecations = config.deprecations!(configDeprecationFactory); + const deprecationMessages: string[] = []; + const _config = { + [CONFIG_PATH]: settings, + }; + const migrated = applyDeprecations( + _config, + deprecations.map((deprecation) => ({ + deprecation, + path: CONFIG_PATH, + })), + (msg) => deprecationMessages.push(msg) + ); + return { + messages: deprecationMessages, + migrated, + }; +}; + +describe('deprecations', () => { + ['.foo', '.kibana_task_manager'].forEach((index) => { + it('logs a warning if index is set', () => { + const { messages } = applyTaskManagerDeprecations({ index }); + expect(messages).toMatchInlineSnapshot(` + Array [ + "\\"xpack.task_manager.index\\" is deprecated. Multitenancy by changing \\"kibana.index\\" will not be supported starting in 8.0. See https://ela.st/kbn-remove-legacy-multitenancy for more details", + ] + `); + }); + }); +}); diff --git a/x-pack/plugins/task_manager/server/index.ts b/x-pack/plugins/task_manager/server/index.ts index cf70e68437cc6..8f96e10430b39 100644 --- a/x-pack/plugins/task_manager/server/index.ts +++ b/x-pack/plugins/task_manager/server/index.ts @@ -4,9 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ -import { PluginInitializerContext } from 'src/core/server'; +import { get } from 'lodash'; +import { PluginConfigDescriptor, PluginInitializerContext } from 'src/core/server'; import { TaskManagerPlugin } from './plugin'; -import { configSchema } from './config'; +import { configSchema, TaskManagerConfig } from './config'; export const plugin = (initContext: PluginInitializerContext) => new TaskManagerPlugin(initContext); @@ -26,6 +27,17 @@ export { TaskManagerStartContract, } from './plugin'; -export const config = { +export const config: PluginConfigDescriptor = { schema: configSchema, + deprecations: () => [ + (settings, fromPath, log) => { + const taskManager = get(settings, fromPath); + if (taskManager?.index) { + log( + `"${fromPath}.index" is deprecated. Multitenancy by changing "kibana.index" will not be supported starting in 8.0. See https://ela.st/kbn-remove-legacy-multitenancy for more details` + ); + } + return settings; + }, + ], }; From 0fe8a65a23c3de3381b58219a1e4f169b1354133 Mon Sep 17 00:00:00 2001 From: Scotty Bollinger Date: Wed, 25 Nov 2020 09:00:10 -0600 Subject: [PATCH 40/49] [Workplace Search] Migrate DisplaySettings tree (#84283) * Initial copy/paste of components Changes for pre-commit hooks were: - Linting - Lodash imports - Fixed warnings for `jsx-a11y/mouse-events-have-key-events` with stubbed onFocus and onBlue events with FIXME comments * Add server routes * Remove reference to shared lib This one-liner appears only once in ent-search so adding it here in the logic file` * Fix paths * Add types and fix TypeScript issues * Replace FlashMessages with global component * More explicit Result type * Remove routes/http in favor of HttpLogic * Fix server routes `urlFieldIsLinkable` was missing and `detailFields` can either be an object or an array of objects * Add base styles These were ported from ent-search. Decided to use spacers where some global styles were missing. * Kibana prefers underscores in URLs Was only going to do display-settings and result-details but decided to YOLO all of them --- .../applications/workplace_search/routes.ts | 36 +- .../applications/workplace_search/types.ts | 23 ++ .../display_settings/custom_source_icon.tsx | 36 ++ .../display_settings/display_settings.scss | 206 +++++++++++ .../display_settings/display_settings.tsx | 141 +++++++ .../display_settings_logic.ts | 350 ++++++++++++++++++ .../display_settings_router.tsx | 31 +- .../example_result_detail_card.tsx | 75 ++++ .../example_search_result_group.tsx | 74 ++++ .../example_standout_result.tsx | 66 ++++ .../display_settings/field_editor_modal.tsx | 101 +++++ .../display_settings/result_detail.tsx | 146 ++++++++ .../display_settings/search_results.tsx | 164 ++++++++ .../display_settings/subtitle_field.tsx | 35 ++ .../display_settings/title_field.tsx | 35 ++ .../routes/workplace_search/sources.test.ts | 152 ++++++++ .../server/routes/workplace_search/sources.ts | 95 +++++ 17 files changed, 1747 insertions(+), 19 deletions(-) create mode 100644 x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/custom_source_icon.tsx create mode 100644 x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings.scss create mode 100644 x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings.tsx create mode 100644 x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings_logic.ts create mode 100644 x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/example_result_detail_card.tsx create mode 100644 x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/example_search_result_group.tsx create mode 100644 x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/example_standout_result.tsx create mode 100644 x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/field_editor_modal.tsx create mode 100644 x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/result_detail.tsx create mode 100644 x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/search_results.tsx create mode 100644 x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/subtitle_field.tsx create mode 100644 x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/title_field.tsx diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/routes.ts b/x-pack/plugins/enterprise_search/public/applications/workplace_search/routes.ts index 3ec22ede888ab..14c288de5a0c8 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/routes.ts +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/routes.ts @@ -46,7 +46,7 @@ export const ENT_SEARCH_LICENSE_MANAGEMENT = `${ENT_SEARCH_DOCS_PREFIX}/license- export const PERSONAL_PATH = '/p'; -export const ROLE_MAPPINGS_PATH = '/role-mappings'; +export const ROLE_MAPPINGS_PATH = '/role_mappings'; export const ROLE_MAPPING_PATH = `${ROLE_MAPPINGS_PATH}/:roleId`; export const ROLE_MAPPING_NEW_PATH = `${ROLE_MAPPINGS_PATH}/new`; @@ -63,18 +63,18 @@ export const PERSONAL_SOURCES_PATH = `${PERSONAL_PATH}${SOURCES_PATH}`; export const SOURCE_ADDED_PATH = `${SOURCES_PATH}/added`; export const ADD_SOURCE_PATH = `${SOURCES_PATH}/add`; export const ADD_BOX_PATH = `${SOURCES_PATH}/add/box`; -export const ADD_CONFLUENCE_PATH = `${SOURCES_PATH}/add/confluence-cloud`; -export const ADD_CONFLUENCE_SERVER_PATH = `${SOURCES_PATH}/add/confluence-server`; +export const ADD_CONFLUENCE_PATH = `${SOURCES_PATH}/add/confluence_cloud`; +export const ADD_CONFLUENCE_SERVER_PATH = `${SOURCES_PATH}/add/confluence_server`; export const ADD_DROPBOX_PATH = `${SOURCES_PATH}/add/dropbox`; -export const ADD_GITHUB_ENTERPRISE_PATH = `${SOURCES_PATH}/add/github-enterprise-server`; +export const ADD_GITHUB_ENTERPRISE_PATH = `${SOURCES_PATH}/add/github_enterprise_server`; export const ADD_GITHUB_PATH = `${SOURCES_PATH}/add/github`; export const ADD_GMAIL_PATH = `${SOURCES_PATH}/add/gmail`; -export const ADD_GOOGLE_DRIVE_PATH = `${SOURCES_PATH}/add/google-drive`; -export const ADD_JIRA_PATH = `${SOURCES_PATH}/add/jira-cloud`; -export const ADD_JIRA_SERVER_PATH = `${SOURCES_PATH}/add/jira-server`; +export const ADD_GOOGLE_DRIVE_PATH = `${SOURCES_PATH}/add/google_drive`; +export const ADD_JIRA_PATH = `${SOURCES_PATH}/add/jira_cloud`; +export const ADD_JIRA_SERVER_PATH = `${SOURCES_PATH}/add/jira_server`; export const ADD_ONEDRIVE_PATH = `${SOURCES_PATH}/add/onedrive`; export const ADD_SALESFORCE_PATH = `${SOURCES_PATH}/add/salesforce`; -export const ADD_SALESFORCE_SANDBOX_PATH = `${SOURCES_PATH}/add/salesforce-sandbox`; +export const ADD_SALESFORCE_SANDBOX_PATH = `${SOURCES_PATH}/add/salesforce_sandbox`; export const ADD_SERVICENOW_PATH = `${SOURCES_PATH}/add/servicenow`; export const ADD_SHAREPOINT_PATH = `${SOURCES_PATH}/add/sharepoint`; export const ADD_SLACK_PATH = `${SOURCES_PATH}/add/slack`; @@ -86,30 +86,30 @@ export const PERSONAL_SETTINGS_PATH = `${PERSONAL_PATH}/settings`; export const SOURCE_DETAILS_PATH = `${SOURCES_PATH}/:sourceId`; export const SOURCE_CONTENT_PATH = `${SOURCES_PATH}/:sourceId/content`; export const SOURCE_SCHEMAS_PATH = `${SOURCES_PATH}/:sourceId/schemas`; -export const SOURCE_DISPLAY_SETTINGS_PATH = `${SOURCES_PATH}/:sourceId/display-settings`; +export const SOURCE_DISPLAY_SETTINGS_PATH = `${SOURCES_PATH}/:sourceId/display_settings`; export const SOURCE_SETTINGS_PATH = `${SOURCES_PATH}/:sourceId/settings`; -export const REINDEX_JOB_PATH = `${SOURCES_PATH}/:sourceId/schema-errors/:activeReindexJobId`; +export const REINDEX_JOB_PATH = `${SOURCES_PATH}/:sourceId/schema_errors/:activeReindexJobId`; export const DISPLAY_SETTINGS_SEARCH_RESULT_PATH = `${SOURCE_DISPLAY_SETTINGS_PATH}/`; -export const DISPLAY_SETTINGS_RESULT_DETAIL_PATH = `${SOURCE_DISPLAY_SETTINGS_PATH}/result-detail`; +export const DISPLAY_SETTINGS_RESULT_DETAIL_PATH = `${SOURCE_DISPLAY_SETTINGS_PATH}/result_detail`; export const ORG_SETTINGS_PATH = '/settings'; export const ORG_SETTINGS_CUSTOMIZE_PATH = `${ORG_SETTINGS_PATH}/customize`; export const ORG_SETTINGS_CONNECTORS_PATH = `${ORG_SETTINGS_PATH}/connectors`; export const ORG_SETTINGS_OAUTH_APPLICATION_PATH = `${ORG_SETTINGS_PATH}/oauth`; export const EDIT_BOX_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/box/edit`; -export const EDIT_CONFLUENCE_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/confluence-cloud/edit`; -export const EDIT_CONFLUENCE_SERVER_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/confluence-server/edit`; +export const EDIT_CONFLUENCE_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/confluence_cloud/edit`; +export const EDIT_CONFLUENCE_SERVER_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/confluence_server/edit`; export const EDIT_DROPBOX_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/dropbox/edit`; -export const EDIT_GITHUB_ENTERPRISE_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/github-enterprise-server/edit`; +export const EDIT_GITHUB_ENTERPRISE_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/github_enterprise_server/edit`; export const EDIT_GITHUB_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/github/edit`; export const EDIT_GMAIL_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/gmail/edit`; -export const EDIT_GOOGLE_DRIVE_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/google-drive/edit`; -export const EDIT_JIRA_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/jira-cloud/edit`; -export const EDIT_JIRA_SERVER_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/jira-server/edit`; +export const EDIT_GOOGLE_DRIVE_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/google_drive/edit`; +export const EDIT_JIRA_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/jira_cloud/edit`; +export const EDIT_JIRA_SERVER_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/jira_server/edit`; export const EDIT_ONEDRIVE_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/onedrive/edit`; export const EDIT_SALESFORCE_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/salesforce/edit`; -export const EDIT_SALESFORCE_SANDBOX_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/salesforce-sandbox/edit`; +export const EDIT_SALESFORCE_SANDBOX_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/salesforce_sandbox/edit`; export const EDIT_SERVICENOW_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/servicenow/edit`; export const EDIT_SHAREPOINT_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/sharepoint/edit`; export const EDIT_SLACK_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/slack/edit`; diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/types.ts b/x-pack/plugins/enterprise_search/public/applications/workplace_search/types.ts index 73e7f7ed701d8..9bda686ebbf00 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/types.ts +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/types.ts @@ -181,3 +181,26 @@ export interface CustomSource { name: string; id: string; } + +export interface Result { + [key: string]: string; +} + +export interface OptionValue { + value: string; + text: string; +} + +export interface DetailField { + fieldName: string; + label: string; +} + +export interface SearchResultConfig { + titleField: string | null; + subtitleField: string | null; + descriptionField: string | null; + urlField: string | null; + color: string; + detailFields: DetailField[]; +} diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/custom_source_icon.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/custom_source_icon.tsx new file mode 100644 index 0000000000000..16129324b56d1 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/custom_source_icon.tsx @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; + +const BLACK_RGB = '#000'; + +interface CustomSourceIconProps { + color?: string; +} + +export const CustomSourceIcon: React.FC = ({ color = BLACK_RGB }) => ( + + + + +); diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings.scss b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings.scss new file mode 100644 index 0000000000000..27935104f4ef6 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings.scss @@ -0,0 +1,206 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +// -------------------------------------------------- +// Custom Source display settings +// -------------------------------------------------- + +@mixin source_name { + font-size: .6875em; + text-transform: uppercase; + font-weight: 600; + letter-spacing: 0.06em; +} + +@mixin example_result_box_shadow { + box-shadow: + 0 1px 3px rgba(black, 0.1), + 0 0 20px $euiColorLightestShade; +} + +// Wrapper +.custom-source-display-settings { + font-size: 16px; +} + +// Example result content +.example-result-content { + & > * { + line-height: 1.5em; + } + + &__title { + font-size: 1em; + font-weight: 600; + color: $euiColorPrimary; + + .example-result-detail-card & { + font-size: 20px; + } + } + + &__subtitle, + &__description { + font-size: .875; + } + + &__subtitle { + color: $euiColorDarkestShade; + } + + &__description { + padding: .1rem 0 .125rem .35rem; + border-left: 3px solid $euiColorLightShade; + color: $euiColorDarkShade; + line-height: 1.8; + word-break: break-word; + + @supports (display: -webkit-box) { + display: -webkit-box; + -webkit-box-orient: vertical; + -webkit-line-clamp: 3; + overflow: hidden; + text-overflow: ellipsis; + } + } + + &__url { + .example-result-detail-card & { + color: $euiColorDarkShade; + } + } +} + +.example-result-content-placeholder { + color: $euiColorMediumShade; +} + +// Example standout result +.example-standout-result { + border-radius: 4px; + overflow: hidden; + @include example_result_box_shadow; + + &__header, + &__content { + padding-left: 1em; + padding-right: 1em; + } + + &__content { + padding-top: 1em; + padding-bottom: 1em; + } + + &__source-name { + line-height: 34px; + @include source_name; + } +} + +// Example result group +.example-result-group { + &__header { + padding: 0 .5em; + border-radius: 4px; + display: inline-flex; + align-items: center; + + .euiIcon { + margin-right: .25rem; + } + } + + &__source-name { + line-height: 1.75em; + @include source_name; + } + + &__content { + display: flex; + align-items: stretch; + padding: .75em 0; + } + + &__border { + width: 4px; + border-radius: 2px; + flex-shrink: 0; + margin-left: .875rem; + } + + &__results { + flex: 1; + max-width: 100%; + } +} + +.example-grouped-result { + padding: 1em; +} + +.example-result-field-hover { + background: lighten($euiColorVis1_behindText, 30%); + position: relative; + + &:before, + &:after { + content: ''; + position: absolute; + height: 100%; + width: 4px; + background: lighten($euiColorVis1_behindText, 30%); + } + + &:before { + right: 100%; + border-radius: 2px 0 0 2px; + } + + &:after { + left: 100%; + border-radius: 0 2px 2px 0; + } + + .example-result-content-placeholder { + color: $euiColorFullShade; + } +} + +.example-result-detail-card { + @include example_result_box_shadow; + + &__header { + position: relative; + padding: 1.25em 1em 0; + } + + &__border { + height: 4px; + position: absolute; + top: 0; + right: 0; + left: 0; + } + + &__source-name { + margin-bottom: 1em; + font-weight: 500; + } + + &__field { + padding: 1em; + + & + & { + border-top: 1px solid $euiColorLightShade; + } + } +} + +.visible-fields-container { + background: $euiColorLightestShade; + border-color: transparent; +} diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings.tsx new file mode 100644 index 0000000000000..e34728beef5e5 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings.tsx @@ -0,0 +1,141 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FormEvent, useEffect } from 'react'; + +import { History } from 'history'; +import { useActions, useValues } from 'kea'; +import { useHistory } from 'react-router-dom'; + +import './display_settings.scss'; + +import { + EuiButton, + EuiEmptyPrompt, + EuiTabbedContent, + EuiPanel, + EuiTabbedContentTab, +} from '@elastic/eui'; + +import { + DISPLAY_SETTINGS_RESULT_DETAIL_PATH, + DISPLAY_SETTINGS_SEARCH_RESULT_PATH, + getContentSourcePath, +} from '../../../../routes'; + +import { AppLogic } from '../../../../app_logic'; + +import { Loading } from '../../../../../shared/loading'; +import { ViewContentHeader } from '../../../../components/shared/view_content_header'; + +import { DisplaySettingsLogic } from './display_settings_logic'; + +import { FieldEditorModal } from './field_editor_modal'; +import { ResultDetail } from './result_detail'; +import { SearchResults } from './search_results'; + +const UNSAVED_MESSAGE = + 'Your display settings have not been saved. Are you sure you want to leave?'; + +interface DisplaySettingsProps { + tabId: number; +} + +export const DisplaySettings: React.FC = ({ tabId }) => { + const history = useHistory() as History; + const { initializeDisplaySettings, setServerData, resetDisplaySettingsState } = useActions( + DisplaySettingsLogic + ); + + const { + dataLoading, + sourceId, + addFieldModalVisible, + unsavedChanges, + exampleDocuments, + } = useValues(DisplaySettingsLogic); + + const { isOrganization } = useValues(AppLogic); + + const hasDocuments = exampleDocuments.length > 0; + + useEffect(() => { + initializeDisplaySettings(); + return resetDisplaySettingsState; + }, []); + + useEffect(() => { + window.onbeforeunload = hasDocuments && unsavedChanges ? () => UNSAVED_MESSAGE : null; + return () => { + window.onbeforeunload = null; + }; + }, [unsavedChanges]); + + if (dataLoading) return ; + + const tabs = [ + { + id: 'search_results', + name: 'Search Results', + content: , + }, + { + id: 'result_detail', + name: 'Result Detail', + content: , + }, + ] as EuiTabbedContentTab[]; + + const onSelectedTabChanged = (tab: EuiTabbedContentTab) => { + const path = + tab.id === tabs[1].id + ? getContentSourcePath(DISPLAY_SETTINGS_RESULT_DETAIL_PATH, sourceId, isOrganization) + : getContentSourcePath(DISPLAY_SETTINGS_SEARCH_RESULT_PATH, sourceId, isOrganization); + + history.push(path); + }; + + const handleFormSubmit = (e: FormEvent) => { + e.preventDefault(); + setServerData(); + }; + + return ( + <> +
+ + Save + + ) : null + } + /> + {hasDocuments ? ( + + ) : ( + + You have no content yet} + body={ +

You need some content to display in order to configure the display settings.

+ } + /> + + )} + + {addFieldModalVisible && } + + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings_logic.ts b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings_logic.ts new file mode 100644 index 0000000000000..c52665524f566 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings_logic.ts @@ -0,0 +1,350 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { cloneDeep, isEqual, differenceBy } from 'lodash'; +import { DropResult } from 'react-beautiful-dnd'; + +import { kea, MakeLogicType } from 'kea'; + +import { HttpLogic } from '../../../../../shared/http'; + +import { + setSuccessMessage, + FlashMessagesLogic, + flashAPIErrors, +} from '../../../../../shared/flash_messages'; + +import { AppLogic } from '../../../../app_logic'; +import { SourceLogic } from '../../source_logic'; + +const SUCCESS_MESSAGE = 'Display Settings have been successfuly updated.'; + +import { DetailField, SearchResultConfig, OptionValue, Result } from '../../../../types'; + +export interface DisplaySettingsResponseProps { + sourceName: string; + searchResultConfig: SearchResultConfig; + schemaFields: object; + exampleDocuments: Result[]; +} + +export interface DisplaySettingsInitialData extends DisplaySettingsResponseProps { + sourceId: string; + serverRoute: string; +} + +interface DisplaySettingsActions { + initializeDisplaySettings(): void; + setServerData(): void; + onInitializeDisplaySettings( + displaySettingsProps: DisplaySettingsInitialData + ): DisplaySettingsInitialData; + setServerResponseData( + displaySettingsProps: DisplaySettingsResponseProps + ): DisplaySettingsResponseProps; + setTitleField(titleField: string | null): string | null; + setUrlField(urlField: string): string; + setSubtitleField(subtitleField: string | null): string | null; + setDescriptionField(descriptionField: string | null): string | null; + setColorField(hex: string): string; + setDetailFields(result: DropResult): { result: DropResult }; + openEditDetailField(editFieldIndex: number | null): number | null; + removeDetailField(index: number): number; + addDetailField(newField: DetailField): DetailField; + updateDetailField( + updatedField: DetailField, + index: number | null + ): { updatedField: DetailField; index: number }; + toggleFieldEditorModal(): void; + toggleTitleFieldHover(): void; + toggleSubtitleFieldHover(): void; + toggleDescriptionFieldHover(): void; + toggleUrlFieldHover(): void; + resetDisplaySettingsState(): void; +} + +interface DisplaySettingsValues { + sourceName: string; + sourceId: string; + schemaFields: object; + exampleDocuments: Result[]; + serverSearchResultConfig: SearchResultConfig; + searchResultConfig: SearchResultConfig; + serverRoute: string; + editFieldIndex: number | null; + dataLoading: boolean; + addFieldModalVisible: boolean; + titleFieldHover: boolean; + urlFieldHover: boolean; + subtitleFieldHover: boolean; + descriptionFieldHover: boolean; + fieldOptions: OptionValue[]; + optionalFieldOptions: OptionValue[]; + availableFieldOptions: OptionValue[]; + unsavedChanges: boolean; +} + +const defaultSearchResultConfig = { + titleField: '', + subtitleField: '', + descriptionField: '', + urlField: '', + color: '#000000', + detailFields: [], +}; + +export const DisplaySettingsLogic = kea< + MakeLogicType +>({ + actions: { + onInitializeDisplaySettings: (displaySettingsProps: DisplaySettingsInitialData) => + displaySettingsProps, + setServerResponseData: (displaySettingsProps: DisplaySettingsResponseProps) => + displaySettingsProps, + setTitleField: (titleField: string) => titleField, + setUrlField: (urlField: string) => urlField, + setSubtitleField: (subtitleField: string | null) => subtitleField, + setDescriptionField: (descriptionField: string) => descriptionField, + setColorField: (hex: string) => hex, + setDetailFields: (result: DropResult) => ({ result }), + openEditDetailField: (editFieldIndex: number | null) => editFieldIndex, + removeDetailField: (index: number) => index, + addDetailField: (newField: DetailField) => newField, + updateDetailField: (updatedField: DetailField, index: number) => ({ updatedField, index }), + toggleFieldEditorModal: () => true, + toggleTitleFieldHover: () => true, + toggleSubtitleFieldHover: () => true, + toggleDescriptionFieldHover: () => true, + toggleUrlFieldHover: () => true, + resetDisplaySettingsState: () => true, + initializeDisplaySettings: () => true, + setServerData: () => true, + }, + reducers: { + sourceName: [ + '', + { + onInitializeDisplaySettings: (_, { sourceName }) => sourceName, + }, + ], + sourceId: [ + '', + { + onInitializeDisplaySettings: (_, { sourceId }) => sourceId, + }, + ], + schemaFields: [ + {}, + { + onInitializeDisplaySettings: (_, { schemaFields }) => schemaFields, + }, + ], + exampleDocuments: [ + [], + { + onInitializeDisplaySettings: (_, { exampleDocuments }) => exampleDocuments, + }, + ], + serverSearchResultConfig: [ + defaultSearchResultConfig, + { + onInitializeDisplaySettings: (_, { searchResultConfig }) => + setDefaultColor(searchResultConfig), + setServerResponseData: (_, { searchResultConfig }) => searchResultConfig, + }, + ], + searchResultConfig: [ + defaultSearchResultConfig, + { + onInitializeDisplaySettings: (_, { searchResultConfig }) => + setDefaultColor(searchResultConfig), + setServerResponseData: (_, { searchResultConfig }) => searchResultConfig, + setTitleField: (searchResultConfig, titleField) => ({ ...searchResultConfig, titleField }), + setSubtitleField: (searchResultConfig, subtitleField) => ({ + ...searchResultConfig, + subtitleField, + }), + setUrlField: (searchResultConfig, urlField) => ({ ...searchResultConfig, urlField }), + setDescriptionField: (searchResultConfig, descriptionField) => ({ + ...searchResultConfig, + descriptionField, + }), + setColorField: (searchResultConfig, color) => ({ ...searchResultConfig, color }), + setDetailFields: (searchResultConfig, { result: { destination, source } }) => { + const detailFields = cloneDeep(searchResultConfig.detailFields); + const element = detailFields[source.index]; + detailFields.splice(source.index, 1); + detailFields.splice(destination!.index, 0, element); + return { + ...searchResultConfig, + detailFields, + }; + }, + addDetailField: (searchResultConfig, newfield) => { + const detailFields = cloneDeep(searchResultConfig.detailFields); + detailFields.push(newfield); + return { + ...searchResultConfig, + detailFields, + }; + }, + removeDetailField: (searchResultConfig, index) => { + const detailFields = cloneDeep(searchResultConfig.detailFields); + detailFields.splice(index, 1); + return { + ...searchResultConfig, + detailFields, + }; + }, + updateDetailField: (searchResultConfig, { updatedField, index }) => { + const detailFields = cloneDeep(searchResultConfig.detailFields); + detailFields[index] = updatedField; + return { + ...searchResultConfig, + detailFields, + }; + }, + }, + ], + serverRoute: [ + '', + { + onInitializeDisplaySettings: (_, { serverRoute }) => serverRoute, + }, + ], + editFieldIndex: [ + null, + { + openEditDetailField: (_, openEditDetailField) => openEditDetailField, + toggleFieldEditorModal: () => null, + }, + ], + dataLoading: [ + true, + { + onInitializeDisplaySettings: () => false, + }, + ], + addFieldModalVisible: [ + false, + { + toggleFieldEditorModal: (addFieldModalVisible) => !addFieldModalVisible, + openEditDetailField: () => true, + updateDetailField: () => false, + addDetailField: () => false, + }, + ], + titleFieldHover: [ + false, + { + toggleTitleFieldHover: (titleFieldHover) => !titleFieldHover, + }, + ], + urlFieldHover: [ + false, + { + toggleUrlFieldHover: (urlFieldHover) => !urlFieldHover, + }, + ], + subtitleFieldHover: [ + false, + { + toggleSubtitleFieldHover: (subtitleFieldHover) => !subtitleFieldHover, + }, + ], + descriptionFieldHover: [ + false, + { + toggleDescriptionFieldHover: (addFieldModalVisible) => !addFieldModalVisible, + }, + ], + }, + selectors: ({ selectors }) => ({ + fieldOptions: [ + () => [selectors.schemaFields], + (schemaFields) => Object.keys(schemaFields).map(euiSelectObjectFromValue), + ], + optionalFieldOptions: [ + () => [selectors.fieldOptions], + (fieldOptions) => { + const optionalFieldOptions = cloneDeep(fieldOptions); + optionalFieldOptions.unshift({ value: '', text: '' }); + return optionalFieldOptions; + }, + ], + // We don't want to let the user add a duplicate detailField. + availableFieldOptions: [ + () => [selectors.fieldOptions, selectors.searchResultConfig], + (fieldOptions, { detailFields }) => { + const usedFields = detailFields.map((usedField: DetailField) => + euiSelectObjectFromValue(usedField.fieldName) + ); + return differenceBy(fieldOptions, usedFields, 'value'); + }, + ], + unsavedChanges: [ + () => [selectors.searchResultConfig, selectors.serverSearchResultConfig], + (uiConfig, serverConfig) => !isEqual(uiConfig, serverConfig), + ], + }), + listeners: ({ actions, values }) => ({ + initializeDisplaySettings: async () => { + const { isOrganization } = AppLogic.values; + const { + contentSource: { id: sourceId }, + } = SourceLogic.values; + + const route = isOrganization + ? `/api/workplace_search/org/sources/${sourceId}/display_settings/config` + : `/api/workplace_search/account/sources/${sourceId}/display_settings/config`; + + try { + const response = await HttpLogic.values.http.get(route); + actions.onInitializeDisplaySettings({ + isOrganization, + sourceId, + serverRoute: route, + ...response, + }); + } catch (e) { + flashAPIErrors(e); + } + }, + setServerData: async () => { + const { searchResultConfig, serverRoute } = values; + + try { + const response = await HttpLogic.values.http.post(serverRoute, { + body: JSON.stringify({ ...searchResultConfig }), + }); + actions.setServerResponseData(response); + } catch (e) { + flashAPIErrors(e); + } + }, + setServerResponseData: () => { + setSuccessMessage(SUCCESS_MESSAGE); + }, + toggleFieldEditorModal: () => { + FlashMessagesLogic.actions.clearFlashMessages(); + }, + resetDisplaySettingsState: () => { + FlashMessagesLogic.actions.clearFlashMessages(); + }, + }), +}); + +const euiSelectObjectFromValue = (value: string) => ({ text: value, value }); + +// By default, the color is `null` on the server. The color is a required field and the +// EuiColorPicker components doesn't allow the field to be required so the form can be +// submitted with no color and this results in a server error. The default should be black +// and this allows the `searchResultConfig` and the `serverSearchResultConfig` reducers to +// stay synced on initialization. +const setDefaultColor = (searchResultConfig: SearchResultConfig) => ({ + ...searchResultConfig, + color: searchResultConfig.color || '#000000', +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings_router.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings_router.tsx index 5cebaad95e3a8..01ac93735b8a8 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings_router.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings_router.tsx @@ -6,4 +6,33 @@ import React from 'react'; -export const DisplaySettingsRouter: React.FC = () => <>Display Settings Placeholder; +import { useValues } from 'kea'; +import { Route, Switch } from 'react-router-dom'; + +import { AppLogic } from '../../../../app_logic'; + +import { + DISPLAY_SETTINGS_RESULT_DETAIL_PATH, + DISPLAY_SETTINGS_SEARCH_RESULT_PATH, + getSourcesPath, +} from '../../../../routes'; + +import { DisplaySettings } from './display_settings'; + +export const DisplaySettingsRouter: React.FC = () => { + const { isOrganization } = useValues(AppLogic); + return ( + + } + /> + } + /> + + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/example_result_detail_card.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/example_result_detail_card.tsx new file mode 100644 index 0000000000000..468f7d2f7ad05 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/example_result_detail_card.tsx @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; + +import classNames from 'classnames'; +import { useValues } from 'kea'; + +import { EuiFlexGroup, EuiFlexItem, EuiSpacer, EuiText, EuiTitle } from '@elastic/eui'; + +import { DisplaySettingsLogic } from './display_settings_logic'; + +import { CustomSourceIcon } from './custom_source_icon'; +import { TitleField } from './title_field'; + +export const ExampleResultDetailCard: React.FC = () => { + const { + sourceName, + searchResultConfig: { titleField, urlField, color, detailFields }, + titleFieldHover, + urlFieldHover, + exampleDocuments, + } = useValues(DisplaySettingsLogic); + + const result = exampleDocuments[0]; + + return ( +
+
+
+
+ + + + + {sourceName} + +
+
+ +
+ {urlField ? ( +
{result[urlField]}
+ ) : ( + URL + )} +
+
+
+
+ {detailFields.length > 0 ? ( + detailFields.map(({ fieldName, label }, index) => ( +
+ +

{label}

+ + +
{result[fieldName]}
+ +
+ )) + ) : ( + + )} +
+
+ ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/example_search_result_group.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/example_search_result_group.tsx new file mode 100644 index 0000000000000..14239b1654308 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/example_search_result_group.tsx @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; + +import { isColorDark, hexToRgb } from '@elastic/eui'; +import classNames from 'classnames'; +import { useValues } from 'kea'; + +import { DisplaySettingsLogic } from './display_settings_logic'; + +import { CustomSourceIcon } from './custom_source_icon'; +import { SubtitleField } from './subtitle_field'; +import { TitleField } from './title_field'; + +export const ExampleSearchResultGroup: React.FC = () => { + const { + sourceName, + searchResultConfig: { titleField, subtitleField, descriptionField, color }, + titleFieldHover, + subtitleFieldHover, + descriptionFieldHover, + exampleDocuments, + } = useValues(DisplaySettingsLogic); + + return ( +
+
+ + + {sourceName} + +
+
+
+
+ {exampleDocuments.map((result, id) => ( +
+
+ + +
+ {descriptionField ? ( +
{result[descriptionField]}
+ ) : ( + Description + )} +
+
+
+ ))} +
+
+
+ ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/example_standout_result.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/example_standout_result.tsx new file mode 100644 index 0000000000000..4ef3b1fe14b93 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/example_standout_result.tsx @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; + +import classNames from 'classnames'; +import { useValues } from 'kea'; + +import { isColorDark, hexToRgb } from '@elastic/eui'; + +import { DisplaySettingsLogic } from './display_settings_logic'; + +import { CustomSourceIcon } from './custom_source_icon'; +import { SubtitleField } from './subtitle_field'; +import { TitleField } from './title_field'; + +export const ExampleStandoutResult: React.FC = () => { + const { + sourceName, + searchResultConfig: { titleField, subtitleField, descriptionField, color }, + titleFieldHover, + subtitleFieldHover, + descriptionFieldHover, + exampleDocuments, + } = useValues(DisplaySettingsLogic); + + const result = exampleDocuments[0]; + + return ( +
+
+ + + {sourceName} + +
+
+
+ + +
+ {descriptionField ? ( + {result[descriptionField]} + ) : ( + Description + )} +
+
+
+
+ ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/field_editor_modal.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/field_editor_modal.tsx new file mode 100644 index 0000000000000..587916a741d66 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/field_editor_modal.tsx @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FormEvent, useState } from 'react'; + +import { useActions, useValues } from 'kea'; + +import { + EuiButton, + EuiButtonEmpty, + EuiFieldText, + EuiForm, + EuiFormRow, + EuiModal, + EuiModalBody, + EuiModalFooter, + EuiModalHeader, + EuiModalHeaderTitle, + EuiOverlayMask, + EuiSelect, +} from '@elastic/eui'; + +import { DisplaySettingsLogic } from './display_settings_logic'; + +const emptyField = { fieldName: '', label: '' }; + +export const FieldEditorModal: React.FC = () => { + const { toggleFieldEditorModal, addDetailField, updateDetailField } = useActions( + DisplaySettingsLogic + ); + + const { + searchResultConfig: { detailFields }, + availableFieldOptions, + fieldOptions, + editFieldIndex, + } = useValues(DisplaySettingsLogic); + + const isEditing = editFieldIndex || editFieldIndex === 0; + const field = isEditing ? detailFields[editFieldIndex || 0] : emptyField; + const [fieldName, setName] = useState(field.fieldName || ''); + const [label, setLabel] = useState(field.label || ''); + + const handleSubmit = (e: FormEvent) => { + e.preventDefault(); + if (isEditing) { + updateDetailField({ fieldName, label }, editFieldIndex); + } else { + addDetailField({ fieldName, label }); + } + }; + + const ACTION_LABEL = isEditing ? 'Update' : 'Add'; + + return ( + +
+ + + {ACTION_LABEL} Field + + + + + setName(e.target.value)} + /> + + + setLabel(e.target.value)} + /> + + + + + Cancel + + {ACTION_LABEL} Field + + + +
+ + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/result_detail.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/result_detail.tsx new file mode 100644 index 0000000000000..cb65d8ef671e6 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/result_detail.tsx @@ -0,0 +1,146 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; + +import { useActions, useValues } from 'kea'; + +import { + EuiButtonEmpty, + EuiButtonIcon, + EuiDragDropContext, + EuiDraggable, + EuiDroppable, + EuiFlexGrid, + EuiFlexGroup, + EuiFlexItem, + EuiForm, + EuiFormRow, + EuiIcon, + EuiPanel, + EuiSpacer, + EuiTextColor, + EuiTitle, +} from '@elastic/eui'; + +import { DisplaySettingsLogic } from './display_settings_logic'; + +import { ExampleResultDetailCard } from './example_result_detail_card'; + +export const ResultDetail: React.FC = () => { + const { + toggleFieldEditorModal, + setDetailFields, + openEditDetailField, + removeDetailField, + } = useActions(DisplaySettingsLogic); + + const { + searchResultConfig: { detailFields }, + availableFieldOptions, + } = useValues(DisplaySettingsLogic); + + return ( + <> + + + + + + + <> + + + +

Visible Fields

+ + + + + Add Field + + + + + {detailFields.length > 0 ? ( + + + <> + {detailFields.map(({ fieldName, label }, index) => ( + + {(provided) => ( + + + +
+ +
+ + + +

{fieldName}

+ + +
“{label || ''}”
+ + + +
+ openEditDetailField(index)} + /> + removeDetailField(index)} + /> +
+ + + + )} + + ))} + + + + ) : ( +

Add fields and move them into the order you want them to appear.

+ )} + + + + + + + +

Preview

+ + + + + + + + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/search_results.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/search_results.tsx new file mode 100644 index 0000000000000..cfe0ddb1533ec --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/search_results.tsx @@ -0,0 +1,164 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; + +import { useActions, useValues } from 'kea'; + +import { + EuiColorPicker, + EuiFlexGrid, + EuiFlexItem, + EuiForm, + EuiFormRow, + EuiPanel, + EuiSelect, + EuiSpacer, + EuiTitle, +} from '@elastic/eui'; + +import { DisplaySettingsLogic } from './display_settings_logic'; + +import { ExampleSearchResultGroup } from './example_search_result_group'; +import { ExampleStandoutResult } from './example_standout_result'; + +export const SearchResults: React.FC = () => { + const { + toggleTitleFieldHover, + toggleSubtitleFieldHover, + toggleDescriptionFieldHover, + setTitleField, + setSubtitleField, + setDescriptionField, + setUrlField, + setColorField, + } = useActions(DisplaySettingsLogic); + + const { + searchResultConfig: { titleField, descriptionField, subtitleField, urlField, color }, + fieldOptions, + optionalFieldOptions, + } = useValues(DisplaySettingsLogic); + + return ( + <> + + + + + +

Search Result Settings

+ + + + null} // FIXME + onBlur={() => null} // FIXME + > + setTitleField(e.target.value)} + /> + + + setUrlField(e.target.value)} + /> + + + null} // FIXME + onBlur={() => null} // FIXME + /> + + null} // FIXME + onBlur={() => null} // FIXME + > + setSubtitleField(value === '' ? null : value)} + /> + + null} // FIXME + onBlur={() => null} // FIXME + > + + setDescriptionField(value === '' ? null : value) + } + /> + + + + + + +

Preview

+ + +
+ +

Featured Results

+ +

+ A matching document will appear as a single bold card. +

+
+ + + +
+ +

Standard Results

+ +

+ Somewhat matching documents will appear as a set. +

+
+ + + + + + + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/subtitle_field.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/subtitle_field.tsx new file mode 100644 index 0000000000000..e27052ddffc04 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/subtitle_field.tsx @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; + +import classNames from 'classnames'; + +import { Result } from '../../../../types'; + +interface SubtitleFieldProps { + result: Result; + subtitleField: string | null; + subtitleFieldHover: boolean; +} + +export const SubtitleField: React.FC = ({ + result, + subtitleField, + subtitleFieldHover, +}) => ( +
+ {subtitleField ? ( +
{result[subtitleField]}
+ ) : ( + Subtitle + )} +
+); diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/title_field.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/title_field.tsx new file mode 100644 index 0000000000000..a54c0977b464f --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/title_field.tsx @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; + +import classNames from 'classnames'; + +import { Result } from '../../../../types'; + +interface TitleFieldProps { + result: Result; + titleField: string | null; + titleFieldHover: boolean; +} + +export const TitleField: React.FC = ({ result, titleField, titleFieldHover }) => { + const title = titleField ? result[titleField] : ''; + const titleDisplay = Array.isArray(title) ? title.join(', ') : title; + return ( +
+ {titleField ? ( +
{titleDisplay}
+ ) : ( + Title + )} +
+ ); +}; diff --git a/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.test.ts b/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.test.ts index 22e2deaace1dc..62f4dceeac363 100644 --- a/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.test.ts +++ b/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.test.ts @@ -18,6 +18,7 @@ import { registerAccountPreSourceRoute, registerAccountPrepareSourcesRoute, registerAccountSourceSearchableRoute, + registerAccountSourceDisplaySettingsConfig, registerOrgSourcesRoute, registerOrgSourcesStatusRoute, registerOrgSourceRoute, @@ -29,6 +30,7 @@ import { registerOrgPreSourceRoute, registerOrgPrepareSourcesRoute, registerOrgSourceSearchableRoute, + registerOrgSourceDisplaySettingsConfig, registerOrgSourceOauthConfigurationsRoute, registerOrgSourceOauthConfigurationRoute, } from './sources'; @@ -446,6 +448,81 @@ describe('sources routes', () => { }); }); + describe('GET /api/workplace_search/account/sources/{id}/display_settings/config', () => { + let mockRouter: MockRouter; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('creates a request handler', () => { + mockRouter = new MockRouter({ + method: 'get', + path: '/api/workplace_search/account/sources/{id}/display_settings/config', + payload: 'params', + }); + + registerAccountSourceDisplaySettingsConfig({ + ...mockDependencies, + router: mockRouter.router, + }); + + const mockRequest = { + params: { + id: '123', + }, + }; + + mockRouter.callRoute(mockRequest); + + expect(mockRequestHandler.createRequest).toHaveBeenCalledWith({ + path: '/ws/sources/123/display_settings/config', + }); + }); + }); + + describe('POST /api/workplace_search/account/sources/{id}/display_settings/config', () => { + let mockRouter: MockRouter; + + beforeEach(() => { + jest.clearAllMocks(); + mockRouter = new MockRouter({ + method: 'post', + path: '/api/workplace_search/account/sources/{id}/display_settings/config', + payload: 'body', + }); + + registerAccountSourceDisplaySettingsConfig({ + ...mockDependencies, + router: mockRouter.router, + }); + }); + + it('creates a request handler', () => { + const mockRequest = { + params: { id: '123' }, + body: { + titleField: 'foo', + subtitleField: 'bar', + descriptionField: 'this is a thing', + urlField: 'http://youknowfor.search', + color: '#aaa', + detailFields: { + fieldName: 'myField', + label: 'My Field', + }, + }, + }; + + mockRouter.callRoute(mockRequest); + + expect(mockRequestHandler.createRequest).toHaveBeenCalledWith({ + path: '/ws/sources/123/display_settings/config', + body: mockRequest.body, + }); + }); + }); + describe('GET /api/workplace_search/org/sources', () => { let mockRouter: MockRouter; @@ -848,6 +925,81 @@ describe('sources routes', () => { }); }); + describe('GET /api/workplace_search/org/sources/{id}/display_settings/config', () => { + let mockRouter: MockRouter; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('creates a request handler', () => { + mockRouter = new MockRouter({ + method: 'get', + path: '/api/workplace_search/org/sources/{id}/display_settings/config', + payload: 'params', + }); + + registerOrgSourceDisplaySettingsConfig({ + ...mockDependencies, + router: mockRouter.router, + }); + + const mockRequest = { + params: { + id: '123', + }, + }; + + mockRouter.callRoute(mockRequest); + + expect(mockRequestHandler.createRequest).toHaveBeenCalledWith({ + path: '/ws/org/sources/123/display_settings/config', + }); + }); + }); + + describe('POST /api/workplace_search/org/sources/{id}/display_settings/config', () => { + let mockRouter: MockRouter; + + beforeEach(() => { + jest.clearAllMocks(); + mockRouter = new MockRouter({ + method: 'post', + path: '/api/workplace_search/org/sources/{id}/display_settings/config', + payload: 'body', + }); + + registerOrgSourceDisplaySettingsConfig({ + ...mockDependencies, + router: mockRouter.router, + }); + }); + + it('creates a request handler', () => { + const mockRequest = { + params: { id: '123' }, + body: { + titleField: 'foo', + subtitleField: 'bar', + descriptionField: 'this is a thing', + urlField: 'http://youknowfor.search', + color: '#aaa', + detailFields: { + fieldName: 'myField', + label: 'My Field', + }, + }, + }; + + mockRouter.callRoute(mockRequest); + + expect(mockRequestHandler.createRequest).toHaveBeenCalledWith({ + path: '/ws/org/sources/123/display_settings/config', + body: mockRequest.body, + }); + }); + }); + describe('GET /api/workplace_search/org/settings/connectors', () => { let mockRouter: MockRouter; diff --git a/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.ts b/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.ts index 24473388c03b1..d43a4252c7e1f 100644 --- a/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.ts +++ b/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.ts @@ -25,6 +25,21 @@ const oAuthConfigSchema = schema.object({ consumer_key: schema.string(), }); +const displayFieldSchema = schema.object({ + fieldName: schema.string(), + label: schema.string(), +}); + +const displaySettingsSchema = schema.object({ + titleField: schema.maybe(schema.string()), + subtitleField: schema.maybe(schema.string()), + descriptionField: schema.maybe(schema.string()), + urlField: schema.maybe(schema.string()), + color: schema.string(), + urlFieldIsLinkable: schema.boolean(), + detailFields: schema.oneOf([schema.arrayOf(displayFieldSchema), displayFieldSchema]), +}); + export function registerAccountSourcesRoute({ router, enterpriseSearchRequestHandler, @@ -285,6 +300,45 @@ export function registerAccountSourceSearchableRoute({ ); } +export function registerAccountSourceDisplaySettingsConfig({ + router, + enterpriseSearchRequestHandler, +}: RouteDependencies) { + router.get( + { + path: '/api/workplace_search/account/sources/{id}/display_settings/config', + validate: { + params: schema.object({ + id: schema.string(), + }), + }, + }, + async (context, request, response) => { + return enterpriseSearchRequestHandler.createRequest({ + path: `/ws/sources/${request.params.id}/display_settings/config`, + })(context, request, response); + } + ); + + router.post( + { + path: '/api/workplace_search/account/sources/{id}/display_settings/config', + validate: { + body: displaySettingsSchema, + params: schema.object({ + id: schema.string(), + }), + }, + }, + async (context, request, response) => { + return enterpriseSearchRequestHandler.createRequest({ + path: `/ws/sources/${request.params.id}/display_settings/config`, + body: request.body, + })(context, request, response); + } + ); +} + export function registerOrgSourcesRoute({ router, enterpriseSearchRequestHandler, @@ -545,6 +599,45 @@ export function registerOrgSourceSearchableRoute({ ); } +export function registerOrgSourceDisplaySettingsConfig({ + router, + enterpriseSearchRequestHandler, +}: RouteDependencies) { + router.get( + { + path: '/api/workplace_search/org/sources/{id}/display_settings/config', + validate: { + params: schema.object({ + id: schema.string(), + }), + }, + }, + async (context, request, response) => { + return enterpriseSearchRequestHandler.createRequest({ + path: `/ws/org/sources/${request.params.id}/display_settings/config`, + })(context, request, response); + } + ); + + router.post( + { + path: '/api/workplace_search/org/sources/{id}/display_settings/config', + validate: { + body: displaySettingsSchema, + params: schema.object({ + id: schema.string(), + }), + }, + }, + async (context, request, response) => { + return enterpriseSearchRequestHandler.createRequest({ + path: `/ws/org/sources/${request.params.id}/display_settings/config`, + body: request.body, + })(context, request, response); + } + ); +} + export function registerOrgSourceOauthConfigurationsRoute({ router, enterpriseSearchRequestHandler, @@ -647,6 +740,7 @@ export const registerSourcesRoutes = (dependencies: RouteDependencies) => { registerAccountPreSourceRoute(dependencies); registerAccountPrepareSourcesRoute(dependencies); registerAccountSourceSearchableRoute(dependencies); + registerAccountSourceDisplaySettingsConfig(dependencies); registerOrgSourcesRoute(dependencies); registerOrgSourcesStatusRoute(dependencies); registerOrgSourceRoute(dependencies); @@ -658,6 +752,7 @@ export const registerSourcesRoutes = (dependencies: RouteDependencies) => { registerOrgPreSourceRoute(dependencies); registerOrgPrepareSourcesRoute(dependencies); registerOrgSourceSearchableRoute(dependencies); + registerOrgSourceDisplaySettingsConfig(dependencies); registerOrgSourceOauthConfigurationsRoute(dependencies); registerOrgSourceOauthConfigurationRoute(dependencies); }; From ea8ea4e4e0c62839ccf3d6c7315b9166b4a3a607 Mon Sep 17 00:00:00 2001 From: Spencer Date: Wed, 25 Nov 2020 08:03:47 -0700 Subject: [PATCH 41/49] [dev/cli] detect worker type using env, not cluster module (#83977) * [dev/cli] detect worker type using env, not cluster module * remove unused property * assume that if process.send is undefined we are not a child * update comment Co-authored-by: spalger Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- packages/kbn-config/src/__mocks__/env.ts | 3 +-- .../kbn-config/src/__snapshots__/env.test.ts.snap | 12 ++++++------ packages/kbn-config/src/env.test.ts | 2 +- packages/kbn-config/src/env.ts | 8 ++++---- .../kbn-legacy-logging/src/log_format_string.ts | 4 ++-- packages/kbn-legacy-logging/src/rotate/index.ts | 7 ------- .../kbn-legacy-logging/src/rotate/log_rotator.ts | 13 ++----------- src/apm.js | 4 ---- src/cli/cluster/cluster_manager.ts | 2 -- src/cli/cluster/worker.ts | 4 +--- src/cli/serve/serve.js | 6 +++--- src/core/server/bootstrap.ts | 11 +++++------ src/core/server/http/http_service.test.ts | 2 +- src/core/server/http/http_service.ts | 2 +- src/core/server/legacy/legacy_service.test.ts | 4 ++-- src/core/server/legacy/legacy_service.ts | 8 +++----- src/core/server/plugins/plugins_service.test.ts | 8 ++++---- src/core/server/plugins/plugins_service.ts | 4 ++-- src/core/server/server.test.ts | 4 ++-- src/core/server/server.ts | 2 +- src/core/test_helpers/kbn_server.ts | 2 +- src/legacy/server/kbn_server.js | 3 +-- 22 files changed, 43 insertions(+), 72 deletions(-) diff --git a/packages/kbn-config/src/__mocks__/env.ts b/packages/kbn-config/src/__mocks__/env.ts index e0f6f6add1686..8b7475680ecf5 100644 --- a/packages/kbn-config/src/__mocks__/env.ts +++ b/packages/kbn-config/src/__mocks__/env.ts @@ -42,7 +42,6 @@ export function getEnvOptions(options: DeepPartial = {}): EnvOptions runExamples: false, ...(options.cliArgs || {}), }, - isDevClusterMaster: - options.isDevClusterMaster !== undefined ? options.isDevClusterMaster : false, + isDevCliParent: options.isDevCliParent !== undefined ? options.isDevCliParent : false, }; } diff --git a/packages/kbn-config/src/__snapshots__/env.test.ts.snap b/packages/kbn-config/src/__snapshots__/env.test.ts.snap index 884896266344c..9236c83f9c921 100644 --- a/packages/kbn-config/src/__snapshots__/env.test.ts.snap +++ b/packages/kbn-config/src/__snapshots__/env.test.ts.snap @@ -22,7 +22,7 @@ Env { "/some/other/path/some-kibana.yml", ], "homeDir": "/test/kibanaRoot", - "isDevClusterMaster": false, + "isDevCliParent": false, "logDir": "/test/kibanaRoot/log", "mode": Object { "dev": true, @@ -67,7 +67,7 @@ Env { "/some/other/path/some-kibana.yml", ], "homeDir": "/test/kibanaRoot", - "isDevClusterMaster": false, + "isDevCliParent": false, "logDir": "/test/kibanaRoot/log", "mode": Object { "dev": false, @@ -111,7 +111,7 @@ Env { "/test/cwd/config/kibana.yml", ], "homeDir": "/test/kibanaRoot", - "isDevClusterMaster": true, + "isDevCliParent": true, "logDir": "/test/kibanaRoot/log", "mode": Object { "dev": true, @@ -155,7 +155,7 @@ Env { "/some/other/path/some-kibana.yml", ], "homeDir": "/test/kibanaRoot", - "isDevClusterMaster": false, + "isDevCliParent": false, "logDir": "/test/kibanaRoot/log", "mode": Object { "dev": false, @@ -199,7 +199,7 @@ Env { "/some/other/path/some-kibana.yml", ], "homeDir": "/test/kibanaRoot", - "isDevClusterMaster": false, + "isDevCliParent": false, "logDir": "/test/kibanaRoot/log", "mode": Object { "dev": false, @@ -243,7 +243,7 @@ Env { "/some/other/path/some-kibana.yml", ], "homeDir": "/some/home/dir", - "isDevClusterMaster": false, + "isDevCliParent": false, "logDir": "/some/home/dir/log", "mode": Object { "dev": false, diff --git a/packages/kbn-config/src/env.test.ts b/packages/kbn-config/src/env.test.ts index 1613a90951d40..5aee33e6fda5e 100644 --- a/packages/kbn-config/src/env.test.ts +++ b/packages/kbn-config/src/env.test.ts @@ -47,7 +47,7 @@ test('correctly creates default environment in dev mode.', () => { REPO_ROOT, getEnvOptions({ configs: ['/test/cwd/config/kibana.yml'], - isDevClusterMaster: true, + isDevCliParent: true, }) ); diff --git a/packages/kbn-config/src/env.ts b/packages/kbn-config/src/env.ts index 7edb4b1c8dad9..4ae8d7b7f9aa5 100644 --- a/packages/kbn-config/src/env.ts +++ b/packages/kbn-config/src/env.ts @@ -26,7 +26,7 @@ import { PackageInfo, EnvironmentMode } from './types'; export interface EnvOptions { configs: string[]; cliArgs: CliArgs; - isDevClusterMaster: boolean; + isDevCliParent: boolean; } /** @internal */ @@ -101,10 +101,10 @@ export class Env { public readonly configs: readonly string[]; /** - * Indicates that this Kibana instance is run as development Node Cluster master. + * Indicates that this Kibana instance is running in the parent process of the dev cli. * @internal */ - public readonly isDevClusterMaster: boolean; + public readonly isDevCliParent: boolean; /** * @internal @@ -122,7 +122,7 @@ export class Env { this.cliArgs = Object.freeze(options.cliArgs); this.configs = Object.freeze(options.configs); - this.isDevClusterMaster = options.isDevClusterMaster; + this.isDevCliParent = options.isDevCliParent; const isDevMode = this.cliArgs.dev || this.cliArgs.envName === 'development'; this.mode = Object.freeze({ diff --git a/packages/kbn-legacy-logging/src/log_format_string.ts b/packages/kbn-legacy-logging/src/log_format_string.ts index 3f024fac55119..b4217c37b960e 100644 --- a/packages/kbn-legacy-logging/src/log_format_string.ts +++ b/packages/kbn-legacy-logging/src/log_format_string.ts @@ -54,7 +54,7 @@ const type = _.memoize((t: string) => { return color(t)(_.pad(t, 7).slice(0, 7)); }); -const workerType = process.env.kbnWorkerType ? `${type(process.env.kbnWorkerType)} ` : ''; +const prefix = process.env.isDevCliChild ? `${type('server')} ` : ''; export class KbnLoggerStringFormat extends BaseLogFormat { format(data: Record) { @@ -71,6 +71,6 @@ export class KbnLoggerStringFormat extends BaseLogFormat { return s + `[${color(t)(t)}]`; }, ''); - return `${workerType}${type(data.type)} [${time}] ${tags} ${msg}`; + return `${prefix}${type(data.type)} [${time}] ${tags} ${msg}`; } } diff --git a/packages/kbn-legacy-logging/src/rotate/index.ts b/packages/kbn-legacy-logging/src/rotate/index.ts index 2387fc530e58b..9a83c625b9431 100644 --- a/packages/kbn-legacy-logging/src/rotate/index.ts +++ b/packages/kbn-legacy-logging/src/rotate/index.ts @@ -17,7 +17,6 @@ * under the License. */ -import { isMaster, isWorker } from 'cluster'; import { Server } from '@hapi/hapi'; import { LogRotator } from './log_rotator'; import { LegacyLoggingConfig } from '../schema'; @@ -30,12 +29,6 @@ export async function setupLoggingRotate(server: Server, config: LegacyLoggingCo return; } - // We just want to start the logging rotate service once - // and we choose to use the master (prod) or the worker server (dev) - if (!isMaster && isWorker && process.env.kbnWorkerType !== 'server') { - return; - } - // We don't want to run logging rotate server if // we are not logging to a file if (config.dest === 'stdout') { diff --git a/packages/kbn-legacy-logging/src/rotate/log_rotator.ts b/packages/kbn-legacy-logging/src/rotate/log_rotator.ts index 54181e30d6007..fc2c088f01dde 100644 --- a/packages/kbn-legacy-logging/src/rotate/log_rotator.ts +++ b/packages/kbn-legacy-logging/src/rotate/log_rotator.ts @@ -18,7 +18,6 @@ */ import * as chokidar from 'chokidar'; -import { isMaster } from 'cluster'; import fs from 'fs'; import { Server } from '@hapi/hapi'; import { throttle } from 'lodash'; @@ -351,22 +350,14 @@ export class LogRotator { } _sendReloadLogConfigSignal() { - if (isMaster) { - (process as NodeJS.EventEmitter).emit('SIGHUP'); + if (!process.env.isDevCliChild || !process.send) { + process.emit('SIGHUP', 'SIGHUP'); return; } // Send a special message to the cluster manager // so it can forward it correctly // It will only run when we are under cluster mode (not under a production environment) - if (!process.send) { - this.log( - ['error', 'logging:rotate'], - 'For some unknown reason process.send is not defined, the rotation was not successful' - ); - return; - } - process.send(['RELOAD_LOGGING_CONFIG_FROM_SERVER_WORKER']); } } diff --git a/src/apm.js b/src/apm.js index bde37fa006c61..4f5fe29cbb5fa 100644 --- a/src/apm.js +++ b/src/apm.js @@ -30,10 +30,6 @@ let apmConfig; const isKibanaDistributable = Boolean(build && build.distributable === true); module.exports = function (serviceName = name) { - if (process.env.kbnWorkerType === 'optmzr') { - return; - } - apmConfig = loadConfiguration(process.argv, ROOT_DIR, isKibanaDistributable); const conf = apmConfig.getConfig(serviceName); const apm = require('elastic-apm-node'); diff --git a/src/cli/cluster/cluster_manager.ts b/src/cli/cluster/cluster_manager.ts index f427c8750912b..7a14f617b5d5a 100644 --- a/src/cli/cluster/cluster_manager.ts +++ b/src/cli/cluster/cluster_manager.ts @@ -33,8 +33,6 @@ import { BasePathProxyServer } from '../../core/server/http'; import { Log } from './log'; import { Worker } from './worker'; -process.env.kbnWorkerType = 'managr'; - export type SomeCliArgs = Pick< CliArgs, | 'quiet' diff --git a/src/cli/cluster/worker.ts b/src/cli/cluster/worker.ts index d28065765070b..26b2a643e5373 100644 --- a/src/cli/cluster/worker.ts +++ b/src/cli/cluster/worker.ts @@ -56,7 +56,6 @@ export class Worker extends EventEmitter { private readonly clusterBinder: BinderFor; private readonly processBinder: BinderFor; - private type: string; private title: string; private log: any; private forkBinder: BinderFor | null = null; @@ -76,7 +75,6 @@ export class Worker extends EventEmitter { super(); this.log = opts.log; - this.type = opts.type; this.title = opts.title || opts.type; this.watch = opts.watch !== false; this.startCount = 0; @@ -88,7 +86,7 @@ export class Worker extends EventEmitter { this.env = { NODE_OPTIONS: process.env.NODE_OPTIONS || '', - kbnWorkerType: this.type, + isDevCliChild: 'true', kbnWorkerArgv: JSON.stringify([...(opts.baseArgv || baseArgv), ...(opts.argv || [])]), ELASTIC_APM_SERVICE_NAME: opts.apmServiceName || '', }; diff --git a/src/cli/serve/serve.js b/src/cli/serve/serve.js index 2fa24cc7f3791..61f880d80633d 100644 --- a/src/cli/serve/serve.js +++ b/src/cli/serve/serve.js @@ -43,7 +43,7 @@ function canRequire(path) { } const CLUSTER_MANAGER_PATH = resolve(__dirname, '../cluster/cluster_manager'); -const CAN_CLUSTER = canRequire(CLUSTER_MANAGER_PATH); +const DEV_MODE_SUPPORTED = canRequire(CLUSTER_MANAGER_PATH); const REPL_PATH = resolve(__dirname, '../repl'); const CAN_REPL = canRequire(REPL_PATH); @@ -189,7 +189,7 @@ export default function (program) { ); } - if (CAN_CLUSTER) { + if (DEV_MODE_SUPPORTED) { command .option('--dev', 'Run the server with development mode defaults') .option('--ssl', 'Run the dev server using HTTPS') @@ -240,7 +240,7 @@ export default function (program) { dist: !!opts.dist, }, features: { - isClusterModeSupported: CAN_CLUSTER, + isCliDevModeSupported: DEV_MODE_SUPPORTED, isReplModeSupported: CAN_REPL, }, applyConfigOverrides: (rawConfig) => applyConfigOverrides(rawConfig, opts, unknownOptions), diff --git a/src/core/server/bootstrap.ts b/src/core/server/bootstrap.ts index ff1a5c0340c46..6711a8b8987e5 100644 --- a/src/core/server/bootstrap.ts +++ b/src/core/server/bootstrap.ts @@ -18,16 +18,15 @@ */ import chalk from 'chalk'; -import { isMaster } from 'cluster'; import { CliArgs, Env, RawConfigService } from './config'; import { Root } from './root'; import { CriticalError } from './errors'; interface KibanaFeatures { - // Indicates whether we can run Kibana in a so called cluster mode in which - // Kibana is run as a "worker" process together with optimizer "worker" process - // that are orchestrated by the "master" process (dev mode only feature). - isClusterModeSupported: boolean; + // Indicates whether we can run Kibana in dev mode in which Kibana is run as + // a child process together with optimizer "worker" processes that are + // orchestrated by a parent process (dev mode only feature). + isCliDevModeSupported: boolean; // Indicates whether we can run Kibana in REPL mode (dev mode only feature). isReplModeSupported: boolean; @@ -71,7 +70,7 @@ export async function bootstrap({ const env = Env.createDefault(REPO_ROOT, { configs, cliArgs, - isDevClusterMaster: isMaster && cliArgs.dev && features.isClusterModeSupported, + isDevCliParent: cliArgs.dev && features.isCliDevModeSupported && !process.env.isDevCliChild, }); const rawConfigService = new RawConfigService(env.configs, applyConfigOverrides); diff --git a/src/core/server/http/http_service.test.ts b/src/core/server/http/http_service.test.ts index 11cea88fa0dd2..3d55322461288 100644 --- a/src/core/server/http/http_service.test.ts +++ b/src/core/server/http/http_service.test.ts @@ -264,7 +264,7 @@ test('does not start http server if process is dev cluster master', async () => const service = new HttpService({ coreId, configService, - env: Env.createDefault(REPO_ROOT, getEnvOptions({ isDevClusterMaster: true })), + env: Env.createDefault(REPO_ROOT, getEnvOptions({ isDevCliParent: true })), logger, }); diff --git a/src/core/server/http/http_service.ts b/src/core/server/http/http_service.ts index 0127a6493e7fd..171a20160d26d 100644 --- a/src/core/server/http/http_service.ts +++ b/src/core/server/http/http_service.ts @@ -158,7 +158,7 @@ export class HttpService * @internal * */ private shouldListen(config: HttpConfig) { - return !this.coreContext.env.isDevClusterMaster && config.autoListen; + return !this.coreContext.env.isDevCliParent && config.autoListen; } public async stop() { diff --git a/src/core/server/legacy/legacy_service.test.ts b/src/core/server/legacy/legacy_service.test.ts index 5cc6fcb133507..fe19ef9d0a774 100644 --- a/src/core/server/legacy/legacy_service.test.ts +++ b/src/core/server/legacy/legacy_service.test.ts @@ -362,7 +362,7 @@ describe('once LegacyService is set up in `devClusterMaster` mode', () => { REPO_ROOT, getEnvOptions({ cliArgs: { silent: true, basePath: false }, - isDevClusterMaster: true, + isDevCliParent: true, }) ), logger, @@ -391,7 +391,7 @@ describe('once LegacyService is set up in `devClusterMaster` mode', () => { REPO_ROOT, getEnvOptions({ cliArgs: { quiet: true, basePath: true }, - isDevClusterMaster: true, + isDevCliParent: true, }) ), logger, diff --git a/src/core/server/legacy/legacy_service.ts b/src/core/server/legacy/legacy_service.ts index 3111c8daf7981..4ae6c9d437576 100644 --- a/src/core/server/legacy/legacy_service.ts +++ b/src/core/server/legacy/legacy_service.ts @@ -144,7 +144,7 @@ export class LegacyService implements CoreService { this.log.debug('starting legacy service'); // Receive initial config and create kbnServer/ClusterManager. - if (this.coreContext.env.isDevClusterMaster) { + if (this.coreContext.env.isDevCliParent) { await this.createClusterManager(this.legacyRawConfig!); } else { this.kbnServer = await this.createKbnServer( @@ -310,10 +310,8 @@ export class LegacyService implements CoreService { logger: this.coreContext.logger, }); - // The kbnWorkerType check is necessary to prevent the repl - // from being started multiple times in different processes. - // We only want one REPL. - if (this.coreContext.env.cliArgs.repl && process.env.kbnWorkerType === 'server') { + // Prevent the repl from being started multiple times in different processes. + if (this.coreContext.env.cliArgs.repl && process.env.isDevCliChild) { // eslint-disable-next-line @typescript-eslint/no-var-requires require('./cli').startRepl(kbnServer); } diff --git a/src/core/server/plugins/plugins_service.test.ts b/src/core/server/plugins/plugins_service.test.ts index 02b82c17ed4fc..601e0038b0cf0 100644 --- a/src/core/server/plugins/plugins_service.test.ts +++ b/src/core/server/plugins/plugins_service.test.ts @@ -102,7 +102,7 @@ const createPlugin = ( }); }; -async function testSetup(options: { isDevClusterMaster?: boolean } = {}) { +async function testSetup(options: { isDevCliParent?: boolean } = {}) { mockPackage.raw = { branch: 'feature-v1', version: 'v1', @@ -116,7 +116,7 @@ async function testSetup(options: { isDevClusterMaster?: boolean } = {}) { coreId = Symbol('core'); env = Env.createDefault(REPO_ROOT, { ...getEnvOptions(), - isDevClusterMaster: options.isDevClusterMaster ?? false, + isDevCliParent: options.isDevCliParent ?? false, }); config$ = new BehaviorSubject>({ plugins: { initialize: true } }); @@ -638,10 +638,10 @@ describe('PluginsService', () => { }); }); -describe('PluginService when isDevClusterMaster is true', () => { +describe('PluginService when isDevCliParent is true', () => { beforeEach(async () => { await testSetup({ - isDevClusterMaster: true, + isDevCliParent: true, }); }); diff --git a/src/core/server/plugins/plugins_service.ts b/src/core/server/plugins/plugins_service.ts index 5967e6d5358de..e1622b1e19231 100644 --- a/src/core/server/plugins/plugins_service.ts +++ b/src/core/server/plugins/plugins_service.ts @@ -90,7 +90,7 @@ export class PluginsService implements CoreService(); - const initialize = config.initialize && !this.coreContext.env.isDevClusterMaster; + const initialize = config.initialize && !this.coreContext.env.isDevCliParent; if (initialize) { contracts = await this.pluginsSystem.setupPlugins(deps); this.registerPluginStaticDirs(deps); diff --git a/src/core/server/server.test.ts b/src/core/server/server.test.ts index 0c7ebbcb527ec..f377bfc321735 100644 --- a/src/core/server/server.test.ts +++ b/src/core/server/server.test.ts @@ -216,10 +216,10 @@ test(`doesn't setup core services if legacy config validation fails`, async () = expect(mockI18nService.setup).not.toHaveBeenCalled(); }); -test(`doesn't validate config if env.isDevClusterMaster is true`, async () => { +test(`doesn't validate config if env.isDevCliParent is true`, async () => { const devParentEnv = Env.createDefault(REPO_ROOT, { ...getEnvOptions(), - isDevClusterMaster: true, + isDevCliParent: true, }); const server = new Server(rawConfigService, devParentEnv, logger); diff --git a/src/core/server/server.ts b/src/core/server/server.ts index 0f7e8cced999c..e253663d8dc8d 100644 --- a/src/core/server/server.ts +++ b/src/core/server/server.ts @@ -124,7 +124,7 @@ export class Server { const legacyConfigSetup = await this.legacy.setupLegacyConfig(); // rely on dev server to validate config, don't validate in the parent process - if (!this.env.isDevClusterMaster) { + if (!this.env.isDevCliParent) { // Immediately terminate in case of invalid configuration // This needs to be done after plugin discovery await this.configService.validate(); diff --git a/src/core/test_helpers/kbn_server.ts b/src/core/test_helpers/kbn_server.ts index f95ea66d3cbc1..3161420b94d22 100644 --- a/src/core/test_helpers/kbn_server.ts +++ b/src/core/test_helpers/kbn_server.ts @@ -82,7 +82,7 @@ export function createRootWithSettings( dist: false, ...cliArgs, }, - isDevClusterMaster: false, + isDevCliParent: false, }); return new Root( diff --git a/src/legacy/server/kbn_server.js b/src/legacy/server/kbn_server.js index b61a86326ca1a..85d75b4e18772 100644 --- a/src/legacy/server/kbn_server.js +++ b/src/legacy/server/kbn_server.js @@ -20,7 +20,6 @@ import { constant, once, compact, flatten } from 'lodash'; import { reconfigureLogging } from '@kbn/legacy-logging'; -import { isWorker } from 'cluster'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { fromRoot, pkg } from '../../core/server/utils'; import { Config } from './config'; @@ -121,7 +120,7 @@ export default class KbnServer { const { server, config } = this; - if (isWorker) { + if (process.env.isDevCliChild) { // help parent process know when we are ready process.send(['WORKER_LISTENING']); } From 280ce7e5fa9256b79e7900c3b3b898d3048d5ddd Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Wed, 25 Nov 2020 16:49:45 +0100 Subject: [PATCH 42/49] [ML] Persisted URL state for Anomalies table (#84314) * [ML] Persisted URL state for Anomalies table * [ML] adjust cell selection according to the time range --- .../anomalies_table/anomalies_table.js | 66 +++++++++++++++---- .../explorer/hooks/use_selected_cells.ts | 53 +++++++++++++-- .../reducers/explorer_reducer/state.ts | 3 +- .../application/routing/routes/explorer.tsx | 7 +- .../ml/public/application/util/url_state.tsx | 2 +- 5 files changed, 111 insertions(+), 20 deletions(-) diff --git a/x-pack/plugins/ml/public/application/components/anomalies_table/anomalies_table.js b/x-pack/plugins/ml/public/application/components/anomalies_table/anomalies_table.js index 0a2c67a3b0dcb..ebc782fe4625b 100644 --- a/x-pack/plugins/ml/public/application/components/anomalies_table/anomalies_table.js +++ b/x-pack/plugins/ml/public/application/components/anomalies_table/anomalies_table.js @@ -25,8 +25,9 @@ import { mlTableService } from '../../services/table_service'; import { RuleEditorFlyout } from '../rule_editor'; import { ml } from '../../services/ml_api_service'; import { INFLUENCERS_LIMIT, ANOMALIES_TABLE_TABS, MAX_CHARS } from './anomalies_table_constants'; +import { usePageUrlState } from '../../util/url_state'; -class AnomaliesTable extends Component { +export class AnomaliesTableInternal extends Component { constructor(props) { super(props); @@ -145,8 +146,20 @@ class AnomaliesTable extends Component { }); }; + onTableChange = ({ page, sort }) => { + const { tableState, updateTableState } = this.props; + const result = { + pageIndex: page && page.index !== undefined ? page.index : tableState.pageIndex, + pageSize: page && page.size !== undefined ? page.size : tableState.pageSize, + sortField: sort && sort.field !== undefined ? sort.field : tableState.sortField, + sortDirection: + sort && sort.direction !== undefined ? sort.direction : tableState.sortDirection, + }; + updateTableState(result); + }; + render() { - const { bounds, tableData, filter, influencerFilter } = this.props; + const { bounds, tableData, filter, influencerFilter, tableState } = this.props; if ( tableData === undefined || @@ -186,8 +199,8 @@ class AnomaliesTable extends Component { const sorting = { sort: { - field: 'severity', - direction: 'desc', + field: tableState.sortField, + direction: tableState.sortDirection, }, }; @@ -199,8 +212,15 @@ class AnomaliesTable extends Component { }; }; + const pagination = { + pageIndex: tableState.pageIndex, + pageSize: tableState.pageSize, + totalItemCount: tableData.anomalies.length, + pageSizeOptions: [10, 25, 100], + }; + return ( - + <> - + ); } } -AnomaliesTable.propTypes = { + +export const getDefaultAnomaliesTableState = () => ({ + pageIndex: 0, + pageSize: 25, + sortField: 'severity', + sortDirection: 'desc', +}); + +export const AnomaliesTable = (props) => { + const [tableState, updateTableState] = usePageUrlState( + 'mlAnomaliesTable', + getDefaultAnomaliesTableState() + ); + return ( + + ); +}; + +AnomaliesTableInternal.propTypes = { bounds: PropTypes.object.isRequired, tableData: PropTypes.object, filter: PropTypes.func, influencerFilter: PropTypes.func, + tableState: PropTypes.object.isRequired, + updateTableState: PropTypes.func.isRequired, }; - -export { AnomaliesTable }; diff --git a/x-pack/plugins/ml/public/application/explorer/hooks/use_selected_cells.ts b/x-pack/plugins/ml/public/application/explorer/hooks/use_selected_cells.ts index 7602954b4c8c3..f940fdc2387e2 100644 --- a/x-pack/plugins/ml/public/application/explorer/hooks/use_selected_cells.ts +++ b/x-pack/plugins/ml/public/application/explorer/hooks/use_selected_cells.ts @@ -4,14 +4,17 @@ * you may not use this file except in compliance with the Elastic License. */ -import { useCallback, useMemo } from 'react'; +import { useCallback, useEffect, useMemo } from 'react'; +import { Duration } from 'moment'; import { SWIMLANE_TYPE } from '../explorer_constants'; -import { AppStateSelectedCells } from '../explorer_utils'; +import { AppStateSelectedCells, TimeRangeBounds } from '../explorer_utils'; import { ExplorerAppState } from '../../../../common/types/ml_url_generator'; export const useSelectedCells = ( appState: ExplorerAppState, - setAppState: (update: Partial) => void + setAppState: (update: Partial) => void, + timeBounds: TimeRangeBounds | undefined, + bucketInterval: Duration | undefined ): [AppStateSelectedCells | undefined, (swimlaneSelectedCells: AppStateSelectedCells) => void] => { // keep swimlane selection, restore selectedCells from AppState const selectedCells = useMemo(() => { @@ -28,7 +31,7 @@ export const useSelectedCells = ( }, [JSON.stringify(appState?.mlExplorerSwimlane)]); const setSelectedCells = useCallback( - (swimlaneSelectedCells: AppStateSelectedCells) => { + (swimlaneSelectedCells?: AppStateSelectedCells) => { const mlExplorerSwimlane = { ...appState.mlExplorerSwimlane, } as ExplorerAppState['mlExplorerSwimlane']; @@ -65,5 +68,47 @@ export const useSelectedCells = ( [appState?.mlExplorerSwimlane, selectedCells, setAppState] ); + /** + * Adjust cell selection with respect to the time boundaries. + * Reset it entirely when it out of range. + */ + useEffect(() => { + if ( + timeBounds === undefined || + selectedCells?.times === undefined || + bucketInterval === undefined + ) + return; + + let [selectedFrom, selectedTo] = selectedCells.times; + + const rangeFrom = timeBounds.min!.unix(); + /** + * Because each cell on the swim lane represent the fixed bucket interval, + * the selection range could be outside of the time boundaries with + * correction within the bucket interval. + */ + const rangeTo = timeBounds.max!.unix() + bucketInterval.asSeconds(); + + selectedFrom = Math.max(selectedFrom, rangeFrom); + + selectedTo = Math.min(selectedTo, rangeTo); + + const isSelectionOutOfRange = rangeFrom > selectedTo || rangeTo < selectedFrom; + + if (isSelectionOutOfRange) { + // reset selection + setSelectedCells(); + return; + } + + if (selectedFrom !== rangeFrom || selectedTo !== rangeTo) { + setSelectedCells({ + ...selectedCells, + times: [selectedFrom, selectedTo], + }); + } + }, [timeBounds, selectedCells, bucketInterval]); + return [selectedCells, setSelectedCells]; }; diff --git a/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/state.ts b/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/state.ts index ea9a8b5c18054..14b0a6033999c 100644 --- a/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/state.ts +++ b/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/state.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { Duration } from 'moment'; import { ML_RESULTS_INDEX_PATTERN } from '../../../../../common/constants/index_patterns'; import { Dictionary } from '../../../../../common/types/common'; @@ -43,7 +44,7 @@ export interface ExplorerState { queryString: string; selectedCells: AppStateSelectedCells | undefined; selectedJobs: ExplorerJob[] | null; - swimlaneBucketInterval: any; + swimlaneBucketInterval: Duration | undefined; swimlaneContainerWidth: number; tableData: AnomaliesTableData; tableQueryString: string; diff --git a/x-pack/plugins/ml/public/application/routing/routes/explorer.tsx b/x-pack/plugins/ml/public/application/routing/routes/explorer.tsx index f8b4de6903ad2..2126cbceae6b1 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/explorer.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/explorer.tsx @@ -205,7 +205,12 @@ const ExplorerUrlStateManager: FC = ({ jobsWithTim const [tableInterval] = useTableInterval(); const [tableSeverity] = useTableSeverity(); - const [selectedCells, setSelectedCells] = useSelectedCells(explorerUrlState, setExplorerUrlState); + const [selectedCells, setSelectedCells] = useSelectedCells( + explorerUrlState, + setExplorerUrlState, + explorerState?.bounds, + explorerState?.swimlaneBucketInterval + ); useEffect(() => { explorerService.setSelectedCells(selectedCells); diff --git a/x-pack/plugins/ml/public/application/util/url_state.tsx b/x-pack/plugins/ml/public/application/util/url_state.tsx index fdc6dd135cd69..6cdc069096dcc 100644 --- a/x-pack/plugins/ml/public/application/util/url_state.tsx +++ b/x-pack/plugins/ml/public/application/util/url_state.tsx @@ -162,7 +162,7 @@ export const useUrlState = (accessor: Accessor) => { return [urlState, setUrlState]; }; -type AppStateKey = 'mlSelectSeverity' | 'mlSelectInterval' | MlPages; +type AppStateKey = 'mlSelectSeverity' | 'mlSelectInterval' | 'mlAnomaliesTable' | MlPages; /** * Hook for managing the URL state of the page. From 5b2e119356c33543a20283a64e1f7ae8555a1fc3 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 25 Nov 2020 16:57:03 +0100 Subject: [PATCH 43/49] add live region for field search (#84310) --- .../datapanel.test.tsx | 19 +++++++++++++++++++ .../indexpattern_datasource/datapanel.tsx | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx index ac82caf9d5227..3d55494fd260c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx @@ -718,6 +718,25 @@ describe('IndexPattern Data Panel', () => { ]); }); + it('should announce filter in live region', () => { + const wrapper = mountWithIntl(); + act(() => { + wrapper.find('[data-test-subj="lnsIndexPatternFieldSearch"]').prop('onChange')!({ + target: { value: 'me' }, + } as ChangeEvent); + }); + + wrapper + .find('[data-test-subj="lnsIndexPatternEmptyFields"]') + .find('button') + .first() + .simulate('click'); + + expect(wrapper.find('[aria-live="polite"]').text()).toEqual( + '1 available field. 1 empty field. 0 meta fields.' + ); + }); + it('should filter down by type', () => { const wrapper = mountWithIntl(); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx index f2c7d7fc20926..ad5509dd88bc9 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx @@ -18,10 +18,12 @@ import { EuiSpacer, EuiFilterGroup, EuiFilterButton, + EuiScreenReaderOnly, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; import { DataPublicPluginStart, EsQueryConfig, Query, Filter } from 'src/plugins/data/public'; +import { htmlIdGenerator } from '@elastic/eui'; import { DatasourceDataPanelProps, DataType, StateSetter } from '../types'; import { ChildDragDropProvider, DragContextState } from '../drag_drop'; import { @@ -222,6 +224,9 @@ const fieldFiltersLabel = i18n.translate('xpack.lens.indexPatterns.fieldFiltersL defaultMessage: 'Field filters', }); +const htmlId = htmlIdGenerator('datapanel'); +const fieldSearchDescriptionId = htmlId(); + export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ currentIndexPatternId, indexPatternRefs, @@ -489,6 +494,7 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ aria-label={i18n.translate('xpack.lens.indexPatterns.filterByNameAriaLabel', { defaultMessage: 'Search fields', })} + aria-describedby={fieldSearchDescriptionId} /> @@ -550,6 +556,19 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ + +
+ {i18n.translate('xpack.lens.indexPatterns.fieldSearchLiveRegion', { + defaultMessage: + '{availableFields} available {availableFields, plural, one {field} other {fields}}. {emptyFields} empty {emptyFields, plural, one {field} other {fields}}. {metaFields} meta {metaFields, plural, one {field} other {fields}}.', + values: { + availableFields: fieldGroups.AvailableFields.fields.length, + emptyFields: fieldGroups.EmptyFields.fields.length, + metaFields: fieldGroups.MetaFields.fields.length, + }, + })} +
+ From 9bef42b2a8078d070b206c6860c65df638d3b323 Mon Sep 17 00:00:00 2001 From: Devon Thomson Date: Wed, 25 Nov 2020 10:59:52 -0500 Subject: [PATCH 44/49] redirect to visualize listing page when by value visualization editor doesn't have a value input (#84287) --- .../application/components/visualize_byvalue_editor.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/plugins/visualize/public/application/components/visualize_byvalue_editor.tsx b/src/plugins/visualize/public/application/components/visualize_byvalue_editor.tsx index a63f597f10135..1c1eb9956a329 100644 --- a/src/plugins/visualize/public/application/components/visualize_byvalue_editor.tsx +++ b/src/plugins/visualize/public/application/components/visualize_byvalue_editor.tsx @@ -33,6 +33,7 @@ import { import { VisualizeServices } from '../types'; import { VisualizeEditorCommon } from './visualize_editor_common'; import { VisualizeAppProps } from '../app'; +import { VisualizeConstants } from '../..'; export const VisualizeByValueEditor = ({ onAppLeave }: VisualizeAppProps) => { const [originatingApp, setOriginatingApp] = useState(); @@ -52,7 +53,8 @@ export const VisualizeByValueEditor = ({ onAppLeave }: VisualizeAppProps) => { setValueInput(valueInputValue); setEmbeddableId(embeddableIdValue); if (!valueInputValue) { - history.back(); + // if there is no value input to load, redirect to the visualize listing page. + services.history.replace(VisualizeConstants.LANDING_PAGE_PATH); } }, [services]); From 351cd6d19d0367ee606f4dd32aac4ee4190844ae Mon Sep 17 00:00:00 2001 From: Devon Thomson Date: Wed, 25 Nov 2020 11:03:00 -0500 Subject: [PATCH 45/49] [Time to Visualize] Fix Unlink Action via Rollback of ReplacePanel (#83873) * Fixed unlink action via rollback of replacePanel changes --- .../actions/add_to_library_action.test.tsx | 18 +++++++--- .../actions/add_to_library_action.tsx | 5 ++- .../unlink_from_library_action.test.tsx | 18 +++++++--- .../actions/unlink_from_library_action.tsx | 5 ++- .../embeddable/dashboard_container.tsx | 33 ++++++++++++++++--- 5 files changed, 60 insertions(+), 19 deletions(-) diff --git a/src/plugins/dashboard/public/application/actions/add_to_library_action.test.tsx b/src/plugins/dashboard/public/application/actions/add_to_library_action.test.tsx index feb30b248c066..5f3945e733527 100644 --- a/src/plugins/dashboard/public/application/actions/add_to_library_action.test.tsx +++ b/src/plugins/dashboard/public/application/actions/add_to_library_action.test.tsx @@ -137,12 +137,17 @@ test('Add to library is not compatible when embeddable is not in a dashboard con test('Add to library replaces embeddableId and retains panel count', async () => { const dashboard = embeddable.getRoot() as IContainer; const originalPanelCount = Object.keys(dashboard.getInput().panels).length; + const originalPanelKeySet = new Set(Object.keys(dashboard.getInput().panels)); const action = new AddToLibraryAction({ toasts: coreStart.notifications.toasts }); await action.execute({ embeddable }); expect(Object.keys(container.getInput().panels).length).toEqual(originalPanelCount); - expect(Object.keys(container.getInput().panels)).toContain(embeddable.id); - const newPanel = container.getInput().panels[embeddable.id!]; + + const newPanelId = Object.keys(container.getInput().panels).find( + (key) => !originalPanelKeySet.has(key) + ); + expect(newPanelId).toBeDefined(); + const newPanel = container.getInput().panels[newPanelId!]; expect(newPanel.type).toEqual(embeddable.type); }); @@ -158,10 +163,15 @@ test('Add to library returns reference type input', async () => { mockedByReferenceInput: { savedObjectId: 'testSavedObjectId', id: embeddable.id }, mockedByValueInput: { attributes: complicatedAttributes, id: embeddable.id } as EmbeddableInput, }); + const dashboard = embeddable.getRoot() as IContainer; + const originalPanelKeySet = new Set(Object.keys(dashboard.getInput().panels)); const action = new AddToLibraryAction({ toasts: coreStart.notifications.toasts }); await action.execute({ embeddable }); - expect(Object.keys(container.getInput().panels)).toContain(embeddable.id); - const newPanel = container.getInput().panels[embeddable.id!]; + const newPanelId = Object.keys(container.getInput().panels).find( + (key) => !originalPanelKeySet.has(key) + ); + expect(newPanelId).toBeDefined(); + const newPanel = container.getInput().panels[newPanelId!]; expect(newPanel.type).toEqual(embeddable.type); expect(newPanel.explicitInput.attributes).toBeUndefined(); expect(newPanel.explicitInput.savedObjectId).toBe('testSavedObjectId'); diff --git a/src/plugins/dashboard/public/application/actions/add_to_library_action.tsx b/src/plugins/dashboard/public/application/actions/add_to_library_action.tsx index 179e5d522a2b3..08cd0c7a15381 100644 --- a/src/plugins/dashboard/public/application/actions/add_to_library_action.tsx +++ b/src/plugins/dashboard/public/application/actions/add_to_library_action.tsx @@ -19,7 +19,6 @@ import { i18n } from '@kbn/i18n'; import _ from 'lodash'; -import uuid from 'uuid'; import { ActionByType, IncompatibleActionError } from '../../ui_actions_plugin'; import { ViewMode, PanelState, IEmbeddable } from '../../embeddable_plugin'; import { @@ -89,9 +88,9 @@ export class AddToLibraryAction implements ActionByType = { type: embeddable.type, - explicitInput: { ...newInput, id: uuid.v4() }, + explicitInput: { ...newInput }, }; - dashboard.replacePanel(panelToReplace, newPanel); + dashboard.replacePanel(panelToReplace, newPanel, true); const title = i18n.translate('dashboard.panel.addToLibrary.successMessage', { defaultMessage: `Panel '{panelTitle}' was added to the visualize library`, diff --git a/src/plugins/dashboard/public/application/actions/unlink_from_library_action.test.tsx b/src/plugins/dashboard/public/application/actions/unlink_from_library_action.test.tsx index f191be6f7baad..6a9769b0c8d16 100644 --- a/src/plugins/dashboard/public/application/actions/unlink_from_library_action.test.tsx +++ b/src/plugins/dashboard/public/application/actions/unlink_from_library_action.test.tsx @@ -135,11 +135,16 @@ test('Unlink is not compatible when embeddable is not in a dashboard container', test('Unlink replaces embeddableId and retains panel count', async () => { const dashboard = embeddable.getRoot() as IContainer; const originalPanelCount = Object.keys(dashboard.getInput().panels).length; + const originalPanelKeySet = new Set(Object.keys(dashboard.getInput().panels)); const action = new UnlinkFromLibraryAction({ toasts: coreStart.notifications.toasts }); await action.execute({ embeddable }); expect(Object.keys(container.getInput().panels).length).toEqual(originalPanelCount); - expect(Object.keys(container.getInput().panels)).toContain(embeddable.id); - const newPanel = container.getInput().panels[embeddable.id!]; + + const newPanelId = Object.keys(container.getInput().panels).find( + (key) => !originalPanelKeySet.has(key) + ); + expect(newPanelId).toBeDefined(); + const newPanel = container.getInput().panels[newPanelId!]; expect(newPanel.type).toEqual(embeddable.type); }); @@ -159,10 +164,15 @@ test('Unlink unwraps all attributes from savedObject', async () => { mockedByReferenceInput: { savedObjectId: 'testSavedObjectId', id: embeddable.id }, mockedByValueInput: { attributes: complicatedAttributes, id: embeddable.id }, }); + const dashboard = embeddable.getRoot() as IContainer; + const originalPanelKeySet = new Set(Object.keys(dashboard.getInput().panels)); const action = new UnlinkFromLibraryAction({ toasts: coreStart.notifications.toasts }); await action.execute({ embeddable }); - expect(Object.keys(container.getInput().panels)).toContain(embeddable.id); - const newPanel = container.getInput().panels[embeddable.id!]; + const newPanelId = Object.keys(container.getInput().panels).find( + (key) => !originalPanelKeySet.has(key) + ); + expect(newPanelId).toBeDefined(); + const newPanel = container.getInput().panels[newPanelId!]; expect(newPanel.type).toEqual(embeddable.type); expect(newPanel.explicitInput.attributes).toEqual(complicatedAttributes); }); diff --git a/src/plugins/dashboard/public/application/actions/unlink_from_library_action.tsx b/src/plugins/dashboard/public/application/actions/unlink_from_library_action.tsx index 5e16145364712..b20bbc6350aaa 100644 --- a/src/plugins/dashboard/public/application/actions/unlink_from_library_action.tsx +++ b/src/plugins/dashboard/public/application/actions/unlink_from_library_action.tsx @@ -19,7 +19,6 @@ import { i18n } from '@kbn/i18n'; import _ from 'lodash'; -import uuid from 'uuid'; import { ActionByType, IncompatibleActionError } from '../../ui_actions_plugin'; import { ViewMode, PanelState, IEmbeddable } from '../../embeddable_plugin'; import { @@ -88,9 +87,9 @@ export class UnlinkFromLibraryAction implements ActionByType = { type: embeddable.type, - explicitInput: { ...newInput, id: uuid.v4() }, + explicitInput: { ...newInput }, }; - dashboard.replacePanel(panelToReplace, newPanel); + dashboard.replacePanel(panelToReplace, newPanel, true); const title = embeddable.getTitle() ? i18n.translate('dashboard.panel.unlinkFromLibrary.successMessageWithTitle', { diff --git a/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx b/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx index 051a7ef8bfb92..e80d387fa3066 100644 --- a/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx +++ b/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx @@ -173,11 +173,30 @@ export class DashboardContainer extends Container, - newPanelState: Partial + newPanelState: Partial, + generateNewId?: boolean ) { - // Because the embeddable type can change, we have to operate at the container level here - return this.updateInput({ - panels: { + let panels; + if (generateNewId) { + // replace panel can be called with generateNewId in order to totally destroy and recreate the embeddable + panels = { ...this.input.panels }; + delete panels[previousPanelState.explicitInput.id]; + const newId = uuid.v4(); + panels[newId] = { + ...previousPanelState, + ...newPanelState, + gridData: { + ...previousPanelState.gridData, + i: newId, + }, + explicitInput: { + ...newPanelState.explicitInput, + id: newId, + }, + }; + } else { + // Because the embeddable type can change, we have to operate at the container level here + panels = { ...this.input.panels, [previousPanelState.explicitInput.id]: { ...previousPanelState, @@ -190,7 +209,11 @@ export class DashboardContainer extends Container Date: Wed, 25 Nov 2020 17:08:15 +0100 Subject: [PATCH 46/49] [APM] Elastic chart issues (#84238) * fixing charts * addressing pr comments --- .../ErrorGroupDetails/Distribution/index.tsx | 7 +- .../TransactionDetails/Distribution/index.tsx | 2 +- .../shared/charts/annotations/index.tsx | 45 ------- .../shared/charts/timeseries_chart.tsx | 38 +++++- .../transaction_breakdown_chart_contents.tsx | 36 +++++- .../charts/transaction_charts/index.tsx | 119 +++++++++--------- .../transaction_error_rate_chart/index.tsx | 1 + .../public/context/annotations_context.tsx | 49 ++++++++ .../apm/public/hooks/use_annotations.ts | 34 ++--- 9 files changed, 194 insertions(+), 137 deletions(-) delete mode 100644 x-pack/plugins/apm/public/components/shared/charts/annotations/index.tsx create mode 100644 x-pack/plugins/apm/public/context/annotations_context.tsx diff --git a/x-pack/plugins/apm/public/components/app/ErrorGroupDetails/Distribution/index.tsx b/x-pack/plugins/apm/public/components/app/ErrorGroupDetails/Distribution/index.tsx index 99316e3520a76..159f111bee04c 100644 --- a/x-pack/plugins/apm/public/components/app/ErrorGroupDetails/Distribution/index.tsx +++ b/x-pack/plugins/apm/public/components/app/ErrorGroupDetails/Distribution/index.tsx @@ -90,7 +90,12 @@ export function ErrorDistribution({ distribution, title }: Props) { showOverlappingTicks tickFormat={xFormatter} /> - + formatYShort(value)} /> ({ - dataValue: annotation['@timestamp'], - header: asAbsoluteDateTime(annotation['@timestamp']), - details: `${i18n.translate('xpack.apm.chart.annotation.version', { - defaultMessage: 'Version', - })} ${annotation.text}`, - }))} - style={{ line: { strokeWidth: 1, stroke: color, opacity: 1 } }} - marker={} - markerPosition={Position.Top} - /> - ); -} diff --git a/x-pack/plugins/apm/public/components/shared/charts/timeseries_chart.tsx b/x-pack/plugins/apm/public/components/shared/charts/timeseries_chart.tsx index c4f5abe104aa9..ea6f2a4a233e5 100644 --- a/x-pack/plugins/apm/public/components/shared/charts/timeseries_chart.tsx +++ b/x-pack/plugins/apm/public/components/shared/charts/timeseries_chart.tsx @@ -5,28 +5,35 @@ */ import { + AnnotationDomainTypes, AreaSeries, Axis, Chart, CurveType, LegendItemListener, + LineAnnotation, LineSeries, niceTimeFormatter, Placement, Position, ScaleType, Settings, + YDomainRange, } from '@elastic/charts'; +import { EuiIcon } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; import moment from 'moment'; import React, { useEffect } from 'react'; import { useHistory } from 'react-router-dom'; import { useChartTheme } from '../../../../../observability/public'; +import { asAbsoluteDateTime } from '../../../../common/utils/formatters'; import { TimeSeries } from '../../../../typings/timeseries'; import { FETCH_STATUS } from '../../../hooks/useFetcher'; +import { useTheme } from '../../../hooks/useTheme'; import { useUrlParams } from '../../../hooks/useUrlParams'; +import { useAnnotations } from '../../../hooks/use_annotations'; import { useChartPointerEvent } from '../../../hooks/use_chart_pointer_event'; import { unit } from '../../../style/variables'; -import { Annotations } from './annotations'; import { ChartContainer } from './chart_container'; import { onBrushEnd } from './helper/helper'; @@ -45,6 +52,7 @@ interface Props { */ yTickFormat?: (y: number) => string; showAnnotations?: boolean; + yDomain?: YDomainRange; } export function TimeseriesChart({ @@ -56,12 +64,16 @@ export function TimeseriesChart({ yLabelFormat, yTickFormat, showAnnotations = true, + yDomain, }: Props) { const history = useHistory(); const chartRef = React.createRef(); + const { annotations } = useAnnotations(); const chartTheme = useChartTheme(); const { pointerEvent, setPointerEvent } = useChartPointerEvent(); const { urlParams } = useUrlParams(); + const theme = useTheme(); + const { start, end } = urlParams; useEffect(() => { @@ -83,6 +95,8 @@ export function TimeseriesChart({ y === null || y === undefined ); + const annotationColor = theme.eui.euiColorSecondary; + return ( @@ -108,17 +122,35 @@ export function TimeseriesChart({ position={Position.Bottom} showOverlappingTicks tickFormat={xFormatter} + gridLine={{ visible: false }} /> - {showAnnotations && } + {showAnnotations && ( + ({ + dataValue: annotation['@timestamp'], + header: asAbsoluteDateTime(annotation['@timestamp']), + details: `${i18n.translate('xpack.apm.chart.annotation.version', { + defaultMessage: 'Version', + })} ${annotation.text}`, + }))} + style={{ + line: { strokeWidth: 1, stroke: annotationColor, opacity: 1 }, + }} + marker={} + markerPosition={Position.Top} + /> + )} {timeseries.map((serie) => { const Series = serie.type === 'area' ? AreaSeries : LineSeries; diff --git a/x-pack/plugins/apm/public/components/shared/charts/transaction_breakdown_chart/transaction_breakdown_chart_contents.tsx b/x-pack/plugins/apm/public/components/shared/charts/transaction_breakdown_chart/transaction_breakdown_chart_contents.tsx index 04c07c01442a4..20056a6831adf 100644 --- a/x-pack/plugins/apm/public/components/shared/charts/transaction_breakdown_chart/transaction_breakdown_chart_contents.tsx +++ b/x-pack/plugins/apm/public/components/shared/charts/transaction_breakdown_chart/transaction_breakdown_chart_contents.tsx @@ -5,27 +5,35 @@ */ import { + AnnotationDomainTypes, AreaSeries, Axis, Chart, CurveType, + LineAnnotation, niceTimeFormatter, Placement, Position, ScaleType, Settings, } from '@elastic/charts'; +import { EuiIcon } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; import moment from 'moment'; import React, { useEffect } from 'react'; import { useHistory } from 'react-router-dom'; import { useChartTheme } from '../../../../../../observability/public'; -import { asPercent } from '../../../../../common/utils/formatters'; +import { + asAbsoluteDateTime, + asPercent, +} from '../../../../../common/utils/formatters'; import { TimeSeries } from '../../../../../typings/timeseries'; import { FETCH_STATUS } from '../../../../hooks/useFetcher'; +import { useTheme } from '../../../../hooks/useTheme'; import { useUrlParams } from '../../../../hooks/useUrlParams'; +import { useAnnotations } from '../../../../hooks/use_annotations'; import { useChartPointerEvent } from '../../../../hooks/use_chart_pointer_event'; import { unit } from '../../../../style/variables'; -import { Annotations } from '../../charts/annotations'; import { ChartContainer } from '../../charts/chart_container'; import { onBrushEnd } from '../../charts/helper/helper'; @@ -44,9 +52,11 @@ export function TransactionBreakdownChartContents({ }: Props) { const history = useHistory(); const chartRef = React.createRef(); + const { annotations } = useAnnotations(); const chartTheme = useChartTheme(); const { pointerEvent, setPointerEvent } = useChartPointerEvent(); const { urlParams } = useUrlParams(); + const theme = useTheme(); const { start, end } = urlParams; useEffect(() => { @@ -64,6 +74,8 @@ export function TransactionBreakdownChartContents({ const xFormatter = niceTimeFormatter([min, max]); + const annotationColor = theme.eui.euiColorSecondary; + return ( @@ -85,6 +97,7 @@ export function TransactionBreakdownChartContents({ position={Position.Bottom} showOverlappingTicks tickFormat={xFormatter} + gridLine={{ visible: false }} /> asPercent(y ?? 0, 1)} /> - {showAnnotations && } + {showAnnotations && ( + ({ + dataValue: annotation['@timestamp'], + header: asAbsoluteDateTime(annotation['@timestamp']), + details: `${i18n.translate('xpack.apm.chart.annotation.version', { + defaultMessage: 'Version', + })} ${annotation.text}`, + }))} + style={{ + line: { strokeWidth: 1, stroke: annotationColor, opacity: 1 }, + }} + marker={} + markerPosition={Position.Top} + /> + )} {timeseries?.length ? ( timeseries.map((serie) => { diff --git a/x-pack/plugins/apm/public/components/shared/charts/transaction_charts/index.tsx b/x-pack/plugins/apm/public/components/shared/charts/transaction_charts/index.tsx index 221f17bb9e1d5..3f8071ec39f0f 100644 --- a/x-pack/plugins/apm/public/components/shared/charts/transaction_charts/index.tsx +++ b/x-pack/plugins/apm/public/components/shared/charts/transaction_charts/index.tsx @@ -20,13 +20,14 @@ import { TRANSACTION_ROUTE_CHANGE, } from '../../../../../common/transaction_types'; import { asTransactionRate } from '../../../../../common/utils/formatters'; +import { AnnotationsContextProvider } from '../../../../context/annotations_context'; import { ChartPointerEventContextProvider } from '../../../../context/chart_pointer_event_context'; import { LicenseContext } from '../../../../context/LicenseContext'; import { IUrlParams } from '../../../../context/UrlParamsContext/types'; import { FETCH_STATUS } from '../../../../hooks/useFetcher'; import { ITransactionChartData } from '../../../../selectors/chart_selectors'; -import { TransactionBreakdownChart } from '../transaction_breakdown_chart'; import { TimeseriesChart } from '../timeseries_chart'; +import { TransactionBreakdownChart } from '../transaction_breakdown_chart'; import { TransactionErrorRateChart } from '../transaction_error_rate_chart/'; import { getResponseTimeTickFormatter } from './helper'; import { MLHeader } from './ml_header'; @@ -51,65 +52,69 @@ export function TransactionCharts({ return ( <> - - - - - - - - {responseTimeLabel(transactionType)} - - - - {(license) => ( - - )} - - - { - if (serie) { - toggleSerie(serie); - } - }} - /> - - + + + + + + + + + {responseTimeLabel(transactionType)} + + + + {(license) => ( + + )} + + + { + if (serie) { + toggleSerie(serie); + } + }} + /> + + - - - - {tpmLabel(transactionType)} - - - - - + + + + {tpmLabel(transactionType)} + + + + + - + - - - - - - - - - + + + + + + + + + + ); } diff --git a/x-pack/plugins/apm/public/components/shared/charts/transaction_error_rate_chart/index.tsx b/x-pack/plugins/apm/public/components/shared/charts/transaction_error_rate_chart/index.tsx index b9028ff2e9e8c..00472df95c4b1 100644 --- a/x-pack/plugins/apm/public/components/shared/charts/transaction_error_rate_chart/index.tsx +++ b/x-pack/plugins/apm/public/components/shared/charts/transaction_error_rate_chart/index.tsx @@ -91,6 +91,7 @@ export function TransactionErrorRateChart({ ]} yLabelFormat={yLabelFormat} yTickFormat={yTickFormat} + yDomain={{ min: 0, max: 1 }} /> ); diff --git a/x-pack/plugins/apm/public/context/annotations_context.tsx b/x-pack/plugins/apm/public/context/annotations_context.tsx new file mode 100644 index 0000000000000..4e09a3d227b11 --- /dev/null +++ b/x-pack/plugins/apm/public/context/annotations_context.tsx @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { createContext } from 'react'; +import { useParams } from 'react-router-dom'; +import { Annotation } from '../../common/annotations'; +import { useFetcher } from '../hooks/useFetcher'; +import { useUrlParams } from '../hooks/useUrlParams'; +import { callApmApi } from '../services/rest/createCallApmApi'; + +export const AnnotationsContext = createContext({ annotations: [] } as { + annotations: Annotation[]; +}); + +const INITIAL_STATE = { annotations: [] }; + +export function AnnotationsContextProvider({ + children, +}: { + children: React.ReactNode; +}) { + const { serviceName } = useParams<{ serviceName?: string }>(); + const { urlParams, uiFilters } = useUrlParams(); + const { start, end } = urlParams; + const { environment } = uiFilters; + + const { data = INITIAL_STATE } = useFetcher(() => { + if (start && end && serviceName) { + return callApmApi({ + endpoint: 'GET /api/apm/services/{serviceName}/annotation/search', + params: { + path: { + serviceName, + }, + query: { + start, + end, + environment, + }, + }, + }); + } + }, [start, end, environment, serviceName]); + + return ; +} diff --git a/x-pack/plugins/apm/public/hooks/use_annotations.ts b/x-pack/plugins/apm/public/hooks/use_annotations.ts index e8f6785706a91..1cd9a7e65dda2 100644 --- a/x-pack/plugins/apm/public/hooks/use_annotations.ts +++ b/x-pack/plugins/apm/public/hooks/use_annotations.ts @@ -3,36 +3,16 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { useParams } from 'react-router-dom'; -import { callApmApi } from '../services/rest/createCallApmApi'; -import { useFetcher } from './useFetcher'; -import { useUrlParams } from './useUrlParams'; -const INITIAL_STATE = { annotations: [] }; +import { useContext } from 'react'; +import { AnnotationsContext } from '../context/annotations_context'; export function useAnnotations() { - const { serviceName } = useParams<{ serviceName?: string }>(); - const { urlParams, uiFilters } = useUrlParams(); - const { start, end } = urlParams; - const { environment } = uiFilters; + const context = useContext(AnnotationsContext); - const { data = INITIAL_STATE } = useFetcher(() => { - if (start && end && serviceName) { - return callApmApi({ - endpoint: 'GET /api/apm/services/{serviceName}/annotation/search', - params: { - path: { - serviceName, - }, - query: { - start, - end, - environment, - }, - }, - }); - } - }, [start, end, environment, serviceName]); + if (!context) { + throw new Error('Missing Annotations context provider'); + } - return data; + return context; } From 284b1046df6981f0da1e743680bf99247523627e Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 25 Nov 2020 10:13:36 -0800 Subject: [PATCH 47/49] [Fleet] Support input-level vars & templates (#83878) * Fix bug creating new policy on the fly * Adjust UI for input with vars but no streams * Revert "Fix bug creating new policy on the fly" This reverts commit 34f7014d6977cf73b792a64bcd9bf7c54b72cf42. * Add `compiled_input` field and compile input template, if any. Make compilation method names more generic (instead of only for streams). Add testts * Add compiled input to generated agent yaml * Don't return empty streams in agent yaml when there aren't any * Update missed assertion Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../package_policies_to_agent_inputs.test.ts | 42 +++++- .../package_policies_to_agent_inputs.ts | 33 +++-- .../fleet/common/types/models/agent_policy.ts | 2 +- .../plugins/fleet/common/types/models/epm.ts | 1 + .../common/types/models/package_policy.ts | 1 + .../package_policy_input_config.tsx | 24 ++-- .../components/package_policy_input_panel.tsx | 32 +++-- x-pack/plugins/fleet/server/mocks.ts | 2 +- .../routes/package_policy/handlers.test.ts | 2 +- .../fleet/server/saved_objects/index.ts | 1 + .../server/services/epm/agent/agent.test.ts | 14 +- .../fleet/server/services/epm/agent/agent.ts | 35 ++--- .../server/services/package_policy.test.ts | 128 +++++++++++++++++- .../fleet/server/services/package_policy.ts | 71 +++++++--- .../apps/endpoint/policy_details.ts | 1 - 15 files changed, 306 insertions(+), 83 deletions(-) diff --git a/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.test.ts b/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.test.ts index f721afb639141..a370f92e97fe1 100644 --- a/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.test.ts +++ b/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.test.ts @@ -100,7 +100,7 @@ describe('Fleet - storedPackagePoliciesToAgentInputs', () => { ).toEqual([]); }); - it('returns agent inputs', () => { + it('returns agent inputs with streams', () => { expect( storedPackagePoliciesToAgentInputs([ { @@ -143,6 +143,46 @@ describe('Fleet - storedPackagePoliciesToAgentInputs', () => { ]); }); + it('returns agent inputs without streams', () => { + expect( + storedPackagePoliciesToAgentInputs([ + { + ...mockPackagePolicy, + package: { + name: 'mock-package', + title: 'Mock package', + version: '0.0.0', + }, + inputs: [ + { + ...mockInput, + compiled_input: { + inputVar: 'input-value', + }, + streams: [], + }, + ], + }, + ]) + ).toEqual([ + { + id: 'some-uuid', + name: 'mock-package-policy', + revision: 1, + type: 'test-logs', + data_stream: { namespace: 'default' }, + use_output: 'default', + meta: { + package: { + name: 'mock-package', + version: '0.0.0', + }, + }, + inputVar: 'input-value', + }, + ]); + }); + it('returns agent inputs without disabled streams', () => { expect( storedPackagePoliciesToAgentInputs([ diff --git a/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.ts b/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.ts index e74256ce732a6..d780fb791aa8e 100644 --- a/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.ts +++ b/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.ts @@ -33,20 +33,25 @@ export const storedPackagePoliciesToAgentInputs = ( acc[key] = value; return acc; }, {} as { [k: string]: any }), - streams: input.streams - .filter((stream) => stream.enabled) - .map((stream) => { - const fullStream: FullAgentPolicyInputStream = { - id: stream.id, - data_stream: stream.data_stream, - ...stream.compiled_stream, - ...Object.entries(stream.config || {}).reduce((acc, [key, { value }]) => { - acc[key] = value; - return acc; - }, {} as { [k: string]: any }), - }; - return fullStream; - }), + ...(input.compiled_input || {}), + ...(input.streams.length + ? { + streams: input.streams + .filter((stream) => stream.enabled) + .map((stream) => { + const fullStream: FullAgentPolicyInputStream = { + id: stream.id, + data_stream: stream.data_stream, + ...stream.compiled_stream, + ...Object.entries(stream.config || {}).reduce((acc, [key, { value }]) => { + acc[key] = value; + return acc; + }, {} as { [k: string]: any }), + }; + return fullStream; + }), + } + : {}), }; if (packagePolicy.package) { diff --git a/x-pack/plugins/fleet/common/types/models/agent_policy.ts b/x-pack/plugins/fleet/common/types/models/agent_policy.ts index f43f65fb317f3..75bb2998f2d92 100644 --- a/x-pack/plugins/fleet/common/types/models/agent_policy.ts +++ b/x-pack/plugins/fleet/common/types/models/agent_policy.ts @@ -49,7 +49,7 @@ export interface FullAgentPolicyInput { package?: Pick; [key: string]: unknown; }; - streams: FullAgentPolicyInputStream[]; + streams?: FullAgentPolicyInputStream[]; [key: string]: any; } diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 7a6f6232b2d4f..53e507f6fb494 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -121,6 +121,7 @@ export interface RegistryInput { title: string; description?: string; vars?: RegistryVarsEntry[]; + template_path?: string; } export interface RegistryStream { diff --git a/x-pack/plugins/fleet/common/types/models/package_policy.ts b/x-pack/plugins/fleet/common/types/models/package_policy.ts index ae16899a4b6f9..6da98a51ef1ff 100644 --- a/x-pack/plugins/fleet/common/types/models/package_policy.ts +++ b/x-pack/plugins/fleet/common/types/models/package_policy.ts @@ -42,6 +42,7 @@ export interface NewPackagePolicyInput { export interface PackagePolicyInput extends Omit { streams: PackagePolicyInputStream[]; + compiled_input?: any; } export interface NewPackagePolicy { diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/package_policy_input_config.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/package_policy_input_config.tsx index 75000ad7e1d3b..9015cd09f78a3 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/package_policy_input_config.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/package_policy_input_config.tsx @@ -27,6 +27,7 @@ const FlexItemWithMaxWidth = styled(EuiFlexItem)` `; export const PackagePolicyInputConfig: React.FunctionComponent<{ + hasInputStreams: boolean; packageInputVars?: RegistryVarsEntry[]; packagePolicyInput: NewPackagePolicyInput; updatePackagePolicyInput: (updatedInput: Partial) => void; @@ -34,6 +35,7 @@ export const PackagePolicyInputConfig: React.FunctionComponent<{ forceShowErrors?: boolean; }> = memo( ({ + hasInputStreams, packageInputVars, packagePolicyInput, updatePackagePolicyInput, @@ -82,15 +84,19 @@ export const PackagePolicyInputConfig: React.FunctionComponent<{ /> - - -

- -

- + {hasInputStreams ? ( + <> + + +

+ +

+ + + ) : null} diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/package_policy_input_panel.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/package_policy_input_panel.tsx index 79ff0cc29850c..8e242980ce807 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/package_policy_input_panel.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/package_policy_input_panel.tsx @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import React, { useState, Fragment, memo } from 'react'; +import React, { useState, Fragment, memo, useMemo } from 'react'; import styled from 'styled-components'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; @@ -85,16 +85,23 @@ export const PackagePolicyInputPanel: React.FunctionComponent<{ const errorCount = countValidationErrors(inputValidationResults); const hasErrors = forceShowErrors && errorCount; - const inputStreams = packageInputStreams - .map((packageInputStream) => { - return { - packageInputStream, - packagePolicyInputStream: packagePolicyInput.streams.find( - (stream) => stream.data_stream.dataset === packageInputStream.data_stream.dataset - ), - }; - }) - .filter((stream) => Boolean(stream.packagePolicyInputStream)); + const hasInputStreams = useMemo(() => packageInputStreams.length > 0, [ + packageInputStreams.length, + ]); + const inputStreams = useMemo( + () => + packageInputStreams + .map((packageInputStream) => { + return { + packageInputStream, + packagePolicyInputStream: packagePolicyInput.streams.find( + (stream) => stream.data_stream.dataset === packageInputStream.data_stream.dataset + ), + }; + }) + .filter((stream) => Boolean(stream.packagePolicyInputStream)), + [packageInputStreams, packagePolicyInput.streams] + ); return ( <> @@ -179,13 +186,14 @@ export const PackagePolicyInputPanel: React.FunctionComponent<{ {isShowingStreams && packageInput.vars && packageInput.vars.length ? ( - + {hasInputStreams ? : } ) : null} diff --git a/x-pack/plugins/fleet/server/mocks.ts b/x-pack/plugins/fleet/server/mocks.ts index 91098c87c312a..bc3e89ef6d3ce 100644 --- a/x-pack/plugins/fleet/server/mocks.ts +++ b/x-pack/plugins/fleet/server/mocks.ts @@ -25,7 +25,7 @@ export const createAppContextStartContractMock = (): FleetAppContext => { export const createPackagePolicyServiceMock = () => { return { - assignPackageStream: jest.fn(), + compilePackagePolicyInputs: jest.fn(), buildPackagePolicyFromPackage: jest.fn(), bulkCreate: jest.fn(), create: jest.fn(), diff --git a/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts b/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts index f47b8499a1b69..fee74e39c833a 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts @@ -22,7 +22,7 @@ jest.mock('../../services/package_policy', (): { } => { return { packagePolicyService: { - assignPackageStream: jest.fn((packageInfo, dataInputs) => Promise.resolve(dataInputs)), + compilePackagePolicyInputs: jest.fn((packageInfo, dataInputs) => Promise.resolve(dataInputs)), buildPackagePolicyFromPackage: jest.fn(), bulkCreate: jest.fn(), create: jest.fn((soClient, callCluster, newData) => diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 9d85a151efbbf..201ca1c7a97bc 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -242,6 +242,7 @@ const getSavedObjectTypes = ( enabled: { type: 'boolean' }, vars: { type: 'flattened' }, config: { type: 'flattened' }, + compiled_input: { type: 'flattened' }, streams: { type: 'nested', properties: { diff --git a/x-pack/plugins/fleet/server/services/epm/agent/agent.test.ts b/x-pack/plugins/fleet/server/services/epm/agent/agent.test.ts index 54b40400bb4e7..dba6f442d76e2 100644 --- a/x-pack/plugins/fleet/server/services/epm/agent/agent.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/agent/agent.test.ts @@ -4,9 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -import { createStream } from './agent'; +import { compileTemplate } from './agent'; -describe('createStream', () => { +describe('compileTemplate', () => { it('should work', () => { const streamTemplate = ` input: log @@ -27,7 +27,7 @@ hidden_password: {{password}} password: { type: 'password', value: '' }, }; - const output = createStream(vars, streamTemplate); + const output = compileTemplate(vars, streamTemplate); expect(output).toEqual({ input: 'log', paths: ['/usr/local/var/log/nginx/access.log'], @@ -67,7 +67,7 @@ foo: bar password: { type: 'password', value: '' }, }; - const output = createStream(vars, streamTemplate); + const output = compileTemplate(vars, streamTemplate); expect(output).toEqual({ input: 'redis/metrics', metricsets: ['key'], @@ -114,7 +114,7 @@ hidden_password: {{password}} tags: { value: ['foo', 'bar', 'forwarded'] }, }; - const output = createStream(vars, streamTemplate); + const output = compileTemplate(vars, streamTemplate); expect(output).toEqual({ input: 'log', paths: ['/usr/local/var/log/nginx/access.log'], @@ -133,7 +133,7 @@ hidden_password: {{password}} tags: { value: ['foo', 'bar'] }, }; - const output = createStream(vars, streamTemplate); + const output = compileTemplate(vars, streamTemplate); expect(output).toEqual({ input: 'log', paths: ['/usr/local/var/log/nginx/access.log'], @@ -157,7 +157,7 @@ input: logs }, }; - const output = createStream(vars, streamTemplate); + const output = compileTemplate(vars, streamTemplate); expect(output).toEqual({ input: 'logs', }); diff --git a/x-pack/plugins/fleet/server/services/epm/agent/agent.ts b/x-pack/plugins/fleet/server/services/epm/agent/agent.ts index eeadac6e168b1..400a688722f99 100644 --- a/x-pack/plugins/fleet/server/services/epm/agent/agent.ts +++ b/x-pack/plugins/fleet/server/services/epm/agent/agent.ts @@ -10,27 +10,30 @@ import { PackagePolicyConfigRecord } from '../../../../common'; const handlebars = Handlebars.create(); -export function createStream(variables: PackagePolicyConfigRecord, streamTemplate: string) { - const { vars, yamlValues } = buildTemplateVariables(variables, streamTemplate); +export function compileTemplate(variables: PackagePolicyConfigRecord, templateStr: string) { + const { vars, yamlValues } = buildTemplateVariables(variables, templateStr); - const template = handlebars.compile(streamTemplate, { noEscape: true }); - let stream = template(vars); - stream = replaceRootLevelYamlVariables(yamlValues, stream); + const template = handlebars.compile(templateStr, { noEscape: true }); + let compiledTemplate = template(vars); + compiledTemplate = replaceRootLevelYamlVariables(yamlValues, compiledTemplate); - const yamlFromStream = safeLoad(stream, {}); + const yamlFromCompiledTemplate = safeLoad(compiledTemplate, {}); // Hack to keep empty string ('') values around in the end yaml because // `safeLoad` replaces empty strings with null - const patchedYamlFromStream = Object.entries(yamlFromStream).reduce((acc, [key, value]) => { - if (value === null && typeof vars[key] === 'string' && vars[key].trim() === '') { - acc[key] = ''; - } else { - acc[key] = value; - } - return acc; - }, {} as { [k: string]: any }); + const patchedYamlFromCompiledTemplate = Object.entries(yamlFromCompiledTemplate).reduce( + (acc, [key, value]) => { + if (value === null && typeof vars[key] === 'string' && vars[key].trim() === '') { + acc[key] = ''; + } else { + acc[key] = value; + } + return acc; + }, + {} as { [k: string]: any } + ); - return replaceVariablesInYaml(yamlValues, patchedYamlFromStream); + return replaceVariablesInYaml(yamlValues, patchedYamlFromCompiledTemplate); } function isValidKey(key: string) { @@ -54,7 +57,7 @@ function replaceVariablesInYaml(yamlVariables: { [k: string]: any }, yaml: any) return yaml; } -function buildTemplateVariables(variables: PackagePolicyConfigRecord, streamTemplate: string) { +function buildTemplateVariables(variables: PackagePolicyConfigRecord, templateStr: string) { const yamlValues: { [k: string]: any } = {}; const vars = Object.entries(variables).reduce((acc, [key, recordEntry]) => { // support variables with . like key.patterns diff --git a/x-pack/plugins/fleet/server/services/package_policy.test.ts b/x-pack/plugins/fleet/server/services/package_policy.test.ts index 6ae76c56436d5..30a980ab07f70 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.test.ts @@ -25,7 +25,16 @@ paths: }, ]; } - return []; + return [ + { + buffer: Buffer.from(` +hosts: +{{#each hosts}} +- {{this}} +{{/each}} +`), + }, + ]; } jest.mock('./epm/packages/assets', () => { @@ -47,9 +56,9 @@ jest.mock('./epm/registry', () => { }); describe('Package policy service', () => { - describe('assignPackageStream', () => { + describe('compilePackagePolicyInputs', () => { it('should work with config variables from the stream', async () => { - const inputs = await packagePolicyService.assignPackageStream( + const inputs = await packagePolicyService.compilePackagePolicyInputs( ({ data_streams: [ { @@ -110,7 +119,7 @@ describe('Package policy service', () => { }); it('should work with config variables at the input level', async () => { - const inputs = await packagePolicyService.assignPackageStream( + const inputs = await packagePolicyService.compilePackagePolicyInputs( ({ data_streams: [ { @@ -169,6 +178,117 @@ describe('Package policy service', () => { }, ]); }); + + it('should work with an input with a template and no streams', async () => { + const inputs = await packagePolicyService.compilePackagePolicyInputs( + ({ + data_streams: [], + policy_templates: [ + { + inputs: [{ type: 'log', template_path: 'some_template_path.yml' }], + }, + ], + } as unknown) as PackageInfo, + [ + { + type: 'log', + enabled: true, + vars: { + hosts: { + value: ['localhost'], + }, + }, + streams: [], + }, + ] + ); + + expect(inputs).toEqual([ + { + type: 'log', + enabled: true, + vars: { + hosts: { + value: ['localhost'], + }, + }, + compiled_input: { + hosts: ['localhost'], + }, + streams: [], + }, + ]); + }); + + it('should work with an input with a template and streams', async () => { + const inputs = await packagePolicyService.compilePackagePolicyInputs( + ({ + data_streams: [ + { + dataset: 'package.dataset1', + type: 'logs', + streams: [{ input: 'log', template_path: 'some_template_path.yml' }], + }, + ], + policy_templates: [ + { + inputs: [{ type: 'log', template_path: 'some_template_path.yml' }], + }, + ], + } as unknown) as PackageInfo, + [ + { + type: 'log', + enabled: true, + vars: { + hosts: { + value: ['localhost'], + }, + paths: { + value: ['/var/log/set.log'], + }, + }, + streams: [ + { + id: 'datastream01', + data_stream: { dataset: 'package.dataset1', type: 'logs' }, + enabled: true, + }, + ], + }, + ] + ); + + expect(inputs).toEqual([ + { + type: 'log', + enabled: true, + vars: { + hosts: { + value: ['localhost'], + }, + paths: { + value: ['/var/log/set.log'], + }, + }, + compiled_input: { + hosts: ['localhost'], + }, + streams: [ + { + id: 'datastream01', + data_stream: { dataset: 'package.dataset1', type: 'logs' }, + enabled: true, + compiled_stream: { + metricset: ['dataset1'], + paths: ['/var/log/set.log'], + type: 'log', + }, + }, + ], + }, + ]); + }); }); describe('update', () => { diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index 0f78c97a6f2bd..7b8952bdea2cd 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -31,7 +31,7 @@ import { outputService } from './output'; import * as Registry from './epm/registry'; import { getPackageInfo, getInstallation, ensureInstalledPackage } from './epm/packages'; import { getAssetsData } from './epm/packages/assets'; -import { createStream } from './epm/agent/agent'; +import { compileTemplate } from './epm/agent/agent'; import { normalizeKuery } from './saved_object'; const SAVED_OBJECT_TYPE = PACKAGE_POLICY_SAVED_OBJECT_TYPE; @@ -92,7 +92,7 @@ class PackagePolicyService { } } - inputs = await this.assignPackageStream(pkgInfo, inputs); + inputs = await this.compilePackagePolicyInputs(pkgInfo, inputs); } const isoDate = new Date().toISOString(); @@ -285,7 +285,7 @@ class PackagePolicyService { pkgVersion: packagePolicy.package.version, }); - inputs = await this.assignPackageStream(pkgInfo, inputs); + inputs = await this.compilePackagePolicyInputs(pkgInfo, inputs); } await soClient.update( @@ -374,14 +374,20 @@ class PackagePolicyService { } } - public async assignPackageStream( + public async compilePackagePolicyInputs( pkgInfo: PackageInfo, inputs: PackagePolicyInput[] ): Promise { const registryPkgInfo = await Registry.fetchInfo(pkgInfo.name, pkgInfo.version); - const inputsPromises = inputs.map((input) => - _assignPackageStreamToInput(registryPkgInfo, pkgInfo, input) - ); + const inputsPromises = inputs.map(async (input) => { + const compiledInput = await _compilePackagePolicyInput(registryPkgInfo, pkgInfo, input); + const compiledStreams = await _compilePackageStreams(registryPkgInfo, pkgInfo, input); + return { + ...input, + compiled_input: compiledInput, + streams: compiledStreams, + }; + }); return Promise.all(inputsPromises); } @@ -396,20 +402,53 @@ function assignStreamIdToInput(packagePolicyId: string, input: NewPackagePolicyI }; } -async function _assignPackageStreamToInput( +async function _compilePackagePolicyInput( + registryPkgInfo: RegistryPackage, + pkgInfo: PackageInfo, + input: PackagePolicyInput +) { + if (!input.enabled || !pkgInfo.policy_templates?.[0].inputs) { + return undefined; + } + + const packageInputs = pkgInfo.policy_templates[0].inputs; + const packageInput = packageInputs.find((pkgInput) => pkgInput.type === input.type); + if (!packageInput) { + throw new Error(`Input template not found, unable to find input type ${input.type}`); + } + + if (!packageInput.template_path) { + return undefined; + } + + const [pkgInputTemplate] = await getAssetsData(registryPkgInfo, (path: string) => + path.endsWith(`/agent/input/${packageInput.template_path!}`) + ); + + if (!pkgInputTemplate || !pkgInputTemplate.buffer) { + throw new Error(`Unable to load input template at /agent/input/${packageInput.template_path!}`); + } + + return compileTemplate( + // Populate template variables from input vars + Object.assign({}, input.vars), + pkgInputTemplate.buffer.toString() + ); +} + +async function _compilePackageStreams( registryPkgInfo: RegistryPackage, pkgInfo: PackageInfo, input: PackagePolicyInput ) { const streamsPromises = input.streams.map((stream) => - _assignPackageStreamToStream(registryPkgInfo, pkgInfo, input, stream) + _compilePackageStream(registryPkgInfo, pkgInfo, input, stream) ); - const streams = await Promise.all(streamsPromises); - return { ...input, streams }; + return await Promise.all(streamsPromises); } -async function _assignPackageStreamToStream( +async function _compilePackageStream( registryPkgInfo: RegistryPackage, pkgInfo: PackageInfo, input: PackagePolicyInput, @@ -442,22 +481,22 @@ async function _assignPackageStreamToStream( throw new Error(`Stream template path not found for dataset ${datasetPath}`); } - const [pkgStream] = await getAssetsData( + const [pkgStreamTemplate] = await getAssetsData( registryPkgInfo, (path: string) => path.endsWith(streamFromPkg.template_path), datasetPath ); - if (!pkgStream || !pkgStream.buffer) { + if (!pkgStreamTemplate || !pkgStreamTemplate.buffer) { throw new Error( `Unable to load stream template ${streamFromPkg.template_path} for dataset ${datasetPath}` ); } - const yaml = createStream( + const yaml = compileTemplate( // Populate template variables from input vars and stream vars Object.assign({}, input.vars, stream.vars), - pkgStream.buffer.toString() + pkgStreamTemplate.buffer.toString() ); stream.compiled_stream = yaml; diff --git a/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts b/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts index b3c130ea1e5dc..46085b0db3063 100644 --- a/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts +++ b/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts @@ -249,7 +249,6 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { }, }, }, - streams: [], type: 'endpoint', use_output: 'default', }, From 497511272308d2aae3f81e26121588aa73dfdd24 Mon Sep 17 00:00:00 2001 From: Spencer Date: Wed, 25 Nov 2020 12:12:28 -0700 Subject: [PATCH 48/49] [cli/dev] log a warning when --no-base-path is used with --dev (#84354) Co-authored-by: spalger --- src/cli/cluster/cluster_manager.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/cli/cluster/cluster_manager.ts b/src/cli/cluster/cluster_manager.ts index 7a14f617b5d5a..b0f7cded938dd 100644 --- a/src/cli/cluster/cluster_manager.ts +++ b/src/cli/cluster/cluster_manager.ts @@ -73,6 +73,19 @@ export class ClusterManager { this.inReplMode = !!opts.repl; this.basePathProxy = basePathProxy; + if (!this.basePathProxy) { + this.log.warn( + '====================================================================================================' + ); + this.log.warn( + 'no-base-path', + 'Running Kibana in dev mode with --no-base-path disables several useful features and is not recommended' + ); + this.log.warn( + '====================================================================================================' + ); + } + // run @kbn/optimizer and write it's state to kbnOptimizerReady$ if (opts.disableOptimizer) { this.kbnOptimizerReady$.next(true); From 5fda30001f69d536bfa7f0f84ecd5f6e7df1dcc8 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com> Date: Wed, 25 Nov 2020 14:47:11 -0500 Subject: [PATCH 49/49] [Security Solution][Resolver] Add support for predefined schemas for endpoint and winlogbeat (#84103) * Refactoring entity route to return schema * Refactoring frontend middleware to pick off id field from entity route * Refactoring schema and adding name and comments * Adding name to schema mocks * Fixing type issue --- .../common/endpoint/types/index.ts | 39 ++++- .../mocks/no_ancestors_two_children.ts | 13 +- ..._children_in_index_called_awesome_index.ts | 13 +- ...ith_related_events_and_cursor_on_origin.ts | 13 +- ..._children_with_related_events_on_origin.ts | 13 +- .../one_node_with_paginated_related_events.ts | 13 +- .../store/middleware/resolver_tree_fetcher.ts | 2 +- .../server/endpoint/routes/resolver/entity.ts | 147 ++++++++++++------ .../resolver/tree/queries/descendants.ts | 8 +- .../routes/resolver/tree/queries/lifecycle.ts | 8 +- .../routes/resolver/tree/queries/stats.ts | 8 +- .../routes/resolver/tree/utils/fetch.test.ts | 9 +- .../routes/resolver/tree/utils/fetch.ts | 25 +-- .../routes/resolver/tree/utils/index.ts | 27 +--- .../apis/resolver/common.ts | 25 +-- .../apis/resolver/entity.ts | 9 +- .../apis/resolver/tree.ts | 12 +- 17 files changed, 259 insertions(+), 125 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/types/index.ts b/x-pack/plugins/security_solution/common/endpoint/types/index.ts index cd5c60e2698cb..d6be83d7cbbe3 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types/index.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types/index.ts @@ -870,10 +870,47 @@ export interface SafeLegacyEndpointEvent { }>; } +/** + * The fields to use to identify nodes within a resolver tree. + */ +export interface ResolverSchema { + /** + * the ancestry field should be set to a field that contains an order array representing + * the ancestors of a node. + */ + ancestry?: string; + /** + * id represents the field to use as the unique ID for a node. + */ + id: string; + /** + * field to use for the name of the node + */ + name?: string; + /** + * parent represents the field that is the edge between two nodes. + */ + parent: string; +} + /** * The response body for the resolver '/entity' index API */ -export type ResolverEntityIndex = Array<{ entity_id: string }>; +export type ResolverEntityIndex = Array<{ + /** + * A name for the schema that is being used (e.g. endpoint, winlogbeat, etc) + */ + name: string; + /** + * The schema to pass to the /tree api and other backend requests, based on the contents of the document found using + * the _id + */ + schema: ResolverSchema; + /** + * Unique ID value for the requested document using the `_id` field passed to the /entity route + */ + id: string; +}>; /** * Takes a @kbn/config-schema 'schema' type and returns a type that represents valid inputs. diff --git a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children.ts b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children.ts index 09625e5726b1d..472fdc79d1f02 100644 --- a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children.ts +++ b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children.ts @@ -99,7 +99,18 @@ export function noAncestorsTwoChildren(): { dataAccessLayer: DataAccessLayer; me * Get entities matching a document. */ entities(): Promise { - return Promise.resolve([{ entity_id: metadata.entityIDs.origin }]); + return Promise.resolve([ + { + name: 'endpoint', + schema: { + id: 'process.entity_id', + parent: 'process.parent.entity_id', + ancestry: 'process.Ext.ancestry', + name: 'process.name', + }, + id: metadata.entityIDs.origin, + }, + ]); }, }, }; diff --git a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_in_index_called_awesome_index.ts b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_in_index_called_awesome_index.ts index 3bbe4bcf51060..b085738d3fd2e 100644 --- a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_in_index_called_awesome_index.ts +++ b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_in_index_called_awesome_index.ts @@ -115,7 +115,18 @@ export function noAncestorsTwoChildenInIndexCalledAwesomeIndex(): { entities({ indices }): Promise { // Only return values if the `indices` array contains exactly `'awesome_index'` if (indices.length === 1 && indices[0] === 'awesome_index') { - return Promise.resolve([{ entity_id: metadata.entityIDs.origin }]); + return Promise.resolve([ + { + name: 'endpoint', + schema: { + id: 'process.entity_id', + parent: 'process.parent.entity_id', + ancestry: 'process.Ext.ancestry', + name: 'process.name', + }, + id: metadata.entityIDs.origin, + }, + ]); } return Promise.resolve([]); }, diff --git a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_and_cursor_on_origin.ts b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_and_cursor_on_origin.ts index 7682165ac5e94..43704db358d7e 100644 --- a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_and_cursor_on_origin.ts +++ b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_and_cursor_on_origin.ts @@ -140,7 +140,18 @@ export function noAncestorsTwoChildrenWithRelatedEventsOnOriginWithOneAfterCurso * Get entities matching a document. */ async entities(): Promise { - return [{ entity_id: metadata.entityIDs.origin }]; + return [ + { + name: 'endpoint', + schema: { + id: 'process.entity_id', + parent: 'process.parent.entity_id', + ancestry: 'process.Ext.ancestry', + name: 'process.name', + }, + id: metadata.entityIDs.origin, + }, + ]; }, }, }; diff --git a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_on_origin.ts b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_on_origin.ts index 837d824db8748..c4d538d2eed94 100644 --- a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_on_origin.ts +++ b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_on_origin.ts @@ -112,7 +112,18 @@ export function noAncestorsTwoChildrenWithRelatedEventsOnOrigin(): { * Get entities matching a document. */ async entities(): Promise { - return [{ entity_id: metadata.entityIDs.origin }]; + return [ + { + name: 'endpoint', + schema: { + id: 'process.entity_id', + parent: 'process.parent.entity_id', + ancestry: 'process.Ext.ancestry', + name: 'process.name', + }, + id: metadata.entityIDs.origin, + }, + ]; }, }, }; diff --git a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/one_node_with_paginated_related_events.ts b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/one_node_with_paginated_related_events.ts index 01477ff16868e..7849776ed1378 100644 --- a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/one_node_with_paginated_related_events.ts +++ b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/one_node_with_paginated_related_events.ts @@ -103,7 +103,18 @@ export function oneNodeWithPaginatedEvents(): { * Get entities matching a document. */ async entities(): Promise { - return [{ entity_id: metadata.entityIDs.origin }]; + return [ + { + name: 'endpoint', + schema: { + id: 'process.entity_id', + parent: 'process.parent.entity_id', + ancestry: 'process.Ext.ancestry', + name: 'process.name', + }, + id: metadata.entityIDs.origin, + }, + ]; }, }, }; diff --git a/x-pack/plugins/security_solution/public/resolver/store/middleware/resolver_tree_fetcher.ts b/x-pack/plugins/security_solution/public/resolver/store/middleware/resolver_tree_fetcher.ts index ef4ca2380ebf4..aecdd6b92a463 100644 --- a/x-pack/plugins/security_solution/public/resolver/store/middleware/resolver_tree_fetcher.ts +++ b/x-pack/plugins/security_solution/public/resolver/store/middleware/resolver_tree_fetcher.ts @@ -56,7 +56,7 @@ export function ResolverTreeFetcher( }); return; } - const entityIDToFetch = matchingEntities[0].entity_id; + const entityIDToFetch = matchingEntities[0].id; result = await dataAccessLayer.resolverTree( entityIDToFetch, lastRequestAbortController.signal diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/entity.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/entity.ts index 510bb6c545558..c731692e6fb89 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/entity.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/entity.ts @@ -3,10 +3,70 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { RequestHandler } from 'kibana/server'; +import _ from 'lodash'; +import { RequestHandler, SearchResponse } from 'kibana/server'; import { TypeOf } from '@kbn/config-schema'; +import { ApiResponse } from '@elastic/elasticsearch'; import { validateEntities } from '../../../../common/endpoint/schema/resolver'; -import { ResolverEntityIndex } from '../../../../common/endpoint/types'; +import { ResolverEntityIndex, ResolverSchema } from '../../../../common/endpoint/types'; + +interface SupportedSchema { + /** + * A name for the schema being used + */ + name: string; + + /** + * A constraint to search for in the documented returned by Elasticsearch + */ + constraint: { field: string; value: string }; + + /** + * Schema to return to the frontend so that it can be passed in to call to the /tree API + */ + schema: ResolverSchema; +} + +/** + * This structure defines the preset supported schemas for a resolver graph. We'll probably want convert this + * implementation to something similar to how row renderers is implemented. + */ +const supportedSchemas: SupportedSchema[] = [ + { + name: 'endpoint', + constraint: { + field: 'agent.type', + value: 'endpoint', + }, + schema: { + id: 'process.entity_id', + parent: 'process.parent.entity_id', + ancestry: 'process.Ext.ancestry', + name: 'process.name', + }, + }, + { + name: 'winlogbeat', + constraint: { + field: 'agent.type', + value: 'winlogbeat', + }, + schema: { + id: 'process.entity_id', + parent: 'process.parent.entity_id', + name: 'process.name', + }, + }, +]; + +function getFieldAsString(doc: unknown, field: string): string | undefined { + const value = _.get(doc, field); + if (value === undefined) { + return undefined; + } + + return String(value); +} /** * This is used to get an 'entity_id' which is an internal-to-Resolver concept, from an `_id`, which @@ -18,61 +78,46 @@ export function handleEntities(): RequestHandler + > = await context.core.elasticsearch.client.asCurrentUser.search({ + ignore_unavailable: true, + index: indices, + body: { + // only return 1 match at most + size: 1, + query: { + bool: { + filter: [ { - _source: { - process?: { - entity_id?: string; - }; - }; - } - ]; - }; - } - - const queryResponse: ExpectedQueryResponse = await context.core.elasticsearch.legacy.client.callAsCurrentUser( - 'search', - { - ignoreUnavailable: true, - index: indices, - body: { - // only return process.entity_id - _source: 'process.entity_id', - // only return 1 match at most - size: 1, - query: { - bool: { - filter: [ - { - // only return documents with the matching _id - ids: { - values: _id, - }, + // only return documents with the matching _id + ids: { + values: _id, }, - ], - }, + }, + ], }, }, - } - ); + }, + }); const responseBody: ResolverEntityIndex = []; - for (const hit of queryResponse.hits.hits) { - // check that the field is defined and that is not an empty string - if (hit._source.process?.entity_id) { - responseBody.push({ - entity_id: hit._source.process.entity_id, - }); + for (const hit of queryResponse.body.hits.hits) { + for (const supportedSchema of supportedSchemas) { + const fieldValue = getFieldAsString(hit._source, supportedSchema.constraint.field); + const id = getFieldAsString(hit._source, supportedSchema.schema.id); + // check that the constraint and id fields are defined and that the id field is not an empty string + if ( + fieldValue?.toLowerCase() === supportedSchema.constraint.value.toLowerCase() && + id !== undefined && + id !== '' + ) { + responseBody.push({ + name: supportedSchema.name, + schema: supportedSchema.schema, + id, + }); + } } } return response.ok({ body: responseBody }); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/descendants.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/descendants.ts index 405429cc24191..3baf3a8667529 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/descendants.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/descendants.ts @@ -6,12 +6,12 @@ import { SearchResponse } from 'elasticsearch'; import { ApiResponse } from '@elastic/elasticsearch'; import { IScopedClusterClient } from 'src/core/server'; -import { FieldsObject } from '../../../../../../common/endpoint/types'; +import { FieldsObject, ResolverSchema } from '../../../../../../common/endpoint/types'; import { JsonObject, JsonValue } from '../../../../../../../../../src/plugins/kibana_utils/common'; -import { NodeID, Schema, Timerange, docValueFields } from '../utils/index'; +import { NodeID, Timerange, docValueFields } from '../utils/index'; interface DescendantsParams { - schema: Schema; + schema: ResolverSchema; indexPatterns: string | string[]; timerange: Timerange; } @@ -20,7 +20,7 @@ interface DescendantsParams { * Builds a query for retrieving descendants of a node. */ export class DescendantsQuery { - private readonly schema: Schema; + private readonly schema: ResolverSchema; private readonly indexPatterns: string | string[]; private readonly timerange: Timerange; private readonly docValueFields: JsonValue[]; diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/lifecycle.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/lifecycle.ts index 606a4538ba88c..5253806be66ba 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/lifecycle.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/lifecycle.ts @@ -6,12 +6,12 @@ import { SearchResponse } from 'elasticsearch'; import { ApiResponse } from '@elastic/elasticsearch'; import { IScopedClusterClient } from 'src/core/server'; -import { FieldsObject } from '../../../../../../common/endpoint/types'; +import { FieldsObject, ResolverSchema } from '../../../../../../common/endpoint/types'; import { JsonObject, JsonValue } from '../../../../../../../../../src/plugins/kibana_utils/common'; -import { NodeID, Schema, Timerange, docValueFields } from '../utils/index'; +import { NodeID, Timerange, docValueFields } from '../utils/index'; interface LifecycleParams { - schema: Schema; + schema: ResolverSchema; indexPatterns: string | string[]; timerange: Timerange; } @@ -20,7 +20,7 @@ interface LifecycleParams { * Builds a query for retrieving descendants of a node. */ export class LifecycleQuery { - private readonly schema: Schema; + private readonly schema: ResolverSchema; private readonly indexPatterns: string | string[]; private readonly timerange: Timerange; private readonly docValueFields: JsonValue[]; diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/stats.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/stats.ts index 33dcdce8987f5..117cc3647dd0e 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/stats.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/stats.ts @@ -7,8 +7,8 @@ import { SearchResponse } from 'elasticsearch'; import { ApiResponse } from '@elastic/elasticsearch'; import { IScopedClusterClient } from 'src/core/server'; import { JsonObject } from '../../../../../../../../../src/plugins/kibana_utils/common'; -import { EventStats } from '../../../../../../common/endpoint/types'; -import { NodeID, Schema, Timerange } from '../utils/index'; +import { EventStats, ResolverSchema } from '../../../../../../common/endpoint/types'; +import { NodeID, Timerange } from '../utils/index'; interface AggBucket { key: string; @@ -26,7 +26,7 @@ interface CategoriesAgg extends AggBucket { } interface StatsParams { - schema: Schema; + schema: ResolverSchema; indexPatterns: string | string[]; timerange: Timerange; } @@ -35,7 +35,7 @@ interface StatsParams { * Builds a query for retrieving descendants of a node. */ export class StatsQuery { - private readonly schema: Schema; + private readonly schema: ResolverSchema; private readonly indexPatterns: string | string[]; private readonly timerange: Timerange; constructor({ schema, indexPatterns, timerange }: StatsParams) { diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.test.ts index 8105f1125d01d..d5e0af9dea239 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.test.ts @@ -18,14 +18,17 @@ import { DescendantsQuery } from '../queries/descendants'; import { StatsQuery } from '../queries/stats'; import { IScopedClusterClient } from 'src/core/server'; import { elasticsearchServiceMock } from 'src/core/server/mocks'; -import { FieldsObject, ResolverNode } from '../../../../../../common/endpoint/types'; -import { Schema } from './index'; +import { + FieldsObject, + ResolverNode, + ResolverSchema, +} from '../../../../../../common/endpoint/types'; jest.mock('../queries/descendants'); jest.mock('../queries/lifecycle'); jest.mock('../queries/stats'); -function formatResponse(results: FieldsObject[], schema: Schema): ResolverNode[] { +function formatResponse(results: FieldsObject[], schema: ResolverSchema): ResolverNode[] { return results.map((node) => { return { id: getIDField(node, schema) ?? '', diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.ts index eaecad6c47970..356357082d6ee 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.ts @@ -8,9 +8,14 @@ import { firstNonNullValue, values, } from '../../../../../../common/endpoint/models/ecs_safety_helpers'; -import { ECSField, ResolverNode, FieldsObject } from '../../../../../../common/endpoint/types'; +import { + ECSField, + ResolverNode, + FieldsObject, + ResolverSchema, +} from '../../../../../../common/endpoint/types'; import { DescendantsQuery } from '../queries/descendants'; -import { Schema, NodeID } from './index'; +import { NodeID } from './index'; import { LifecycleQuery } from '../queries/lifecycle'; import { StatsQuery } from '../queries/stats'; @@ -26,7 +31,7 @@ export interface TreeOptions { from: string; to: string; }; - schema: Schema; + schema: ResolverSchema; nodes: NodeID[]; indexPatterns: string[]; } @@ -98,7 +103,7 @@ export class Fetcher { private static getNextAncestorsToFind( results: FieldsObject[], - schema: Schema, + schema: ResolverSchema, levelsLeft: number ): NodeID[] { const nodesByID = results.reduce((accMap: Map, result: FieldsObject) => { @@ -216,7 +221,7 @@ export class Fetcher { export function getLeafNodes( results: FieldsObject[], nodes: Array, - schema: Schema + schema: ResolverSchema ): NodeID[] { let largestAncestryArray = 0; const nodesToQueryNext: Map> = new Map(); @@ -269,7 +274,7 @@ export function getLeafNodes( * @param obj the doc value fields retrieved from a document returned by Elasticsearch * @param schema the schema used for identifying connections between documents */ -export function getIDField(obj: FieldsObject, schema: Schema): NodeID | undefined { +export function getIDField(obj: FieldsObject, schema: ResolverSchema): NodeID | undefined { const id: ECSField = obj[schema.id]; return firstNonNullValue(id); } @@ -281,7 +286,7 @@ export function getIDField(obj: FieldsObject, schema: Schema): NodeID | undefine * @param obj the doc value fields retrieved from a document returned by Elasticsearch * @param schema the schema used for identifying connections between documents */ -export function getNameField(obj: FieldsObject, schema: Schema): string | undefined { +export function getNameField(obj: FieldsObject, schema: ResolverSchema): string | undefined { if (!schema.name) { return undefined; } @@ -297,12 +302,12 @@ export function getNameField(obj: FieldsObject, schema: Schema): string | undefi * @param obj the doc value fields retrieved from a document returned by Elasticsearch * @param schema the schema used for identifying connections between documents */ -export function getParentField(obj: FieldsObject, schema: Schema): NodeID | undefined { +export function getParentField(obj: FieldsObject, schema: ResolverSchema): NodeID | undefined { const parent: ECSField = obj[schema.parent]; return firstNonNullValue(parent); } -function getAncestryField(obj: FieldsObject, schema: Schema): NodeID[] | undefined { +function getAncestryField(obj: FieldsObject, schema: ResolverSchema): NodeID[] | undefined { if (!schema.ancestry) { return undefined; } @@ -324,7 +329,7 @@ function getAncestryField(obj: FieldsObject, schema: Schema): NodeID[] | undefin * @param obj the doc value fields retrieved from a document returned by Elasticsearch * @param schema the schema used for identifying connections between documents */ -export function getAncestryAsArray(obj: FieldsObject, schema: Schema): NodeID[] { +export function getAncestryAsArray(obj: FieldsObject, schema: ResolverSchema): NodeID[] { const ancestry = getAncestryField(obj, schema); if (!ancestry || ancestry.length <= 0) { const parentField = getParentField(obj, schema); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/index.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/index.ts index 21a49e268310b..be08b4390a69c 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/index.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/index.ts @@ -4,6 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ +import { ResolverSchema } from '../../../../../../common/endpoint/types'; + /** * Represents a time range filter */ @@ -17,29 +19,6 @@ export interface Timerange { */ export type NodeID = string | number; -/** - * The fields to use to identify nodes within a resolver tree. - */ -export interface Schema { - /** - * the ancestry field should be set to a field that contains an order array representing - * the ancestors of a node. - */ - ancestry?: string; - /** - * id represents the field to use as the unique ID for a node. - */ - id: string; - /** - * field to use for the name of the node - */ - name?: string; - /** - * parent represents the field that is the edge between two nodes. - */ - parent: string; -} - /** * Returns the doc value fields filter to use in queries to limit the number of fields returned in the * query response. @@ -49,7 +28,7 @@ export interface Schema { * @param schema is the node schema information describing how relationships are formed between nodes * in the resolver graph. */ -export function docValueFields(schema: Schema): Array<{ field: string }> { +export function docValueFields(schema: ResolverSchema): Array<{ field: string }> { const filter = [{ field: '@timestamp' }, { field: schema.id }, { field: schema.parent }]; if (schema.ancestry) { filter.push({ field: schema.ancestry }); diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/common.ts b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/common.ts index b4e98d7d4b95e..3cc833c6a2475 100644 --- a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/common.ts +++ b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/common.ts @@ -6,16 +6,14 @@ import _ from 'lodash'; import expect from '@kbn/expect'; import { firstNonNullValue } from '../../../../plugins/security_solution/common/endpoint/models/ecs_safety_helpers'; -import { - NodeID, - Schema, -} from '../../../../plugins/security_solution/server/endpoint/routes/resolver/tree/utils'; +import { NodeID } from '../../../../plugins/security_solution/server/endpoint/routes/resolver/tree/utils'; import { SafeResolverChildNode, SafeResolverLifecycleNode, SafeResolverEvent, ResolverNodeStats, ResolverNode, + ResolverSchema, } from '../../../../plugins/security_solution/common/endpoint/types'; import { parentEntityIDSafeVersion, @@ -41,7 +39,7 @@ const createLevels = ({ descendantsByParent: Map>; levels: Array>; currentNodes: Map | undefined; - schema: Schema; + schema: ResolverSchema; }): Array> => { if (!currentNodes || currentNodes.size === 0) { return levels; @@ -98,7 +96,7 @@ export interface APIResponse { * @param node a resolver node * @param schema the schema that was used to retrieve this resolver node */ -export const getID = (node: ResolverNode | undefined, schema: Schema): NodeID => { +export const getID = (node: ResolverNode | undefined, schema: ResolverSchema): NodeID => { const id = firstNonNullValue(node?.data[schema.id]); if (!id) { throw new Error(`Unable to find id ${schema.id} in node: ${JSON.stringify(node)}`); @@ -106,7 +104,10 @@ export const getID = (node: ResolverNode | undefined, schema: Schema): NodeID => return id; }; -const getParentInternal = (node: ResolverNode | undefined, schema: Schema): NodeID | undefined => { +const getParentInternal = ( + node: ResolverNode | undefined, + schema: ResolverSchema +): NodeID | undefined => { if (node) { return firstNonNullValue(node?.data[schema.parent]); } @@ -119,7 +120,7 @@ const getParentInternal = (node: ResolverNode | undefined, schema: Schema): Node * @param node a resolver node * @param schema the schema that was used to retrieve this resolver node */ -export const getParent = (node: ResolverNode | undefined, schema: Schema): NodeID => { +export const getParent = (node: ResolverNode | undefined, schema: ResolverSchema): NodeID => { const parent = getParentInternal(node, schema); if (!parent) { throw new Error(`Unable to find parent ${schema.parent} in node: ${JSON.stringify(node)}`); @@ -138,7 +139,7 @@ export const getParent = (node: ResolverNode | undefined, schema: Schema): NodeI const createTreeFromResponse = ( treeExpectations: TreeExpectation[], nodes: ResolverNode[], - schema: Schema + schema: ResolverSchema ) => { const nodesByID = new Map(); const nodesByParent = new Map>(); @@ -206,7 +207,7 @@ const verifyAncestry = ({ genTree, }: { responseTrees: APIResponse; - schema: Schema; + schema: ResolverSchema; genTree: Tree; }) => { const allGenNodes = new Map([ @@ -277,7 +278,7 @@ const verifyChildren = ({ genTree, }: { responseTrees: APIResponse; - schema: Schema; + schema: ResolverSchema; genTree: Tree; }) => { const allGenNodes = new Map([ @@ -358,7 +359,7 @@ export const verifyTree = ({ }: { expectations: TreeExpectation[]; response: ResolverNode[]; - schema: Schema; + schema: ResolverSchema; genTree: Tree; relatedEventsCategories?: RelatedEventInfo[]; }) => { diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/entity.ts b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/entity.ts index 7fbba4e04798d..2607b934e7df2 100644 --- a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/entity.ts +++ b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/entity.ts @@ -29,8 +29,15 @@ export default function ({ getService }: FtrProviderContext) { ); expect(body).eql([ { + name: 'endpoint', + schema: { + id: 'process.entity_id', + parent: 'process.parent.entity_id', + ancestry: 'process.Ext.ancestry', + name: 'process.name', + }, // this value is from the es archive - entity_id: + id: 'MTIwNWY1NWQtODRkYS00MzkxLWIyNWQtYTNkNGJmNDBmY2E1LTc1NTItMTMyNDM1NDY1MTQuNjI0MjgxMDA=', }, ]); diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/tree.ts b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/tree.ts index 646a666629ac9..7a1210c6b762f 100644 --- a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/tree.ts +++ b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/tree.ts @@ -5,8 +5,10 @@ */ import expect from '@kbn/expect'; import { getNameField } from '../../../../plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch'; -import { Schema } from '../../../../plugins/security_solution/server/endpoint/routes/resolver/tree/utils'; -import { ResolverNode } from '../../../../plugins/security_solution/common/endpoint/types'; +import { + ResolverNode, + ResolverSchema, +} from '../../../../plugins/security_solution/common/endpoint/types'; import { parentEntityIDSafeVersion, timestampSafeVersion, @@ -44,18 +46,18 @@ export default function ({ getService }: FtrProviderContext) { ancestryArraySize: 2, }; - const schemaWithAncestry: Schema = { + const schemaWithAncestry: ResolverSchema = { ancestry: 'process.Ext.ancestry', id: 'process.entity_id', parent: 'process.parent.entity_id', }; - const schemaWithoutAncestry: Schema = { + const schemaWithoutAncestry: ResolverSchema = { id: 'process.entity_id', parent: 'process.parent.entity_id', }; - const schemaWithName: Schema = { + const schemaWithName: ResolverSchema = { id: 'process.entity_id', parent: 'process.parent.entity_id', name: 'process.name',