diff --git a/.github/ISSUE_TEMPLATE/Bug_report_security_solution.md b/.github/ISSUE_TEMPLATE/Bug_report_security_solution.md index 059e1d267c286..7a0514bca621d 100644 --- a/.github/ISSUE_TEMPLATE/Bug_report_security_solution.md +++ b/.github/ISSUE_TEMPLATE/Bug_report_security_solution.md @@ -1,38 +1,38 @@ ---- -name: Bug report for Security Solution -about: Help us identify bugs in Elastic Security, SIEM, and Endpoint so we can fix them! -title: '[Security Solution]' -labels: 'Team: SecuritySolution' ---- - -**Describe the bug:** - -**Kibana/Elasticsearch Stack version:** - -**Server OS version:** - -**Browser and Browser OS versions:** - -**Elastic Endpoint version:** - -**Original install method (e.g. download page, yum, from source, etc.):** - -**Functional Area (e.g. Endpoint management, timelines, resolver, etc.):** - -**Steps to reproduce:** - -1. -2. -3. - -**Current behavior:** - -**Expected behavior:** - -**Screenshots (if relevant):** - -**Errors in browser console (if relevant):** - -**Provide logs and/or server output (if relevant):** - -**Any additional context (logs, chat logs, magical formulas, etc.):** +--- +name: Bug report for Security Solution +about: Help us identify bugs in Elastic Security, SIEM, and Endpoint so we can fix them! +title: '[Security Solution]' +labels: 'bug, Team: SecuritySolution' +--- + +**Describe the bug:** + +**Kibana/Elasticsearch Stack version:** + +**Server OS version:** + +**Browser and Browser OS versions:** + +**Elastic Endpoint version:** + +**Original install method (e.g. download page, yum, from source, etc.):** + +**Functional Area (e.g. Endpoint management, timelines, resolver, etc.):** + +**Steps to reproduce:** + +1. +2. +3. + +**Current behavior:** + +**Expected behavior:** + +**Screenshots (if relevant):** + +**Errors in browser console (if relevant):** + +**Provide logs and/or server output (if relevant):** + +**Any additional context (logs, chat logs, magical formulas, etc.):** diff --git a/src/plugins/discover/public/application/angular/context_app.html b/src/plugins/discover/public/application/angular/context_app.html index d609a497c4ba1..ef7bc09206176 100644 --- a/src/plugins/discover/public/application/angular/context_app.html +++ b/src/plugins/discover/public/application/angular/context_app.html @@ -10,7 +10,6 @@ > - - - - - - - + predecessor-count="contextApp.state.queryParameters.predecessorCount" + predecessor-available="contextApp.state.rows.predecessors.length" + predecessor-status="contextApp.state.loadingStatus.predecessors.status" + on-change-predecessor-count="contextApp.actions.fetchGivenPredecessorRows" + successor-count="contextApp.state.queryParameters.successorCount" + successor-available="contextApp.state.rows.successors.length" + successor-status="contextApp.state.loadingStatus.successors.status" + on-change-successor-count="contextApp.actions.fetchGivenSuccessorRows" + > diff --git a/src/plugins/discover/public/application/components/context_app/__snapshots__/context_app_legacy.test.tsx.snap b/src/plugins/discover/public/application/components/context_app/__snapshots__/context_app_legacy.test.tsx.snap deleted file mode 100644 index 58305ee23cb21..0000000000000 --- a/src/plugins/discover/public/application/components/context_app/__snapshots__/context_app_legacy.test.tsx.snap +++ /dev/null @@ -1,741 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`ContextAppLegacy test renders correctly 1`] = ` - - - - - -
-
- -
- -
-
- - - - - -`; - -exports[`ContextAppLegacy test renders loading indicator 1`] = ` - - - - - -
- -
- -
- - Loading... - -
- -
- -
- - - - - -`; diff --git a/src/plugins/discover/public/application/components/context_app/context_app_legacy.test.tsx b/src/plugins/discover/public/application/components/context_app/context_app_legacy.test.tsx index 16d8cd78004f9..25576a9072944 100644 --- a/src/plugins/discover/public/application/components/context_app/context_app_legacy.test.tsx +++ b/src/plugins/discover/public/application/components/context_app/context_app_legacy.test.tsx @@ -23,6 +23,7 @@ import { IIndexPattern } from '../../../../../data/common/index_patterns'; import { mountWithIntl } from 'test_utils/enzyme_helpers'; import { DocTableLegacy } from '../../angular/doc_table/create_doc_table_react'; import { findTestSubject } from '@elastic/eui/lib/test'; +import { ActionBar } from '../../angular/context/components/action_bar/action_bar'; describe('ContextAppLegacy test', () => { const hit = { @@ -48,28 +49,36 @@ describe('ContextAppLegacy test', () => { columns: ['_source'], filter: () => {}, hits: [hit], - infiniteScroll: true, sorting: ['order_date', 'desc'], minimumVisibleRows: 5, indexPattern, status: 'loaded', + defaultStepSize: 5, + predecessorCount: 10, + successorCount: 10, + predecessorAvailable: 10, + successorAvailable: 10, + onChangePredecessorCount: jest.fn(), + onChangeSuccessorCount: jest.fn(), + predecessorStatus: 'loaded', + successorStatus: 'loaded', }; it('renders correctly', () => { const component = mountWithIntl(); - expect(component).toMatchSnapshot(); expect(component.find(DocTableLegacy).length).toBe(1); const loadingIndicator = findTestSubject(component, 'contextApp_loadingIndicator'); expect(loadingIndicator.length).toBe(0); + expect(component.find(ActionBar).length).toBe(2); }); it('renders loading indicator', () => { const props = { ...defaultProps }; props.status = 'loading'; const component = mountWithIntl(); - expect(component).toMatchSnapshot(); expect(component.find('DocTableLegacy').length).toBe(0); const loadingIndicator = findTestSubject(component, 'contextApp_loadingIndicator'); expect(loadingIndicator.length).toBe(1); + expect(component.find(ActionBar).length).toBe(2); }); }); diff --git a/src/plugins/discover/public/application/components/context_app/context_app_legacy.tsx b/src/plugins/discover/public/application/components/context_app/context_app_legacy.tsx index ee8b2f590f71c..afb4a9a981e21 100644 --- a/src/plugins/discover/public/application/components/context_app/context_app_legacy.tsx +++ b/src/plugins/discover/public/application/components/context_app/context_app_legacy.tsx @@ -17,15 +17,15 @@ * under the License. */ import React from 'react'; -import { FormattedMessage } from '@kbn/i18n/react'; +import { FormattedMessage, I18nProvider } from '@kbn/i18n/react'; import { EuiPanel, EuiText } from '@elastic/eui'; -import { I18nProvider } from '@kbn/i18n/react'; import { DocTableLegacy, DocTableLegacyProps, } from '../../angular/doc_table/create_doc_table_react'; import { IIndexPattern, IndexPatternField } from '../../../../../data/common/index_patterns'; import { LOADING_STATUS } from './constants'; +import { ActionBar, ActionBarProps } from '../../angular/context/components/action_bar/action_bar'; export interface ContextAppProps { columns: string[]; @@ -35,15 +35,64 @@ export interface ContextAppProps { minimumVisibleRows: number; sorting: string[]; status: string; + defaultStepSize: number; + predecessorCount: number; + successorCount: number; + predecessorAvailable: number; + successorAvailable: number; + onChangePredecessorCount: (count: number) => void; + onChangeSuccessorCount: (count: number) => void; + predecessorStatus: string; + successorStatus: string; +} + +const PREDECESSOR_TYPE = 'predecessors'; +const SUCCESSOR_TYPE = 'successors'; + +function isLoading(status: string) { + return status !== LOADING_STATUS.LOADED && status !== LOADING_STATUS.FAILED; } export function ContextAppLegacy(renderProps: ContextAppProps) { - const { hits, filter, sorting, status } = renderProps; - const props = ({ ...renderProps } as unknown) as DocTableLegacyProps; - props.rows = hits; - props.onFilter = filter; - props.sort = sorting.map((el) => [el]); + const status = renderProps.status; const isLoaded = status === LOADING_STATUS.LOADED; + + const actionBarProps = (type: string) => { + const { + defaultStepSize, + successorCount, + predecessorCount, + predecessorAvailable, + successorAvailable, + predecessorStatus, + successorStatus, + onChangePredecessorCount, + onChangeSuccessorCount, + } = renderProps; + const isPredecessorType = type === PREDECESSOR_TYPE; + return { + defaultStepSize, + docCount: isPredecessorType ? predecessorCount : successorCount, + docCountAvailable: isPredecessorType ? predecessorAvailable : successorAvailable, + onChangeCount: isPredecessorType ? onChangePredecessorCount : onChangeSuccessorCount, + isLoading: isPredecessorType ? isLoading(predecessorStatus) : isLoading(successorStatus), + type, + isDisabled: !isLoaded, + } as ActionBarProps; + }; + + const docTableProps = () => { + const { hits, filter, sorting, columns, indexPattern, minimumVisibleRows } = renderProps; + return { + columns, + indexPattern, + minimumVisibleRows, + rows: hits, + onFilter: filter, + sort: sorting.map((el) => [el]), + } as DocTableLegacyProps; + }; + const loadingFeedback = () => { if (status === LOADING_STATUS.UNINITIALIZED || status === LOADING_STATUS.LOADING) { return ( @@ -59,17 +108,20 @@ export function ContextAppLegacy(renderProps: ContextAppProps) { } return null; }; + return ( + {loadingFeedback()} {isLoaded ? (
- +
) : null} + ); diff --git a/src/plugins/discover/public/application/components/context_app/context_app_legacy_directive.ts b/src/plugins/discover/public/application/components/context_app/context_app_legacy_directive.ts index af94c5537da28..4a315be513a0d 100644 --- a/src/plugins/discover/public/application/components/context_app/context_app_legacy_directive.ts +++ b/src/plugins/discover/public/application/components/context_app/context_app_legacy_directive.ts @@ -25,8 +25,16 @@ export function createContextAppLegacy(reactDirective: any) { ['indexPattern', { watchDepth: 'reference' }], ['sorting', { watchDepth: 'reference' }], ['columns', { watchDepth: 'collection' }], - ['infiniteScroll', { watchDepth: 'reference' }], ['minimumVisibleRows', { watchDepth: 'reference' }], ['status', { watchDepth: 'reference' }], + ['defaultStepSize', { watchDepth: 'reference' }], + ['predecessorCount', { watchDepth: 'reference' }], + ['predecessorAvailable', { watchDepth: 'reference' }], + ['predecessorStatus', { watchDepth: 'reference' }], + ['onChangePredecessorCount', { watchDepth: 'reference' }], + ['successorCount', { watchDepth: 'reference' }], + ['successorAvailable', { watchDepth: 'reference' }], + ['successorStatus', { watchDepth: 'reference' }], + ['onChangeSuccessorCount', { watchDepth: 'reference' }], ]); } diff --git a/src/plugins/vis_default_editor/public/components/agg_params_helper.test.ts b/src/plugins/vis_default_editor/public/components/agg_params_helper.test.ts index 834ad8b70ad0d..a56155db02f6b 100644 --- a/src/plugins/vis_default_editor/public/components/agg_params_helper.test.ts +++ b/src/plugins/vis_default_editor/public/components/agg_params_helper.test.ts @@ -33,6 +33,7 @@ import { FieldParamEditor, OrderByParamEditor } from './controls'; import { EditorConfig } from './utils'; import { Schema } from '../schemas'; import { EditorVisState } from './sidebar/state/reducers'; +import { groupAndSortBy } from '../utils'; jest.mock('../utils', () => ({ groupAndSortBy: jest.fn(() => ['indexedFields']), @@ -169,6 +170,9 @@ describe('DefaultEditorAggParams helpers', () => { ], advanced: [], }); + + // Should be grouped using displayName as label + expect(groupAndSortBy).toHaveBeenCalledWith(expect.anything(), 'type', 'displayName', 'name'); }); }); diff --git a/src/plugins/vis_default_editor/public/components/agg_params_helper.ts b/src/plugins/vis_default_editor/public/components/agg_params_helper.ts index b13ca32601aa9..271fc75a0853e 100644 --- a/src/plugins/vis_default_editor/public/components/agg_params_helper.ts +++ b/src/plugins/vis_default_editor/public/components/agg_params_helper.ts @@ -93,7 +93,7 @@ function getAggParamsToRender({ } } fields = filterAggTypeFields(availableFields, agg); - indexedFields = groupAndSortBy(fields, 'type', 'name'); + indexedFields = groupAndSortBy(fields, 'type', 'displayName', 'name'); if (fields && !indexedFields.length && index > 0) { // don't draw the rest of the options if there are no indexed fields and it's an extra param (index > 0). diff --git a/src/plugins/vis_default_editor/public/components/controls/field.tsx b/src/plugins/vis_default_editor/public/components/controls/field.tsx index bfc4f881f8458..41d6db25da5e2 100644 --- a/src/plugins/vis_default_editor/public/components/controls/field.tsx +++ b/src/plugins/vis_default_editor/public/components/controls/field.tsx @@ -52,7 +52,7 @@ function FieldParamEditor({ }: FieldParamEditorProps) { const [isDirty, setIsDirty] = useState(false); const selectedOptions: ComboBoxGroupedOptions = value - ? [{ label: value.displayName || value.name, target: value }] + ? [{ label: value.displayName, target: value, key: value.name }] : []; const onChange = (options: EuiComboBoxOptionOption[]) => { diff --git a/src/plugins/vis_default_editor/public/utils.ts b/src/plugins/vis_default_editor/public/utils.ts index d0a9c067e9da2..11b7c07acc2c2 100644 --- a/src/plugins/vis_default_editor/public/utils.ts +++ b/src/plugins/vis_default_editor/public/utils.ts @@ -18,10 +18,12 @@ */ interface ComboBoxOption { + key?: string; label: string; target: T; } interface ComboBoxGroupedOption { + key?: string; label: string; options: Array>; } @@ -40,15 +42,22 @@ export type ComboBoxGroupedOptions = Array>; * @returns An array of grouped and sorted alphabetically `objects` that are compatible with EuiComboBox options. */ export function groupAndSortBy< - T extends Record, + T extends Record, TGroupBy extends string = 'type', - TLabelName extends string = 'title' ->(objects: T[], groupBy: TGroupBy, labelName: TLabelName): ComboBoxGroupedOptions { + TLabelName extends string = 'title', + TKeyName extends string = never +>( + objects: T[], + groupBy: TGroupBy, + labelName: TLabelName, + keyName?: TKeyName +): ComboBoxGroupedOptions { const groupedOptions = objects.reduce((array, obj) => { const group = array.find((element) => element.label === obj[groupBy]); const option = { label: obj[labelName], target: obj, + ...(keyName ? { key: obj[keyName] } : {}), }; if (group && group.options) { diff --git a/x-pack/plugins/lens/public/app_plugin/mounter.tsx b/x-pack/plugins/lens/public/app_plugin/mounter.tsx index 3943cbc54f0b3..ac5d145eedd5b 100644 --- a/x-pack/plugins/lens/public/app_plugin/mounter.tsx +++ b/x-pack/plugins/lens/public/app_plugin/mounter.tsx @@ -97,9 +97,12 @@ export async function mountApp( const redirectTo = (routeProps: RouteComponentProps<{ id?: string }>, savedObjectId?: string) => { if (!savedObjectId) { - routeProps.history.push('/'); + routeProps.history.push({ pathname: '/', search: routeProps.history.location.search }); } else { - routeProps.history.push(`/edit/${savedObjectId}`); + routeProps.history.push({ + pathname: `/edit/${savedObjectId}`, + search: routeProps.history.location.search, + }); } }; diff --git a/x-pack/plugins/lens/server/plugin.tsx b/x-pack/plugins/lens/server/plugin.tsx index b801d30f5ba9b..a8f9bef92349c 100644 --- a/x-pack/plugins/lens/server/plugin.tsx +++ b/x-pack/plugins/lens/server/plugin.tsx @@ -29,13 +29,13 @@ export class LensServerPlugin implements Plugin<{}, {}, {}, {}> { private readonly kibanaIndexConfig: Observable<{ kibana: { index: string } }>; private readonly telemetryLogger: Logger; - constructor(initializerContext: PluginInitializerContext) { + constructor(private initializerContext: PluginInitializerContext) { this.kibanaIndexConfig = initializerContext.config.legacy.globalConfig$; this.telemetryLogger = initializerContext.logger.get('usage'); } setup(core: CoreSetup, plugins: PluginSetupContract) { setupSavedObjects(core); - setupRoutes(core); + setupRoutes(core, this.initializerContext.logger.get()); if (plugins.usageCollection && plugins.taskManager) { registerLensUsageCollector( plugins.usageCollection, diff --git a/x-pack/plugins/lens/server/routes/existing_fields.ts b/x-pack/plugins/lens/server/routes/existing_fields.ts index c925517b572da..d3b2314a199cb 100644 --- a/x-pack/plugins/lens/server/routes/existing_fields.ts +++ b/x-pack/plugins/lens/server/routes/existing_fields.ts @@ -7,10 +7,14 @@ import Boom from 'boom'; import { schema } from '@kbn/config-schema'; import { ILegacyScopedClusterClient, SavedObject, RequestHandlerContext } from 'src/core/server'; -import { CoreSetup } from 'src/core/server'; +import { CoreSetup, Logger } from 'src/core/server'; import { BASE_API_URL } from '../../common'; import { IndexPatternAttributes, UI_SETTINGS } from '../../../../../src/plugins/data/server'; +export function isBoomError(error: { isBoom?: boolean }): error is Boom { + return error.isBoom === true; +} + /** * The number of docs to sample to determine field empty status. */ @@ -24,7 +28,7 @@ export interface Field { script?: string; } -export async function existingFieldsRoute(setup: CoreSetup) { +export async function existingFieldsRoute(setup: CoreSetup, logger: Logger) { const router = setup.http.createRouter(); router.post( @@ -52,14 +56,17 @@ export async function existingFieldsRoute(setup: CoreSetup) { }), }); } catch (e) { + logger.info( + `Field existence check failed: ${isBoomError(e) ? e.output.payload.message : e.message}` + ); if (e.status === 404) { - return res.notFound(); + return res.notFound({ body: e.message }); } - if (e.isBoom) { + if (isBoomError(e)) { if (e.output.statusCode === 404) { - return res.notFound(); + return res.notFound({ body: e.output.payload.message }); } - return res.internalError(e.output.message); + return res.internalError({ body: e.output.payload.message }); } else { return res.internalError({ body: Boom.internal(e.message || e.name), diff --git a/x-pack/plugins/lens/server/routes/index.ts b/x-pack/plugins/lens/server/routes/index.ts index 8bc04a56e16d5..01018d8cd7fe5 100644 --- a/x-pack/plugins/lens/server/routes/index.ts +++ b/x-pack/plugins/lens/server/routes/index.ts @@ -4,13 +4,13 @@ * you may not use this file except in compliance with the Elastic License. */ -import { CoreSetup } from 'src/core/server'; +import { CoreSetup, Logger } from 'src/core/server'; import { existingFieldsRoute } from './existing_fields'; import { initFieldsRoute } from './field_stats'; import { initLensUsageRoute } from './telemetry'; -export function setupRoutes(setup: CoreSetup) { - existingFieldsRoute(setup); +export function setupRoutes(setup: CoreSetup, logger: Logger) { + existingFieldsRoute(setup, logger); initFieldsRoute(setup); initLensUsageRoute(setup); } diff --git a/x-pack/plugins/maps/server/routes.js b/x-pack/plugins/maps/server/routes.js index 6b19103b59722..5feacfee4d4d2 100644 --- a/x-pack/plugins/maps/server/routes.js +++ b/x-pack/plugins/maps/server/routes.js @@ -189,18 +189,28 @@ export function initRoutes(router, licenseUid, mapConfig, kbnVersion, logger) { return badRequest('map.proxyElasticMapsServiceInMaps disabled'); } - const file = await emsClient.getDefaultFileManifest(); - const layers = file.layers.map((layer) => { - const newLayer = { ...layer }; - const id = encodeURIComponent(layer.layer_id); + const file = await emsClient.getDefaultFileManifest(); //need raw manifest + const fileLayers = await emsClient.getFileLayers(); + + const layers = file.layers.map((layerJson) => { + const newLayerJson = { ...layerJson }; + const id = encodeURIComponent(layerJson.layer_id); + + const fileLayer = fileLayers.find((fileLayer) => fileLayer.getId() === layerJson.layer_id); + const defaultFormat = layerJson.formats.find( + (format) => format.type === fileLayer.getDefaultFormatType() + ); + const newUrl = `${EMS_FILES_DEFAULT_JSON_PATH}?id=${id}`; - newLayer.formats = [ + + //Only proxy default-format. Others are unused in Maps-app + newLayerJson.formats = [ { - ...layer.formats[0], + ...defaultFormat, url: newUrl, }, ]; - return newLayer; + return newLayerJson; }); //rewrite return ok({ diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/malware.tsx b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/malware.tsx index 1da832fb081ef..6773ed6541927 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/malware.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/malware.tsx @@ -11,6 +11,7 @@ import { EuiRadio, EuiSwitch, EuiTitle, + EuiText, EuiSpacer, htmlIdGenerator, EuiCallOut, @@ -28,6 +29,7 @@ import { policyConfig } from '../../../store/policy_details/selectors'; import { usePolicyDetailsSelector } from '../../policy_hooks'; import { clone } from '../../../models/policy_details_config'; import { LinkToApp } from '../../../../../../common/components/endpoint/link_to_app'; +import { popupVersionsMap } from './popup_options_to_versions'; const ProtectionRadioGroup = styled.div` display: flex; @@ -83,6 +85,25 @@ const ProtectionRadio = React.memo(({ id, label }: { id: ProtectionModes; label: ProtectionRadio.displayName = 'ProtectionRadio'; +const SupportedVersionNotice = ({ optionName }: { optionName: string }) => { + const version = popupVersionsMap.get(optionName); + if (!version) { + return null; + } + + return ( + + + + + + ); +}; + /** The Malware Protections form for policy details * which will configure for all relevant OSes. */ @@ -189,14 +210,15 @@ export const MalwareProtections = React.memo(() => { /> + + ); diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/popup_options_to_versions.ts b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/popup_options_to_versions.ts new file mode 100644 index 0000000000000..d4c7d0102ebd4 --- /dev/null +++ b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/popup_options_to_versions.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +const popupVersions: Array<[string, string]> = [['malware', '7.11+']]; + +export const popupVersionsMap: ReadonlyMap = new Map(popupVersions); diff --git a/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.test.ts b/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.test.ts index 7cdbd8b11bb06..8a94ae4ed82f5 100644 --- a/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.test.ts +++ b/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.test.ts @@ -5,21 +5,12 @@ */ import _ from 'lodash'; -import { - asUpdateByQuery, - shouldBeOneOf, - mustBeAllOf, - ExistsFilter, - TermFilter, - RangeFilter, -} from './query_clauses'; +import { asUpdateByQuery, shouldBeOneOf, mustBeAllOf } from './query_clauses'; import { - updateFields, + updateFieldsAndMarkAsFailed, IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt, - TaskWithSchedule, - taskWithLessThanMaxAttempts, SortByRunAtAndRetryAt, } from './mark_available_tasks_as_claimed'; @@ -40,29 +31,29 @@ describe('mark_available_tasks_as_claimed', () => { createTaskRunner: () => ({ run: () => Promise.resolve() }), }, }); + const claimTasksById = undefined; const defaultMaxAttempts = 1; const taskManagerId = '3478fg6-82374f6-83467gf5-384g6f'; const claimOwnershipUntil = '2019-02-12T21:01:22.479Z'; + const fieldUpdates = { + ownerId: taskManagerId, + retryAt: claimOwnershipUntil, + }; expect( asUpdateByQuery({ query: mustBeAllOf( // Either a task with idle status and runAt <= now or // status running or claiming with a retryAt <= now. - shouldBeOneOf(IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt), - // Either task has an schedule or the attempts < the maximum configured - shouldBeOneOf( - TaskWithSchedule, - ...Array.from(definitions).map(([type, { maxAttempts }]) => - taskWithLessThanMaxAttempts(type, maxAttempts || defaultMaxAttempts) - ) - ) + shouldBeOneOf(IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt) + ), + update: updateFieldsAndMarkAsFailed( + fieldUpdates, + claimTasksById || [], + Array.from(definitions).reduce((accumulator, [type, { maxAttempts }]) => { + return { ...accumulator, [type]: maxAttempts || defaultMaxAttempts }; + }, {}) ), - update: updateFields({ - ownerId: taskManagerId, - status: 'claiming', - retryAt: claimOwnershipUntil, - }), sort: SortByRunAtAndRetryAt, }) ).toEqual({ @@ -100,42 +91,6 @@ describe('mark_available_tasks_as_claimed', () => { ], }, }, - // Either task has an recurring schedule or the attempts < the maximum configured - { - bool: { - should: [ - { exists: { field: 'task.schedule' } }, - { - bool: { - must: [ - { term: { 'task.taskType': 'sampleTask' } }, - { - range: { - 'task.attempts': { - lt: 5, - }, - }, - }, - ], - }, - }, - { - bool: { - must: [ - { term: { 'task.taskType': 'otherTask' } }, - { - range: { - 'task.attempts': { - lt: 1, - }, - }, - }, - ], - }, - }, - ], - }, - }, ], }, }, @@ -158,12 +113,26 @@ if (doc['task.runAt'].size()!=0) { }, seq_no_primary_term: true, script: { - source: `ctx._source.task.ownerId=params.ownerId; ctx._source.task.status=params.status; ctx._source.task.retryAt=params.retryAt;`, + source: ` + if (ctx._source.task.schedule != null || ctx._source.task.attempts < params.taskMaxAttempts[ctx._source.task.taskType] || params.claimTasksById.contains(ctx._id)) { + ctx._source.task.status = "claiming"; ${Object.keys(fieldUpdates) + .map((field) => `ctx._source.task.${field}=params.fieldUpdates.${field};`) + .join(' ')} + } else { + ctx._source.task.status = "failed"; + } + `, lang: 'painless', params: { - ownerId: taskManagerId, - retryAt: claimOwnershipUntil, - status: 'claiming', + fieldUpdates: { + ownerId: taskManagerId, + retryAt: claimOwnershipUntil, + }, + claimTasksById: [], + taskMaxAttempts: { + sampleTask: 5, + otherTask: 1, + }, }, }, }); diff --git a/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.ts b/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.ts index 699af3ed07d5d..072ec4648201a 100644 --- a/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.ts +++ b/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.ts @@ -100,12 +100,26 @@ if (doc['task.runAt'].size()!=0) { }, }; -export const updateFields = (fieldUpdates: { - [field: string]: string | number | Date; -}): ScriptClause => ({ - source: Object.keys(fieldUpdates) - .map((field) => `ctx._source.task.${field}=params.${field};`) - .join(' '), +export const updateFieldsAndMarkAsFailed = ( + fieldUpdates: { + [field: string]: string | number | Date; + }, + claimTasksById: string[], + taskMaxAttempts: { [field: string]: number } +): ScriptClause => ({ + source: ` + if (ctx._source.task.schedule != null || ctx._source.task.attempts < params.taskMaxAttempts[ctx._source.task.taskType] || params.claimTasksById.contains(ctx._id)) { + ctx._source.task.status = "claiming"; ${Object.keys(fieldUpdates) + .map((field) => `ctx._source.task.${field}=params.fieldUpdates.${field};`) + .join(' ')} + } else { + ctx._source.task.status = "failed"; + } + `, lang: 'painless', - params: fieldUpdates, + params: { + fieldUpdates, + claimTasksById, + taskMaxAttempts, + }, }); diff --git a/x-pack/plugins/task_manager/server/queries/query_clauses.ts b/x-pack/plugins/task_manager/server/queries/query_clauses.ts index f16ee302d6c83..5503b9cd94105 100644 --- a/x-pack/plugins/task_manager/server/queries/query_clauses.ts +++ b/x-pack/plugins/task_manager/server/queries/query_clauses.ts @@ -165,7 +165,12 @@ export interface ScriptClause { source: string; lang: string; params: { - [field: string]: string | number | Date; + [field: string]: + | string + | number + | Date + | string[] + | { [field: string]: string | number | Date }; }; } diff --git a/x-pack/plugins/task_manager/server/task_store.test.ts b/x-pack/plugins/task_manager/server/task_store.test.ts index a40df3b84132e..46e55df4ee1e6 100644 --- a/x-pack/plugins/task_manager/server/task_store.test.ts +++ b/x-pack/plugins/task_manager/server/task_store.test.ts @@ -415,41 +415,6 @@ describe('TaskStore', () => { ], }, }, - { - bool: { - should: [ - { exists: { field: 'task.schedule' } }, - { - bool: { - must: [ - { term: { 'task.taskType': 'foo' } }, - { - range: { - 'task.attempts': { - lt: maxAttempts, - }, - }, - }, - ], - }, - }, - { - bool: { - must: [ - { term: { 'task.taskType': 'bar' } }, - { - range: { - 'task.attempts': { - lt: customMaxAttempts, - }, - }, - }, - ], - }, - }, - ], - }, - }, ], }, }, @@ -501,6 +466,11 @@ if (doc['task.runAt'].size()!=0) { const maxAttempts = _.random(2, 43); const customMaxAttempts = _.random(44, 100); const definitions = new TaskTypeDictionary(mockLogger()); + const taskManagerId = uuid.v1(); + const fieldUpdates = { + ownerId: taskManagerId, + retryAt: new Date(Date.now()), + }; definitions.registerTaskDefinitions({ foo: { title: 'foo', @@ -514,10 +484,11 @@ if (doc['task.runAt'].size()!=0) { }); const { args: { - updateByQuery: { body: { query, sort } = {} }, + updateByQuery: { body: { query, script, sort } = {} }, }, } = await testClaimAvailableTasks({ opts: { + taskManagerId, maxAttempts, definitions, }, @@ -576,41 +547,6 @@ if (doc['task.runAt'].size()!=0) { ], }, }, - { - bool: { - should: [ - { exists: { field: 'task.schedule' } }, - { - bool: { - must: [ - { term: { 'task.taskType': 'foo' } }, - { - range: { - 'task.attempts': { - lt: maxAttempts, - }, - }, - }, - ], - }, - }, - { - bool: { - must: [ - { term: { 'task.taskType': 'bar' } }, - { - range: { - 'task.attempts': { - lt: customMaxAttempts, - }, - }, - }, - ], - }, - }, - ], - }, - }, ], }, }, @@ -640,6 +576,30 @@ if (doc['task.runAt'].size()!=0) { }, }); + expect(script).toMatchObject({ + source: ` + if (ctx._source.task.schedule != null || ctx._source.task.attempts < params.taskMaxAttempts[ctx._source.task.taskType] || params.claimTasksById.contains(ctx._id)) { + ctx._source.task.status = "claiming"; ${Object.keys(fieldUpdates) + .map((field) => `ctx._source.task.${field}=params.fieldUpdates.${field};`) + .join(' ')} + } else { + ctx._source.task.status = "failed"; + } + `, + lang: 'painless', + params: { + fieldUpdates, + claimTasksById: [ + 'task:33c6977a-ed6d-43bd-98d9-3f827f7b7cd8', + 'task:a208b22c-14ec-4fb4-995f-d2ff7a3b03b8', + ], + taskMaxAttempts: { + bar: customMaxAttempts, + foo: maxAttempts, + }, + }, + }); + expect(sort).toMatchObject([ '_score', { @@ -665,6 +625,10 @@ if (doc['task.runAt'].size()!=0) { test('it claims tasks by setting their ownerId, status and retryAt', async () => { const taskManagerId = uuid.v1(); const claimOwnershipUntil = new Date(Date.now()); + const fieldUpdates = { + ownerId: taskManagerId, + retryAt: claimOwnershipUntil, + }; const { args: { updateByQuery: { body: { script } = {} }, @@ -679,12 +643,24 @@ if (doc['task.runAt'].size()!=0) { }, }); expect(script).toMatchObject({ - source: `ctx._source.task.ownerId=params.ownerId; ctx._source.task.status=params.status; ctx._source.task.retryAt=params.retryAt;`, + source: ` + if (ctx._source.task.schedule != null || ctx._source.task.attempts < params.taskMaxAttempts[ctx._source.task.taskType] || params.claimTasksById.contains(ctx._id)) { + ctx._source.task.status = "claiming"; ${Object.keys(fieldUpdates) + .map((field) => `ctx._source.task.${field}=params.fieldUpdates.${field};`) + .join(' ')} + } else { + ctx._source.task.status = "failed"; + } + `, lang: 'painless', params: { - ownerId: taskManagerId, - retryAt: claimOwnershipUntil, - status: 'claiming', + fieldUpdates, + claimTasksById: [], + taskMaxAttempts: { + dernstraight: 2, + report: 2, + yawn: 2, + }, }, }); }); diff --git a/x-pack/plugins/task_manager/server/task_store.ts b/x-pack/plugins/task_manager/server/task_store.ts index 4f60a01c14da9..8c0d7764e009f 100644 --- a/x-pack/plugins/task_manager/server/task_store.ts +++ b/x-pack/plugins/task_manager/server/task_store.ts @@ -41,21 +41,16 @@ import { shouldBeOneOf, mustBeAllOf, filterDownBy, - ExistsFilter, - TermFilter, - RangeFilter, asPinnedQuery, matchesClauses, SortOptions, } from './queries/query_clauses'; import { - updateFields, + updateFieldsAndMarkAsFailed, IdleTaskWithExpiredRunAt, InactiveTasks, RunningOrClaimingTaskWithExpiredRetryAt, - TaskWithSchedule, - taskWithLessThanMaxAttempts, SortByRunAtAndRetryAt, tasksClaimedByOwner, } from './queries/mark_available_tasks_as_claimed'; @@ -264,18 +259,13 @@ export class TaskStore { claimTasksById: OwnershipClaimingOpts['claimTasksById'], size: OwnershipClaimingOpts['size'] ): Promise { - const tasksWithRemainingAttempts = [...this.definitions].map(([type, { maxAttempts }]) => - taskWithLessThanMaxAttempts(type, maxAttempts || this.maxAttempts) - ); + const taskMaxAttempts = [...this.definitions].reduce((accumulator, [type, { maxAttempts }]) => { + return { ...accumulator, [type]: maxAttempts || this.maxAttempts }; + }, {}); const queryForScheduledTasks = mustBeAllOf( // Either a task with idle status and runAt <= now or // status running or claiming with a retryAt <= now. - shouldBeOneOf(IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt), - // Either task has a schedule or the attempts < the maximum configured - shouldBeOneOf( - TaskWithSchedule, - ...tasksWithRemainingAttempts - ) + shouldBeOneOf(IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt) ); // The documents should be sorted by runAt/retryAt, unless there are pinned @@ -300,11 +290,14 @@ export class TaskStore { ), filterDownBy(InactiveTasks) ), - update: updateFields({ - ownerId: this.taskManagerId, - status: 'claiming', - retryAt: claimOwnershipUntil, - }), + update: updateFieldsAndMarkAsFailed( + { + ownerId: this.taskManagerId, + retryAt: claimOwnershipUntil, + }, + claimTasksById || [], + taskMaxAttempts + ), sort, }), { diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_form.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_form.tsx index 8f19db6fc827d..404f9aeec7c68 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_form.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_form.tsx @@ -102,11 +102,6 @@ export const StepDefineForm: FC = React.memo((props) => { toastNotifications, }; - // TODO This should use the actual value of `indices.query.bool.max_clause_count` - const maxIndexFields = 1024; - const numIndexFields = indexPattern.fields.length; - const disabledQuery = numIndexFields > maxIndexFields; - const copyToClipboardSource = getIndexDevConsoleStatement(pivotQuery, indexPattern.title); const copyToClipboardSourceDescription = i18n.translate( 'xpack.transform.indexPreview.copyClipboardTooltip', @@ -181,18 +176,6 @@ export const StepDefineForm: FC = React.memo((props) => { label={i18n.translate('xpack.transform.stepDefineForm.indexPatternLabel', { defaultMessage: 'Index pattern', })} - helpText={ - disabledQuery - ? i18n.translate('xpack.transform.stepDefineForm.indexPatternHelpText', { - defaultMessage: - 'An optional query for this index pattern is not supported. The number of supported index fields is {maxIndexFields} whereas this index has {numIndexFields} fields.', - values: { - maxIndexFields, - numIndexFields, - }, - }) - : '' - } > {indexPattern.title} @@ -214,7 +197,7 @@ export const StepDefineForm: FC = React.memo((props) => { {/* Flex Column #1: Search Bar / Advanced Search Editor */} {searchItems.savedSearch === undefined && ( <> - {!disabledQuery && !isAdvancedSourceEditorEnabled && ( + {!isAdvancedSourceEditorEnabled && ( { + it('should correctly rewrite url and format', async () => { + const resp = await supertest + .get(`/api/maps/ems/files/v7.10/manifest`) + .set('kbn-xsrf', 'kibana') + .expect(200); + + expect(resp.body.layers.length).to.be.greaterThan(0); + + //Check world-layer + const worldLayer = resp.body.layers.find((layer) => layer.layer_id === 'world_countries'); + expect(worldLayer.formats.length).to.be.greaterThan(0); + expect(worldLayer.formats[0].type).to.be('geojson'); + expect(worldLayer.formats[0].url).to.be('file?id=world_countries'); + }); + }); +} diff --git a/x-pack/test/api_integration/config.ts b/x-pack/test/api_integration/config.ts index 97fd968ce7992..fc021ffa0ca5b 100644 --- a/x-pack/test/api_integration/config.ts +++ b/x-pack/test/api_integration/config.ts @@ -25,6 +25,7 @@ export async function getApiIntegrationConfig({ readConfigFile }: FtrConfigProvi ...xPackFunctionalTestsConfig.get('kbnTestServer'), serverArgs: [ ...xPackFunctionalTestsConfig.get('kbnTestServer.serverArgs'), + '--map.proxyElasticMapsServiceInMaps=true', '--xpack.security.session.idleTimeout=3600000', // 1 hour '--telemetry.optIn=true', '--xpack.fleet.enabled=true', diff --git a/x-pack/test/functional/apps/lens/persistent_context.ts b/x-pack/test/functional/apps/lens/persistent_context.ts index 15515df92a89f..8d536aac3f795 100644 --- a/x-pack/test/functional/apps/lens/persistent_context.ts +++ b/x-pack/test/functional/apps/lens/persistent_context.ts @@ -66,5 +66,28 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(timeRange.end).to.equal('Sep 19, 2025 @ 06:31:44.000'); await filterBar.hasFilter('ip', '97.220.3.248', false, true); }); + + it('keeps time range and pinned filters after refreshing directly after saving', async () => { + // restore defaults so visualization becomes saveable + await security.testUser.restoreDefaults(); + await PageObjects.lens.configureDimension({ + dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', + operation: 'date_histogram', + field: '@timestamp', + }); + + await PageObjects.lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', + operation: 'avg', + field: 'bytes', + }); + await PageObjects.lens.save('persistentcontext'); + await browser.refresh(); + await PageObjects.header.waitUntilLoadingHasFinished(); + const timeRange = await PageObjects.timePicker.getTimeConfig(); + expect(timeRange.start).to.equal('Sep 7, 2015 @ 06:31:44.000'); + expect(timeRange.end).to.equal('Sep 19, 2025 @ 06:31:44.000'); + await filterBar.hasFilter('ip', '97.220.3.248', false, true); + }); }); } diff --git a/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/plugin.ts b/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/plugin.ts index 803df6a66ea58..b5d2c98d8cbcd 100644 --- a/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/plugin.ts +++ b/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/plugin.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import _ from 'lodash'; import { Plugin, CoreSetup, CoreStart } from 'src/core/server'; import { EventEmitter } from 'events'; import { Subject } from 'rxjs'; @@ -103,6 +104,29 @@ export class SampleTaskManagerFixturePlugin // fail after the first failed run maxAttempts: 1, }, + sampleRecurringTaskTimingOut: { + title: 'Sample Recurring Task that Times Out', + description: 'A sample task that times out each run.', + maxAttempts: 3, + timeout: '1s', + createTaskRunner: () => ({ + async run() { + return await new Promise((resolve) => {}); + }, + }), + }, + sampleOneTimeTaskTimingOut: { + title: 'Sample One-Time Task that Times Out', + description: 'A sample task that times out each run.', + maxAttempts: 3, + timeout: '1s', + getRetry: (attempts: number, error: object) => new Date(Date.now() + _.random(2, 5) * 1000), + createTaskRunner: () => ({ + async run() { + return await new Promise((resolve) => {}); + }, + }), + }, }); taskManager.addMiddleware({ diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts index 2434f05b5403f..1fd313c1ac437 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts @@ -646,5 +646,34 @@ export default function ({ getService }: FtrProviderContext) { expect(getTaskById(tasks, longRunningTask.id).state.count).to.eql(1); }); }); + + it('should mark non-recurring task as failed if task is still running but maxAttempts has been reached', async () => { + const task = await scheduleTask({ + taskType: 'sampleOneTimeTaskTimingOut', + params: {}, + }); + + await retry.try(async () => { + const [scheduledTask] = (await currentTasks()).docs; + expect(scheduledTask.id).to.eql(task.id); + expect(scheduledTask.status).to.eql('failed'); + expect(scheduledTask.attempts).to.eql(3); + }); + }); + + it('should continue claiming recurring task even if maxAttempts has been reached', async () => { + const task = await scheduleTask({ + taskType: 'sampleRecurringTaskTimingOut', + schedule: { interval: '1s' }, + params: {}, + }); + + await retry.try(async () => { + const [scheduledTask] = (await currentTasks()).docs; + expect(scheduledTask.id).to.eql(task.id); + expect(scheduledTask.status).to.eql('claiming'); + expect(scheduledTask.attempts).to.eql(4); + }); + }); }); }