diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 5a9e8bc585119..525da9d832b53 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -381,8 +381,9 @@ x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @elastic/kib **/*.scss @elastic/kibana-design #CC# /packages/kbn-ui-framework/ @elastic/kibana-design -# Core design +# Core UI design /src/plugins/dashboard/**/*.scss @elastic/kibana-core-ui-designers +/src/plugins/embeddable/**/*.scss @elastic/kibana-core-ui-designers /x-pack/plugins/canvas/**/*.scss @elastic/kibana-core-ui-designers /x-pack/plugins/spaces/**/*.scss @elastic/kibana-core-ui-designers /x-pack/plugins/security/**/*.scss @elastic/kibana-core-ui-designers diff --git a/.github/ISSUE_TEMPLATE/Bug_report_security_solution.md b/.github/ISSUE_TEMPLATE/Bug_report_security_solution.md index 059e1d267c286..7a0514bca621d 100644 --- a/.github/ISSUE_TEMPLATE/Bug_report_security_solution.md +++ b/.github/ISSUE_TEMPLATE/Bug_report_security_solution.md @@ -1,38 +1,38 @@ ---- -name: Bug report for Security Solution -about: Help us identify bugs in Elastic Security, SIEM, and Endpoint so we can fix them! -title: '[Security Solution]' -labels: 'Team: SecuritySolution' ---- - -**Describe the bug:** - -**Kibana/Elasticsearch Stack version:** - -**Server OS version:** - -**Browser and Browser OS versions:** - -**Elastic Endpoint version:** - -**Original install method (e.g. download page, yum, from source, etc.):** - -**Functional Area (e.g. Endpoint management, timelines, resolver, etc.):** - -**Steps to reproduce:** - -1. -2. -3. - -**Current behavior:** - -**Expected behavior:** - -**Screenshots (if relevant):** - -**Errors in browser console (if relevant):** - -**Provide logs and/or server output (if relevant):** - -**Any additional context (logs, chat logs, magical formulas, etc.):** +--- +name: Bug report for Security Solution +about: Help us identify bugs in Elastic Security, SIEM, and Endpoint so we can fix them! +title: '[Security Solution]' +labels: 'bug, Team: SecuritySolution' +--- + +**Describe the bug:** + +**Kibana/Elasticsearch Stack version:** + +**Server OS version:** + +**Browser and Browser OS versions:** + +**Elastic Endpoint version:** + +**Original install method (e.g. download page, yum, from source, etc.):** + +**Functional Area (e.g. Endpoint management, timelines, resolver, etc.):** + +**Steps to reproduce:** + +1. +2. +3. + +**Current behavior:** + +**Expected behavior:** + +**Screenshots (if relevant):** + +**Errors in browser console (if relevant):** + +**Provide logs and/or server output (if relevant):** + +**Any additional context (logs, chat logs, magical formulas, etc.):** diff --git a/.github/ISSUE_TEMPLATE/v8_breaking_change.md b/.github/ISSUE_TEMPLATE/v8_breaking_change.md index 99f779c288f5b..c91b937586a09 100644 --- a/.github/ISSUE_TEMPLATE/v8_breaking_change.md +++ b/.github/ISSUE_TEMPLATE/v8_breaking_change.md @@ -2,7 +2,7 @@ name: 8.0 Breaking change about: Breaking changes from 7.x -> 8.0 title: "[Breaking change]" -labels: Team:Elasticsearch UI, Feature:Upgrade Assistant +labels: Team:Elasticsearch UI, Feature:Upgrade Assistant, Breaking Change assignees: '' --- @@ -11,15 +11,16 @@ assignees: '' **Which release will ship the breaking change?** - +8.0 **Describe the change. How will it manifest to users?** -**What percentage of users will be affected?** +**How many users will be affected?** - + + -**What can users to do to address the change manually?** +**What can users do to address the change manually?** diff --git a/.telemetryrc.json b/.telemetryrc.json index 3d1b0df1d8f93..0f1530c6225d6 100644 --- a/.telemetryrc.json +++ b/.telemetryrc.json @@ -7,5 +7,10 @@ "src/plugins/testbed/", "src/plugins/kibana_utils/" ] + }, + { + "output": "src/plugins/telemetry/schema/legacy_plugins.json", + "root": "src/legacy/server/", + "exclude": [] } ] diff --git a/docs/apm/images/advanced-discover.png b/docs/apm/images/advanced-discover.png index 56ba58b2c1d41..5291526783a6b 100644 Binary files a/docs/apm/images/advanced-discover.png and b/docs/apm/images/advanced-discover.png differ diff --git a/docs/apm/troubleshooting.asciidoc b/docs/apm/troubleshooting.asciidoc index 6c52c021fc0fc..7084777cbb6f9 100644 --- a/docs/apm/troubleshooting.asciidoc +++ b/docs/apm/troubleshooting.asciidoc @@ -157,7 +157,7 @@ the values in `http.request.cookies` are not indexed and thus not searchable. *Ensure an index pattern exists* As a first step, you should ensure the correct index pattern exists. -In Kibana, navigate to *Management > Kibana > Index Patterns*. +Open the main menu, then click *Stack Management > Index Patterns*. In the pattern list, you should see an apm index pattern; The default is `apm-*`. If you don't, the index pattern doesn't exist. See <> for information on how to fix this problem. diff --git a/docs/canvas/canvas-tutorial.asciidoc b/docs/canvas/canvas-tutorial.asciidoc index 312391541a777..6456ba02bb8a8 100644 --- a/docs/canvas/canvas-tutorial.asciidoc +++ b/docs/canvas/canvas-tutorial.asciidoc @@ -14,7 +14,7 @@ For this tutorial, you'll need to add the <>, and work with data in other contexts. -To get started, open the menu, go to *Dev Tools*, then click *Painless Lab*. +To get started, open the main menu, click *Dev Tools*, then click *Painless Lab*. image::dev-tools/painlesslab/images/painless-lab.png[Painless Lab] diff --git a/docs/dev-tools/searchprofiler/getting-started.asciidoc b/docs/dev-tools/searchprofiler/getting-started.asciidoc index eaa7fea6c7f8d..7cd54db5562b7 100644 --- a/docs/dev-tools/searchprofiler/getting-started.asciidoc +++ b/docs/dev-tools/searchprofiler/getting-started.asciidoc @@ -2,7 +2,7 @@ [[profiler-getting-started]] === Getting Started -The {searchprofiler} is automatically enabled in {kib}. From the menu, go to *Dev Tools*, then click *Search Profiler* +The {searchprofiler} is automatically enabled in {kib}. Open the main menu, click *Dev Tools*, then click *Search Profiler* to get started. {searchprofiler} displays the names of the indices searched, the shards in each index, diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.painlesserror._constructor_.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.painlesserror._constructor_.md index f8966572afbb6..051414eac7585 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.painlesserror._constructor_.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.painlesserror._constructor_.md @@ -9,13 +9,13 @@ Constructs a new instance of the `PainlessError` class Signature: ```typescript -constructor(err: EsError, request: IKibanaSearchRequest); +constructor(err: IEsError, request: IKibanaSearchRequest); ``` ## Parameters | Parameter | Type | Description | | --- | --- | --- | -| err | EsError | | +| err | IEsError | | | request | IKibanaSearchRequest | | diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.painlesserror.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.painlesserror.md index 306211cd60259..6ab32f3fb1dfa 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.painlesserror.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.painlesserror.md @@ -7,7 +7,7 @@ Signature: ```typescript -export declare class PainlessError extends KbnError +export declare class PainlessError extends EsError ``` ## Constructors diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsfetcher._constructor_.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsfetcher._constructor_.md index d36ebd0745e8d..214c795fda9d1 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsfetcher._constructor_.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsfetcher._constructor_.md @@ -9,12 +9,13 @@ Constructs a new instance of the `IndexPatternsFetcher` class Signature: ```typescript -constructor(callDataCluster: LegacyAPICaller); +constructor(elasticsearchClient: ElasticsearchClient, allowNoIndices?: boolean); ``` ## Parameters | Parameter | Type | Description | | --- | --- | --- | -| callDataCluster | LegacyAPICaller | | +| elasticsearchClient | ElasticsearchClient | | +| allowNoIndices | boolean | | diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsfetcher.getfieldsforwildcard.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsfetcher.getfieldsforwildcard.md index 52382372d6d96..addd29916d81d 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsfetcher.getfieldsforwildcard.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsfetcher.getfieldsforwildcard.md @@ -13,7 +13,7 @@ getFieldsForWildcard(options: { pattern: string | string[]; metaFields?: string[]; fieldCapsOptions?: { - allowNoIndices: boolean; + allow_no_indices: boolean; }; }): Promise; ``` @@ -22,7 +22,7 @@ getFieldsForWildcard(options: { | Parameter | Type | Description | | --- | --- | --- | -| options | {
pattern: string | string[];
metaFields?: string[];
fieldCapsOptions?: {
allowNoIndices: boolean;
};
} | | +| options | {
pattern: string | string[];
metaFields?: string[];
fieldCapsOptions?: {
allow_no_indices: boolean;
};
} | | Returns: diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsfetcher.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsfetcher.md index f71a702f3381d..3ba3c862bf16a 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsfetcher.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsfetcher.md @@ -14,7 +14,7 @@ export declare class IndexPatternsFetcher | Constructor | Modifiers | Description | | --- | --- | --- | -| [(constructor)(callDataCluster)](./kibana-plugin-plugins-data-server.indexpatternsfetcher._constructor_.md) | | Constructs a new instance of the IndexPatternsFetcher class | +| [(constructor)(elasticsearchClient, allowNoIndices)](./kibana-plugin-plugins-data-server.indexpatternsfetcher._constructor_.md) | | Constructs a new instance of the IndexPatternsFetcher class | ## Methods diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution._constructor_.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution._constructor_.md index 1d0c9f99169db..14a0f8818e903 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution._constructor_.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution._constructor_.md @@ -9,12 +9,12 @@ Constructs a new instance of the `Execution` class Signature: ```typescript -constructor(params: ExecutionParams); +constructor(execution: ExecutionParams); ``` ## Parameters | Parameter | Type | Description | | --- | --- | --- | -| params | ExecutionParams<ExtraContext> | | +| execution | ExecutionParams | | diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.context.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.context.md index 732fe94d65617..e884db46563b5 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.context.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.context.md @@ -9,5 +9,5 @@ Execution context - object that allows to do side-effects. Context is passed to Signature: ```typescript -readonly context: ExecutionContext & ExtraContext; +readonly context: ExecutionContext; ``` diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.contract.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.contract.md index fa03297ea22a7..383e9ee3e81b8 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.contract.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.contract.md @@ -9,5 +9,5 @@ Contract is a public representation of `Execution` instances. Contract we can re Signature: ```typescript -readonly contract: ExecutionContract; +readonly contract: ExecutionContract; ``` diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.params.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.execution.md similarity index 67% rename from docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.params.md rename to docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.execution.md index cd90bf6adab47..eebb5cf5440d5 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.params.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.execution.md @@ -1,11 +1,11 @@ -[Home](./index.md) > [kibana-plugin-plugins-expressions-public](./kibana-plugin-plugins-expressions-public.md) > [Execution](./kibana-plugin-plugins-expressions-public.execution.md) > [params](./kibana-plugin-plugins-expressions-public.execution.params.md) +[Home](./index.md) > [kibana-plugin-plugins-expressions-public](./kibana-plugin-plugins-expressions-public.md) > [Execution](./kibana-plugin-plugins-expressions-public.execution.md) > [execution](./kibana-plugin-plugins-expressions-public.execution.execution.md) -## Execution.params property +## Execution.execution property Signature: ```typescript -readonly params: ExecutionParams; +readonly execution: ExecutionParams; ``` diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.interpret.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.interpret.md index 31f38b7069812..24dee04861b4e 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.interpret.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.interpret.md @@ -7,7 +7,7 @@ Signature: ```typescript -interpret(ast: ExpressionAstNode, input: T, options?: ExpressionExecOptions): Promise; +interpret(ast: ExpressionAstNode, input: T): Promise; ``` ## Parameters @@ -16,7 +16,6 @@ interpret(ast: ExpressionAstNode, input: T, options?: ExpressionExecOptions): | --- | --- | --- | | ast | ExpressionAstNode | | | input | T | | -| options | ExpressionExecOptions | | Returns: diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.md index 4d227e6ab85b8..56b14e005adfb 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.md @@ -7,25 +7,25 @@ Signature: ```typescript -export declare class Execution = Record, Input = unknown, Output = unknown, InspectorAdapters extends Adapters = ExtraContext['inspectorAdapters'] extends object ? ExtraContext['inspectorAdapters'] : DefaultInspectorAdapters> +export declare class Execution ``` ## Constructors | Constructor | Modifiers | Description | | --- | --- | --- | -| [(constructor)(params)](./kibana-plugin-plugins-expressions-public.execution._constructor_.md) | | Constructs a new instance of the Execution class | +| [(constructor)(execution)](./kibana-plugin-plugins-expressions-public.execution._constructor_.md) | | Constructs a new instance of the Execution class | ## Properties | Property | Modifiers | Type | Description | | --- | --- | --- | --- | -| [context](./kibana-plugin-plugins-expressions-public.execution.context.md) | | ExecutionContext<Input, InspectorAdapters> & ExtraContext | Execution context - object that allows to do side-effects. Context is passed to every function. | -| [contract](./kibana-plugin-plugins-expressions-public.execution.contract.md) | | ExecutionContract<ExtraContext, Input, Output, InspectorAdapters> | Contract is a public representation of Execution instances. Contract we can return to other plugins for their consumption. | +| [context](./kibana-plugin-plugins-expressions-public.execution.context.md) | | ExecutionContext<InspectorAdapters> | Execution context - object that allows to do side-effects. Context is passed to every function. | +| [contract](./kibana-plugin-plugins-expressions-public.execution.contract.md) | | ExecutionContract<Input, Output, InspectorAdapters> | Contract is a public representation of Execution instances. Contract we can return to other plugins for their consumption. | +| [execution](./kibana-plugin-plugins-expressions-public.execution.execution.md) | | ExecutionParams | | | [expression](./kibana-plugin-plugins-expressions-public.execution.expression.md) | | string | | | [input](./kibana-plugin-plugins-expressions-public.execution.input.md) | | Input | Initial input of the execution.N.B. It is initialized to null rather than undefined for legacy reasons, because in legacy interpreter it was set to null by default. | | [inspectorAdapters](./kibana-plugin-plugins-expressions-public.execution.inspectoradapters.md) | | InspectorAdapters | | -| [params](./kibana-plugin-plugins-expressions-public.execution.params.md) | | ExecutionParams<ExtraContext> | | | [result](./kibana-plugin-plugins-expressions-public.execution.result.md) | | Promise<Output | ExpressionValueError> | | | [state](./kibana-plugin-plugins-expressions-public.execution.state.md) | | ExecutionContainer<Output | ExpressionValueError> | Dynamic state of the execution. | @@ -35,7 +35,7 @@ export declare class Execution = Re | --- | --- | --- | | [cancel()](./kibana-plugin-plugins-expressions-public.execution.cancel.md) | | Stop execution of expression. | | [cast(value, toTypeNames)](./kibana-plugin-plugins-expressions-public.execution.cast.md) | | | -| [interpret(ast, input, options)](./kibana-plugin-plugins-expressions-public.execution.interpret.md) | | | +| [interpret(ast, input)](./kibana-plugin-plugins-expressions-public.execution.interpret.md) | | | | [invokeChain(chainArr, input)](./kibana-plugin-plugins-expressions-public.execution.invokechain.md) | | | | [invokeFunction(fn, input, args)](./kibana-plugin-plugins-expressions-public.execution.invokefunction.md) | | | | [resolveArgs(fnDef, input, argAsts)](./kibana-plugin-plugins-expressions-public.execution.resolveargs.md) | | | diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.getinitialinput.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.getsearchcontext.md similarity index 55% rename from docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.getinitialinput.md rename to docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.getsearchcontext.md index 460b1622c6fbd..471e18ee6a7eb 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.getinitialinput.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.getsearchcontext.md @@ -1,13 +1,13 @@ -[Home](./index.md) > [kibana-plugin-plugins-expressions-public](./kibana-plugin-plugins-expressions-public.md) > [ExecutionContext](./kibana-plugin-plugins-expressions-public.executioncontext.md) > [getInitialInput](./kibana-plugin-plugins-expressions-public.executioncontext.getinitialinput.md) +[Home](./index.md) > [kibana-plugin-plugins-expressions-public](./kibana-plugin-plugins-expressions-public.md) > [ExecutionContext](./kibana-plugin-plugins-expressions-public.executioncontext.md) > [getSearchContext](./kibana-plugin-plugins-expressions-public.executioncontext.getsearchcontext.md) -## ExecutionContext.getInitialInput property +## ExecutionContext.getSearchContext property -Get initial input with which execution started. +Get search context of the expression. Signature: ```typescript -getInitialInput: () => Input; +getSearchContext: () => ExecutionContextSearch; ``` diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.search.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.getsearchsessionid.md similarity index 63% rename from docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.search.md rename to docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.getsearchsessionid.md index 05501a475cbd4..107ae16dc8901 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.search.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.getsearchsessionid.md @@ -1,13 +1,13 @@ -[Home](./index.md) > [kibana-plugin-plugins-expressions-public](./kibana-plugin-plugins-expressions-public.md) > [ExecutionContext](./kibana-plugin-plugins-expressions-public.executioncontext.md) > [search](./kibana-plugin-plugins-expressions-public.executioncontext.search.md) +[Home](./index.md) > [kibana-plugin-plugins-expressions-public](./kibana-plugin-plugins-expressions-public.md) > [ExecutionContext](./kibana-plugin-plugins-expressions-public.executioncontext.md) > [getSearchSessionId](./kibana-plugin-plugins-expressions-public.executioncontext.getsearchsessionid.md) -## ExecutionContext.search property +## ExecutionContext.getSearchSessionId property Search context in which expression should operate. Signature: ```typescript -search?: ExecutionContextSearch; +getSearchSessionId: () => string | undefined; ``` diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.md index 786e94455c600..2a1a78b8fcb1a 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.md @@ -9,7 +9,7 @@ Signature: ```typescript -export interface ExecutionContext +export interface ExecutionContext ``` ## Properties @@ -17,10 +17,10 @@ export interface ExecutionContextAbortSignal | Adds ability to abort current execution. | -| [getInitialInput](./kibana-plugin-plugins-expressions-public.executioncontext.getinitialinput.md) | () => Input | Get initial input with which execution started. | | [getSavedObject](./kibana-plugin-plugins-expressions-public.executioncontext.getsavedobject.md) | <T extends SavedObjectAttributes = SavedObjectAttributes>(type: string, id: string) => Promise<SavedObject<T>> | Allows to fetch saved objects from ElasticSearch. In browser getSavedObject function is provided automatically by the Expressions plugin. On the server the caller of the expression has to provide this context function. The reason is because on the browser we always know the user who tries to fetch a saved object, thus saved object client is scoped automatically to that user. However, on the server we can scope that saved object client to any user, or even not scope it at all and execute it as an "internal" user. | +| [getSearchContext](./kibana-plugin-plugins-expressions-public.executioncontext.getsearchcontext.md) | () => ExecutionContextSearch | Get search context of the expression. | +| [getSearchSessionId](./kibana-plugin-plugins-expressions-public.executioncontext.getsearchsessionid.md) | () => string | undefined | Search context in which expression should operate. | | [inspectorAdapters](./kibana-plugin-plugins-expressions-public.executioncontext.inspectoradapters.md) | InspectorAdapters | Adapters for inspector plugin. | -| [search](./kibana-plugin-plugins-expressions-public.executioncontext.search.md) | ExecutionContextSearch | Search context in which expression should operate. | | [types](./kibana-plugin-plugins-expressions-public.executioncontext.types.md) | Record<string, ExpressionType> | A map of available expression types. | | [variables](./kibana-plugin-plugins-expressions-public.executioncontext.variables.md) | Record<string, unknown> | Context variables that can be consumed using var and var_set functions. | diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontract._constructor_.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontract._constructor_.md index 89a99ef2f8ef8..ee8b113881a05 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontract._constructor_.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontract._constructor_.md @@ -9,12 +9,12 @@ Constructs a new instance of the `ExecutionContract` class Signature: ```typescript -constructor(execution: Execution); +constructor(execution: Execution); ``` ## Parameters | Parameter | Type | Description | | --- | --- | --- | -| execution | Execution<ExtraContext, Input, Output, InspectorAdapters> | | +| execution | Execution<Input, Output, InspectorAdapters> | | diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontract.execution.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontract.execution.md index f7911250488f2..aa058c71c12df 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontract.execution.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontract.execution.md @@ -7,5 +7,5 @@ Signature: ```typescript -protected readonly execution: Execution; +protected readonly execution: Execution; ``` diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontract.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontract.md index d05620eace208..f2c050bbfe0ba 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontract.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontract.md @@ -9,7 +9,7 @@ Signature: ```typescript -export declare class ExecutionContract = Record, Input = unknown, Output = unknown, InspectorAdapters = unknown> +export declare class ExecutionContract ``` ## Constructors @@ -23,7 +23,7 @@ export declare class ExecutionContract() => void | Cancel the execution of the expression. This will set abort signal (available in execution context) to aborted state, letting expression functions to stop their execution. | -| [execution](./kibana-plugin-plugins-expressions-public.executioncontract.execution.md) | | Execution<ExtraContext, Input, Output, InspectorAdapters> | | +| [execution](./kibana-plugin-plugins-expressions-public.executioncontract.execution.md) | | Execution<Input, Output, InspectorAdapters> | | | [getAst](./kibana-plugin-plugins-expressions-public.executioncontract.getast.md) | | () => ExpressionAstExpression | Get AST used to execute the expression. | | [getData](./kibana-plugin-plugins-expressions-public.executioncontract.getdata.md) | | () => Promise<Output | ExpressionValueError> | Returns the final output of expression, if any error happens still wraps that error into ExpressionValueError type and returns that. This function never throws. | | [getExpression](./kibana-plugin-plugins-expressions-public.executioncontract.getexpression.md) | | () => string | Get string representation of the expression. Returns the original string if execution was started from a string. If execution was started from an AST this method returns a string generated from AST. | diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executionparams.debug.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executionparams.debug.md deleted file mode 100644 index 61ec72465f55e..0000000000000 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executionparams.debug.md +++ /dev/null @@ -1,13 +0,0 @@ - - -[Home](./index.md) > [kibana-plugin-plugins-expressions-public](./kibana-plugin-plugins-expressions-public.md) > [ExecutionParams](./kibana-plugin-plugins-expressions-public.executionparams.md) > [debug](./kibana-plugin-plugins-expressions-public.executionparams.debug.md) - -## ExecutionParams.debug property - -Whether to execute expression in \*debug mode\*. In \*debug mode\* inputs and outputs as well as all resolved arguments and time it took to execute each function are saved and are available for introspection. - -Signature: - -```typescript -debug?: boolean; -``` diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executionparams.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executionparams.md index e39dc231fbf96..6e5d70c61ead6 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executionparams.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executionparams.md @@ -7,7 +7,7 @@ Signature: ```typescript -export interface ExecutionParams = Record> +export interface ExecutionParams ``` ## Properties @@ -15,8 +15,7 @@ export interface ExecutionParams = | Property | Type | Description | | --- | --- | --- | | [ast](./kibana-plugin-plugins-expressions-public.executionparams.ast.md) | ExpressionAstExpression | | -| [context](./kibana-plugin-plugins-expressions-public.executionparams.context.md) | ExtraContext | | -| [debug](./kibana-plugin-plugins-expressions-public.executionparams.debug.md) | boolean | Whether to execute expression in \*debug mode\*. In \*debug mode\* inputs and outputs as well as all resolved arguments and time it took to execute each function are saved and are available for introspection. | | [executor](./kibana-plugin-plugins-expressions-public.executionparams.executor.md) | Executor<any> | | | [expression](./kibana-plugin-plugins-expressions-public.executionparams.expression.md) | string | | +| [params](./kibana-plugin-plugins-expressions-public.executionparams.params.md) | ExpressionExecutionParams | | diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executionparams.context.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executionparams.params.md similarity index 65% rename from docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executionparams.context.md rename to docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executionparams.params.md index b6013162ef2ae..0dbe87bfda79e 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executionparams.context.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executionparams.params.md @@ -1,11 +1,11 @@ -[Home](./index.md) > [kibana-plugin-plugins-expressions-public](./kibana-plugin-plugins-expressions-public.md) > [ExecutionParams](./kibana-plugin-plugins-expressions-public.executionparams.md) > [context](./kibana-plugin-plugins-expressions-public.executionparams.context.md) +[Home](./index.md) > [kibana-plugin-plugins-expressions-public](./kibana-plugin-plugins-expressions-public.md) > [ExecutionParams](./kibana-plugin-plugins-expressions-public.executionparams.md) > [params](./kibana-plugin-plugins-expressions-public.executionparams.params.md) -## ExecutionParams.context property +## ExecutionParams.params property Signature: ```typescript -context?: ExtraContext; +params: ExpressionExecutionParams; ``` diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executor.createexecution.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executor.createexecution.md index e6765064d4a27..2832ba92262f2 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executor.createexecution.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executor.createexecution.md @@ -7,7 +7,7 @@ Signature: ```typescript -createExecution = Record, Input = unknown, Output = unknown>(ast: string | ExpressionAstExpression, context?: ExtraContext, { debug }?: ExpressionExecOptions): Execution; +createExecution(ast: string | ExpressionAstExpression, params?: ExpressionExecutionParams): Execution; ``` ## Parameters @@ -15,10 +15,9 @@ createExecution = Recordstring | ExpressionAstExpression | | -| context | ExtraContext | | -| { debug } | ExpressionExecOptions | | +| params | ExpressionExecutionParams | | Returns: -`Execution` +`Execution` diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executor.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executor.md index 013624f30b45a..aefd04112dc1c 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executor.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executor.md @@ -29,7 +29,7 @@ export declare class Executor = Recordstatic | | | [extendContext(extraContext)](./kibana-plugin-plugins-expressions-public.executor.extendcontext.md) | | | | [extract(ast)](./kibana-plugin-plugins-expressions-public.executor.extract.md) | | | @@ -41,6 +41,6 @@ export declare class Executor = RecordSignature: ```typescript -run = Record>(ast: string | ExpressionAstExpression, input: Input, context?: ExtraContext, options?: ExpressionExecOptions): Promise; +run(ast: string | ExpressionAstExpression, input: Input, params?: ExpressionExecutionParams): Promise; ``` ## Parameters @@ -18,8 +18,7 @@ run = Recordstring | ExpressionAstExpression | | | input | Input | | -| context | ExtraContext | | -| options | ExpressionExecOptions | | +| params | ExpressionExecutionParams | | Returns: diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionsservicestart.execute.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionsservicestart.execute.md index 18b856b946da4..043d3472228a2 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionsservicestart.execute.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionsservicestart.execute.md @@ -9,5 +9,5 @@ Starts expression execution and immediately returns `ExecutionContract` instance Signature: ```typescript -execute: = Record>(ast: string | ExpressionAstExpression, input: Input, context?: ExtraContext, options?: ExpressionExecOptions) => ExecutionContract; +execute: (ast: string | ExpressionAstExpression, input: Input, params?: ExpressionExecutionParams) => ExecutionContract; ``` diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionsservicestart.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionsservicestart.md index def572abead22..6b678fc4fbc26 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionsservicestart.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionsservicestart.md @@ -16,12 +16,12 @@ export interface ExpressionsServiceStart | Property | Type | Description | | --- | --- | --- | -| [execute](./kibana-plugin-plugins-expressions-public.expressionsservicestart.execute.md) | <Input = unknown, Output = unknown, ExtraContext extends Record<string, unknown> = Record<string, unknown>>(ast: string | ExpressionAstExpression, input: Input, context?: ExtraContext, options?: ExpressionExecOptions) => ExecutionContract<ExtraContext, Input, Output> | Starts expression execution and immediately returns ExecutionContract instance that tracks the progress of the execution and can be used to interact with the execution. | +| [execute](./kibana-plugin-plugins-expressions-public.expressionsservicestart.execute.md) | <Input = unknown, Output = unknown>(ast: string | ExpressionAstExpression, input: Input, params?: ExpressionExecutionParams) => ExecutionContract<Input, Output> | Starts expression execution and immediately returns ExecutionContract instance that tracks the progress of the execution and can be used to interact with the execution. | | [fork](./kibana-plugin-plugins-expressions-public.expressionsservicestart.fork.md) | () => ExpressionsService | Create a new instance of ExpressionsService. The new instance inherits all state of the original ExpressionsService, including all expression types, expression functions and context. Also, all new types and functions registered in the original services AFTER the forking event will be available in the forked instance. However, all new types and functions registered in the forked instances will NOT be available to the original service. | | [getFunction](./kibana-plugin-plugins-expressions-public.expressionsservicestart.getfunction.md) | (name: string) => ReturnType<Executor['getFunction']> | Get a registered ExpressionFunction by its name, which was registered using the registerFunction method. The returned ExpressionFunction instance is an internal representation of the function in Expressions service - do not mutate that object. | | [getRenderer](./kibana-plugin-plugins-expressions-public.expressionsservicestart.getrenderer.md) | (name: string) => ReturnType<ExpressionRendererRegistry['get']> | Get a registered ExpressionRenderer by its name, which was registered using the registerRenderer method. The returned ExpressionRenderer instance is an internal representation of the renderer in Expressions service - do not mutate that object. | | [getType](./kibana-plugin-plugins-expressions-public.expressionsservicestart.gettype.md) | (name: string) => ReturnType<Executor['getType']> | Get a registered ExpressionType by its name, which was registered using the registerType method. The returned ExpressionType instance is an internal representation of the type in Expressions service - do not mutate that object. | -| [run](./kibana-plugin-plugins-expressions-public.expressionsservicestart.run.md) | <Input, Output, ExtraContext extends Record<string, unknown> = Record<string, unknown>>(ast: string | ExpressionAstExpression, input: Input, context?: ExtraContext, options?: ExpressionExecOptions) => Promise<Output> | Executes expression string or a parsed expression AST and immediately returns the result.Below example will execute sleep 100 | clog expression with 123 initial input to the first function. +| [run](./kibana-plugin-plugins-expressions-public.expressionsservicestart.run.md) | <Input, Output>(ast: string | ExpressionAstExpression, input: Input, params?: ExpressionExecutionParams) => Promise<Output> | Executes expression string or a parsed expression AST and immediately returns the result.Below example will execute sleep 100 | clog expression with 123 initial input to the first function. ```ts expressions.run('sleep 100 | clog', 123); diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionsservicestart.run.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionsservicestart.run.md index d717af51a00fa..9efca0011174c 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionsservicestart.run.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionsservicestart.run.md @@ -24,5 +24,5 @@ expressions.run('...', null, { elasticsearchClient }); Signature: ```typescript -run: = Record>(ast: string | ExpressionAstExpression, input: Input, context?: ExtraContext, options?: ExpressionExecOptions) => Promise; +run: (ast: string | ExpressionAstExpression, input: Input, params?: ExpressionExecutionParams) => Promise; ``` diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iexpressionloaderparams.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iexpressionloaderparams.md index d6e02350bae3f..e2ad6215e25d0 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iexpressionloaderparams.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iexpressionloaderparams.md @@ -22,6 +22,7 @@ export interface IExpressionLoaderParams | [inspectorAdapters](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.inspectoradapters.md) | Adapters | | | [onRenderError](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.onrendererror.md) | RenderErrorHandlerFnType | | | [searchContext](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.searchcontext.md) | ExecutionContextSearch | | +| [searchSessionId](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.searchsessionid.md) | string | | | [uiState](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.uistate.md) | unknown | | | [variables](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.variables.md) | Record<string, any> | | diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iexpressionloaderparams.searchsessionid.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iexpressionloaderparams.searchsessionid.md new file mode 100644 index 0000000000000..bb021b003f0d3 --- /dev/null +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iexpressionloaderparams.searchsessionid.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-expressions-public](./kibana-plugin-plugins-expressions-public.md) > [IExpressionLoaderParams](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.md) > [searchSessionId](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.searchsessionid.md) + +## IExpressionLoaderParams.searchSessionId property + +Signature: + +```typescript +searchSessionId?: string; +``` diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution._constructor_.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution._constructor_.md index 75f4cc4c2a017..f24aae8603b7d 100644 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution._constructor_.md +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution._constructor_.md @@ -9,12 +9,12 @@ Constructs a new instance of the `Execution` class Signature: ```typescript -constructor(params: ExecutionParams); +constructor(execution: ExecutionParams); ``` ## Parameters | Parameter | Type | Description | | --- | --- | --- | -| params | ExecutionParams<ExtraContext> | | +| execution | ExecutionParams | | diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.context.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.context.md index d1969fb0859b7..65c7bdca0fe5d 100644 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.context.md +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.context.md @@ -9,5 +9,5 @@ Execution context - object that allows to do side-effects. Context is passed to Signature: ```typescript -readonly context: ExecutionContext & ExtraContext; +readonly context: ExecutionContext; ``` diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.contract.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.contract.md index 149b5a7ced9cb..2fc6a38997f77 100644 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.contract.md +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.contract.md @@ -9,5 +9,5 @@ Contract is a public representation of `Execution` instances. Contract we can re Signature: ```typescript -readonly contract: ExecutionContract; +readonly contract: ExecutionContract; ``` diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.params.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.execution.md similarity index 67% rename from docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.params.md rename to docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.execution.md index 498f9bbfccfa4..acaccdeab7321 100644 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.params.md +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.execution.md @@ -1,11 +1,11 @@ -[Home](./index.md) > [kibana-plugin-plugins-expressions-server](./kibana-plugin-plugins-expressions-server.md) > [Execution](./kibana-plugin-plugins-expressions-server.execution.md) > [params](./kibana-plugin-plugins-expressions-server.execution.params.md) +[Home](./index.md) > [kibana-plugin-plugins-expressions-server](./kibana-plugin-plugins-expressions-server.md) > [Execution](./kibana-plugin-plugins-expressions-server.execution.md) > [execution](./kibana-plugin-plugins-expressions-server.execution.execution.md) -## Execution.params property +## Execution.execution property Signature: ```typescript -readonly params: ExecutionParams; +readonly execution: ExecutionParams; ``` diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.interpret.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.interpret.md index cf59e796e6120..e425bdc70e349 100644 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.interpret.md +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.interpret.md @@ -7,7 +7,7 @@ Signature: ```typescript -interpret(ast: ExpressionAstNode, input: T, options?: ExpressionExecOptions): Promise; +interpret(ast: ExpressionAstNode, input: T): Promise; ``` ## Parameters @@ -16,7 +16,6 @@ interpret(ast: ExpressionAstNode, input: T, options?: ExpressionExecOptions): | --- | --- | --- | | ast | ExpressionAstNode | | | input | T | | -| options | ExpressionExecOptions | | Returns: diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.md index fc663dd115580..c94ae9bcfe946 100644 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.md +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.md @@ -7,25 +7,25 @@ Signature: ```typescript -export declare class Execution = Record, Input = unknown, Output = unknown, InspectorAdapters extends Adapters = ExtraContext['inspectorAdapters'] extends object ? ExtraContext['inspectorAdapters'] : DefaultInspectorAdapters> +export declare class Execution ``` ## Constructors | Constructor | Modifiers | Description | | --- | --- | --- | -| [(constructor)(params)](./kibana-plugin-plugins-expressions-server.execution._constructor_.md) | | Constructs a new instance of the Execution class | +| [(constructor)(execution)](./kibana-plugin-plugins-expressions-server.execution._constructor_.md) | | Constructs a new instance of the Execution class | ## Properties | Property | Modifiers | Type | Description | | --- | --- | --- | --- | -| [context](./kibana-plugin-plugins-expressions-server.execution.context.md) | | ExecutionContext<Input, InspectorAdapters> & ExtraContext | Execution context - object that allows to do side-effects. Context is passed to every function. | -| [contract](./kibana-plugin-plugins-expressions-server.execution.contract.md) | | ExecutionContract<ExtraContext, Input, Output, InspectorAdapters> | Contract is a public representation of Execution instances. Contract we can return to other plugins for their consumption. | +| [context](./kibana-plugin-plugins-expressions-server.execution.context.md) | | ExecutionContext<InspectorAdapters> | Execution context - object that allows to do side-effects. Context is passed to every function. | +| [contract](./kibana-plugin-plugins-expressions-server.execution.contract.md) | | ExecutionContract<Input, Output, InspectorAdapters> | Contract is a public representation of Execution instances. Contract we can return to other plugins for their consumption. | +| [execution](./kibana-plugin-plugins-expressions-server.execution.execution.md) | | ExecutionParams | | | [expression](./kibana-plugin-plugins-expressions-server.execution.expression.md) | | string | | | [input](./kibana-plugin-plugins-expressions-server.execution.input.md) | | Input | Initial input of the execution.N.B. It is initialized to null rather than undefined for legacy reasons, because in legacy interpreter it was set to null by default. | | [inspectorAdapters](./kibana-plugin-plugins-expressions-server.execution.inspectoradapters.md) | | InspectorAdapters | | -| [params](./kibana-plugin-plugins-expressions-server.execution.params.md) | | ExecutionParams<ExtraContext> | | | [result](./kibana-plugin-plugins-expressions-server.execution.result.md) | | Promise<Output | ExpressionValueError> | | | [state](./kibana-plugin-plugins-expressions-server.execution.state.md) | | ExecutionContainer<Output | ExpressionValueError> | Dynamic state of the execution. | @@ -35,7 +35,7 @@ export declare class Execution = Re | --- | --- | --- | | [cancel()](./kibana-plugin-plugins-expressions-server.execution.cancel.md) | | Stop execution of expression. | | [cast(value, toTypeNames)](./kibana-plugin-plugins-expressions-server.execution.cast.md) | | | -| [interpret(ast, input, options)](./kibana-plugin-plugins-expressions-server.execution.interpret.md) | | | +| [interpret(ast, input)](./kibana-plugin-plugins-expressions-server.execution.interpret.md) | | | | [invokeChain(chainArr, input)](./kibana-plugin-plugins-expressions-server.execution.invokechain.md) | | | | [invokeFunction(fn, input, args)](./kibana-plugin-plugins-expressions-server.execution.invokefunction.md) | | | | [resolveArgs(fnDef, input, argAsts)](./kibana-plugin-plugins-expressions-server.execution.resolveargs.md) | | | diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.getinitialinput.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.getsearchcontext.md similarity index 55% rename from docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.getinitialinput.md rename to docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.getsearchcontext.md index b5f9b91e1c7b7..783841fbafa92 100644 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.getinitialinput.md +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.getsearchcontext.md @@ -1,13 +1,13 @@ -[Home](./index.md) > [kibana-plugin-plugins-expressions-server](./kibana-plugin-plugins-expressions-server.md) > [ExecutionContext](./kibana-plugin-plugins-expressions-server.executioncontext.md) > [getInitialInput](./kibana-plugin-plugins-expressions-server.executioncontext.getinitialinput.md) +[Home](./index.md) > [kibana-plugin-plugins-expressions-server](./kibana-plugin-plugins-expressions-server.md) > [ExecutionContext](./kibana-plugin-plugins-expressions-server.executioncontext.md) > [getSearchContext](./kibana-plugin-plugins-expressions-server.executioncontext.getsearchcontext.md) -## ExecutionContext.getInitialInput property +## ExecutionContext.getSearchContext property -Get initial input with which execution started. +Get search context of the expression. Signature: ```typescript -getInitialInput: () => Input; +getSearchContext: () => ExecutionContextSearch; ``` diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.search.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.getsearchsessionid.md similarity index 63% rename from docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.search.md rename to docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.getsearchsessionid.md index 641e50696f6e0..a69f0ed32bd51 100644 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.search.md +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.getsearchsessionid.md @@ -1,13 +1,13 @@ -[Home](./index.md) > [kibana-plugin-plugins-expressions-server](./kibana-plugin-plugins-expressions-server.md) > [ExecutionContext](./kibana-plugin-plugins-expressions-server.executioncontext.md) > [search](./kibana-plugin-plugins-expressions-server.executioncontext.search.md) +[Home](./index.md) > [kibana-plugin-plugins-expressions-server](./kibana-plugin-plugins-expressions-server.md) > [ExecutionContext](./kibana-plugin-plugins-expressions-server.executioncontext.md) > [getSearchSessionId](./kibana-plugin-plugins-expressions-server.executioncontext.getsearchsessionid.md) -## ExecutionContext.search property +## ExecutionContext.getSearchSessionId property Search context in which expression should operate. Signature: ```typescript -search?: ExecutionContextSearch; +getSearchSessionId: () => string | undefined; ``` diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.md index 0128ba934da73..047879fd99255 100644 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.md +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.md @@ -9,7 +9,7 @@ Signature: ```typescript -export interface ExecutionContext +export interface ExecutionContext ``` ## Properties @@ -17,10 +17,10 @@ export interface ExecutionContextAbortSignal | Adds ability to abort current execution. | -| [getInitialInput](./kibana-plugin-plugins-expressions-server.executioncontext.getinitialinput.md) | () => Input | Get initial input with which execution started. | | [getSavedObject](./kibana-plugin-plugins-expressions-server.executioncontext.getsavedobject.md) | <T extends SavedObjectAttributes = SavedObjectAttributes>(type: string, id: string) => Promise<SavedObject<T>> | Allows to fetch saved objects from ElasticSearch. In browser getSavedObject function is provided automatically by the Expressions plugin. On the server the caller of the expression has to provide this context function. The reason is because on the browser we always know the user who tries to fetch a saved object, thus saved object client is scoped automatically to that user. However, on the server we can scope that saved object client to any user, or even not scope it at all and execute it as an "internal" user. | +| [getSearchContext](./kibana-plugin-plugins-expressions-server.executioncontext.getsearchcontext.md) | () => ExecutionContextSearch | Get search context of the expression. | +| [getSearchSessionId](./kibana-plugin-plugins-expressions-server.executioncontext.getsearchsessionid.md) | () => string | undefined | Search context in which expression should operate. | | [inspectorAdapters](./kibana-plugin-plugins-expressions-server.executioncontext.inspectoradapters.md) | InspectorAdapters | Adapters for inspector plugin. | -| [search](./kibana-plugin-plugins-expressions-server.executioncontext.search.md) | ExecutionContextSearch | Search context in which expression should operate. | | [types](./kibana-plugin-plugins-expressions-server.executioncontext.types.md) | Record<string, ExpressionType> | A map of available expression types. | | [variables](./kibana-plugin-plugins-expressions-server.executioncontext.variables.md) | Record<string, unknown> | Context variables that can be consumed using var and var_set functions. | diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executionparams.debug.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executionparams.debug.md deleted file mode 100644 index b3631e0aeebe6..0000000000000 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executionparams.debug.md +++ /dev/null @@ -1,13 +0,0 @@ - - -[Home](./index.md) > [kibana-plugin-plugins-expressions-server](./kibana-plugin-plugins-expressions-server.md) > [ExecutionParams](./kibana-plugin-plugins-expressions-server.executionparams.md) > [debug](./kibana-plugin-plugins-expressions-server.executionparams.debug.md) - -## ExecutionParams.debug property - -Whether to execute expression in \*debug mode\*. In \*debug mode\* inputs and outputs as well as all resolved arguments and time it took to execute each function are saved and are available for introspection. - -Signature: - -```typescript -debug?: boolean; -``` diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executionparams.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executionparams.md index a7594bff48c1a..6a901c91ffff1 100644 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executionparams.md +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executionparams.md @@ -7,7 +7,7 @@ Signature: ```typescript -export interface ExecutionParams = Record> +export interface ExecutionParams ``` ## Properties @@ -15,8 +15,7 @@ export interface ExecutionParams = | Property | Type | Description | | --- | --- | --- | | [ast](./kibana-plugin-plugins-expressions-server.executionparams.ast.md) | ExpressionAstExpression | | -| [context](./kibana-plugin-plugins-expressions-server.executionparams.context.md) | ExtraContext | | -| [debug](./kibana-plugin-plugins-expressions-server.executionparams.debug.md) | boolean | Whether to execute expression in \*debug mode\*. In \*debug mode\* inputs and outputs as well as all resolved arguments and time it took to execute each function are saved and are available for introspection. | | [executor](./kibana-plugin-plugins-expressions-server.executionparams.executor.md) | Executor<any> | | | [expression](./kibana-plugin-plugins-expressions-server.executionparams.expression.md) | string | | +| [params](./kibana-plugin-plugins-expressions-server.executionparams.params.md) | ExpressionExecutionParams | | diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executionparams.context.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executionparams.params.md similarity index 65% rename from docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executionparams.context.md rename to docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executionparams.params.md index 8b9a210416dd6..fec60af33e0c4 100644 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executionparams.context.md +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executionparams.params.md @@ -1,11 +1,11 @@ -[Home](./index.md) > [kibana-plugin-plugins-expressions-server](./kibana-plugin-plugins-expressions-server.md) > [ExecutionParams](./kibana-plugin-plugins-expressions-server.executionparams.md) > [context](./kibana-plugin-plugins-expressions-server.executionparams.context.md) +[Home](./index.md) > [kibana-plugin-plugins-expressions-server](./kibana-plugin-plugins-expressions-server.md) > [ExecutionParams](./kibana-plugin-plugins-expressions-server.executionparams.md) > [params](./kibana-plugin-plugins-expressions-server.executionparams.params.md) -## ExecutionParams.context property +## ExecutionParams.params property Signature: ```typescript -context?: ExtraContext; +params: ExpressionExecutionParams; ``` diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executor.createexecution.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executor.createexecution.md index 8ed228d70ff37..34e920a04fd02 100644 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executor.createexecution.md +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executor.createexecution.md @@ -7,7 +7,7 @@ Signature: ```typescript -createExecution = Record, Input = unknown, Output = unknown>(ast: string | ExpressionAstExpression, context?: ExtraContext, { debug }?: ExpressionExecOptions): Execution; +createExecution(ast: string | ExpressionAstExpression, params?: ExpressionExecutionParams): Execution; ``` ## Parameters @@ -15,10 +15,9 @@ createExecution = Recordstring | ExpressionAstExpression | | -| context | ExtraContext | | -| { debug } | ExpressionExecOptions | | +| params | ExpressionExecutionParams | | Returns: -`Execution` +`Execution` diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executor.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executor.md index 46ad60ae07126..97bb3ac895084 100644 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executor.md +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executor.md @@ -29,7 +29,7 @@ export declare class Executor = Recordstatic | | | [extendContext(extraContext)](./kibana-plugin-plugins-expressions-server.executor.extendcontext.md) | | | | [extract(ast)](./kibana-plugin-plugins-expressions-server.executor.extract.md) | | | @@ -41,6 +41,6 @@ export declare class Executor = RecordSignature: ```typescript -run = Record>(ast: string | ExpressionAstExpression, input: Input, context?: ExtraContext, options?: ExpressionExecOptions): Promise; +run(ast: string | ExpressionAstExpression, input: Input, params?: ExpressionExecutionParams): Promise; ``` ## Parameters @@ -18,8 +18,7 @@ run = Recordstring | ExpressionAstExpression | | | input | Input | | -| context | ExtraContext | | -| options | ExpressionExecOptions | | +| params | ExpressionExecutionParams | | Returns: diff --git a/docs/discover/images/Discover-Start.png b/docs/discover/images/Discover-Start.png index fb885c20c1cf7..12ec2f9889bbd 100644 Binary files a/docs/discover/images/Discover-Start.png and b/docs/discover/images/Discover-Start.png differ diff --git a/docs/discover/images/time-filter.png b/docs/discover/images/time-filter.png new file mode 100644 index 0000000000000..f6d1d5809d7eb Binary files /dev/null and b/docs/discover/images/time-filter.png differ diff --git a/docs/discover/search.asciidoc b/docs/discover/search.asciidoc index ee1e1526f9d6f..3720a5b457d84 100644 --- a/docs/discover/search.asciidoc +++ b/docs/discover/search.asciidoc @@ -104,9 +104,7 @@ To save the current search: . Click *Save* in the Kibana toolbar. . Enter a name for the search and click *Save*. -To import, export and delete saved searches: -. Open the menu, then click *Stack Management. -. From the {kib} menu, click *Saved Ojbects*. +To import, export, and delete saved searches, open the main menu, then click *Stack Management > Saved Ojbects*. ==== Open a saved search To load a saved search into Discover: diff --git a/docs/discover/set-time-filter.asciidoc b/docs/discover/set-time-filter.asciidoc index 93fdf9ffd695a..dcdc8ee791e83 100644 --- a/docs/discover/set-time-filter.asciidoc +++ b/docs/discover/set-time-filter.asciidoc @@ -30,7 +30,7 @@ to the last 15 minutes. * *Refresh every* to specify an automatic refresh rate. + [role="screenshot"] -image::images/Timepicker-View.png[Time filter menu] +image::images/time-filter.png[Time filter menu] . To set the start and end times, click the bar next to the time filter. In the popup, select *Absolute*, *Relative* or *Now*, then specify the required diff --git a/docs/fleet/images/fleet-start.png b/docs/fleet/images/fleet-start.png index 60e5416fde127..0d0f7b8feec9c 100644 Binary files a/docs/fleet/images/fleet-start.png and b/docs/fleet/images/fleet-start.png differ diff --git a/docs/getting-started/images/add-sample-data.png b/docs/getting-started/images/add-sample-data.png index b8c2002b9c4cd..9dee27dcde71b 100644 Binary files a/docs/getting-started/images/add-sample-data.png and b/docs/getting-started/images/add-sample-data.png differ diff --git a/docs/getting-started/images/tutorial-sample-dashboard.png b/docs/getting-started/images/tutorial-sample-dashboard.png index 9f287640f201c..4c95c04c5e43e 100644 Binary files a/docs/getting-started/images/tutorial-sample-dashboard.png and b/docs/getting-started/images/tutorial-sample-dashboard.png differ diff --git a/docs/getting-started/images/tutorial-sample-filter.png b/docs/getting-started/images/tutorial-sample-filter.png index 7c1d041448557..56ebacadbef45 100644 Binary files a/docs/getting-started/images/tutorial-sample-filter.png and b/docs/getting-started/images/tutorial-sample-filter.png differ diff --git a/docs/getting-started/quick-start-guide.asciidoc b/docs/getting-started/quick-start-guide.asciidoc index 6386feac5ab49..f239b7ae6ca88 100644 --- a/docs/getting-started/quick-start-guide.asciidoc +++ b/docs/getting-started/quick-start-guide.asciidoc @@ -10,8 +10,9 @@ When you've finished, you'll know how to: * <> [float] -=== Before you begin -When security is enabled, you must have `read`, `write`, and `manage` privileges on the `kibana_sample_data_*` indices. For more information, refer to {ref}/security-privileges.html[Security privileges]. +=== Required privileges +When security is enabled, you must have `read`, `write`, and `manage` privileges on the `kibana_sample_data_*` indices. +For more information, refer to {ref}/security-privileges.html[Security privileges]. [float] [[set-up-on-cloud]] @@ -30,7 +31,7 @@ Sample data sets come with sample visualizations, dashboards, and more to help y . On the *Sample eCommerce orders* card, click *Add data*. + [role="screenshot"] -image::getting-started/images/add-sample-data.png[] +image::getting-started/images/add-sample-data.png[Add data UI] [float] [[explore-the-data]] @@ -38,7 +39,7 @@ image::getting-started/images/add-sample-data.png[] *Discover* displays an interactive histogram that shows the distribution of of data, or documents, over time, and a table that lists the fields for each document that matches the index. By default, all fields are shown for each matching document. -. Open the menu, then click *Discover*. +. Open the main menu, then click *Discover*. . Change the <> to *Last 7 days*. + @@ -70,7 +71,7 @@ For more information, refer to <>. A dashboard is a collection of panels that you can use to view and analyze the data. Panels contain visualizations, interactive controls, Markdown, and more. -. Open the menu, then click *Dashboard*. +. Open the main menu, then click *Dashboard*. . Click *[eCommerce] Revenue Dashboard*. + @@ -83,7 +84,7 @@ image::getting-started/images/tutorial-sample-dashboard.png[] To focus in on the data you want to view on the dashboard, use filters. -. From the *Controls* visualization, make a selection from the *Manufacturer* and *Category* dropdowns, then click *Apply changes*. +. From the *[eCommerce] Controls* panel, make a selection from the *Manufacturer* and *Category* dropdowns, then click *Apply changes*. + For example, the following dashboard shows the data for women's clothing from Gnomehouse. + @@ -103,11 +104,11 @@ For more information, refer to <>. [float] [[create-a-visualization]] -=== Create a visualization +=== Create a visualization panel -To create a treemap that shows the top regions and manufacturers, use *Lens*, then add the treemap to the dashboard. +To create a treemap panel that shows the top regions and manufacturers, use *Lens*, then add the treemap panel to the dashboard. -. From the {kib} toolbar, click *Edit*, then click *Create new*. +. From the toolbar, click *Edit*, then click *Create new*. . On the *New Visualization* window, click *Lens*. @@ -126,7 +127,7 @@ image::getting-started/images/tutorial-visualization-dropdown.png[Visualization . On the *Save Lens visualization*, enter a title and make sure *Add to Dashboard after saving* is selected, then click *Save and return*. + -The treemap appears as the last visualization on the dashboard. +The treemap appears as the last visualization panel on the dashboard. + [role="screenshot"] image::getting-started/images/tutorial-final-dashboard.gif[Final dashboard with new treemap visualization] diff --git a/docs/management/advanced-options.asciidoc b/docs/management/advanced-options.asciidoc index 8e8d0e5bf996e..dbac6997ff433 100644 --- a/docs/management/advanced-options.asciidoc +++ b/docs/management/advanced-options.asciidoc @@ -6,11 +6,18 @@ behavior of Kibana. For example, you can change the format used to display dates specify the default index pattern, and set the precision for displayed decimal values. -. Open the menu, then go to *Stack Management > {kib} > Advanced Settings*. +. Open the main menu, then click *Stack Management > Advanced Settings*. . Scroll or search for the setting you want to modify. . Enter a new value for the setting. . Click *Save changes*. +[float] +=== Required permissions + +The `Advanced Settings` {kib} privilege is required to access *Advanced Settings*. + +To add the privilege, open the menu, then click *Stack Management > Roles*. + [float] [[settings-read-only-access]] diff --git a/docs/management/alerting/alerts-and-actions-intro.asciidoc b/docs/management/alerting/alerts-and-actions-intro.asciidoc index 429d7915cc1c3..35a2452e99951 100644 --- a/docs/management/alerting/alerts-and-actions-intro.asciidoc +++ b/docs/management/alerting/alerts-and-actions-intro.asciidoc @@ -6,8 +6,8 @@ beta[] The *Alerts and Actions* UI lets you <> in a space, and provides tools to <> so that alerts can trigger actions like notification, indexing, and ticketing. -To manage alerting and connectors, open the menu, -then go to *Stack Management > Alerts and Insights > Alerts and Actions*. +To manage alerting and connectors, open the main menu, +then click *Stack Management > Alerts and Insights > Alerts and Actions*. [role="screenshot"] image:management/alerting/images/alerts-and-actions-ui.png[Example alert listing in the Alerts and Actions UI] @@ -24,3 +24,8 @@ The *Alerts and Actions* UI only shows alerts and connectors for the current spa can be managed through the <>. See <> for more information. ============================================================================ + +[float] +=== Required permissions + +Access to alerts and actions is granted based on your privileges to alerting-enabled features. See <> for more information. diff --git a/docs/management/index-patterns.asciidoc b/docs/management/index-patterns.asciidoc index 7de2a042160e9..e83e6d262f26c 100644 --- a/docs/management/index-patterns.asciidoc +++ b/docs/management/index-patterns.asciidoc @@ -25,8 +25,8 @@ image::images/management-index-read-only-badge.png[Example of Index Pattern Mana [[settings-create-pattern]] === Create an index pattern -When you don't have an index pattern, {kib} prompts you to create one. Or, you can open the menu, -then go to *Stack Management > {kib} > Index Patterns* to go directly to the *Index Patterns* UI. +When you don't have an index pattern, {kib} prompts you to create one. Or, you can open the main menu, +then click *Stack Management > Index Patterns*. [role="screenshot"] image:management/index-patterns/images/rollup-index-pattern.png["Menu with rollup index pattern"] diff --git a/docs/management/index-patterns/images/index-pattern-ui.png b/docs/management/index-patterns/images/index-pattern-ui.png new file mode 100644 index 0000000000000..7d16540aa03a2 Binary files /dev/null and b/docs/management/index-patterns/images/index-pattern-ui.png differ diff --git a/docs/management/ingest-pipelines/ingest-pipelines.asciidoc b/docs/management/ingest-pipelines/ingest-pipelines.asciidoc index 7986e4e56279a..d9745bfef524a 100644 --- a/docs/management/ingest-pipelines/ingest-pipelines.asciidoc +++ b/docs/management/ingest-pipelines/ingest-pipelines.asciidoc @@ -7,7 +7,7 @@ pipelines that perform common transformations and enrichments on your data. For example, you might remove a field, rename an existing field, or set a new field. -You’ll find *Ingest Node Pipelines* in *Stack Management > Ingest*. With this feature, you can: +To begin, open the main menu, then click *Stack Management > Ingest Node Pipelines*. With *Ingest Node Pipelines*, you can: * View a list of your pipelines and drill down into details. * Create a pipeline that defines a series of tasks, known as processors. @@ -23,7 +23,7 @@ image:management/ingest-pipelines/images/ingest-pipeline-list.png["Ingest node p The minimum required permissions to access *Ingest Node Pipelines* are the `manage_pipeline` and `cluster:monitor/nodes/info` cluster privileges. -You can add these privileges in *Stack Management > Security > Roles*. +To add privileges, open the main menu, then click *Stack Management > Roles*. [role="screenshot"] image:management/ingest-pipelines/images/ingest-pipeline-privileges.png["Privileges required for Ingest Node Pipelines"] diff --git a/docs/management/managing-beats.asciidoc b/docs/management/managing-beats.asciidoc index 678e160b99af0..232efb60cadd3 100644 --- a/docs/management/managing-beats.asciidoc +++ b/docs/management/managing-beats.asciidoc @@ -4,7 +4,7 @@ include::{asciidoc-dir}/../../shared/discontinued.asciidoc[tag=cm-discontinued] -To use {beats} Central Management UI, open the menu, go to *Stack Management > Ingest > +To use {beats} Central Management, open the main menu, click *Stack Management > {beats} Central Management*, then define and manage configurations in a central location in {kib} and quickly deploy configuration changes to all {beats} running across your enterprise. For more @@ -18,8 +18,8 @@ about central management, see the related {beats} documentation: This feature requires an Elastic license that includes {beats} central management. -Don't have a license? You can start a 30-day trial. Open the menu, -go to *Stack Management > Stack > License Management*. At the end of the trial +Don't have a license? You can start a 30-day trial. Open the main menu, then +click *Stack Management > License Management*. At the end of the trial period, you can purchase a subscription to keep using central management. For more information, see https://www.elastic.co/subscriptions and <>. @@ -29,6 +29,13 @@ more information, see https://www.elastic.co/subscriptions and enrollment and configuration process step by step the first time you use the Central Management UI. +[float] +=== Required permissions + +You must have the `beats_admin` role assigned to use **{beats} Central Management** + +To assign the role, open the menu, then click *Stack Management > Users*. + [float] === Enroll {beats} diff --git a/docs/management/managing-fields.asciidoc b/docs/management/managing-fields.asciidoc index ad3a0ef0fcdd1..3734655edd91b 100644 --- a/docs/management/managing-fields.asciidoc +++ b/docs/management/managing-fields.asciidoc @@ -7,6 +7,13 @@ the index patterns that retrieve your data from {es}. [role="screenshot"] image::images/management-index-patterns.png[] +[float] +=== Required permissions + +The `Index Pattern Management` {kib} privilege is required to access the *Index patterns* UI. + +To add the privilege, open the menu, then click *Stack Management > Roles*. + [float] === Create an index pattern @@ -134,7 +141,7 @@ https://www.elastic.co/blog/using-painless-kibana-scripted-fields[Using Painless [[create-scripted-field]] === Create a scripted field -. Open the menu, then go to *Stack Management > {kib} > Index Patterns* +. Open the main menu, then click *Stack Management > Index Patterns*. . Select the index pattern you want to add a scripted field to. . Go to the *Scripted fields* tab for the index pattern, then click *Add scripted field*. . Enter a name for the scripted field. diff --git a/docs/management/managing-indices.asciidoc b/docs/management/managing-indices.asciidoc index b199e076443ab..8416c164c6c51 100644 --- a/docs/management/managing-indices.asciidoc +++ b/docs/management/managing-indices.asciidoc @@ -12,15 +12,7 @@ way possible. This page shows you how to use *Index Management* features to: -* View and edit index settings. -* View mappings and statistics for an index. -* Perform index-level operations, such as refreshes and freezes. -* View and manage data streams. -* Create index templates to automatically configure new data streams and -indices. - -To manage your indices, open the menu, then click *Stack Management > Index -Management*. +To manage your indices, open the main menu, then click *Stack Management > Index Management*. [role="screenshot"] image::images/management_index_labels.png[Index Management UI] diff --git a/docs/management/managing-licenses.asciidoc b/docs/management/managing-licenses.asciidoc index b53bda95466dc..8944414f6bfbc 100644 --- a/docs/management/managing-licenses.asciidoc +++ b/docs/management/managing-licenses.asciidoc @@ -7,7 +7,7 @@ with no expiration date. For the full list of features, refer to If you want to try out the full set of features, you can activate a free 30-day trial. To view the status of your license, start a trial, or install a new -license, open the menu, then go to *Stack Management > Stack > License Management*. +license, open the main menu, then click *Stack Management > License Management*. NOTE: You can start a trial only if your cluster has not already activated a trial license for the current major product version. For example, if you have @@ -34,7 +34,7 @@ the features that will no longer be supported if you revert to a basic license. The `manage` cluster privilege is required to access *License Management*. -You can add this privilege in *Stack Management > Security > Roles*. +To add the privilege, open the main menu, then click *Stack Management > Roles*. [discrete] [[update-license]] diff --git a/docs/management/managing-saved-objects.asciidoc b/docs/management/managing-saved-objects.asciidoc index 8c885ddca52e5..2e081c09e0e70 100644 --- a/docs/management/managing-saved-objects.asciidoc +++ b/docs/management/managing-saved-objects.asciidoc @@ -5,17 +5,21 @@ The *Saved Objects* UI helps you keep track of and manage your saved objects. Th store data for later use, including dashboards, visualizations, maps, index patterns, Canvas workpads, and more. -To get started, open the menu, then go to *Stack Management > {kib} > Saved Objects*. With this UI, you can: - -* <> -* <> -* <> -* <> - +To get started, open the main menu, then click *Stack Management > Saved Objects*. [role="screenshot"] image::images/management-saved-objects.png[Saved Objects] +[float] +=== Required permissions + +The `Saved Objects Management` {kib} privilege is required to access the *Saved Objects* UI. + +To add the privilege, open the menu, then click *Stack Management > Roles*. + +NOTE: +Granting access to Saved Objects Management will authorize users to manage all saved objects in {kib}, including objects that are managed by applications they may not otherwise be authorized to access. + [float] [[managing-saved-objects-view]] diff --git a/docs/management/rollups/create_and_manage_rollups.asciidoc b/docs/management/rollups/create_and_manage_rollups.asciidoc index 7324f45594bd7..bc876ab67bc62 100644 --- a/docs/management/rollups/create_and_manage_rollups.asciidoc +++ b/docs/management/rollups/create_and_manage_rollups.asciidoc @@ -8,11 +8,7 @@ by an index pattern, and then rolls it into a new index. Rollup indices are a go compactly store months or years of historical data for use in visualizations and reports. -To get started, open the menu, then go to *Stack Management > Data > Rollup Jobs*. With this UI, -you can: - -* <> -* <> +To get started, open the main menu, then click *Stack Management > Rollup Jobs*. [role="screenshot"] image::images/management_rollup_list.png[][List of currently active rollup jobs] @@ -25,7 +21,7 @@ Before using this feature, you should be familiar with how rollups work. The `manage_rollup` cluster privilege is required to access *Rollup jobs*. -You can add this privilege in *Stack Management > Security > Roles*. +To add the privilege, open the main menu, then click *Stack Management > Roles*. [float] [[create-and-manage-rollup-job]] @@ -137,7 +133,7 @@ Your next step is to visualize your rolled up data in a vertical bar chart. Most visualizations support rolled up data, with the exception of Timelion and Vega visualizations. -. Go to *Stack Management > {kib} > Index Patterns*. +. Open the main menu, then click *Stack Management > Index Patterns*. . Click *Create index pattern*, and select *Rollup index pattern* from the dropdown. + @@ -152,7 +148,7 @@ is `rollup_logstash,kibana_sample_data_logs`. In this index pattern, `rollup_log matches the rolled up index pattern and `kibana_sample_data_logs` matches the index pattern for raw data. -. Go to *Dashboard* and create a vertical bar chart. +. Open the main menu, click *Dashboard*, then create and add a vertical bar chart. . Choose `rollup_logstash,kibana_sample_data_logs` as your source to see both the raw and rolled up data. diff --git a/docs/management/snapshot-restore/index.asciidoc b/docs/management/snapshot-restore/index.asciidoc index 1bf62522e245c..62633441ef161 100644 --- a/docs/management/snapshot-restore/index.asciidoc +++ b/docs/management/snapshot-restore/index.asciidoc @@ -8,7 +8,7 @@ Snapshots are important because they provide a copy of your data in case something goes wrong. If you need to roll back to an older version of your data, you can restore a snapshot from the repository. -To get started, open the menu, then go to *Stack Management > Data > Snapshot and Restore*. +To get started, open the main menu, then click *Stack Management > Snapshot and Restore*. With this UI, you can: * Register a repository for storing your snapshots @@ -32,7 +32,7 @@ The minimum required permissions to access *Snapshot and Restore* include: * Cluster privileges: `monitor`, `manage_slm`, `cluster:admin/snapshot`, and `cluster:admin/repository` * Index privileges: `all` on the `monitor` index if you want to access content in the *Restore Status* tab -To add privileges, open the menu, then go to *Stack Management > Security > Roles*. +To add privileges, open the main menu, then click *Stack Management > Roles*. [role="screenshot"] image:management/snapshot-restore/images/snapshot_permissions.png["Edit Role"] @@ -191,7 +191,7 @@ your master and data nodes. You can do this in one of two ways: Use *Snapshot and Restore* to register the repository where your snapshots will live. -. Open the menu, then go to *Stack Management > Data > Snapshot and Restore*. +. Open the main menu, then click *Stack Management > Snapshot and Restore*. . Click *Register a repository* in either the introductory message or *Repository view*. . Enter a name for your repository, for example, `my_backup`. . Select *Shared file system*. @@ -212,7 +212,7 @@ The repository currently doesn’t have any snapshots. ==== Add a snapshot to the repository Use the {ref}/snapshots-take-snapshot.html[snapshot API] to create a snapshot. -. Open the menu, go to *Dev Tools*, then select *Console*. +. Open the main menu, click *Dev Tools*, then select *Console*. . Create the snapshot: + [source,js] diff --git a/docs/management/upgrade-assistant/index.asciidoc b/docs/management/upgrade-assistant/index.asciidoc index 2b8c2da2ef577..61df6457a9bde 100644 --- a/docs/management/upgrade-assistant/index.asciidoc +++ b/docs/management/upgrade-assistant/index.asciidoc @@ -4,7 +4,7 @@ The Upgrade Assistant helps you prepare for your upgrade to the next major {es} version. For example, if you are using 6.8, the Upgrade Assistant helps you to upgrade to 7.0. -To access the assistant, open the menu, then go to *Stack Management > Stack > Upgrade Assistant*. +To access the assistant, open the main menu, then click *Stack Management > Upgrade Assistant*. The assistant identifies the deprecated settings in your cluster and indices and guides you through the process of resolving issues, including reindexing. @@ -19,7 +19,7 @@ For example, if you want to upgrade to to 7.0, make sure that you are using 6.8. The `manage` cluster privilege is required to access the *Upgrade assistant*. Additional privileges may be needed to perform certain actions. -You can add this privilege in *Stack Management > Security > Roles*. +To add the privilege, open the main menu, then click *Stack Management > Roles*. [float] === Reindexing diff --git a/docs/management/watcher-ui/index.asciidoc b/docs/management/watcher-ui/index.asciidoc index 23a0acbff5718..69c33aa7a1dac 100644 --- a/docs/management/watcher-ui/index.asciidoc +++ b/docs/management/watcher-ui/index.asciidoc @@ -8,8 +8,8 @@ Watches are helpful for analyzing mission-critical and business-critical streaming data. For example, you might watch application logs for performance outages or audit access logs for security threats. -To get started with the Watcher UI, open then menu, -then go to *Stack Management > Alerts and Insights > Watcher*. +To get started, open then main menu, +then click *Stack Management > Watcher*. With this UI, you can: * <> @@ -41,7 +41,7 @@ and either of these watcher roles: * `watcher_admin`. You can perform all Watcher actions, including create and edit watches. * `watcher_user`. You can view watches, but not create or edit them. -To manage roles, open then menu, then go to *Stack Management > Security > Roles*, or use the +To manage roles, open then main menu, then click *Stack Management > Roles*, or use the <>. Watches are shared between all users with the same role. diff --git a/docs/maps/geojson-upload.asciidoc b/docs/maps/geojson-upload.asciidoc new file mode 100644 index 0000000000000..3c9bea11176cc --- /dev/null +++ b/docs/maps/geojson-upload.asciidoc @@ -0,0 +1,44 @@ +[role="xpack"] +[[geojson-upload]] +== Upload GeoJSON data + +Maps makes it easy to import geospatial data into the Elastic Stack. +Using the GeoJSON Upload feature, you can drag and drop your point and shape +data files directly into {es}, and then use them as layers +in the map. You can also use the GeoJSON data in the broader Kibana ecosystem, +for example, in visualizations and Canvas workpads. + +[float] +=== Why GeoJSON? +GeoJSON is an open-standard file format for storing geospatial vector data. +Although many vector data formats are available in the GIS community, +GeoJSON is the most commonly used and flexible option. +[float] + +=== Upload a GeoJSON file +Follow these instructions to upload a GeoJSON data file, or try the +<>. + +. Open the main menu, click *Maps*, and then click *Add layer*. +. Click *Uploaded GeoJSON*. ++ +[role="screenshot"] +image::maps/images/fu_gs_select_source_file_upload.png[] + +. Use the file chooser to select a valid GeoJSON file. The file will load +a preview of the data on the map. +. Use the default *Index type* of {ref}/geo-point.html[geo_point] for point data, +or override it and select {ref}/geo-shape.html[geo_shape]. +All other shapes will default to a type of `geo_shape`. +. Leave the default *Index name* and *Index pattern* names (the name of the uploaded +file minus its extension). You might need to change the index name if it is invalid. +. Click *Import file*. ++ +Upon completing the indexing process and creating the associated index pattern, +the Elasticsearch responses are shown on the *Layer add panel* and the indexed data +appears on the map. The geospatial data on the map +should be identical to the locally-previewed data, but now it's indexed data from Elasticsearch. + +. To continue adding data to the map, click *Add layer*. +. In *Layer settings*, adjust any settings or <> as needed. +. Click *Save & close*. diff --git a/docs/maps/import-geospatial-data.asciidoc b/docs/maps/import-geospatial-data.asciidoc index 194d09c491cee..ff0c9bf1f72ba 100644 --- a/docs/maps/import-geospatial-data.asciidoc +++ b/docs/maps/import-geospatial-data.asciidoc @@ -11,7 +11,7 @@ Choose an import tool based on the format of your geospatial data. *File Data Visualizer* indexes CSV files with latitude and longitude columns as a geo_point. -. Open the side navigation menu, and click *Machine Learning*. +. Open the main menu, then click *Machine Learning*. . Select the *Data Visualizer* tab, then click *Upload file*. . Use the file chooser to select a CSV file. . Click *Import*. diff --git a/docs/maps/maps-getting-started.asciidoc b/docs/maps/maps-getting-started.asciidoc index f48ff268755d2..5c6cd87b235e1 100644 --- a/docs/maps/maps-getting-started.asciidoc +++ b/docs/maps/maps-getting-started.asciidoc @@ -50,7 +50,7 @@ In this tutorial, you'll learn to: The first thing to do is to create a new map. -. If you haven't already, click *{kib} > Maps* from the side navigation. +. If you haven't already, open the main menu, then click *Maps*. . On the maps list page, click *Create map*. . Set the time range to *Last 7 days*. + @@ -188,7 +188,7 @@ You have completed the steps for re-creating the sample data map. === Add the map to a dashboard You can add your saved map to a {kibana-ref}/dashboard.html[dashboard] and view your geospatial data alongside bar charts, pie charts, and other visualizations. -. Open the menu, then go to *Dashboard*. +. Open the main menu, then click *Dashboard*. . Click *Create dashboard*. . Set the time range to *Last 7 days*. . Click *Add*. diff --git a/docs/settings/apm-settings.asciidoc b/docs/settings/apm-settings.asciidoc index b396c40aa21f9..9054a97c90496 100644 --- a/docs/settings/apm-settings.asciidoc +++ b/docs/settings/apm-settings.asciidoc @@ -18,7 +18,7 @@ It is enabled by default. // Any changes made in this file will be seen there as well. // tag::apm-indices-settings[] -Index defaults can be changed in Kibana. Navigate to *APM* > *Settings* > *Indices*. +Index defaults can be changed in Kibana. Open the main menu, then click *APM > Settings > Indices*. Index settings in the APM app take precedence over those set in `kibana.yml`. [role="screenshot"] @@ -44,7 +44,7 @@ Changing these settings may disable features of the APM App. | Set to `false` to disable the APM app. Defaults to `true`. | `xpack.apm.ui.enabled` {ess-icon} - | Set to `false` to hide the APM app from the menu. Defaults to `true`. + | Set to `false` to hide the APM app from the main menu. Defaults to `true`. | `xpack.apm.ui.transactionGroupBucketSize` | Number of top transaction groups displayed in the APM app. Defaults to `1000`. diff --git a/docs/setup/access.asciidoc b/docs/setup/access.asciidoc index 49aa411e91512..edf936fe54267 100644 --- a/docs/setup/access.asciidoc +++ b/docs/setup/access.asciidoc @@ -1,24 +1,37 @@ [[access]] == Access {kib} -Kibana is a web application that you access through port 5601. All you need to do is point your web browser at the -machine where Kibana is running and specify the port number. For example, `localhost:5601` or `http://YOURDOMAIN.com:5601`. -If you want to allow remote users to connect, set the parameter `server.host` in `kibana.yml` to a non-loopback address. +The fastest way to access {kib} is to use our hosted {es} Service. If you <>, access {kib} through the web application. -When you access Kibana, the <> page loads by default with the default index pattern selected. The -time filter is set to the last 15 minutes and the search query is set to match-all (\*). +[float] +=== Set up on cloud -If you don't see any documents, try setting the time filter to a wider time range. -If you still don't see any results, it's possible that you don't *have* any documents. +include::{docs-root}/shared/cloud/ess-getting-started.asciidoc[] + +[float] +[[log-on-to-the-web-application]] +=== Log on to the web application + +If you are using a self-managed deployment, you access {kib} through the web application on port 5601. + +. Point your web browser to the machine where you are running {kib} and specify the port number. For example, `localhost:5601` or `http://YOURDOMAIN.com:5601`. + +. To allow remote users to connect to {kib}, set the parameter `server.host` in kibana.yml to a non-loopback address. + +. On the home page, click *{kib}*. ++ +To make the {kib} page your landing page, click *Make this my landing page*. [float] [[status]] -=== Check {kib} status +=== Check the {kib} status -You can reach the Kibana server's status page by navigating to the status endpoint, for example, `localhost:5601/status`. The status page displays -information about the server's resource usage and lists the installed plugins. +To view the {kib} status page, use the status endpoint. For example, `localhost:5601/status`. The status page displays +information about the server resource usage and installed plugins. [role="screenshot"] image::images/kibana-status-page-7_5_0.png[] -NOTE: For JSON-formatted server status details, use the API endpoint at `localhost:5601/api/status` +For JSON-formatted server status details, use the `localhost:5601/api/status` API endpoint. + + diff --git a/docs/setup/connect-to-elasticsearch.asciidoc b/docs/setup/connect-to-elasticsearch.asciidoc index 3db562319641c..c968ca6f35029 100644 --- a/docs/setup/connect-to-elasticsearch.asciidoc +++ b/docs/setup/connect-to-elasticsearch.asciidoc @@ -20,14 +20,15 @@ to see all that you can do in {kib}. experimental[] -To visualize data in a CSV, JSON, or log file, you can upload it using the File -Data Visualizer. On the home page, click *Import a CSV, NDSON, or log file*, and -then drag your file into the File Data Visualizer. Alternatively, you can open -it by navigating to *Machine Learning* from the side navigation and selecting +To visualize data in a CSV, JSON, or log file, you can upload it using the File +Data Visualizer. On the home page, click *Upload a file*, and +then drag your file onto the *File Data Visualizer*. Alternatively, you can open +it by navigating to *Machine Learning* from the side navigation and selecting + *Data Visualizer*. [role="screenshot"] -image::images/data-viz-homepage.jpg[File Data Visualizer on the home page] +image::images/ingest-data.png[File Data Visualizer on the home page] You can upload a file up to 100 MB. This value is configurable up to 1 GB in <>. @@ -78,7 +79,7 @@ create an index pattern that matches the names of the indices that you want to e When you add data with the File Data Visualizer, GeoJSON Upload feature, or built-in tutorial, an index pattern is created for you. -. Go to *Stack Management*, and then click *Index Patterns*. +. Open the main menu, then click *Stack Management > Index Patterns*. . Click *Create index pattern*. diff --git a/docs/setup/images/data-viz-homepage.jpg b/docs/setup/images/data-viz-homepage.jpg deleted file mode 100644 index f7a952b65d41f..0000000000000 Binary files a/docs/setup/images/data-viz-homepage.jpg and /dev/null differ diff --git a/docs/setup/images/ingest-data.png b/docs/setup/images/ingest-data.png new file mode 100644 index 0000000000000..b1943d6de27d2 Binary files /dev/null and b/docs/setup/images/ingest-data.png differ diff --git a/docs/setup/upgrade.asciidoc b/docs/setup/upgrade.asciidoc index 6d69b6921b612..92cd6e9ead5a1 100644 --- a/docs/setup/upgrade.asciidoc +++ b/docs/setup/upgrade.asciidoc @@ -4,29 +4,24 @@ Depending on the {kib} version you're upgrading from, the upgrade process to 7.0 varies. -NOTE: {kib} upgrades automatically when starting a new version, as described in -<>. -Although you do not need to manually back up {kib} before upgrading, we recommend -that you have a backup on hand. You can use -<> to back up {kib} -data by targeting `.kibana*` indices. If you are using the Reporting plugin, -you can also target `.reporting*` indices. - [float] [[upgrade-before-you-begin]] === Before you begin +WARNING: {kib} automatically runs upgrade migrations when required. To roll back to an earlier version in case of an upgrade failure, you **must** have a backup snapshot available. Use <> to back up {kib} data by targeting the `.kibana*` indices. For more information see <>. + Before you upgrade {kib}: * Consult the <>. +* Back up your data with <>. To roll back to an earlier version, you **must** have a snapshot of the `.kibana*` indices. +* Although not a requirement for rollbacks, we recommend taking a snapshot of all {kib} indices created by the plugins you use such as the `.reporting*` indices created by the reporting plugin. * Before you upgrade production servers, test the upgrades in a dev environment. -* Back up your data with {es} {ref}/modules-snapshots.html[snapshots]. - To roll back to an earlier version, you **must** have a backup of your data. +* See <> for common reasons upgrades fail and how to prevent these. * If you are using custom plugins, check that a compatible version is available. -* Shut down all {kib} nodes. Running more than one {kib} version against the - same Elasticseach index is unsupported. If you upgrade while older {kib} nodes are - running, the upgrade can fail. +* Shut down all {kib} instances. Running more than one {kib} version against + the same Elasticseach index is unsupported. Upgrading while older {kib} + instances are running can cause data loss or upgrade failures. To identify the changes you need to make to upgrade, and to enable you to perform an Elasticsearch rolling upgrade with no downtime, you must upgrade to diff --git a/docs/setup/upgrade/upgrade-migrations.asciidoc b/docs/setup/upgrade/upgrade-migrations.asciidoc index cbe56b9e65894..74d097164c4a7 100644 --- a/docs/setup/upgrade/upgrade-migrations.asciidoc +++ b/docs/setup/upgrade/upgrade-migrations.asciidoc @@ -1,54 +1,127 @@ [[upgrade-migrations]] -=== Migrate saved objects +=== Upgrade migrations -Every time {kib} is upgraded it checks to see if all saved objects, such as dashboards, visualizations, and index patterns, are compatible with the new version. If any objects need to be updated, then the automatic saved object migration process is kicked off. +Every time {kib} is upgraded it checks to see if all saved objects, such as dashboards, visualizations, and index patterns, are compatible with the new version. If any saved objects need to be updated, then the automatic saved object migration process is kicked off. NOTE: 6.7 includes an https://www.elastic.co/guide/en/kibana/6.7/upgrade-assistant.html[Upgrade Assistant] to help you prepare for your upgrade to 7.0. To access the assistant, go to *Management > 7.0 Upgrade Assistant*. +WARNING: The following instructions assumes {kib} is using the default index names. If the `kibana.index` or `xpack.tasks.index` configuration settings were changed these instructions will have to be adapted accordingly. + [float] [[upgrade-migrations-process]] -==== How the process works +==== Background -Saved objects are stored in an index named `.kibana_N`, where `N` is a number that increments over time as {kib} is upgraded. The index alias `.kibana` points to the latest up-to-date index for a given install. +Saved objects are stored in two indices: -NOTE: Prior to 6.5.0, saved objects were stored directly in an index named `.kibana`, so the first time you upgrade to {kib} version 6.5+, {kib} will migrate into `.kibana_1` and set `.kibana` up as an index alias. +* `.kibana_N`, or if set, the `kibana.index` configuration setting +* `.kibana_task_manager_N`, or if set, the `xpack.tasks.index` configuration setting + +For each of these indices, `N` is a number that increments every time {kib} runs an upgrade migration on that index. The index aliases `.kibana` and `.kibana_task_manager` point to the most up-to-date index. While {kib} is starting up and before serving any HTTP traffic, it checks to see if any internal mapping changes or data transformations for existing saved objects are required. -When changes are necessary, a new incremental `.kibana_N` index is created with updated mappings, then the saved objects are loaded in batches from the existing index, transformed to whatever extent necessary, and added to this new index. +When changes are necessary, a new migration is started. To ensure that only one {kib} instance performs the migration, each instance will attempt to obtain a migration lock by creating a new `.kibana_N+1` index. The instance that succeeds in creating the index will then read batches of documents from the existing index, migrate them, and write them to the new index. Once the objects are migrated, the lock is released by pointing the `.kibana` index alias the new upgraded `.kibana_N+1` index. + +Instances that failed to acquire a lock will log `Another Kibana instance appears to be migrating the index. Waiting for that migration to complete`. The instance will then wait until `.kibana` points to an upgraded index before starting up and serving HTTP traffic. -Once the objects are migrated, the `.kibana` index alias is updated to point to the new index, and {kib} finishes starting up and serving HTTP traffic. +NOTE: Prior to 6.5.0, saved objects were stored directly in an index named `.kibana`. After upgrading to version 6.5+, {kib} will migrate this index into `.kibana_N` and set `.kibana` up as an index alias. + +Prior to 7.4.0, task manager tasks were stored directly in an index name `.kibana_task_manager`. After upgrading to version 7.4+, {kib} will migrate this index into `.kibana_task_manager_N` and set `.kibana_task_manager` up as an index alias. [float] -[[upgrade-migrations-old-indices]] -==== Handling old `.kibana` indices +[[preventing-migration-failures]] +==== Preventing migration failures +This section highlights common causes of {kib} upgrade failures and how to prevent them. + +[float] +===== Corrupt saved objects +We highly recommend testing your {kib} upgrade in a development cluster to discover and remedy problems caused by corrupt documents, especially when there are custom integrations creating saved objects in your environment. Saved objects that were corrupted through manual editing or integrations will cause migration failures with a log message like `Failed to transform document. Transform: index-pattern:7.0.0\n Doc: {...}` or `Unable to migrate the corrupt Saved Object document ...`. Corrupt documents will have to be fixed or deleted before an upgrade migration can succeed. + +[float] +===== User defined index templates that causes new `.kibana*` indices to have incompatible settings or mappings +Matching index templates which specify `settings.refresh_interval` or `mappings` are known to interfere with {kib} upgrades. + +Prevention: narrow down the index patterns of any user-defined index templates to ensure that these won't apply to new `.kibana*` indices. -After migrations have run, there will be multiple {kib} indices in {es}: (`.kibana_1`, `.kibana_2`, etc). {kib} only uses the index that the `.kibana` alias points to. The other {kib} indices can be safely deleted, but are left around as a matter of historical record, and to facilitate rolling {kib} back to a previous version. +Note: {kib} < 6.5 creates it's own index template called `kibana_index_template:.kibana` and index pattern `.kibana`. This index template will not interfere and does not need to be changed or removed. + +[float] +===== An unhealthy {es} cluster +Problems with your {es} cluster can prevent {kib} upgrades from succeeding. Ensure that your cluster has: + + * enough free disk space, at least twice the amount of storage taken up by the `.kibana` and `.kibana_task_manager` indices + * sufficient heap size + * a "green" cluster status + +[float] +===== Running different versions of {kib} connected to the same {es} index +Kibana does not support rolling upgrades. Stop all {kib} instances before starting a newer version to prevent upgrade failures and data loss. + +[float] +===== Incompatible `xpack.tasks.index` configuration setting +For {kib} < 7.5.1, if the task manager index is set to `.tasks` with the configuration setting `xpack.tasks.index: ".tasks"`, upgrade migrations will fail. {kib} 7.5.1 and later prevents this by refusing to start with an incompatible configuration setting. [float] -[[upgrade-migrations-errors]] -==== Handling errors during saved object migrations +[[resolve-migrations-failures]] +==== Resolving migration failures -If {kib} terminates unexpectedly while migrating a saved object index, some additional work may be required in order to get {kib} to re-attempt the migration. +If {kib} terminates unexpectedly while migrating a saved object index, manual intervention is required before {kib} will attempt to perform the migration again. Follow the advice in (preventing migration failures)[preventing-migration-failures] before retrying a migration upgrade. -For example, if the `.kibana` alias is pointing to `.kibana_4`, and there is a `.kibana_5` index in {es}, the `.kibana_5` index will need to be deleted. {kib} will never attempt to overwrite an existing index. +As mentioned above, {kib} will create a migration lock for each index that requires a migration by creating a new `.kibana_N+1` index. For example: if the `.kibana_task_manager` alias is pointing to `.kibana_task_manager_5` then the first {kib} that succeeds in creating `.kibana_task_manager_6` will obtain the lock to start migrations. + +However, if the instance that obtained the lock fails to migrate the index, all other {kib} instances will be blocked from performing this migration. This includes the instance that originally obtained the lock, it will be blocked from retrying the migration even when restarted. [float] -[[upgrade-migrations-multiple-instances]] -==== Support for multiple {kib} instances +===== Retry a migration by restoring a backup snapshot: + +1. Before proceeding ensure that you have a recent and successful backup snapshot of all `.kibana*` indices. +2. Shutdown all {kib} instances to be 100% sure that there are no instances currently performing a migration. +3. Delete all saved object indices with `DELETE /.kibana*` +4. Restore the `.kibana* indices and their aliases from the backup snapshot. See {es} {ref}/modules-snapshots.html[snapshots] +5. Start up all {kib} instances to retry the upgrade migration. -If you're running multiple {kib} instances for a single index behind a load balancer, it's important that you stop all instances before upgrading, so you do not have multiple different versions of {kib} trying to perform saved object migrations. +[float] +===== (Not recommended) Retry a migration without a backup snapshot: -The first instance that triggers saved object migrations will run the entire process. Any other instances started up while a migration is running will log a message and then wait until saved object migration has completed before they start serving HTTP traffic. +1. Shutdown all {kib} instances to be 100% sure that there are no instances currently performing a migration. +2. Identify any migration locks by comparing the output of `GET /_cat/aliases` and `GET /_cat/indices`. If e.g. `.kibana` is pointing to `.kibana_4` and there is a `.kibana_5` index, the `.kibana_5` index will act like a migration lock blocking further attempts. Be sure to check both the `.kibana` and `.kibana_task_manager` aliases and their indices. +3. Remove any migration locks e.g. `DELETE /.kibana_5`. +4. Start up all {kib} instances. [float] [[upgrade-migrations-rolling-back]] ==== Rolling back to a previous version of {kib} -When rolling {kib} back to a previous version, point the `.kibana` alias to -the appropriate {kib} index. When you have the previous version running again, -delete the more recent `.kibana_N` index or indices so that future upgrades are -based on the current {kib} index. You must restart {kib} to re-trigger the migration. +If you've followed the advice in (preventing migration failures)[preventing-migration-failures] and (resolving migration failures)[resolve-migrations-failures] and {kib} is still not able to upgrade successfully, you might choose to rollback {kib} until you're able to identify the root cause. + +WARNING: Before rolling back {kib}, ensure that the version you wish to rollback to is compatible with your {es} cluster. If the version you're rolling back to is not compatible, you will have to also rollback {es}. + +Any changes made after an upgrade will be lost when rolling back to a previous version. + +In order to rollback after a failed upgrade migration, the saved object indices might also have to be rolled back to be compatible with the previous {kibana} version. + +[float] +===== Rollback by restoring a backup snapshot: + +1. Before proceeding ensure that you have a recent and successful backup snapshot of all `.kibana*` indices. +2. Shutdown all {kib} instances to be 100% sure that there are no instances currently performing a migration. +3. Delete all saved object indices with `DELETE /.kibana*` +4. Restore the `.kibana* indices and their aliases from the backup snapshot. See {es} {ref}/modules-snapshots.html[snapshots] +5. Start up all {kib} instances on the older version you wish to rollback to. + +[float] +===== (Not recommended) Rollback without a backup snapshot: + +WARNING: {kib} does not run a migration for every saved object index on every upgrade. A {kib} version upgrade can cause no migrations, migrate only the `.kibana` or the `.kibana_task_manager` index or both. Carefully read the logs to ensure that you're only deleting indices created by a later version of {kib} to avoid data loss. + +1. Shutdown all {kib} instances to be 100% sure that there are no {kib} instances currently performing a migration. +2. Create a backup snapshot of the `.kibana*` indices. +3. Use the logs from the upgraded instances to identify which indices {kib} attempted to upgrade. The server logs will contain an entry like `[savedobjects-service] Creating index .kibana_4.` and/or `[savedobjects-service] Creating index .kibana_task_manager_2.` If no indices were created after upgrading {kib} then no further action is required to perform a rollback, skip ahead to step (5). If you're running multiple {kib} instances, be sure to inspect all instances' logs. +4. Delete each of the indices identified in step (2). e.g. `DELETE /.kibana_task_manager_2` +5. Inspect the output of `GET /_cat/aliases`. If either the `.kibana` and/or `.kibana_task_manager` alias is missing, these will have to be created manually. Find the latest index from the output of `GET /_cat/indices` and create the missing alias to point to the latest index. E.g. if the `.kibana` alias was missing and the latest index is `.kibana_3` create a new alias with `POST /.kibana_3/_aliases/.kibana`. +6. Start up {kib} on the older version you wish to rollback to. + +[float] +[[upgrade-migrations-old-indices]] +==== Handling old `.kibana_N` indices -WARNING: Rolling back to a previous {kib} version can result in saved object data loss if you had successfully upgraded and made changes to saved objects before rolling back. +After migrations have completed, there will be multiple {kib} indices in {es}: (`.kibana_1`, `.kibana_2`, etc). {kib} only uses the index that the `.kibana` alias points to. The other {kib} indices can be safely deleted, but are left around as a matter of historical record, and to facilitate rolling {kib} back to a previous version. \ No newline at end of file diff --git a/docs/setup/upgrade/upgrade-standard.asciidoc b/docs/setup/upgrade/upgrade-standard.asciidoc index df38427881d65..b27bb8867e624 100644 --- a/docs/setup/upgrade/upgrade-standard.asciidoc +++ b/docs/setup/upgrade/upgrade-standard.asciidoc @@ -12,11 +12,20 @@ If you've saved and/or exported objects in {kib} that rely on the necessary remediation steps as per those instructions. =========================================== +[float] +==== Upgrading multiple {kib} instances + +WARNING: Kibana does not support rolling upgrades. If you're running multiple {kib} instances, all instances should be stopped before upgrading. + +Different versions of {kib} running against the same {es} index, such as during a rolling upgrade, can cause upgrade migration failures and data loss. This is because acknowledged writes from the older instances could be written into the _old_ index while the migration is in progress. To prevent this from happening ensure that all old {kib} instances are shutdown before starting up instances on a newer version. + +The first instance that triggers saved object migrations will run the entire process. Any other instances started up while a migration is running will log a message and then wait until saved object migrations has completed before they start serving HTTP traffic. + [float] ==== Upgrade using a `deb` or `rpm` package . Stop the existing {kib} process using the appropriate command for your - system. + system. If you have multiple {kib} instances connecting to the same {es} cluster ensure that all instances are stopped before proceeding to the next step to avoid data loss. . Use `rpm` or `dpkg` to install the new package. All files should be placed in their proper locations and config files should not be overwritten. + @@ -43,8 +52,7 @@ otherwise {kib} will fail to start. don't overwrite the `config` or `data` directories. + + -- -IMPORTANT: If you use {monitor-features}, you must re-use the data directory when you -upgrade {kib}. Otherwise, the {kib} instance is assigned a new persistent UUID +IMPORTANT: If you use {monitor-features}, you must re-use the data directory when you upgrade {kib}. Otherwise, the {kib} instance is assigned a new persistent UUID and becomes a new instance in the monitoring data. -- @@ -57,5 +65,5 @@ and becomes a new instance in the monitoring data. . Install the appropriate versions of all your plugins for your new installation using the `kibana-plugin` script. Check out the <> documentation for more information. -. Stop the old {kib} process. +. Stop the old {kib} process. If you have multiple {kib} instances connecting to the same {es} cluster ensure that all instances are stopped before proceeding to the next step to avoid data loss. . Start the new {kib} process. diff --git a/docs/spaces/images/edit-space-feature-visibility.png b/docs/spaces/images/edit-space-feature-visibility.png index bc983bde2679c..3a74c21fe2635 100644 Binary files a/docs/spaces/images/edit-space-feature-visibility.png and b/docs/spaces/images/edit-space-feature-visibility.png differ diff --git a/docs/spaces/images/edit-space.png b/docs/spaces/images/edit-space.png index 68ffea23c4ac4..c4565a3ba0f4a 100644 Binary files a/docs/spaces/images/edit-space.png and b/docs/spaces/images/edit-space.png differ diff --git a/docs/spaces/images/space-selector.png b/docs/spaces/images/space-selector.png index 908c5360acd39..b2576cbc9acc4 100644 Binary files a/docs/spaces/images/space-selector.png and b/docs/spaces/images/space-selector.png differ diff --git a/docs/spaces/images/spaces-roles.png b/docs/spaces/images/spaces-roles.png old mode 100755 new mode 100644 index 2ecbfed6017c4..407926895daa8 Binary files a/docs/spaces/images/spaces-roles.png and b/docs/spaces/images/spaces-roles.png differ diff --git a/docs/spaces/index.asciidoc b/docs/spaces/index.asciidoc index 9e505b8bfe045..81f3945779503 100644 --- a/docs/spaces/index.asciidoc +++ b/docs/spaces/index.asciidoc @@ -25,11 +25,16 @@ Kibana supports spaces in several ways. You can: * <> * <> +[float] +==== Required permissions + +The `kibana_admin` role or equivilent is required to manage **Spaces**. + [float] [[spaces-managing]] === View, create, and delete spaces -Open the menu, then go to *Stack Management > {kib} > Spaces* for an overview of your spaces. This view provides actions +Open the main menu, then click *Stack Management > Spaces* for an overview of your spaces. This view provides actions for you to create, edit, and delete spaces. [role="screenshot"] @@ -94,8 +99,8 @@ image::spaces/images/spaces-roles.png["Controlling features visiblity"] [[spaces-moving-objects]] === Move saved objects between spaces -To <> from one space to another, open the menu, -then go to *Stack Management > {kib} > Saved objects*. +To <> from one space to another, open the main menu, +then click *Stack Management > Saved Objects*. Alternately, you can move objects using {kib}'s <> interface. diff --git a/docs/user/alerting/action-types/pagerduty.asciidoc b/docs/user/alerting/action-types/pagerduty.asciidoc index 9301224e6df48..aad192dbddb30 100644 --- a/docs/user/alerting/action-types/pagerduty.asciidoc +++ b/docs/user/alerting/action-types/pagerduty.asciidoc @@ -89,8 +89,8 @@ image::user/alerting/images/pagerduty-integration.png[PagerDuty Integrations tab + * Create a connector as part of creating an alert by selecting PagerDuty in the *Actions* section of the alert configuration and selecting *Add new*. -* Alternatively, create a connector by navigating to *Management* from the {kib} navbar and selecting -*Alerts and Actions*. Then, select the *Connectors* tab, click the *Create connector* button, and select the PagerDuty option. +* Alternatively, create a connector. To create a connector, open the main menu, click *Stack Management* > +Alerts and Actions*, select *Connectors*, click *Create connector*, then select the PagerDuty option. . Configure the connector by giving it a name and entering the Integration Key, optionally entering a custom API URL. + @@ -99,7 +99,7 @@ See <> for how to obtain the endpoint and . Save the Connector. -. Create an alert using *Management > Alerts and Actions* or the application of your choice. +. To create an alert, open the main menu, then click *Stack Management > Alerts and Actions* or the application of your choice. . Set up an action using your PagerDuty connector, by determining: + @@ -120,7 +120,7 @@ To remove a PagerDuty connector from an alert, simply remove it from the *Actions* section of that alert, using the remove (x) icon. This will disable the integration for the particular alert. -To delete the connector entirely, go to *Management > Alerts and Actions*. +To delete the connector entirely, open the main menu, then click *Stack Management > Alerts and Actions*. Select the *Connectors* tab, and then click on the delete icon. This is an irreversible action and impacts all alerts that use this connector. diff --git a/docs/user/alerting/action-types/pre-configured-connectors.asciidoc b/docs/user/alerting/action-types/pre-configured-connectors.asciidoc index e3f1703f08e88..722607ac05f87 100644 --- a/docs/user/alerting/action-types/pre-configured-connectors.asciidoc +++ b/docs/user/alerting/action-types/pre-configured-connectors.asciidoc @@ -61,7 +61,7 @@ Sensitive properties, such as passwords, can also be stored in the < {kib} > Alerts and Actions*, preconfigured connectors +When you open the main menu, click *Stack Management > Alerts and Actions*. Preconfigured connectors appear on the <>, regardless of which space you are in. They are tagged as “preconfigured”, and you cannot delete them. @@ -101,7 +101,7 @@ This example shows a preconfigured action type with one out-of-the box connector [[managing-pre-configured-action-types]] To attach a preconfigured action to an alert: -. Open the menu, then go to *Stack Management > {kib} > Alerts and Actions*, open the *Connectors* tab. +. Open the main menu, click *Stack Management > Alerts and Actions*, then open the *Connectors* tab. . Click *Create connector.* diff --git a/docs/user/canvas.asciidoc b/docs/user/canvas.asciidoc index 297dfac5b10bd..c10641bb3a6b9 100644 --- a/docs/user/canvas.asciidoc +++ b/docs/user/canvas.asciidoc @@ -17,7 +17,7 @@ With Canvas, you can: * Focus the data you want to display with filters. -To begin, open the menu, then go to *Canvas*. +To begin, open the main menu, then click *Canvas*. [role="screenshot"] image::images/canvas-gs-example.png[Getting started example] diff --git a/docs/user/dashboard/dashboard.asciidoc b/docs/user/dashboard/dashboard.asciidoc index 4fa4f9860c2bd..5fda1af55c7fe 100644 --- a/docs/user/dashboard/dashboard.asciidoc +++ b/docs/user/dashboard/dashboard.asciidoc @@ -8,7 +8,7 @@ A _dashboard_ is a collection of panels that you use to analyze your data. On a you can rearrange and tell a story about your data. Panels contain everything you need, including visualizations, interactive controls, markdown, and more. -With *Dashboard*s, you can: +With *Dashboard*, you can: * Add multiple panels to see many aspects and views of your data in one place. @@ -22,7 +22,7 @@ With *Dashboard*s, you can: * Generate reports based on your findings. -To begin, open the menu, go to *Dashboard*, then click *Create dashboard*. +To begin, open the main menu, click *Dashboard*, then click *Create dashboard*. [role="screenshot"] image:images/Dashboard_example.png[Example dashboard] @@ -424,33 +424,37 @@ Ready to try out Timelion? For step-by-step tutorials, refer to: [[timelion-deprecation]] ==== Timelion app deprecation -Deprecated since 7.0, the Timelion app will be removed in 8.0. If you have any Timelion worksheets, you must migrate them to a dashboard. +In 7.0 and later, *Timelion* app is deprecated. In 8.0 and later, *Timelion* app is removed from {kib}. To prepare for the removal of *Timelion* app, you must migrate *Timelion* app worksheets to a dashboard. -NOTE: Only the Timelion app is deprecated. {kib} continues to support Timelion -visualizations on dashboards and in Visualize and Canvas. +NOTE: Only *Timelion* app is deprecated. {kib} continues to support *Timelion* +visualizations in *Dashboard*, *Visualize*, and *Canvas*. -To migrate a Timelion worksheet to a dashboard: +To migrate a *Timelion* worksheet to a dashboard: -. Open the menu, click **Dashboard**, then click **Create dashboard**. +. Open the main menu, click *Dashboard*, then click *Create dashboard*. -. On the dashboard, click **Create New**, then select the Timelion visualization. +. For each *Timelion* app worksheet, complete the following steps. -. On a new tab, open the Timelion app, select the chart you want to copy, and copy its expression. +.. On the dashboard, click *Create New*, then click *Timelion* on the *New Visualization* window. + +.. Open a new tab, open the *Timelion* app, select the chart you want to copy, then copy the chart expression. + [role="screenshot"] -image::images/timelion-copy-expression.png[] +image::images/timelion-copy-expression.png[Timelion app chart] -. Return to the other tab and paste the copied expression to the *Timelion Expression* field and click **Update**. +.. Go to *Timelion*, paste the chart expression in the *Timelion expression* field, then click *Update*. + [role="screenshot"] -image::images/timelion-vis-paste-expression.png[] +image::images/timelion-vis-paste-expression.png[Timelion advanced editor UI] + +.. In the toolbar, click *Save*. -. Save the new visualization, give it a name, and click **Save and Return**. +.. On the *Save visualization* window, enter the visualization *Title*, then click *Save and return*. + -Your Timelion visualization will appear on the dashboard. Repeat this for all your charts on each worksheet. +The Timelion visualization panel appears on the dashboard. + [role="screenshot"] -image::images/timelion-dashboard.png[] +image::images/timelion-dashboard.png[Final dashboard with saved Timelion app worksheet] [float] [[save-panels]] @@ -458,7 +462,7 @@ image::images/timelion-dashboard.png[] When you’ve finished making changes, save the panels. -. Click *Save*. +. In the toolbar, click *Save*. . Add the *Title* and optional *Description*. . Click *Save and return*. diff --git a/docs/user/dashboard/edit-dashboards.asciidoc b/docs/user/dashboard/edit-dashboards.asciidoc index 7534ea1e9e9fb..7b712b355b315 100644 --- a/docs/user/dashboard/edit-dashboards.asciidoc +++ b/docs/user/dashboard/edit-dashboards.asciidoc @@ -78,7 +78,7 @@ Put the dashboard in *Edit* mode, then use the following options: * To resize, click the resize control, then drag to the new dimensions. -* To delete, open the panel menu, then select Delete from dashboard. When you delete a panel from the dashboard, the +* To delete, open the panel menu, then select *Delete from dashboard*. When you delete a panel from the dashboard, the visualization or saved search from the panel is still available in Kibana. [float] diff --git a/docs/user/dashboard/images/Dashboard_add_new_visualization.png b/docs/user/dashboard/images/Dashboard_add_new_visualization.png index 3685f9c5c9a74..5f73b2f1adde2 100644 Binary files a/docs/user/dashboard/images/Dashboard_add_new_visualization.png and b/docs/user/dashboard/images/Dashboard_add_new_visualization.png differ diff --git a/docs/user/dashboard/images/Dashboard_add_visualization.png b/docs/user/dashboard/images/Dashboard_add_visualization.png index b1b86d47e5982..4caa34ef3d082 100644 Binary files a/docs/user/dashboard/images/Dashboard_add_visualization.png and b/docs/user/dashboard/images/Dashboard_add_visualization.png differ diff --git a/docs/user/dashboard/images/Dashboard_example.png b/docs/user/dashboard/images/Dashboard_example.png index 1a80f4b3bdf07..c2e338d0fd31b 100644 Binary files a/docs/user/dashboard/images/Dashboard_example.png and b/docs/user/dashboard/images/Dashboard_example.png differ diff --git a/docs/user/dashboard/images/Dashboard_inspect.png b/docs/user/dashboard/images/Dashboard_inspect.png index d65b968e043a6..635eef4a017f6 100644 Binary files a/docs/user/dashboard/images/Dashboard_inspect.png and b/docs/user/dashboard/images/Dashboard_inspect.png differ diff --git a/docs/user/dashboard/images/drilldown_on_piechart.gif b/docs/user/dashboard/images/drilldown_on_piechart.gif index c9b3311df0325..c438e14371887 100644 Binary files a/docs/user/dashboard/images/drilldown_on_piechart.gif and b/docs/user/dashboard/images/drilldown_on_piechart.gif differ diff --git a/docs/user/dashboard/images/timelion-copy-expression.png b/docs/user/dashboard/images/timelion-copy-expression.png new file mode 100644 index 0000000000000..a9c3afe9b060f Binary files /dev/null and b/docs/user/dashboard/images/timelion-copy-expression.png differ diff --git a/docs/visualize/images/timelion-vis-paste-expression.png b/docs/user/dashboard/images/timelion-vis-paste-expression.png similarity index 100% rename from docs/visualize/images/timelion-vis-paste-expression.png rename to docs/user/dashboard/images/timelion-vis-paste-expression.png diff --git a/docs/user/dashboard/images/url_drilldown_go_to_github.gif b/docs/user/dashboard/images/url_drilldown_go_to_github.gif index 7cca3f72d5a68..3a3b00dc0e2ce 100644 Binary files a/docs/user/dashboard/images/url_drilldown_go_to_github.gif and b/docs/user/dashboard/images/url_drilldown_go_to_github.gif differ diff --git a/docs/user/dashboard/share-dashboards.asciidoc b/docs/user/dashboard/share-dashboards.asciidoc index cfa146d60fdac..6c05240c934e8 100644 --- a/docs/user/dashboard/share-dashboards.asciidoc +++ b/docs/user/dashboard/share-dashboards.asciidoc @@ -23,5 +23,5 @@ tools. To create a short URL, you must have write access to {kib}. [[import-dashboards]] === Export the dashboard -To export the dashboard, open the menu, then click *Stack Management > Saved Objects*. For more information, +To export the dashboard, open the main menu, then click *Stack Management > Saved Objects*. For more information, refer to <>. \ No newline at end of file diff --git a/docs/user/graph/getting-started.asciidoc b/docs/user/graph/getting-started.asciidoc index aca6d40a3532e..086c0707b3c2c 100644 --- a/docs/user/graph/getting-started.asciidoc +++ b/docs/user/graph/getting-started.asciidoc @@ -9,7 +9,7 @@ You must index data into {es} before you can create a graph. [[exploring-connections]] === Graph a data connection -. Open the menu, then go to *Graph*. +. Open the main menu, then click *Graph*. + If this is your first graph, follow the prompts to create it. For subsequent graphs, click *New*. diff --git a/docs/user/graph/images/graph-add-query.png b/docs/user/graph/images/graph-add-query.png index 1b233e3ef8b69..93ddf6a6132f4 100644 Binary files a/docs/user/graph/images/graph-add-query.png and b/docs/user/graph/images/graph-add-query.png differ diff --git a/docs/user/graph/images/graph-link-summary.png b/docs/user/graph/images/graph-link-summary.png index 4c75be00de0f5..a3dfdc0f79d96 100644 Binary files a/docs/user/graph/images/graph-link-summary.png and b/docs/user/graph/images/graph-link-summary.png differ diff --git a/docs/user/graph/images/graph-url-connections.png b/docs/user/graph/images/graph-url-connections.png index 4f8c163ab764b..34b57d489b048 100644 Binary files a/docs/user/graph/images/graph-url-connections.png and b/docs/user/graph/images/graph-url-connections.png differ diff --git a/docs/user/introduction.asciidoc b/docs/user/introduction.asciidoc index 7e5dc59b03a2c..91f149d5cdb3c 100644 --- a/docs/user/introduction.asciidoc +++ b/docs/user/introduction.asciidoc @@ -20,16 +20,16 @@ and more — all from the convenience of a {kib} UI. document discovery to SIEM, {kib} is the portal for accessing these and other capabilities. [role="screenshot"] -image::images/intro-kibana.png[] +image::images/intro-kibana.png[Kibana home page] [float] [[get-data-into-kibana]] -=== Add data +=== Ingest data -{kib} is designed to use {es} as a data source. Think of Elasticsearch as the engine that stores +{kib} is designed to use {es} as a data source. Think of {es} as the engine that stores and processes the data, with {kib} sitting on top. -From the home page, {kib} provides these options for adding data: +From the home page, {kib} provides these options for ingesting data: * Import data using the https://www.elastic.co/blog/importing-csv-and-log-data-into-elasticsearch-with-file-data-visualizer[File Data visualizer]. @@ -60,7 +60,7 @@ search for hidden insights and relationships. Ask your questions, and then narrow the results to just the data you want. [role="screenshot"] -image::images/intro-discover.png[] +image::images/intro-discover.png[Discover UI] [float] [[visualize-and-analyze]] @@ -79,7 +79,7 @@ use <> to collect them in one place. A dashboard provides insights into your data from multiple perspectives. [role="screenshot"] -image::images/intro-dashboard.png[] +image::images/intro-dashboard.png[Sample eCommerce data set dashboard] {kib} also offers these visualization features: @@ -112,7 +112,7 @@ You can even choose which features to enable within each space. Don’t need Machine learning in your “Executive” space? Simply turn it off. [role="screenshot"] -image::images/intro-spaces.jpg[] +image::images/intro-spaces.png[Space selector screen] You can take this all one step further with Kibana’s security features, and control which users have access to each space. {kib} allows for fine-grained @@ -156,5 +156,4 @@ You can also <> — no code, no addi infrastructure required. Our <> and in-product guidance can -help you get up and running, faster. Click the help icon image:images/intro-help-icon.png[] -in the top navigation bar for help with questions or to provide feedback. +help you get up and running, faster. Click the help icon image:images/intro-help-icon.png[Help icon in navigation bar] for help with questions or to provide feedback. diff --git a/docs/user/introduction/images/intro-dashboard.png b/docs/user/introduction/images/intro-dashboard.png index fe4e6f620d19c..bb4e98a516fb7 100644 Binary files a/docs/user/introduction/images/intro-dashboard.png and b/docs/user/introduction/images/intro-dashboard.png differ diff --git a/docs/user/introduction/images/intro-data-tutorial.png b/docs/user/introduction/images/intro-data-tutorial.png index 2882a092fbb0b..781e134605b87 100644 Binary files a/docs/user/introduction/images/intro-data-tutorial.png and b/docs/user/introduction/images/intro-data-tutorial.png differ diff --git a/docs/user/introduction/images/intro-discover.png b/docs/user/introduction/images/intro-discover.png index 54e5725596421..134804941a356 100644 Binary files a/docs/user/introduction/images/intro-discover.png and b/docs/user/introduction/images/intro-discover.png differ diff --git a/docs/user/introduction/images/intro-kibana.png b/docs/user/introduction/images/intro-kibana.png index 62c2c99826131..3d10a31d7e380 100644 Binary files a/docs/user/introduction/images/intro-kibana.png and b/docs/user/introduction/images/intro-kibana.png differ diff --git a/docs/user/introduction/images/intro-spaces.jpg b/docs/user/introduction/images/intro-spaces.jpg deleted file mode 100755 index 7569dfc16b4f7..0000000000000 Binary files a/docs/user/introduction/images/intro-spaces.jpg and /dev/null differ diff --git a/docs/user/introduction/images/intro-spaces.png b/docs/user/introduction/images/intro-spaces.png new file mode 100644 index 0000000000000..b2576cbc9acc4 Binary files /dev/null and b/docs/user/introduction/images/intro-spaces.png differ diff --git a/docs/user/management.asciidoc b/docs/user/management.asciidoc index c371aa695c475..ee85819b4fd98 100644 --- a/docs/user/management.asciidoc +++ b/docs/user/management.asciidoc @@ -6,6 +6,10 @@ *Stack Management* is home to UIs for managing all things Elastic Stack— indices, clusters, licenses, UI settings, index patterns, spaces, and more. + +Access to individual features is governed by {es} and {kib} privileges. +Consult your administrator if you do not have the appropriate access. + [float] [[manage-ingest]] == Ingest diff --git a/docs/user/monitoring/monitoring-kibana.asciidoc b/docs/user/monitoring/monitoring-kibana.asciidoc index 9d735ea1fe3db..047fcc08775e6 100644 --- a/docs/user/monitoring/monitoring-kibana.asciidoc +++ b/docs/user/monitoring/monitoring-kibana.asciidoc @@ -48,7 +48,7 @@ By default, if you are running {kib} locally, go to `http://localhost:5601/`. If {security-features} are enabled, log in. -- -... Open the menu, then go to *Stack Monitoring*. If data collection is +... Open the main menu, then click *Stack Monitoring*. If data collection is disabled, you are prompted to turn it on. ** From the Console or command line, set `xpack.monitoring.collection.enabled` diff --git a/docs/user/monitoring/viewing-metrics.asciidoc b/docs/user/monitoring/viewing-metrics.asciidoc index 0c48e3b7d011d..9507b70c4f72e 100644 --- a/docs/user/monitoring/viewing-metrics.asciidoc +++ b/docs/user/monitoring/viewing-metrics.asciidoc @@ -80,7 +80,7 @@ By default, if you are running {kib} locally, go to `http://localhost:5601/`. If the Elastic {security-features} are enabled, log in. -- -. Open *Stack Monitoring*. +. Open the main menu, then click *Stack Monitoring*. + -- If data collection is disabled, you are prompted to turn on data collection. diff --git a/docs/user/reporting/automating-report-generation.asciidoc b/docs/user/reporting/automating-report-generation.asciidoc index 371855deb2f3c..413573e7ec182 100644 --- a/docs/user/reporting/automating-report-generation.asciidoc +++ b/docs/user/reporting/automating-report-generation.asciidoc @@ -13,7 +13,7 @@ URL that triggers a report to generate. To create the POST URL for PDF reports: -. Go to *Dashboard*, then open the visualization or dashboard. +. Open then main menu, click *Dashboard*, then open a dashboard. + To specify a relative or absolute time period, use the time filter. diff --git a/docs/user/reporting/images/preserve-layout-switch.png b/docs/user/reporting/images/preserve-layout-switch.png index 9cfbdaafc3ac5..0aaefb14d7ee5 100644 Binary files a/docs/user/reporting/images/preserve-layout-switch.png and b/docs/user/reporting/images/preserve-layout-switch.png differ diff --git a/docs/user/reporting/images/share-button.png b/docs/user/reporting/images/share-button.png deleted file mode 100644 index 0b307d947935e..0000000000000 Binary files a/docs/user/reporting/images/share-button.png and /dev/null differ diff --git a/docs/user/reporting/images/share-menu.png b/docs/user/reporting/images/share-menu.png new file mode 100644 index 0000000000000..7f1d9eda0b5bc Binary files /dev/null and b/docs/user/reporting/images/share-menu.png differ diff --git a/docs/user/reporting/images/shareable-container.png b/docs/user/reporting/images/shareable-container.png index e114f63e2fe12..829fe15706a52 100644 Binary files a/docs/user/reporting/images/shareable-container.png and b/docs/user/reporting/images/shareable-container.png differ diff --git a/docs/user/reporting/index.asciidoc b/docs/user/reporting/index.asciidoc index 50ae92382fb24..cd93389bb5fde 100644 --- a/docs/user/reporting/index.asciidoc +++ b/docs/user/reporting/index.asciidoc @@ -14,7 +14,7 @@ Reporting is available from the *Share* menu in *Discover*, *Dashboard*, and *Canvas*. [role="screenshot"] -image::user/reporting/images/share-button.png["Share"] +image::user/reporting/images/share-menu.png["Share"] [float] == Setup @@ -94,7 +94,7 @@ image::user/reporting/images/preserve-layout-switch.png["Share"] [[manage-report-history]] == View and manage report history -For a list of your reports, open the menu, then go to *Stack Management > Alerts and Insights > Reporting*. +For a list of your reports, open the main menu, then click *Stack Management > Reporting*. From this view, you can monitor the generation of a report and download reports that you previously generated. diff --git a/docs/user/security/api-keys/index.asciidoc b/docs/user/security/api-keys/index.asciidoc index 7cf1b964082d9..8b59115859622 100644 --- a/docs/user/security/api-keys/index.asciidoc +++ b/docs/user/security/api-keys/index.asciidoc @@ -15,7 +15,7 @@ Or, you might create API keys to automate ingestion of new data from remote sources, without a live user interaction. You can create API keys from the {kib} Console. To view and invalidate -API keys, open the menu, then go to *Stack Management > Security > API Keys*. +API keys, open the main menu, then click *Stack Management > API Keys*. [role="screenshot"] image:user/security/api-keys/images/api-keys.png["API Keys UI"] @@ -39,8 +39,8 @@ or contact your system administrator. === Security privileges You must have the `manage_security`, `manage_api_key`, or the `manage_own_api_key` -cluster privileges to use API keys in {kib}. To manage roles, open the menu, then go to -*Stack Management > Security > Roles*, or use the <>. +cluster privileges to use API keys in {kib}. To manage roles, open the main menu, then click +*Stack Management > Roles*, or use the <>. [float] diff --git a/docs/user/security/authorization/index.asciidoc b/docs/user/security/authorization/index.asciidoc index 3af49753db664..af5fd34b3b350 100644 --- a/docs/user/security/authorization/index.asciidoc +++ b/docs/user/security/authorization/index.asciidoc @@ -12,7 +12,12 @@ NOTE: When running multiple tenants of {kib} by changing the `kibana.index` in y [[xpack-kibana-role-management]] === {kib} role management -To create a role that grants {kib} privileges, open the menu, go to *Stack Management > Security > Roles* and click **Create role**. +To create a role that grants {kib} privileges, open the menu, then click *Stack Management > Roles* and click **Create role**. + +[float] +==== Required permissions + +The `manage_security` cluster privilege is required to access role management. [[adding_kibana_privileges]] ==== Adding {kib} privileges diff --git a/docs/user/security/images/add-space-privileges.png b/docs/user/security/images/add-space-privileges.png old mode 100755 new mode 100644 index 7739332c33b60..d2fcbe76c1a06 Binary files a/docs/user/security/images/add-space-privileges.png and b/docs/user/security/images/add-space-privileges.png differ diff --git a/docs/user/security/images/assign_base_privilege.png b/docs/user/security/images/assign_base_privilege.png index 34e2bcf81d618..93bed0de05555 100644 Binary files a/docs/user/security/images/assign_base_privilege.png and b/docs/user/security/images/assign_base_privilege.png differ diff --git a/docs/user/security/images/assign_feature_privilege.png b/docs/user/security/images/assign_feature_privilege.png index c9449f6390253..d42ec208325a3 100644 Binary files a/docs/user/security/images/assign_feature_privilege.png and b/docs/user/security/images/assign_feature_privilege.png differ diff --git a/docs/user/security/images/privilege-example-1.png b/docs/user/security/images/privilege-example-1.png old mode 100755 new mode 100644 index 68ba716437240..b8fb4d15b8f77 Binary files a/docs/user/security/images/privilege-example-1.png and b/docs/user/security/images/privilege-example-1.png differ diff --git a/docs/user/security/images/role-space-visualization.png b/docs/user/security/images/role-space-visualization.png index 746af89c66e85..0de94d81065fe 100644 Binary files a/docs/user/security/images/role-space-visualization.png and b/docs/user/security/images/role-space-visualization.png differ diff --git a/docs/user/security/images/view-privilege-summary.png b/docs/user/security/images/view-privilege-summary.png old mode 100755 new mode 100644 index d93d55c93fd12..7d2f3018d7de9 Binary files a/docs/user/security/images/view-privilege-summary.png and b/docs/user/security/images/view-privilege-summary.png differ diff --git a/docs/user/security/index.asciidoc b/docs/user/security/index.asciidoc index e1a46a415fe68..18ace452ce00c 100644 --- a/docs/user/security/index.asciidoc +++ b/docs/user/security/index.asciidoc @@ -10,10 +10,16 @@ auditing. For more information, see {ref}/secure-cluster.html[Secure a cluster] and <>. +[float] +=== Required permissions + +The `manage_security` cluster privilege is required to access all Security features. + + [float] === Users -To create and manage users, open the menu, then go to *Stack Management > Security > Users*. +To create and manage users, open the main menu, then click *Stack Management > Users*. You can also change their passwords and roles. For more information about authentication and built-in users, see {ref}/setting-up-authentication.html[Setting up user authentication]. @@ -21,7 +27,7 @@ authentication and built-in users, see [float] === Roles -To manage roles, open the menu, then go to *Stack Management > Security > Roles*, or use +To manage roles, open the main menu, then click *Stack Management > Roles*, or use the <>. For more information on configuring roles for {kib}, see <>. For a more holistic overview of configuring roles for the entire stack, diff --git a/docs/user/security/rbac_tutorial.asciidoc b/docs/user/security/rbac_tutorial.asciidoc index bf7be6284b1a9..2088110f6de21 100644 --- a/docs/user/security/rbac_tutorial.asciidoc +++ b/docs/user/security/rbac_tutorial.asciidoc @@ -45,7 +45,7 @@ through in this tutorial: [float] ==== Create a role -Open the menu, then go to *Stack Management > Security > Roles* +Open the main menu, then click *Stack Management > Roles* for an overview of your roles. This view provides actions for you to create, edit, and delete roles. @@ -90,7 +90,7 @@ image::security/images/role-space-visualization.png["Associate space"] [float] ==== Create the developer user account with the proper roles -. Open the menu, then go to *Stack Management > Security > Users*. +. Open the main menu, then click *Stack Management > Users*. . Click **Create user**, then give the user the `dev-mortgage` and `monitoring-user` roles, which are required for *Stack Monitoring* users. diff --git a/docs/user/security/reporting.asciidoc b/docs/user/security/reporting.asciidoc index daf9720a0f1d8..6e7fc0c212f07 100644 --- a/docs/user/security/reporting.asciidoc +++ b/docs/user/security/reporting.asciidoc @@ -24,11 +24,11 @@ to report on and the {es} indices. [[reporting-roles-management-ui]] === If you are using the `native` realm -To assign roles, open the menu, then go to *Stack Management > Security > Roles*, use the <>. +To assign roles, use the *Roles* UI or <>. This example shows how to use *Roles* page to create a user who has a custom role and the `reporting_user` role. -. Open the menu, then go to *Stack Management > Security > Roles*. +. Open the main menu, then click *Stack Management > Roles*. . Click *Create role*, then give the role a name, for example, `custom_reporting_user`. @@ -51,7 +51,7 @@ that provides read and write privileges in . Save your new role. -. Open the menu, then go to *Stack Management > Security > Users*, add a new user, and assign the user the built-in +. Open the main menu, then click *Stack Management > Users*, add a new user, and assign the user the built-in `reporting_user` role and your new custom role, `custom_reporting_user`. [float] @@ -69,10 +69,10 @@ If you use a different pattern for the `xpack.reporting.index` setting, you must create a custom role with appropriate access to the index, similar to the following: -. Open the menu, then go to *Stack Management >Security > Roles*. +. Open the main menu, then click *Stack Management > Roles*. . Click *Create role*, then name the role `custom-reporting-user`. . Specify the custom index and assign it the `all` index privilege. -. Open the menu, then go to *Stack Management > Security > Users* and create a new user with +. Open the main menu, then click *Stack Management > Users* and create a new user with the `kibana_system` role and the `custom-reporting-user` role. . Configure {kib} to use the new account: [source,js] diff --git a/docs/user/security/role-mappings/index.asciidoc b/docs/user/security/role-mappings/index.asciidoc index 661c319af827f..ca3ca9a686892 100644 --- a/docs/user/security/role-mappings/index.asciidoc +++ b/docs/user/security/role-mappings/index.asciidoc @@ -9,7 +9,7 @@ or SAML. Role mappings have no effect for users inside the `native` or `file` realms. -To manage your role mappings, open the menu, then go to *Stack Management > Security > Role Mappings*. +To manage your role mappings, open the main menu, then click *Stack Management > Role Mappings*. With *Role mappings*, you can: @@ -19,11 +19,16 @@ With *Role mappings*, you can: [role="screenshot"] image:user/security/role-mappings/images/role-mappings-grid.png["Role mappings"] +[float] +==== Required permissions + +The `manage_security` cluster privilege is required to manage Role Mappings. + [float] === Create a role mapping -. Open the menu, then go to *Stack Management > Security > Role Mappings*. +. Open the main menu, then click *Stack Management > Role Mappings*. . Click *Create role mapping*. . Give your role mapping a unique name, and choose which roles you wish to assign to your users. + diff --git a/docs/user/security/securing-kibana.asciidoc b/docs/user/security/securing-kibana.asciidoc index e7bd297a3ebb5..613ec88ed0edc 100644 --- a/docs/user/security/securing-kibana.asciidoc +++ b/docs/user/security/securing-kibana.asciidoc @@ -81,10 +81,10 @@ use {kib}. For more information on Basic Authentication and additional methods of authenticating {kib} users, see <>. -To manage privileges, open the menu, then go to *Stack Management > Security > Roles*. +To manage privileges, open the main menu, then click *Stack Management > Roles*. -If you're using the native realm with Basic Authentication, open then menu, -then go to *Stack Management > Security > Users* to assign roles, or use the +If you're using the native realm with Basic Authentication, open then main menu, +then click *Stack Management > Users* to assign roles, or use the {ref}/security-api.html#security-user-apis[user management APIs]. For example, the following creates a user named `jacknich` and assigns it the `kibana_admin` role: diff --git a/docs/user/setup.asciidoc b/docs/user/setup.asciidoc index 31e7d157d1bc7..54bdfff8e0bbb 100644 --- a/docs/user/setup.asciidoc +++ b/docs/user/setup.asciidoc @@ -1,5 +1,5 @@ [[setup]] -= Set up Kibana += Set up [partintro] -- diff --git a/docs/visualize/images/timelion-copy-expression.png b/docs/visualize/images/timelion-copy-expression.png deleted file mode 100644 index 376bf7919166e..0000000000000 Binary files a/docs/visualize/images/timelion-copy-expression.png and /dev/null differ diff --git a/package.json b/package.json index 84e9c0e2762eb..3a2d13fd5ef3b 100644 --- a/package.json +++ b/package.json @@ -228,7 +228,7 @@ "@babel/register": "^7.10.5", "@babel/types": "^7.11.0", "@elastic/apm-rum": "^5.6.1", - "@elastic/charts": "23.2.1", + "@elastic/charts": "24.0.0", "@elastic/ems-client": "7.10.0", "@elastic/eslint-config-kibana": "0.15.0", "@elastic/eslint-plugin-eui": "0.0.2", diff --git a/packages/kbn-apm-config-loader/src/config.test.ts b/packages/kbn-apm-config-loader/src/config.test.ts index 83438215716ac..8adc4d84635af 100644 --- a/packages/kbn-apm-config-loader/src/config.test.ts +++ b/packages/kbn-apm-config-loader/src/config.test.ts @@ -28,6 +28,8 @@ import { import { ApmConfiguration } from './config'; +const initialEnv = { ...process.env }; + describe('ApmConfiguration', () => { beforeEach(() => { packageMock.raw = { @@ -39,6 +41,7 @@ describe('ApmConfiguration', () => { }); afterEach(() => { + process.env = { ...initialEnv }; resetAllMocks(); }); @@ -90,13 +93,16 @@ describe('ApmConfiguration', () => { let config = new ApmConfiguration(mockedRootDir, {}, false); expect(config.getConfig('serviceName')).toEqual( expect.objectContaining({ - serverUrl: expect.any(String), - secretToken: expect.any(String), + breakdownMetrics: true, }) ); config = new ApmConfiguration(mockedRootDir, {}, true); - expect(Object.keys(config.getConfig('serviceName'))).not.toContain('serverUrl'); + expect(config.getConfig('serviceName')).toEqual( + expect.objectContaining({ + breakdownMetrics: false, + }) + ); }); it('loads the configuration from the kibana config file', () => { @@ -156,4 +162,32 @@ describe('ApmConfiguration', () => { }) ); }); + + it('correctly sets environment', () => { + delete process.env.ELASTIC_APM_ENVIRONMENT; + delete process.env.NODE_ENV; + + let config = new ApmConfiguration(mockedRootDir, {}, false); + expect(config.getConfig('serviceName')).toEqual( + expect.objectContaining({ + environment: 'development', + }) + ); + + process.env.NODE_ENV = 'production'; + config = new ApmConfiguration(mockedRootDir, {}, false); + expect(config.getConfig('serviceName')).toEqual( + expect.objectContaining({ + environment: 'production', + }) + ); + + process.env.ELASTIC_APM_ENVIRONMENT = 'ci'; + config = new ApmConfiguration(mockedRootDir, {}, false); + expect(config.getConfig('serviceName')).toEqual( + expect.objectContaining({ + environment: 'ci', + }) + ); + }); }); diff --git a/packages/kbn-apm-config-loader/src/config.ts b/packages/kbn-apm-config-loader/src/config.ts index 897e7fd7ca610..a611e205ec83a 100644 --- a/packages/kbn-apm-config-loader/src/config.ts +++ b/packages/kbn-apm-config-loader/src/config.ts @@ -26,32 +26,26 @@ import { readFileSync } from 'fs'; import { ApmAgentConfig } from './types'; const getDefaultConfig = (isDistributable: boolean): ApmAgentConfig => { - if (isDistributable) { - return { - active: false, - globalLabels: {}, - // Do not use a centralized controlled config - centralConfig: false, - // Capture all exceptions that are not caught - logUncaughtExceptions: true, - // Can be performance intensive, disabling by default - breakdownMetrics: false, - }; - } - + // https://www.elastic.co/guide/en/apm/agent/nodejs/current/configuration.html return { - active: false, - serverUrl: 'https://f1542b814f674090afd914960583265f.apm.us-central1.gcp.cloud.es.io:443', + active: process.env.ELASTIC_APM_ACTIVE || false, + environment: process.env.ELASTIC_APM_ENVIRONMENT || process.env.NODE_ENV || 'development', + + serverUrl: 'https://b1e3b4b4233e44cdad468c127d0af8d8.apm.europe-west1.gcp.cloud.es.io:443', + // The secretToken below is intended to be hardcoded in this file even though // it makes it public. This is not a security/privacy issue. Normally we'd // instead disable the need for a secretToken in the APM Server config where // the data is transmitted to, but due to how it's being hosted, it's easier, // for now, to simply leave it in. - secretToken: 'R0Gjg46pE9K9wGestd', + secretToken: '2OyjjaI6RVkzx2O5CV', + + logUncaughtExceptions: true, globalLabels: {}, - breakdownMetrics: true, centralConfig: false, - logUncaughtExceptions: true, + + // Can be performance intensive, disabling by default + breakdownMetrics: isDistributable ? false : true, }; }; @@ -84,7 +78,8 @@ export class ApmConfiguration { getDefaultConfig(this.isDistributable), this.getConfigFromKibanaConfig(), this.getDevConfig(), - this.getDistConfig() + this.getDistConfig(), + this.getCIConfig() ); const rev = this.getGitRev(); @@ -146,6 +141,21 @@ export class ApmConfiguration { }; } + private getCIConfig(): ApmAgentConfig { + if (process.env.ELASTIC_APM_ENVIRONMENT !== 'ci') { + return {}; + } + + return { + globalLabels: { + branch: process.env.ghprbSourceBranch || '', + targetBranch: process.env.ghprbTargetBranch || '', + ciJobName: process.env.JOB_NAME || '', + ciBuildNumber: process.env.BUILD_NUMBER || '', + }, + }; + } + private getGitRev() { if (this.isDistributable) { return this.pkgBuild.sha; diff --git a/packages/kbn-monaco/src/esql/constants.ts b/packages/kbn-monaco/src/esql/constants.ts new file mode 100644 index 0000000000000..59bf9a94d05b2 --- /dev/null +++ b/packages/kbn-monaco/src/esql/constants.ts @@ -0,0 +1,20 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export const ID = 'esql'; diff --git a/packages/kbn-monaco/src/esql/index.ts b/packages/kbn-monaco/src/esql/index.ts new file mode 100644 index 0000000000000..b0e25af760a26 --- /dev/null +++ b/packages/kbn-monaco/src/esql/index.ts @@ -0,0 +1,23 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { ID } from './constants'; +import { lexerRules } from './lexer_rules'; + +export const EsqlLang = { ID, lexerRules }; diff --git a/packages/kbn-monaco/src/xjson/lexer_rules/esql.ts b/packages/kbn-monaco/src/esql/lexer_rules/esql.ts similarity index 98% rename from packages/kbn-monaco/src/xjson/lexer_rules/esql.ts rename to packages/kbn-monaco/src/esql/lexer_rules/esql.ts index e75b1013d3727..8badc8ffc4184 100644 --- a/packages/kbn-monaco/src/xjson/lexer_rules/esql.ts +++ b/packages/kbn-monaco/src/esql/lexer_rules/esql.ts @@ -17,9 +17,7 @@ * under the License. */ -import { monaco } from '../../monaco'; - -export const ID = 'esql'; +import { monaco } from '../../monaco_imports'; const brackets = [ { open: '[', close: ']', token: 'delimiter.square' }, diff --git a/packages/kbn-monaco/src/esql/lexer_rules/index.ts b/packages/kbn-monaco/src/esql/lexer_rules/index.ts new file mode 100644 index 0000000000000..5210bc2411716 --- /dev/null +++ b/packages/kbn-monaco/src/esql/lexer_rules/index.ts @@ -0,0 +1,20 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export { lexerRules } from './esql'; diff --git a/packages/kbn-monaco/src/index.ts b/packages/kbn-monaco/src/index.ts index 9213a1bfe1327..2a8467d6ef8fd 100644 --- a/packages/kbn-monaco/src/index.ts +++ b/packages/kbn-monaco/src/index.ts @@ -17,8 +17,12 @@ * under the License. */ -export { monaco } from './monaco'; +// global setup for supported languages +import './register_globals'; + +export { monaco } from './monaco_imports'; export { XJsonLang } from './xjson'; +export { PainlessLang } from './painless'; /* eslint-disable-next-line @kbn/eslint/module_migration */ import * as BarePluginApi from 'monaco-editor/esm/vs/editor/editor.api'; diff --git a/packages/kbn-monaco/src/monaco.ts b/packages/kbn-monaco/src/monaco_imports.ts similarity index 100% rename from packages/kbn-monaco/src/monaco.ts rename to packages/kbn-monaco/src/monaco_imports.ts diff --git a/packages/kbn-monaco/src/painless/constants.ts b/packages/kbn-monaco/src/painless/constants.ts new file mode 100644 index 0000000000000..32bbc0aaaa0be --- /dev/null +++ b/packages/kbn-monaco/src/painless/constants.ts @@ -0,0 +1,20 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export const ID = 'painless'; diff --git a/packages/kbn-monaco/src/painless/index.ts b/packages/kbn-monaco/src/painless/index.ts new file mode 100644 index 0000000000000..2ff1f4a19f9bd --- /dev/null +++ b/packages/kbn-monaco/src/painless/index.ts @@ -0,0 +1,23 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { ID } from './constants'; +import { lexerRules } from './lexer_rules'; + +export const PainlessLang = { ID, lexerRules }; diff --git a/packages/kbn-monaco/src/painless/lexer_rules/index.ts b/packages/kbn-monaco/src/painless/lexer_rules/index.ts new file mode 100644 index 0000000000000..7cf9064c6aa51 --- /dev/null +++ b/packages/kbn-monaco/src/painless/lexer_rules/index.ts @@ -0,0 +1,20 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export { lexerRules } from './painless'; diff --git a/packages/kbn-monaco/src/xjson/lexer_rules/painless.ts b/packages/kbn-monaco/src/painless/lexer_rules/painless.ts similarity index 87% rename from packages/kbn-monaco/src/xjson/lexer_rules/painless.ts rename to packages/kbn-monaco/src/painless/lexer_rules/painless.ts index 676eb3134026a..2f4383911c9ad 100644 --- a/packages/kbn-monaco/src/xjson/lexer_rules/painless.ts +++ b/packages/kbn-monaco/src/painless/lexer_rules/painless.ts @@ -17,16 +17,9 @@ * under the License. */ -import { monaco } from '../../monaco'; +import { monaco } from '../../monaco_imports'; -export const ID = 'painless'; - -/** - * Extends the default type for a Monarch language so we can use - * attribute references (like @keywords to reference the keywords list) - * in the defined tokenizer - */ -interface Language extends monaco.languages.IMonarchLanguage { +export interface Language extends monaco.languages.IMonarchLanguage { default: string; brackets: any; keywords: string[]; @@ -41,8 +34,7 @@ interface Language extends monaco.languages.IMonarchLanguage { } export const lexerRules = { - default: 'invalid', - tokenPostfix: '', + default: '', // painless does not use < >, so we define our own brackets: [ ['{', '}', 'delimiter.curly'], @@ -136,9 +128,9 @@ export const lexerRules = { }, ], // whitespace - [/[ \t\r\n]+/, { token: 'whitespace' }], + [/[ \t\r\n]+/, '@whitespace'], // comments - [/\/\*/, 'comment', '@comment'], + // [/\/\*/, 'comment', '@comment'], [/\/\/.*$/, 'comment'], // brackets [/[{}()\[\]]/, '@brackets'], @@ -168,7 +160,6 @@ export const lexerRules = { // strings single quoted [/'([^'\\]|\\.)*$/, 'string.invalid'], // string without termination [/'/, 'string', '@string_sq'], - [/"""/, { token: 'punctuation.end_triple_quote', nextEmbedded: '@pop' }], ], comment: [ [/[^\/*]+/, 'comment'], @@ -189,6 +180,3 @@ export const lexerRules = { ], }, } as Language; - -monaco.languages.register({ id: ID }); -monaco.languages.setMonarchTokensProvider(ID, lexerRules); diff --git a/packages/kbn-monaco/src/register_globals.ts b/packages/kbn-monaco/src/register_globals.ts new file mode 100644 index 0000000000000..b9e94803b7542 --- /dev/null +++ b/packages/kbn-monaco/src/register_globals.ts @@ -0,0 +1,55 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { XJsonLang } from './xjson'; +import { PainlessLang } from './painless'; +import { EsqlLang } from './esql'; +import { monaco } from './monaco_imports'; +// @ts-ignore +import xJsonWorkerSrc from '!!raw-loader!../target/public/xjson.editor.worker.js'; +// @ts-ignore +import defaultWorkerSrc from '!!raw-loader!../target/public/default.editor.worker.js'; + +/** + * Register languages and lexer rules + */ +monaco.languages.register({ id: XJsonLang.ID }); +monaco.languages.setMonarchTokensProvider(XJsonLang.ID, XJsonLang.lexerRules); +monaco.languages.setLanguageConfiguration(XJsonLang.ID, XJsonLang.languageConfiguration); +monaco.languages.register({ id: PainlessLang.ID }); +monaco.languages.setMonarchTokensProvider(PainlessLang.ID, PainlessLang.lexerRules); +monaco.languages.register({ id: EsqlLang.ID }); +monaco.languages.setMonarchTokensProvider(EsqlLang.ID, EsqlLang.lexerRules); + +/** + * Create web workers by language ID + */ +const mapLanguageIdToWorker: { [key: string]: any } = { + [XJsonLang.ID]: xJsonWorkerSrc, +}; + +// @ts-ignore +window.MonacoEnvironment = { + getWorker: (module: string, languageId: string) => { + const workerSrc = mapLanguageIdToWorker[languageId] || defaultWorkerSrc; + + const blob = new Blob([workerSrc], { type: 'application/javascript' }); + return new Worker(URL.createObjectURL(blob)); + }, +}; diff --git a/packages/kbn-monaco/src/xjson/index.ts b/packages/kbn-monaco/src/xjson/index.ts index 8a4644a3792d2..c372f02c09c76 100644 --- a/packages/kbn-monaco/src/xjson/index.ts +++ b/packages/kbn-monaco/src/xjson/index.ts @@ -22,5 +22,6 @@ */ import './language'; import { ID } from './constants'; +import { lexerRules, languageConfiguration } from './lexer_rules'; -export const XJsonLang = { ID }; +export const XJsonLang = { ID, lexerRules, languageConfiguration }; diff --git a/packages/kbn-monaco/src/xjson/language.ts b/packages/kbn-monaco/src/xjson/language.ts index 4ae7f2402ed2f..9759dc1b24401 100644 --- a/packages/kbn-monaco/src/xjson/language.ts +++ b/packages/kbn-monaco/src/xjson/language.ts @@ -19,32 +19,12 @@ // This file contains a lot of single setup logic for registering a language globally -import { monaco } from '../monaco'; +import { monaco } from '../monaco_imports'; import { WorkerProxyService } from './worker_proxy_service'; -import { registerLexerRules } from './lexer_rules'; import { ID } from './constants'; -// @ts-ignore -import workerSrc from '!!raw-loader!../../target/public/xjson.editor.worker.js'; const wps = new WorkerProxyService(); -// Register rules against shared monaco instance. -registerLexerRules(monaco); - -// In future we will need to make this map languages to workers using "id" and/or "label" values -// that get passed in. Also this should not live inside the "xjson" dir directly. We can update this -// once we have another worker. -// @ts-ignore -window.MonacoEnvironment = { - getWorker: (module: string, languageId: string) => { - if (languageId === ID) { - // In kibana we will probably build this once and then load with raw-loader - const blob = new Blob([workerSrc], { type: 'application/javascript' }); - return new Worker(URL.createObjectURL(blob)); - } - }, -}; - monaco.languages.onLanguage(ID, async () => { return wps.setup(); }); diff --git a/packages/kbn-monaco/src/xjson/lexer_rules/index.ts b/packages/kbn-monaco/src/xjson/lexer_rules/index.ts index 515de09510a61..7393c6a68c1bf 100644 --- a/packages/kbn-monaco/src/xjson/lexer_rules/index.ts +++ b/packages/kbn-monaco/src/xjson/lexer_rules/index.ts @@ -17,17 +17,4 @@ * under the License. */ -/* eslint-disable-next-line @kbn/eslint/module_migration */ -import * as monaco from 'monaco-editor/esm/vs/editor/editor.api'; -import * as xJson from './xjson'; -import * as esql from './esql'; -import * as painless from './painless'; - -export const registerLexerRules = (m: typeof monaco) => { - m.languages.register({ id: xJson.ID }); - m.languages.setMonarchTokensProvider(xJson.ID, xJson.lexerRules); - m.languages.register({ id: painless.ID }); - m.languages.setMonarchTokensProvider(painless.ID, painless.lexerRules); - m.languages.register({ id: esql.ID }); - m.languages.setMonarchTokensProvider(esql.ID, esql.lexerRules); -}; +export { lexerRules, languageConfiguration } from './xjson'; diff --git a/packages/kbn-monaco/src/xjson/lexer_rules/xjson.ts b/packages/kbn-monaco/src/xjson/lexer_rules/xjson.ts index d6fea9e91acfb..e0c566fd3b0f2 100644 --- a/packages/kbn-monaco/src/xjson/lexer_rules/xjson.ts +++ b/packages/kbn-monaco/src/xjson/lexer_rules/xjson.ts @@ -17,15 +17,10 @@ * under the License. */ -import { monaco } from '../../monaco'; -import { ID } from '../constants'; -import './painless'; -import './esql'; +import { monaco } from '../../monaco_imports'; import { globals } from './shared'; -export { ID }; - export const lexerRules: monaco.languages.IMonarchLanguage = { ...(globals as any), @@ -124,11 +119,7 @@ export const lexerRules: monaco.languages.IMonarchLanguage = { }, }; -monaco.languages.register({ - id: ID, -}); -monaco.languages.setMonarchTokensProvider(ID, lexerRules); -monaco.languages.setLanguageConfiguration(ID, { +export const languageConfiguration: monaco.languages.LanguageConfiguration = { brackets: [ ['{', '}'], ['[', ']'], @@ -138,4 +129,4 @@ monaco.languages.setLanguageConfiguration(ID, { { open: '[', close: ']' }, { open: '"', close: '"' }, ], -}); +}; diff --git a/packages/kbn-monaco/src/xjson/worker_proxy_service.ts b/packages/kbn-monaco/src/xjson/worker_proxy_service.ts index 548a413a483d9..c0e735b294484 100644 --- a/packages/kbn-monaco/src/xjson/worker_proxy_service.ts +++ b/packages/kbn-monaco/src/xjson/worker_proxy_service.ts @@ -18,7 +18,7 @@ */ import { ParseResult } from './grammar'; -import { monaco } from '../monaco'; +import { monaco } from '../monaco_imports'; import { XJsonWorker } from './worker'; import { ID } from './constants'; diff --git a/packages/kbn-monaco/webpack.config.js b/packages/kbn-monaco/webpack.config.js index 1a7d8c031670c..53f440689a233 100644 --- a/packages/kbn-monaco/webpack.config.js +++ b/packages/kbn-monaco/webpack.config.js @@ -19,33 +19,40 @@ const path = require('path'); -const createLangWorkerConfig = (lang) => ({ - mode: 'production', - entry: path.resolve(__dirname, 'src', lang, 'worker', `${lang}.worker.ts`), - output: { - path: path.resolve(__dirname, 'target/public'), - filename: `${lang}.editor.worker.js`, - }, - resolve: { - modules: ['node_modules'], - extensions: ['.js', '.ts', '.tsx'], - }, - stats: 'errors-only', - module: { - rules: [ - { - test: /\.(js|ts)$/, - exclude: /node_modules/, - use: { - loader: 'babel-loader', - options: { - babelrc: false, - presets: [require.resolve('@kbn/babel-preset/webpack_preset')], +const createLangWorkerConfig = (lang) => { + const entry = + lang === 'default' + ? 'monaco-editor/esm/vs/editor/editor.worker.js' + : path.resolve(__dirname, 'src', lang, 'worker', `${lang}.worker.ts`); + + return { + mode: 'production', + entry, + output: { + path: path.resolve(__dirname, 'target/public'), + filename: `${lang}.editor.worker.js`, + }, + resolve: { + modules: ['node_modules'], + extensions: ['.js', '.ts', '.tsx'], + }, + stats: 'errors-only', + module: { + rules: [ + { + test: /\.(js|ts)$/, + exclude: /node_modules/, + use: { + loader: 'babel-loader', + options: { + babelrc: false, + presets: [require.resolve('@kbn/babel-preset/webpack_preset')], + }, }, }, - }, - ], - }, -}); + ], + }, + }; +}; -module.exports = [createLangWorkerConfig('xjson')]; +module.exports = [createLangWorkerConfig('xjson'), createLangWorkerConfig('default')]; diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index fd0be15affab3..c660d37222504 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -14,7 +14,7 @@ pageLoadAssetSize: dashboard: 374194 dashboardEnhanced: 65646 dashboardMode: 22716 - data: 1170713 + data: 1287839 dataEnhanced: 50420 devTools: 38637 discover: 105145 diff --git a/packages/kbn-optimizer/src/cli.ts b/packages/kbn-optimizer/src/cli.ts index 28b3e37380b4e..976ffe1f5b1b5 100644 --- a/packages/kbn-optimizer/src/cli.ts +++ b/packages/kbn-optimizer/src/cli.ts @@ -146,7 +146,11 @@ run( await lastValueFrom(update$.pipe(logOptimizerState(log, config))); if (updateLimits) { - updateBundleLimits(log, config); + updateBundleLimits({ + log, + config, + dropMissing: !(focus || filter), + }); } }, { diff --git a/packages/kbn-optimizer/src/limits.ts b/packages/kbn-optimizer/src/limits.ts index b0fae0901251d..64ec92657fc5b 100644 --- a/packages/kbn-optimizer/src/limits.ts +++ b/packages/kbn-optimizer/src/limits.ts @@ -79,10 +79,18 @@ export function validateLimitsForAllBundles(log: ToolingLog, config: OptimizerCo log.success('limits.yml file valid'); } -export function updateBundleLimits(log: ToolingLog, config: OptimizerConfig) { +interface UpdateBundleLimitsOptions { + log: ToolingLog; + config: OptimizerConfig; + dropMissing: boolean; +} + +export function updateBundleLimits({ log, config, dropMissing }: UpdateBundleLimitsOptions) { const metrics = getMetrics(log, config); - const pageLoadAssetSize: NonNullable = {}; + const pageLoadAssetSize: NonNullable = dropMissing + ? {} + : config.limits.pageLoadAssetSize ?? {}; for (const metric of metrics.sort((a, b) => a.id.localeCompare(b.id))) { if (metric.group === 'page load bundle size') { diff --git a/packages/kbn-plugin-generator/README.md b/packages/kbn-plugin-generator/README.md index 9ff9a8aa95ca2..bee8e6c2ca783 100644 --- a/packages/kbn-plugin-generator/README.md +++ b/packages/kbn-plugin-generator/README.md @@ -51,7 +51,7 @@ yarn kbn bootstrap Generated plugins receive a handful of scripts that can be used during development. Those scripts are detailed in the [README.md](template/README.md) file in each newly generated plugin, and expose the scripts provided by the [Kibana plugin helpers](../kbn-plugin-helpers), but here is a quick reference in case you need it: -> ***NOTE:*** All of these scripts should be run from the generated plugin. +> ***NOTE:*** The following scripts should be run from the generated plugin. - `yarn kbn bootstrap` @@ -59,14 +59,6 @@ Generated plugins receive a handful of scripts that can be used during developme > ***IMPORTANT:*** Use this script instead of `yarn` to install dependencies when switching branches, and re-run it whenever your dependencies change. - - `yarn start` - - Start kibana and have it include this plugin. You can pass any arguments that you would normally send to `bin/kibana` - - ``` - yarn start --elasticsearch.hosts http://localhost:9220 - ``` - - `yarn build` Build a distributable archive of your plugin. @@ -75,4 +67,15 @@ Generated plugins receive a handful of scripts that can be used during developme Run the server tests using mocha. + +To start kibana run the following command from Kibana root. + + - `yarn start` + + Start kibana and it will automatically include this plugin. You can pass any arguments that you would normally send to `bin/kibana` + + ``` + yarn start --elasticsearch.hosts http://localhost:9220 + ``` + For more information about any of these commands run `yarn ${task} --help`. For a full list of tasks run `yarn run` or take a look in the `package.json` file. diff --git a/packages/kbn-ui-shared-deps/package.json b/packages/kbn-ui-shared-deps/package.json index 980d9d02317b6..b1b5d6e2b419e 100644 --- a/packages/kbn-ui-shared-deps/package.json +++ b/packages/kbn-ui-shared-deps/package.json @@ -9,7 +9,7 @@ "kbn:watch": "node scripts/build --dev --watch" }, "dependencies": { - "@elastic/charts": "23.2.1", + "@elastic/charts": "24.0.0", "@elastic/eui": "29.5.0", "@elastic/numeral": "^2.5.0", "@kbn/i18n": "1.0.0", diff --git a/src/core/server/elasticsearch/client/client_config.test.ts b/src/core/server/elasticsearch/client/client_config.test.ts index e8083836d3c1e..2beb07e6da5bc 100644 --- a/src/core/server/elasticsearch/client/client_config.test.ts +++ b/src/core/server/elasticsearch/client/client_config.test.ts @@ -216,28 +216,14 @@ describe('parseClientOptions', () => { ); }); - it('adds auth to the nodes if both `username` and `password` are set', () => { - let options = parseClientOptions( + it('does not add auth to the nodes', () => { + const options = parseClientOptions( createConfig({ username: 'user', - hosts: ['http://node-A:9200'], - }), - false - ); - expect(options.nodes).toMatchInlineSnapshot(` - Array [ - Object { - "url": "http://node-a:9200/", - }, - ] - `); - - options = parseClientOptions( - createConfig({ password: 'pass', hosts: ['http://node-A:9200'], }), - false + true ); expect(options.nodes).toMatchInlineSnapshot(` Array [ @@ -246,22 +232,6 @@ describe('parseClientOptions', () => { }, ] `); - - options = parseClientOptions( - createConfig({ - username: 'user', - password: 'pass', - hosts: ['http://node-A:9200'], - }), - false - ); - expect(options.nodes).toMatchInlineSnapshot(` - Array [ - Object { - "url": "http://user:pass@node-a:9200/", - }, - ] - `); }); }); describe('when `scoped` is true', () => { diff --git a/src/core/server/elasticsearch/client/client_config.ts b/src/core/server/elasticsearch/client/client_config.ts index f24c0d86550b8..dab20edfc61ff 100644 --- a/src/core/server/elasticsearch/client/client_config.ts +++ b/src/core/server/elasticsearch/client/client_config.ts @@ -93,7 +93,7 @@ export function parseClientOptions( }; } - clientOptions.nodes = config.hosts.map((host) => convertHost(host, !scoped, config)); + clientOptions.nodes = config.hosts.map((host) => convertHost(host)); if (config.ssl) { clientOptions.ssl = generateSslConfig( @@ -140,18 +140,10 @@ const generateSslConfig = ( return ssl; }; -const convertHost = ( - host: string, - needAuth: boolean, - { username, password }: ElasticsearchClientConfig -): NodeOptions => { +const convertHost = (host: string): NodeOptions => { const url = new URL(host); const isHTTPS = url.protocol === 'https:'; url.port = url.port || (isHTTPS ? '443' : '80'); - if (needAuth && username && password) { - url.username = username; - url.password = password; - } return { url, diff --git a/src/dev/build/tasks/os_packages/docker_generator/resources/bin/kibana-docker b/src/dev/build/tasks/os_packages/docker_generator/resources/bin/kibana-docker index f5cf6c85fcbef..274d7a4e5a488 100755 --- a/src/dev/build/tasks/os_packages/docker_generator/resources/bin/kibana-docker +++ b/src/dev/build/tasks/os_packages/docker_generator/resources/bin/kibana-docker @@ -138,6 +138,7 @@ kibana_vars=( tilemap.url timelion.enabled vega.enableExternalUrls + xpack.actions.proxyUrl xpack.apm.enabled xpack.apm.serviceMapEnabled xpack.apm.ui.enabled diff --git a/src/dev/ci_setup/setup_env.sh b/src/dev/ci_setup/setup_env.sh index 8ec80ac295c73..5dac270239c4a 100644 --- a/src/dev/ci_setup/setup_env.sh +++ b/src/dev/ci_setup/setup_env.sh @@ -24,6 +24,10 @@ export NODE_OPTIONS="$NODE_OPTIONS --max-old-space-size=4096" ### export FORCE_COLOR=1 +### APM tracking +### +export ELASTIC_APM_ENVIRONMENT=ci + ### ### check that we seem to be in a kibana project ### diff --git a/src/fixtures/telemetry_collectors/externally_defined_collector.ts b/src/fixtures/telemetry_collectors/externally_defined_collector.ts index 00a8d643e27b3..decee9df28185 100644 --- a/src/fixtures/telemetry_collectors/externally_defined_collector.ts +++ b/src/fixtures/telemetry_collectors/externally_defined_collector.ts @@ -16,7 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -import { CollectorSet, CollectorOptions } from '../../plugins/usage_collection/server/collector'; +import { + CollectorSet, + UsageCollectorOptions, +} from '../../plugins/usage_collection/server/collector'; import { loggerMock } from '../../core/server/logging/logger.mock'; const collectorSet = new CollectorSet({ @@ -28,7 +31,7 @@ interface Usage { locale: string; } -function createCollector(): CollectorOptions { +function createCollector(): UsageCollectorOptions { return { type: 'from_fn_collector', isReady: () => true, @@ -46,7 +49,7 @@ function createCollector(): CollectorOptions { } export function defineCollectorFromVariable() { - const fromVarCollector: CollectorOptions = { + const fromVarCollector: UsageCollectorOptions = { type: 'from_variable_collector', isReady: () => true, fetch(): Usage { diff --git a/src/fixtures/telemetry_collectors/unmapped_collector.ts b/src/fixtures/telemetry_collectors/unmapped_collector.ts index 1ea360fcd9e96..143bb168aff35 100644 --- a/src/fixtures/telemetry_collectors/unmapped_collector.ts +++ b/src/fixtures/telemetry_collectors/unmapped_collector.ts @@ -28,6 +28,7 @@ interface Usage { locale: string; } +// @ts-expect-error Intentionally not specifying `schema` export const myCollector = makeUsageCollector({ type: 'unmapped_collector', isReady: () => true, diff --git a/src/legacy/server/i18n/constants.ts b/src/legacy/server/i18n/constants.ts index 96fa420d4c6e1..a7a410dbcb5b3 100644 --- a/src/legacy/server/i18n/constants.ts +++ b/src/legacy/server/i18n/constants.ts @@ -18,8 +18,3 @@ */ export const I18N_RC = '.i18nrc.json'; - -/** - * The type name used within the Monitoring index to publish localization stats. - */ -export const KIBANA_LOCALIZATION_STATS_TYPE = 'localization'; diff --git a/src/legacy/server/i18n/get_kibana_translation_paths.test.ts b/src/legacy/server/i18n/get_kibana_translation_paths.test.ts new file mode 100644 index 0000000000000..0f202c4d433c0 --- /dev/null +++ b/src/legacy/server/i18n/get_kibana_translation_paths.test.ts @@ -0,0 +1,58 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { I18N_RC } from './constants'; +import { fromRoot } from '../../../core/server/utils'; + +jest.mock('./get_translation_paths', () => ({ getTranslationPaths: jest.fn() })); +import { getKibanaTranslationPaths } from './get_kibana_translation_paths'; +import { getTranslationPaths as mockGetTranslationPaths } from './get_translation_paths'; + +describe('getKibanaTranslationPaths', () => { + const mockConfig = { get: jest.fn() }; + + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('calls getTranslationPaths against kibana root and kibana-extra', async () => { + mockConfig.get.mockReturnValue([]); + await getKibanaTranslationPaths(mockConfig); + expect(mockGetTranslationPaths).toHaveBeenNthCalledWith(1, { + cwd: fromRoot('.'), + glob: `*/${I18N_RC}`, + }); + + expect(mockGetTranslationPaths).toHaveBeenNthCalledWith(2, { + cwd: fromRoot('../kibana-extra'), + glob: `*/${I18N_RC}`, + }); + }); + + it('calls getTranslationPaths for each config returned in plugin.paths and plugins.scanDirs', async () => { + mockConfig.get.mockReturnValueOnce(['a', 'b']).mockReturnValueOnce(['c']); + await getKibanaTranslationPaths(mockConfig); + expect(mockConfig.get).toHaveBeenNthCalledWith(1, 'plugins.paths'); + expect(mockConfig.get).toHaveBeenNthCalledWith(2, 'plugins.scanDirs'); + + expect(mockGetTranslationPaths).toHaveBeenNthCalledWith(2, { cwd: 'a', glob: I18N_RC }); + expect(mockGetTranslationPaths).toHaveBeenNthCalledWith(3, { cwd: 'b', glob: I18N_RC }); + expect(mockGetTranslationPaths).toHaveBeenNthCalledWith(4, { cwd: 'c', glob: `*/${I18N_RC}` }); + }); +}); diff --git a/src/legacy/server/i18n/get_kibana_translation_paths.ts b/src/legacy/server/i18n/get_kibana_translation_paths.ts new file mode 100644 index 0000000000000..d7f77d3185ba4 --- /dev/null +++ b/src/legacy/server/i18n/get_kibana_translation_paths.ts @@ -0,0 +1,42 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { KibanaConfig } from '../kbn_server'; +import { fromRoot } from '../../../core/server/utils'; +import { I18N_RC } from './constants'; +import { getTranslationPaths } from './get_translation_paths'; + +export async function getKibanaTranslationPaths(config: Pick) { + return await Promise.all([ + getTranslationPaths({ + cwd: fromRoot('.'), + glob: `*/${I18N_RC}`, + }), + ...(config.get('plugins.paths') as string[]).map((cwd) => + getTranslationPaths({ cwd, glob: I18N_RC }) + ), + ...(config.get('plugins.scanDirs') as string[]).map((cwd) => + getTranslationPaths({ cwd, glob: `*/${I18N_RC}` }) + ), + getTranslationPaths({ + cwd: fromRoot('../kibana-extra'), + glob: `*/${I18N_RC}`, + }), + ]); +} diff --git a/src/legacy/server/i18n/get_translations_path.ts b/src/legacy/server/i18n/get_translation_paths.ts similarity index 100% rename from src/legacy/server/i18n/get_translations_path.ts rename to src/legacy/server/i18n/get_translation_paths.ts diff --git a/src/legacy/server/i18n/i18n_mixin.ts b/src/legacy/server/i18n/i18n_mixin.ts new file mode 100644 index 0000000000000..4f77fa8df96cd --- /dev/null +++ b/src/legacy/server/i18n/i18n_mixin.ts @@ -0,0 +1,63 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { i18n, i18nLoader } from '@kbn/i18n'; +import { basename } from 'path'; +import { Server } from 'hapi'; +import type { UsageCollectionSetup } from '../../../plugins/usage_collection/server'; +import { getKibanaTranslationPaths } from './get_kibana_translation_paths'; +import KbnServer, { KibanaConfig } from '../kbn_server'; +import { registerLocalizationUsageCollector } from './localization'; + +export async function i18nMixin( + kbnServer: KbnServer, + server: Server, + config: Pick +) { + const locale = config.get('i18n.locale') as string; + + const translationPaths = await getKibanaTranslationPaths(config); + + const currentTranslationPaths = ([] as string[]) + .concat(...translationPaths) + .filter((translationPath) => basename(translationPath, '.json') === locale); + i18nLoader.registerTranslationFiles(currentTranslationPaths); + + const translations = await i18nLoader.getTranslationsByLocale(locale); + i18n.init( + Object.freeze({ + locale, + ...translations, + }) + ); + + const getTranslationsFilePaths = () => currentTranslationPaths; + + server.decorate('server', 'getTranslationsFilePaths', getTranslationsFilePaths); + + if (kbnServer.newPlatform.setup.plugins.usageCollection) { + const { usageCollection } = kbnServer.newPlatform.setup.plugins as { + usageCollection: UsageCollectionSetup; + }; + registerLocalizationUsageCollector(usageCollection, { + getLocale: () => config.get('i18n.locale') as string, + getTranslationsFilePaths, + }); + } +} diff --git a/src/legacy/server/i18n/index.ts b/src/legacy/server/i18n/index.ts index 61caefb2fb599..a7ef49f44532c 100644 --- a/src/legacy/server/i18n/index.ts +++ b/src/legacy/server/i18n/index.ts @@ -17,60 +17,4 @@ * under the License. */ -import { i18n, i18nLoader } from '@kbn/i18n'; -import { basename } from 'path'; -import { Server } from 'hapi'; -import { fromRoot } from '../../../core/server/utils'; -import type { UsageCollectionSetup } from '../../../plugins/usage_collection/server'; -import { getTranslationPaths } from './get_translations_path'; -import { I18N_RC } from './constants'; -import KbnServer, { KibanaConfig } from '../kbn_server'; -import { registerLocalizationUsageCollector } from './localization'; - -export async function i18nMixin(kbnServer: KbnServer, server: Server, config: KibanaConfig) { - const locale = config.get('i18n.locale') as string; - - const translationPaths = await Promise.all([ - getTranslationPaths({ - cwd: fromRoot('.'), - glob: `*/${I18N_RC}`, - }), - ...(config.get('plugins.paths') as string[]).map((cwd) => - getTranslationPaths({ cwd, glob: I18N_RC }) - ), - ...(config.get('plugins.scanDirs') as string[]).map((cwd) => - getTranslationPaths({ cwd, glob: `*/${I18N_RC}` }) - ), - getTranslationPaths({ - cwd: fromRoot('../kibana-extra'), - glob: `*/${I18N_RC}`, - }), - ]); - - const currentTranslationPaths = ([] as string[]) - .concat(...translationPaths) - .filter((translationPath) => basename(translationPath, '.json') === locale); - i18nLoader.registerTranslationFiles(currentTranslationPaths); - - const translations = await i18nLoader.getTranslationsByLocale(locale); - i18n.init( - Object.freeze({ - locale, - ...translations, - }) - ); - - const getTranslationsFilePaths = () => currentTranslationPaths; - - server.decorate('server', 'getTranslationsFilePaths', getTranslationsFilePaths); - - if (kbnServer.newPlatform.setup.plugins.usageCollection) { - const { usageCollection } = kbnServer.newPlatform.setup.plugins as { - usageCollection: UsageCollectionSetup; - }; - registerLocalizationUsageCollector(usageCollection, { - getLocale: () => config.get('i18n.locale') as string, - getTranslationsFilePaths, - }); - } -} +export { i18nMixin } from './i18n_mixin'; diff --git a/src/legacy/server/i18n/localization/telemetry_localization_collector.ts b/src/legacy/server/i18n/localization/telemetry_localization_collector.ts index 89566dfd4ef68..fb837f5ae28df 100644 --- a/src/legacy/server/i18n/localization/telemetry_localization_collector.ts +++ b/src/legacy/server/i18n/localization/telemetry_localization_collector.ts @@ -21,7 +21,6 @@ import { i18nLoader } from '@kbn/i18n'; import { size } from 'lodash'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { getIntegrityHashes, Integrities } from './file_integrity'; -import { KIBANA_LOCALIZATION_STATS_TYPE } from '../constants'; export interface UsageStats { locale: string; @@ -63,14 +62,20 @@ export function createCollectorFetch({ }; } +// TODO: Migrate out of the Legacy dir export function registerLocalizationUsageCollector( usageCollection: UsageCollectionSetup, helpers: LocalizationUsageCollectorHelpers ) { - const collector = usageCollection.makeUsageCollector({ - type: KIBANA_LOCALIZATION_STATS_TYPE, + const collector = usageCollection.makeUsageCollector({ + type: 'localization', isReady: () => true, fetch: createCollectorFetch(helpers), + schema: { + locale: { type: 'keyword' }, + integrities: { DYNAMIC_KEY: { type: 'text' } }, + labelsCount: { type: 'long' }, + }, }); usageCollection.registerCollector(collector); diff --git a/src/plugins/advanced_settings/public/management_app/components/field/field.tsx b/src/plugins/advanced_settings/public/management_app/components/field/field.tsx index 794168132abb2..e9fa2833c3db5 100644 --- a/src/plugins/advanced_settings/public/management_app/components/field/field.tsx +++ b/src/plugins/advanced_settings/public/management_app/components/field/field.tsx @@ -22,6 +22,7 @@ import classNames from 'classnames'; import 'brace/theme/textmate'; import 'brace/mode/markdown'; +import 'brace/mode/json'; import { EuiBadge, diff --git a/src/plugins/dashboard/public/application/actions/library_notification_popover.tsx b/src/plugins/dashboard/public/application/actions/library_notification_popover.tsx index 8bc81b3296c3d..6ec5b0d637556 100644 --- a/src/plugins/dashboard/public/application/actions/library_notification_popover.tsx +++ b/src/plugins/dashboard/public/application/actions/library_notification_popover.tsx @@ -72,7 +72,7 @@ export function LibraryNotificationPopover({

{i18n.translate('dashboard.panel.libraryNotification.toolTip', { defaultMessage: - 'This panel is linked to a library item. Editing the panel might affect other dashboards.', + 'Editing this panel might affect other dashboards. To change to this panel only, unlink it from the library.', })}

diff --git a/src/plugins/data/common/index_patterns/fields/index_pattern_field.test.ts b/src/plugins/data/common/index_patterns/fields/index_pattern_field.test.ts index 3c4fac81c2c7c..be7836de31246 100644 --- a/src/plugins/data/common/index_patterns/fields/index_pattern_field.test.ts +++ b/src/plugins/data/common/index_patterns/fields/index_pattern_field.test.ts @@ -91,6 +91,17 @@ describe('Field', function () { expect(fieldC.searchable).toEqual(false); }); + it('calculates visualizable', () => { + const field = getField({ type: 'unknown' }); + expect(field.visualizable).toEqual(false); + + const fieldB = getField({ type: 'conflict' }); + expect(fieldB.visualizable).toEqual(false); + + const fieldC = getField({ aggregatable: false, scripted: false }); + expect(fieldC.visualizable).toEqual(false); + }); + it('calculates aggregatable', () => { const field = getField({ aggregatable: true, scripted: false }); expect(field.aggregatable).toEqual(true); diff --git a/src/plugins/data/common/index_patterns/fields/index_pattern_field.ts b/src/plugins/data/common/index_patterns/fields/index_pattern_field.ts index 808afc3449c2a..4a22508f7fef3 100644 --- a/src/plugins/data/common/index_patterns/fields/index_pattern_field.ts +++ b/src/plugins/data/common/index_patterns/fields/index_pattern_field.ts @@ -18,6 +18,7 @@ */ import { KbnFieldType, getKbnFieldType } from '../../kbn_field_types'; +import { KBN_FIELD_TYPES } from '../../kbn_field_types/types'; import { IFieldType } from './types'; import { FieldSpec, IndexPattern } from '../..'; @@ -129,7 +130,8 @@ export class IndexPatternField implements IFieldType { } public get visualizable() { - return this.aggregatable; + const notVisualizableFieldTypes: string[] = [KBN_FIELD_TYPES.UNKNOWN, KBN_FIELD_TYPES.CONFLICT]; + return this.aggregatable && !notVisualizableFieldTypes.includes(this.spec.type); } public toJSON() { diff --git a/src/plugins/data/common/search/aggs/utils/get_format_with_aggs.test.ts b/src/plugins/data/common/search/aggs/utils/get_format_with_aggs.test.ts index 28646c092c01c..826e402b14682 100644 --- a/src/plugins/data/common/search/aggs/utils/get_format_with_aggs.test.ts +++ b/src/plugins/data/common/search/aggs/utils/get_format_with_aggs.test.ts @@ -79,6 +79,33 @@ describe('getFormatWithAggs', () => { expect(getFormat).toHaveBeenCalledTimes(1); }); + test('creates alternative format for range using the template parameter', () => { + const mapping = { id: 'range', params: { template: 'arrow_right' } }; + const getFieldFormat = getFormatWithAggs(getFormat); + const format = getFieldFormat(mapping); + + expect(format.convert({ gte: 1, lt: 20 })).toBe('1 → 20'); + expect(getFormat).toHaveBeenCalledTimes(1); + }); + + test('handles Infinity values internally when no nestedFormatter is passed', () => { + const mapping = { id: 'range', params: { replaceInfinity: true } }; + const getFieldFormat = getFormatWithAggs(getFormat); + const format = getFieldFormat(mapping); + + expect(format.convert({ gte: -Infinity, lt: Infinity })).toBe('≥ −∞ and < +∞'); + expect(getFormat).toHaveBeenCalledTimes(1); + }); + + test('lets Infinity values handling to nestedFormatter even when flag is on', () => { + const mapping = { id: 'range', params: { replaceInfinity: true, id: 'any' } }; + const getFieldFormat = getFormatWithAggs(getFormat); + const format = getFieldFormat(mapping); + + expect(format.convert({ gte: -Infinity, lt: Infinity })).toBe('≥ -Infinity and < Infinity'); + expect(getFormat).toHaveBeenCalledTimes(1); + }); + test('returns custom label for range if provided', () => { const mapping = { id: 'range', params: {} }; const getFieldFormat = getFormatWithAggs(getFormat); diff --git a/src/plugins/data/common/search/aggs/utils/get_format_with_aggs.ts b/src/plugins/data/common/search/aggs/utils/get_format_with_aggs.ts index a8134619fec0d..6b03dc5f70edc 100644 --- a/src/plugins/data/common/search/aggs/utils/get_format_with_aggs.ts +++ b/src/plugins/data/common/search/aggs/utils/get_format_with_aggs.ts @@ -56,15 +56,35 @@ export function getFormatWithAggs(getFieldFormat: GetFieldFormat): GetFieldForma id: nestedFormatter.id, params: nestedFormatter.params, }); + const gte = '\u2265'; const lt = '\u003c'; + let fromValue = format.convert(range.gte); + let toValue = format.convert(range.lt); + // In case of identity formatter and a specific flag, replace Infinity values by specific strings + if (params.replaceInfinity && nestedFormatter.id == null) { + const FROM_PLACEHOLDER = '\u2212\u221E'; + const TO_PLACEHOLDER = '+\u221E'; + fromValue = isFinite(range.gte) ? fromValue : FROM_PLACEHOLDER; + toValue = isFinite(range.lt) ? toValue : TO_PLACEHOLDER; + } + + if (params.template === 'arrow_right') { + return i18n.translate('data.aggTypes.buckets.ranges.rangesFormatMessageArrowRight', { + defaultMessage: '{from} → {to}', + values: { + from: fromValue, + to: toValue, + }, + }); + } return i18n.translate('data.aggTypes.buckets.ranges.rangesFormatMessage', { defaultMessage: '{gte} {from} and {lt} {to}', values: { gte, - from: format.convert(range.gte), + from: fromValue, lt, - to: format.convert(range.lt), + to: toValue, }, }); }); diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index d2439e3f1573c..81fa6d4ba20db 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -1598,13 +1598,13 @@ export interface OptionedValueProp { value: string; } -// Warning: (ae-forgotten-export) The symbol "KbnError" needs to be exported by the entry point index.d.ts +// Warning: (ae-forgotten-export) The symbol "EsError" needs to be exported by the entry point index.d.ts // Warning: (ae-missing-release-tag) "PainlessError" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export class PainlessError extends KbnError { - // Warning: (ae-forgotten-export) The symbol "EsError" needs to be exported by the entry point index.d.ts - constructor(err: EsError, request: IKibanaSearchRequest); +export class PainlessError extends EsError { + // Warning: (ae-forgotten-export) The symbol "IEsError" needs to be exported by the entry point index.d.ts + constructor(err: IEsError, request: IKibanaSearchRequest); // (undocumented) getErrorMessage(application: ApplicationStart): JSX.Element; // (undocumented) @@ -2134,6 +2134,7 @@ export interface SearchSourceFields { version?: boolean; } +// Warning: (ae-forgotten-export) The symbol "KbnError" needs to be exported by the entry point index.d.ts // Warning: (ae-missing-release-tag) "SearchTimeoutError" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public diff --git a/src/plugins/data/public/search/errors/es_error.tsx b/src/plugins/data/public/search/errors/es_error.tsx new file mode 100644 index 0000000000000..53d00159b836b --- /dev/null +++ b/src/plugins/data/public/search/errors/es_error.tsx @@ -0,0 +1,46 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import React from 'react'; +import { EuiCodeBlock, EuiSpacer } from '@elastic/eui'; +import { ApplicationStart } from 'kibana/public'; +import { KbnError } from '../../../../kibana_utils/common'; +import { IEsError } from './types'; +import { getRootCause } from './utils'; + +export class EsError extends KbnError { + constructor(protected readonly err: IEsError) { + super('EsError'); + } + + public getErrorMessage(application: ApplicationStart) { + const rootCause = getRootCause(this.err)?.reason; + + return ( + <> + + {rootCause ? ( + + {rootCause} + + ) : null} + + ); + } +} diff --git a/src/plugins/data/public/search/errors/http_error.tsx b/src/plugins/data/public/search/errors/http_error.tsx new file mode 100644 index 0000000000000..58ae3148804a2 --- /dev/null +++ b/src/plugins/data/public/search/errors/http_error.tsx @@ -0,0 +1,38 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { EuiCodeBlock, EuiSpacer } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import React from 'react'; + +export function getHttpError(message: string) { + return ( + <> + {i18n.translate('data.errors.fetchError', { + defaultMessage: + 'Check your network and proxy configuration. If the problem persists, contact your network administrator.', + })} + + + + {message} + + + ); +} diff --git a/src/plugins/data/public/search/errors/index.ts b/src/plugins/data/public/search/errors/index.ts index 6082e758a8bad..01357d25334a3 100644 --- a/src/plugins/data/public/search/errors/index.ts +++ b/src/plugins/data/public/search/errors/index.ts @@ -17,5 +17,9 @@ * under the License. */ +export * from './es_error'; export * from './painless_error'; export * from './timeout_error'; +export * from './utils'; +export * from './types'; +export * from './http_error'; diff --git a/src/plugins/data/public/search/errors/painless_error.tsx b/src/plugins/data/public/search/errors/painless_error.tsx index 244f205469a2f..282a602d358c7 100644 --- a/src/plugins/data/public/search/errors/painless_error.tsx +++ b/src/plugins/data/public/search/errors/painless_error.tsx @@ -22,22 +22,15 @@ import { i18n } from '@kbn/i18n'; import { EuiButton, EuiSpacer, EuiText, EuiCodeBlock } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; import { ApplicationStart } from 'kibana/public'; -import { KbnError } from '../../../../kibana_utils/common'; -import { EsError, isEsError } from './types'; +import { IEsError, isEsError } from './types'; +import { EsError } from './es_error'; +import { getRootCause } from './utils'; import { IKibanaSearchRequest } from '..'; -export class PainlessError extends KbnError { +export class PainlessError extends EsError { painlessStack?: string; - constructor(err: EsError, request: IKibanaSearchRequest) { - const rootCause = getRootCause(err as EsError); - - super( - i18n.translate('data.painlessError.painlessScriptedFieldErrorMessage', { - defaultMessage: "Error executing Painless script: '{script}'.", - values: { script: rootCause?.script }, - }) - ); - this.painlessStack = rootCause?.script_stack ? rootCause?.script_stack.join('\n') : undefined; + constructor(err: IEsError, request: IKibanaSearchRequest) { + super(err); } public getErrorMessage(application: ApplicationStart) { @@ -47,14 +40,20 @@ export class PainlessError extends KbnError { }); } + const rootCause = getRootCause(this.err); + const painlessStack = rootCause?.script_stack ? rootCause?.script_stack.join('\n') : undefined; + return ( <> - {this.message} + {i18n.translate('data.painlessError.painlessScriptedFieldErrorMessage', { + defaultMessage: "Error executing Painless script: '{script}'.", + values: { script: rootCause?.script }, + })} - {this.painlessStack ? ( + {painlessStack ? ( - {this.painlessStack} + {painlessStack} ) : null} @@ -67,21 +66,10 @@ export class PainlessError extends KbnError { } } -function getFailedShards(err: EsError) { - const failedShards = - err.body?.attributes?.error?.failed_shards || - err.body?.attributes?.error?.caused_by?.failed_shards; - return failedShards ? failedShards[0] : undefined; -} - -function getRootCause(err: EsError) { - return getFailedShards(err)?.reason; -} - -export function isPainlessError(err: Error | EsError) { +export function isPainlessError(err: Error | IEsError) { if (!isEsError(err)) return false; - const rootCause = getRootCause(err as EsError); + const rootCause = getRootCause(err as IEsError); if (!rootCause) return false; const { lang } = rootCause; diff --git a/src/plugins/data/public/search/errors/types.ts b/src/plugins/data/public/search/errors/types.ts index 4182209eb68a5..011af015318a7 100644 --- a/src/plugins/data/public/search/errors/types.ts +++ b/src/plugins/data/public/search/errors/types.ts @@ -17,7 +17,7 @@ * under the License. */ -interface FailedShard { +export interface FailedShard { shard: number; index: string; node: string; @@ -39,7 +39,7 @@ interface FailedShard { }; } -export interface EsError { +export interface IEsError { body: { statusCode: number; error: string; @@ -68,6 +68,6 @@ export interface EsError { }; } -export function isEsError(e: any): e is EsError { +export function isEsError(e: any): e is IEsError { return !!e.body?.attributes; } diff --git a/src/plugins/data/public/search/errors/utils.ts b/src/plugins/data/public/search/errors/utils.ts new file mode 100644 index 0000000000000..d07d9b05e91e9 --- /dev/null +++ b/src/plugins/data/public/search/errors/utils.ts @@ -0,0 +1,31 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { IEsError } from './types'; + +export function getFailedShards(err: IEsError) { + const failedShards = + err.body?.attributes?.error?.failed_shards || + err.body?.attributes?.error?.caused_by?.failed_shards; + return failedShards ? failedShards[0] : undefined; +} + +export function getRootCause(err: IEsError) { + return getFailedShards(err)?.reason; +} diff --git a/src/plugins/data/public/search/search_interceptor.ts b/src/plugins/data/public/search/search_interceptor.ts index e3c6dd3e287d4..1afcf4615ab5a 100644 --- a/src/plugins/data/public/search/search_interceptor.ts +++ b/src/plugins/data/public/search/search_interceptor.ts @@ -21,6 +21,7 @@ import { get, memoize, trimEnd } from 'lodash'; import { BehaviorSubject, throwError, timer, defer, from, Observable, NEVER } from 'rxjs'; import { catchError, finalize } from 'rxjs/operators'; import { CoreStart, CoreSetup, ToastsSetup } from 'kibana/public'; +import { i18n } from '@kbn/i18n'; import { getCombinedSignal, AbortError, @@ -31,7 +32,15 @@ import { ISessionService, } from '../../common'; import { SearchUsageCollector } from './collectors'; -import { SearchTimeoutError, PainlessError, isPainlessError, TimeoutErrorMode } from './errors'; +import { + SearchTimeoutError, + PainlessError, + isPainlessError, + TimeoutErrorMode, + isEsError, + EsError, + getHttpError, +} from './errors'; import { toMountPoint } from '../../../kibana_react/public'; export interface SearchInterceptorDeps { @@ -101,8 +110,12 @@ export class SearchInterceptor { } else if (options?.abortSignal?.aborted) { // In the case an application initiated abort, throw the existing AbortError. return e; - } else if (isPainlessError(e)) { - return new PainlessError(e, request); + } else if (isEsError(e)) { + if (isPainlessError(e)) { + return new PainlessError(e, request); + } else { + return new EsError(e); + } } else { return e; } @@ -236,24 +249,28 @@ export class SearchInterceptor { * */ public showError(e: Error) { - if (e instanceof AbortError) return; - - if (e instanceof SearchTimeoutError) { + if (e instanceof AbortError || e instanceof SearchTimeoutError) { // The SearchTimeoutError is shown by the interceptor in getSearchError (regardless of how the app chooses to handle errors) return; - } - - if (e instanceof PainlessError) { + } else if (e instanceof EsError) { this.deps.toasts.addDanger({ - title: 'Search Error', + title: i18n.translate('data.search.esErrorTitle', { + defaultMessage: 'Cannot retrieve search results', + }), text: toMountPoint(e.getErrorMessage(this.application)), }); - return; + } else if (e.constructor.name === 'HttpFetchError') { + this.deps.toasts.addDanger({ + title: i18n.translate('data.search.httpErrorTitle', { + defaultMessage: 'Cannot retrieve your data', + }), + text: toMountPoint(getHttpError(e.message)), + }); + } else { + this.deps.toasts.addError(e, { + title: 'Search Error', + }); } - - this.deps.toasts.addError(e, { - title: 'Search Error', - }); } } diff --git a/src/plugins/data/public/ui/query_string_input/query_string_input.tsx b/src/plugins/data/public/ui/query_string_input/query_string_input.tsx index c17872028ea8d..bc9e2ed6a83ce 100644 --- a/src/plugins/data/public/ui/query_string_input/query_string_input.tsx +++ b/src/plugins/data/public/ui/query_string_input/query_string_input.tsx @@ -111,6 +111,7 @@ export default class QueryStringInputUI extends Component { private persistedLog: PersistedLog | undefined; private abortController?: AbortController; + private fetchIndexPatternsAbortController?: AbortController; private services = this.props.kibana.services; private componentIsUnmounting = false; private queryBarInputDivRefInstance: RefObject = createRef(); @@ -119,7 +120,7 @@ export default class QueryStringInputUI extends Component { return toUser(this.props.query.query); }; - private fetchIndexPatterns = async () => { + private fetchIndexPatterns = debounce(async () => { const stringPatterns = this.props.indexPatterns.filter( (indexPattern) => typeof indexPattern === 'string' ) as string[]; @@ -127,16 +128,26 @@ export default class QueryStringInputUI extends Component { (indexPattern) => typeof indexPattern !== 'string' ) as IIndexPattern[]; + // abort the previous fetch to avoid overriding with outdated data + // issue https://github.com/elastic/kibana/issues/80831 + if (this.fetchIndexPatternsAbortController) this.fetchIndexPatternsAbortController.abort(); + this.fetchIndexPatternsAbortController = new AbortController(); + const currentAbortController = this.fetchIndexPatternsAbortController; + const objectPatternsFromStrings = (await fetchIndexPatterns( this.services.savedObjects!.client, stringPatterns, this.services.uiSettings! )) as IIndexPattern[]; - this.setState({ - indexPatterns: [...objectPatterns, ...objectPatternsFromStrings], - }); - }; + if (!currentAbortController.signal.aborted) { + this.setState({ + indexPatterns: [...objectPatterns, ...objectPatternsFromStrings], + }); + + this.updateSuggestions(); + } + }, 200); private getSuggestions = async () => { if (!this.inputRef) { @@ -506,7 +517,7 @@ export default class QueryStringInputUI extends Component { } this.initPersistedLog(); - this.fetchIndexPatterns().then(this.updateSuggestions); + this.fetchIndexPatterns(); this.handleListUpdate(); window.addEventListener('resize', this.handleAutoHeight); @@ -525,7 +536,7 @@ export default class QueryStringInputUI extends Component { this.initPersistedLog(); if (!isEqual(prevProps.indexPatterns, this.props.indexPatterns)) { - this.fetchIndexPatterns().then(this.updateSuggestions); + this.fetchIndexPatterns(); } else if (!isEqual(prevProps.query, this.props.query)) { this.updateSuggestions(); } diff --git a/src/plugins/data/server/index_patterns/fetcher/index_patterns_fetcher.ts b/src/plugins/data/server/index_patterns/fetcher/index_patterns_fetcher.ts index 57c636a9e3c69..e75b8761984ec 100644 --- a/src/plugins/data/server/index_patterns/fetcher/index_patterns_fetcher.ts +++ b/src/plugins/data/server/index_patterns/fetcher/index_patterns_fetcher.ts @@ -17,7 +17,7 @@ * under the License. */ -import { LegacyAPICaller } from 'kibana/server'; +import { ElasticsearchClient } from 'kibana/server'; import { getFieldCapabilities, resolveTimePattern, createNoMatchingIndicesError } from './lib'; @@ -37,10 +37,12 @@ interface FieldSubType { } export class IndexPatternsFetcher { - private _callDataCluster: LegacyAPICaller; + private elasticsearchClient: ElasticsearchClient; + private allowNoIndices: boolean; - constructor(callDataCluster: LegacyAPICaller) { - this._callDataCluster = callDataCluster; + constructor(elasticsearchClient: ElasticsearchClient, allowNoIndices: boolean = false) { + this.elasticsearchClient = elasticsearchClient; + this.allowNoIndices = allowNoIndices; } /** @@ -55,10 +57,12 @@ export class IndexPatternsFetcher { async getFieldsForWildcard(options: { pattern: string | string[]; metaFields?: string[]; - fieldCapsOptions?: { allowNoIndices: boolean }; + fieldCapsOptions?: { allow_no_indices: boolean }; }): Promise { const { pattern, metaFields, fieldCapsOptions } = options; - return await getFieldCapabilities(this._callDataCluster, pattern, metaFields, fieldCapsOptions); + return await getFieldCapabilities(this.elasticsearchClient, pattern, metaFields, { + allow_no_indices: fieldCapsOptions ? fieldCapsOptions.allow_no_indices : this.allowNoIndices, + }); } /** @@ -78,11 +82,11 @@ export class IndexPatternsFetcher { interval: string; }) { const { pattern, lookBack, metaFields } = options; - const { matches } = await resolveTimePattern(this._callDataCluster, pattern); + const { matches } = await resolveTimePattern(this.elasticsearchClient, pattern); const indices = matches.slice(0, lookBack); if (indices.length === 0) { throw createNoMatchingIndicesError(pattern); } - return await getFieldCapabilities(this._callDataCluster, indices, metaFields); + return await getFieldCapabilities(this.elasticsearchClient, indices, metaFields); } } diff --git a/src/plugins/data/server/index_patterns/fetcher/lib/es_api.test.js b/src/plugins/data/server/index_patterns/fetcher/lib/es_api.test.js index 8078ea32187b3..fad20a8f0be06 100644 --- a/src/plugins/data/server/index_patterns/fetcher/lib/es_api.test.js +++ b/src/plugins/data/server/index_patterns/fetcher/lib/es_api.test.js @@ -32,36 +32,60 @@ describe('server/index_patterns/service/lib/es_api', () => { afterEach(() => sandbox.restore()); it('calls indices.getAlias() via callCluster', async () => { - const callCluster = sinon.stub(); + const getAlias = sinon.stub(); + const callCluster = { + indices: { + getAlias, + }, + fieldCaps: sinon.stub(), + }; + await callIndexAliasApi(callCluster); - sinon.assert.calledOnce(callCluster); - sinon.assert.calledWith(callCluster, 'indices.getAlias'); + sinon.assert.calledOnce(getAlias); }); it('passes indices directly to es api', async () => { const football = {}; - const callCluster = sinon.stub(); + const getAlias = sinon.stub(); + const callCluster = { + indices: { + getAlias, + }, + fieldCaps: sinon.stub(), + }; await callIndexAliasApi(callCluster, football); - sinon.assert.calledOnce(callCluster); - expect(callCluster.args[0][1].index).toBe(football); + sinon.assert.calledOnce(getAlias); + expect(getAlias.args[0][0].index).toBe(football); }); it('returns the es response directly', async () => { const football = {}; - const callCluster = sinon.stub().returns(football); + const getAlias = sinon.stub().returns(football); + const callCluster = { + indices: { + getAlias, + }, + fieldCaps: sinon.stub(), + }; const resp = await callIndexAliasApi(callCluster); - sinon.assert.calledOnce(callCluster); + sinon.assert.calledOnce(getAlias); expect(resp).toBe(football); }); it('sets ignoreUnavailable and allowNoIndices params', async () => { - const callCluster = sinon.stub(); + const getAlias = sinon.stub(); + const callCluster = { + indices: { + getAlias, + }, + fieldCaps: sinon.stub(), + }; await callIndexAliasApi(callCluster); - sinon.assert.calledOnce(callCluster); + sinon.assert.calledOnce(getAlias); - const passedOpts = callCluster.args[0][1]; - expect(passedOpts).toHaveProperty('ignoreUnavailable', true); - expect(passedOpts).toHaveProperty('allowNoIndices', false); + const passedOpts = getAlias.args[0][0]; + expect(passedOpts).toHaveProperty('ignore_unavailable', true); + expect(passedOpts).toHaveProperty('allow_no_indices', false); }); it('handles errors with convertEsError()', async () => { @@ -70,9 +94,15 @@ describe('server/index_patterns/service/lib/es_api', () => { const convertedError = new Error('convertedError'); sandbox.stub(convertEsErrorNS, 'convertEsError').throws(convertedError); - const callCluster = sinon.spy(async () => { + const getAlias = sinon.stub(async () => { throw esError; }); + const callCluster = { + indices: { + getAlias, + }, + fieldCaps: sinon.stub(), + }; try { await callIndexAliasApi(callCluster, indices); throw new Error('expected callIndexAliasApi() to throw'); @@ -91,37 +121,60 @@ describe('server/index_patterns/service/lib/es_api', () => { afterEach(() => sandbox.restore()); it('calls fieldCaps() via callCluster', async () => { - const callCluster = sinon.stub(); + const fieldCaps = sinon.stub(); + const callCluster = { + indices: { + getAlias: sinon.stub(), + }, + fieldCaps, + }; await callFieldCapsApi(callCluster); - sinon.assert.calledOnce(callCluster); - sinon.assert.calledWith(callCluster, 'fieldCaps'); + sinon.assert.calledOnce(fieldCaps); }); it('passes indices directly to es api', async () => { const football = {}; - const callCluster = sinon.stub(); + const fieldCaps = sinon.stub(); + const callCluster = { + indices: { + getAlias: sinon.stub(), + }, + fieldCaps, + }; await callFieldCapsApi(callCluster, football); - sinon.assert.calledOnce(callCluster); - expect(callCluster.args[0][1].index).toBe(football); + sinon.assert.calledOnce(fieldCaps); + expect(fieldCaps.args[0][0].index).toBe(football); }); it('returns the es response directly', async () => { const football = {}; - const callCluster = sinon.stub().returns(football); + const fieldCaps = sinon.stub().returns(football); + const callCluster = { + indices: { + getAlias: sinon.stub(), + }, + fieldCaps, + }; const resp = await callFieldCapsApi(callCluster); - sinon.assert.calledOnce(callCluster); + sinon.assert.calledOnce(fieldCaps); expect(resp).toBe(football); }); it('sets ignoreUnavailable, allowNoIndices, and fields params', async () => { - const callCluster = sinon.stub(); + const fieldCaps = sinon.stub(); + const callCluster = { + indices: { + getAlias: sinon.stub(), + }, + fieldCaps, + }; await callFieldCapsApi(callCluster); - sinon.assert.calledOnce(callCluster); + sinon.assert.calledOnce(fieldCaps); - const passedOpts = callCluster.args[0][1]; + const passedOpts = fieldCaps.args[0][0]; expect(passedOpts).toHaveProperty('fields', '*'); - expect(passedOpts).toHaveProperty('ignoreUnavailable', true); - expect(passedOpts).toHaveProperty('allowNoIndices', false); + expect(passedOpts).toHaveProperty('ignore_unavailable', true); + expect(passedOpts).toHaveProperty('allow_no_indices', false); }); it('handles errors with convertEsError()', async () => { @@ -130,9 +183,15 @@ describe('server/index_patterns/service/lib/es_api', () => { const convertedError = new Error('convertedError'); sandbox.stub(convertEsErrorNS, 'convertEsError').throws(convertedError); - const callCluster = sinon.spy(async () => { + const fieldCaps = sinon.spy(async () => { throw esError; }); + const callCluster = { + indices: { + getAlias: sinon.stub(), + }, + fieldCaps, + }; try { await callFieldCapsApi(callCluster, indices); throw new Error('expected callFieldCapsApi() to throw'); diff --git a/src/plugins/data/server/index_patterns/fetcher/lib/es_api.ts b/src/plugins/data/server/index_patterns/fetcher/lib/es_api.ts index 27ce14f9a3597..7969324943a9f 100644 --- a/src/plugins/data/server/index_patterns/fetcher/lib/es_api.ts +++ b/src/plugins/data/server/index_patterns/fetcher/lib/es_api.ts @@ -17,7 +17,7 @@ * under the License. */ -import { LegacyAPICaller } from 'kibana/server'; +import { ElasticsearchClient } from 'kibana/server'; import { convertEsError } from './errors'; import { FieldCapsResponse } from './field_capabilities'; @@ -46,15 +46,15 @@ export interface IndexAliasResponse { * @return {Promise} */ export async function callIndexAliasApi( - callCluster: LegacyAPICaller, + callCluster: ElasticsearchClient, indices: string[] | string -): Promise { +) { try { - return (await callCluster('indices.getAlias', { + return await callCluster.indices.getAlias({ index: indices, - ignoreUnavailable: true, - allowNoIndices: false, - })) as Promise; + ignore_unavailable: true, + allow_no_indices: false, + }); } catch (error) { throw convertEsError(indices, error); } @@ -73,17 +73,17 @@ export async function callIndexAliasApi( * @return {Promise} */ export async function callFieldCapsApi( - callCluster: LegacyAPICaller, + callCluster: ElasticsearchClient, indices: string[] | string, - fieldCapsOptions: { allowNoIndices: boolean } = { allowNoIndices: false } + fieldCapsOptions: { allow_no_indices: boolean } = { allow_no_indices: false } ) { try { - return (await callCluster('fieldCaps', { + return await callCluster.fieldCaps({ index: indices, fields: '*', - ignoreUnavailable: true, + ignore_unavailable: true, ...fieldCapsOptions, - })) as FieldCapsResponse; + }); } catch (error) { throw convertEsError(indices, error); } diff --git a/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.test.js b/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.test.js index 0e5757b7b782b..2d860dc8b1843 100644 --- a/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.test.js +++ b/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.test.js @@ -48,9 +48,11 @@ describe('index_patterns/field_capabilities/field_capabilities', () => { }; const stubDeps = (options = {}) => { - const { esResponse = {}, fieldsFromFieldCaps = [], mergeOverrides = identity } = options; + const { esResponse = [], fieldsFromFieldCaps = [], mergeOverrides = identity } = options; - sandbox.stub(callFieldCapsApiNS, 'callFieldCapsApi').callsFake(async () => esResponse); + sandbox + .stub(callFieldCapsApiNS, 'callFieldCapsApi') + .callsFake(async () => ({ body: esResponse })); sandbox.stub(readFieldCapsResponseNS, 'readFieldCapsResponse').returns(fieldsFromFieldCaps); sandbox.stub(mergeOverridesNS, 'mergeOverrides').callsFake(mergeOverrides); }; diff --git a/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.ts b/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.ts index 62e77e0adad66..b9e3e8aae0899 100644 --- a/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.ts +++ b/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.ts @@ -19,9 +19,9 @@ import { defaults, keyBy, sortBy } from 'lodash'; -import { LegacyAPICaller } from 'kibana/server'; +import { ElasticsearchClient } from 'kibana/server'; import { callFieldCapsApi } from '../es_api'; -import { FieldCapsResponse, readFieldCapsResponse } from './field_caps_response'; +import { readFieldCapsResponse } from './field_caps_response'; import { mergeOverrides } from './overrides'; import { FieldDescriptor } from '../../index_patterns_fetcher'; @@ -36,17 +36,13 @@ import { FieldDescriptor } from '../../index_patterns_fetcher'; * @return {Promise>} */ export async function getFieldCapabilities( - callCluster: LegacyAPICaller, + callCluster: ElasticsearchClient, indices: string | string[] = [], metaFields: string[] = [], - fieldCapsOptions?: { allowNoIndices: boolean } + fieldCapsOptions?: { allow_no_indices: boolean } ) { - const esFieldCaps: FieldCapsResponse = await callFieldCapsApi( - callCluster, - indices, - fieldCapsOptions - ); - const fieldsFromFieldCapsByName = keyBy(readFieldCapsResponse(esFieldCaps), 'name'); + const esFieldCaps = await callFieldCapsApi(callCluster, indices, fieldCapsOptions); + const fieldsFromFieldCapsByName = keyBy(readFieldCapsResponse(esFieldCaps.body), 'name'); const allFieldsUnsorted = Object.keys(fieldsFromFieldCapsByName) .filter((name) => !name.startsWith('_')) diff --git a/src/plugins/data/server/index_patterns/fetcher/lib/resolve_time_pattern.test.js b/src/plugins/data/server/index_patterns/fetcher/lib/resolve_time_pattern.test.js index 660e9ec30db6a..87f222aaad89d 100644 --- a/src/plugins/data/server/index_patterns/fetcher/lib/resolve_time_pattern.test.js +++ b/src/plugins/data/server/index_patterns/fetcher/lib/resolve_time_pattern.test.js @@ -32,6 +32,11 @@ const TIME_PATTERN = '[logs-]dddd-YYYY.w'; describe('server/index_patterns/service/lib/resolve_time_pattern', () => { let sandbox; + const esClientMock = { + indices: { + getAlias: () => ({}), + }, + }; beforeEach(() => (sandbox = sinon.createSandbox())); afterEach(() => sandbox.restore()); @@ -39,7 +44,7 @@ describe('server/index_patterns/service/lib/resolve_time_pattern', () => { describe('pre request', () => { it('uses callIndexAliasApi() fn', async () => { sandbox.stub(callIndexAliasApiNS, 'callIndexAliasApi').returns({}); - await resolveTimePattern(noop, TIME_PATTERN); + await resolveTimePattern(esClientMock, TIME_PATTERN); sinon.assert.calledOnce(callIndexAliasApi); }); @@ -49,7 +54,7 @@ describe('server/index_patterns/service/lib/resolve_time_pattern', () => { sandbox.stub(timePatternToWildcardNS, 'timePatternToWildcard').returns(wildcard); - await resolveTimePattern(noop, timePattern); + await resolveTimePattern(esClientMock, timePattern); sinon.assert.calledOnce(timePatternToWildcard); expect(timePatternToWildcard.firstCall.args).toEqual([timePattern]); }); @@ -61,7 +66,7 @@ describe('server/index_patterns/service/lib/resolve_time_pattern', () => { sandbox.stub(callIndexAliasApiNS, 'callIndexAliasApi').returns({}); sandbox.stub(timePatternToWildcardNS, 'timePatternToWildcard').returns(wildcard); - await resolveTimePattern(noop, timePattern); + await resolveTimePattern(esClientMock, timePattern); sinon.assert.calledOnce(callIndexAliasApi); expect(callIndexAliasApi.firstCall.args[1]).toBe(wildcard); }); @@ -70,13 +75,15 @@ describe('server/index_patterns/service/lib/resolve_time_pattern', () => { describe('read response', () => { it('returns all aliases names in result.all, ordered by time desc', async () => { sandbox.stub(callIndexAliasApiNS, 'callIndexAliasApi').returns({ - 'logs-2016.2': {}, - 'logs-Saturday-2017.1': {}, - 'logs-2016.1': {}, - 'logs-Sunday-2017.1': {}, - 'logs-2015': {}, - 'logs-2016.3': {}, - 'logs-Friday-2017.1': {}, + body: { + 'logs-2016.2': {}, + 'logs-Saturday-2017.1': {}, + 'logs-2016.1': {}, + 'logs-Sunday-2017.1': {}, + 'logs-2015': {}, + 'logs-2016.3': {}, + 'logs-Friday-2017.1': {}, + }, }); const resp = await resolveTimePattern(noop, TIME_PATTERN); @@ -94,13 +101,15 @@ describe('server/index_patterns/service/lib/resolve_time_pattern', () => { it('returns all indices matching the time pattern in matches, ordered by time desc', async () => { sandbox.stub(callIndexAliasApiNS, 'callIndexAliasApi').returns({ - 'logs-2016.2': {}, - 'logs-Saturday-2017.1': {}, - 'logs-2016.1': {}, - 'logs-Sunday-2017.1': {}, - 'logs-2015': {}, - 'logs-2016.3': {}, - 'logs-Friday-2017.1': {}, + body: { + 'logs-2016.2': {}, + 'logs-Saturday-2017.1': {}, + 'logs-2016.1': {}, + 'logs-Sunday-2017.1': {}, + 'logs-2015': {}, + 'logs-2016.3': {}, + 'logs-Friday-2017.1': {}, + }, }); const resp = await resolveTimePattern(noop, TIME_PATTERN); diff --git a/src/plugins/data/server/index_patterns/fetcher/lib/resolve_time_pattern.ts b/src/plugins/data/server/index_patterns/fetcher/lib/resolve_time_pattern.ts index 2e408d7569be5..95ec06dd9c6e6 100644 --- a/src/plugins/data/server/index_patterns/fetcher/lib/resolve_time_pattern.ts +++ b/src/plugins/data/server/index_patterns/fetcher/lib/resolve_time_pattern.ts @@ -20,7 +20,7 @@ import { chain } from 'lodash'; import moment from 'moment'; -import { LegacyAPICaller } from 'kibana/server'; +import { ElasticsearchClient } from 'kibana/server'; import { timePatternToWildcard } from './time_pattern_to_wildcard'; import { callIndexAliasApi, IndicesAliasResponse } from './es_api'; @@ -36,10 +36,10 @@ import { callIndexAliasApi, IndicesAliasResponse } from './es_api'; * and the indices that actually match the time * pattern (matches); */ -export async function resolveTimePattern(callCluster: LegacyAPICaller, timePattern: string) { +export async function resolveTimePattern(callCluster: ElasticsearchClient, timePattern: string) { const aliases = await callIndexAliasApi(callCluster, timePatternToWildcard(timePattern)); - const allIndexDetails = chain(aliases) + const allIndexDetails = chain(aliases.body) .reduce( (acc: string[], index: any, indexName: string) => acc.concat(indexName, Object.keys(index.aliases || {})), diff --git a/src/plugins/data/server/index_patterns/routes.ts b/src/plugins/data/server/index_patterns/routes.ts index 428e7fef6deea..041eb235d01e0 100644 --- a/src/plugins/data/server/index_patterns/routes.ts +++ b/src/plugins/data/server/index_patterns/routes.ts @@ -46,8 +46,8 @@ export function registerRoutes(http: HttpServiceSetup) { }, }, async (context, request, response) => { - const { callAsCurrentUser } = context.core.elasticsearch.legacy.client; - const indexPatterns = new IndexPatternsFetcher(callAsCurrentUser); + const { asCurrentUser } = context.core.elasticsearch.client; + const indexPatterns = new IndexPatternsFetcher(asCurrentUser); const { pattern, meta_fields: metaFields } = request.query; let parsedFields: string[] = []; @@ -105,8 +105,8 @@ export function registerRoutes(http: HttpServiceSetup) { }, }, async (context: RequestHandlerContext, request: any, response: any) => { - const { callAsCurrentUser } = context.core.elasticsearch.legacy.client; - const indexPatterns = new IndexPatternsFetcher(callAsCurrentUser); + const { asCurrentUser } = context.core.elasticsearch.client; + const indexPatterns = new IndexPatternsFetcher(asCurrentUser); const { pattern, interval, look_back: lookBack, meta_fields: metaFields } = request.query; let parsedFields: string[] = []; diff --git a/src/plugins/data/server/server.api.md b/src/plugins/data/server/server.api.md index 3143d5baa5b77..97cbb40c13db0 100644 --- a/src/plugins/data/server/server.api.md +++ b/src/plugins/data/server/server.api.md @@ -26,6 +26,7 @@ import { ISearchSource } from 'src/plugins/data/public'; import { KibanaRequest } from 'src/core/server'; import { LegacyAPICaller } from 'kibana/server'; import { Logger } from 'kibana/server'; +import { Logger as Logger_2 } from 'src/core/server'; import { LoggerFactory } from '@kbn/logging'; import { Moment } from 'moment'; import moment from 'moment'; @@ -658,7 +659,7 @@ export const indexPatterns: { // // @public (undocumented) export class IndexPatternsFetcher { - constructor(callDataCluster: LegacyAPICaller); + constructor(elasticsearchClient: ElasticsearchClient, allowNoIndices?: boolean); getFieldsForTimePattern(options: { pattern: string; metaFields: string[]; @@ -669,7 +670,7 @@ export class IndexPatternsFetcher { pattern: string | string[]; metaFields?: string[]; fieldCapsOptions?: { - allowNoIndices: boolean; + allow_no_indices: boolean; }; }): Promise; } diff --git a/src/plugins/discover/public/application/angular/context_app.html b/src/plugins/discover/public/application/angular/context_app.html index d609a497c4ba1..ef7bc09206176 100644 --- a/src/plugins/discover/public/application/angular/context_app.html +++ b/src/plugins/discover/public/application/angular/context_app.html @@ -10,7 +10,6 @@ > - - - - - - - + predecessor-count="contextApp.state.queryParameters.predecessorCount" + predecessor-available="contextApp.state.rows.predecessors.length" + predecessor-status="contextApp.state.loadingStatus.predecessors.status" + on-change-predecessor-count="contextApp.actions.fetchGivenPredecessorRows" + successor-count="contextApp.state.queryParameters.successorCount" + successor-available="contextApp.state.rows.successors.length" + successor-status="contextApp.state.loadingStatus.successors.status" + on-change-successor-count="contextApp.actions.fetchGivenSuccessorRows" + > diff --git a/src/plugins/discover/public/application/components/context_app/__snapshots__/context_app_legacy.test.tsx.snap b/src/plugins/discover/public/application/components/context_app/__snapshots__/context_app_legacy.test.tsx.snap deleted file mode 100644 index 58305ee23cb21..0000000000000 --- a/src/plugins/discover/public/application/components/context_app/__snapshots__/context_app_legacy.test.tsx.snap +++ /dev/null @@ -1,741 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`ContextAppLegacy test renders correctly 1`] = ` - - - - - -
-
- -
- -
-
- - - - - -`; - -exports[`ContextAppLegacy test renders loading indicator 1`] = ` - - - - - -
- -
- -
- - Loading... - -
- -
- -
- - - - - -`; diff --git a/src/plugins/discover/public/application/components/context_app/context_app_legacy.test.tsx b/src/plugins/discover/public/application/components/context_app/context_app_legacy.test.tsx index 16d8cd78004f9..25576a9072944 100644 --- a/src/plugins/discover/public/application/components/context_app/context_app_legacy.test.tsx +++ b/src/plugins/discover/public/application/components/context_app/context_app_legacy.test.tsx @@ -23,6 +23,7 @@ import { IIndexPattern } from '../../../../../data/common/index_patterns'; import { mountWithIntl } from 'test_utils/enzyme_helpers'; import { DocTableLegacy } from '../../angular/doc_table/create_doc_table_react'; import { findTestSubject } from '@elastic/eui/lib/test'; +import { ActionBar } from '../../angular/context/components/action_bar/action_bar'; describe('ContextAppLegacy test', () => { const hit = { @@ -48,28 +49,36 @@ describe('ContextAppLegacy test', () => { columns: ['_source'], filter: () => {}, hits: [hit], - infiniteScroll: true, sorting: ['order_date', 'desc'], minimumVisibleRows: 5, indexPattern, status: 'loaded', + defaultStepSize: 5, + predecessorCount: 10, + successorCount: 10, + predecessorAvailable: 10, + successorAvailable: 10, + onChangePredecessorCount: jest.fn(), + onChangeSuccessorCount: jest.fn(), + predecessorStatus: 'loaded', + successorStatus: 'loaded', }; it('renders correctly', () => { const component = mountWithIntl(); - expect(component).toMatchSnapshot(); expect(component.find(DocTableLegacy).length).toBe(1); const loadingIndicator = findTestSubject(component, 'contextApp_loadingIndicator'); expect(loadingIndicator.length).toBe(0); + expect(component.find(ActionBar).length).toBe(2); }); it('renders loading indicator', () => { const props = { ...defaultProps }; props.status = 'loading'; const component = mountWithIntl(); - expect(component).toMatchSnapshot(); expect(component.find('DocTableLegacy').length).toBe(0); const loadingIndicator = findTestSubject(component, 'contextApp_loadingIndicator'); expect(loadingIndicator.length).toBe(1); + expect(component.find(ActionBar).length).toBe(2); }); }); diff --git a/src/plugins/discover/public/application/components/context_app/context_app_legacy.tsx b/src/plugins/discover/public/application/components/context_app/context_app_legacy.tsx index ee8b2f590f71c..afb4a9a981e21 100644 --- a/src/plugins/discover/public/application/components/context_app/context_app_legacy.tsx +++ b/src/plugins/discover/public/application/components/context_app/context_app_legacy.tsx @@ -17,15 +17,15 @@ * under the License. */ import React from 'react'; -import { FormattedMessage } from '@kbn/i18n/react'; +import { FormattedMessage, I18nProvider } from '@kbn/i18n/react'; import { EuiPanel, EuiText } from '@elastic/eui'; -import { I18nProvider } from '@kbn/i18n/react'; import { DocTableLegacy, DocTableLegacyProps, } from '../../angular/doc_table/create_doc_table_react'; import { IIndexPattern, IndexPatternField } from '../../../../../data/common/index_patterns'; import { LOADING_STATUS } from './constants'; +import { ActionBar, ActionBarProps } from '../../angular/context/components/action_bar/action_bar'; export interface ContextAppProps { columns: string[]; @@ -35,15 +35,64 @@ export interface ContextAppProps { minimumVisibleRows: number; sorting: string[]; status: string; + defaultStepSize: number; + predecessorCount: number; + successorCount: number; + predecessorAvailable: number; + successorAvailable: number; + onChangePredecessorCount: (count: number) => void; + onChangeSuccessorCount: (count: number) => void; + predecessorStatus: string; + successorStatus: string; +} + +const PREDECESSOR_TYPE = 'predecessors'; +const SUCCESSOR_TYPE = 'successors'; + +function isLoading(status: string) { + return status !== LOADING_STATUS.LOADED && status !== LOADING_STATUS.FAILED; } export function ContextAppLegacy(renderProps: ContextAppProps) { - const { hits, filter, sorting, status } = renderProps; - const props = ({ ...renderProps } as unknown) as DocTableLegacyProps; - props.rows = hits; - props.onFilter = filter; - props.sort = sorting.map((el) => [el]); + const status = renderProps.status; const isLoaded = status === LOADING_STATUS.LOADED; + + const actionBarProps = (type: string) => { + const { + defaultStepSize, + successorCount, + predecessorCount, + predecessorAvailable, + successorAvailable, + predecessorStatus, + successorStatus, + onChangePredecessorCount, + onChangeSuccessorCount, + } = renderProps; + const isPredecessorType = type === PREDECESSOR_TYPE; + return { + defaultStepSize, + docCount: isPredecessorType ? predecessorCount : successorCount, + docCountAvailable: isPredecessorType ? predecessorAvailable : successorAvailable, + onChangeCount: isPredecessorType ? onChangePredecessorCount : onChangeSuccessorCount, + isLoading: isPredecessorType ? isLoading(predecessorStatus) : isLoading(successorStatus), + type, + isDisabled: !isLoaded, + } as ActionBarProps; + }; + + const docTableProps = () => { + const { hits, filter, sorting, columns, indexPattern, minimumVisibleRows } = renderProps; + return { + columns, + indexPattern, + minimumVisibleRows, + rows: hits, + onFilter: filter, + sort: sorting.map((el) => [el]), + } as DocTableLegacyProps; + }; + const loadingFeedback = () => { if (status === LOADING_STATUS.UNINITIALIZED || status === LOADING_STATUS.LOADING) { return ( @@ -59,17 +108,20 @@ export function ContextAppLegacy(renderProps: ContextAppProps) { } return null; }; + return ( + {loadingFeedback()} {isLoaded ? (
- +
) : null} + ); diff --git a/src/plugins/discover/public/application/components/context_app/context_app_legacy_directive.ts b/src/plugins/discover/public/application/components/context_app/context_app_legacy_directive.ts index af94c5537da28..4a315be513a0d 100644 --- a/src/plugins/discover/public/application/components/context_app/context_app_legacy_directive.ts +++ b/src/plugins/discover/public/application/components/context_app/context_app_legacy_directive.ts @@ -25,8 +25,16 @@ export function createContextAppLegacy(reactDirective: any) { ['indexPattern', { watchDepth: 'reference' }], ['sorting', { watchDepth: 'reference' }], ['columns', { watchDepth: 'collection' }], - ['infiniteScroll', { watchDepth: 'reference' }], ['minimumVisibleRows', { watchDepth: 'reference' }], ['status', { watchDepth: 'reference' }], + ['defaultStepSize', { watchDepth: 'reference' }], + ['predecessorCount', { watchDepth: 'reference' }], + ['predecessorAvailable', { watchDepth: 'reference' }], + ['predecessorStatus', { watchDepth: 'reference' }], + ['onChangePredecessorCount', { watchDepth: 'reference' }], + ['successorCount', { watchDepth: 'reference' }], + ['successorAvailable', { watchDepth: 'reference' }], + ['successorStatus', { watchDepth: 'reference' }], + ['onChangeSuccessorCount', { watchDepth: 'reference' }], ]); } diff --git a/src/plugins/discover/public/application/components/sidebar/discover_field_details.test.tsx b/src/plugins/discover/public/application/components/sidebar/discover_field_details.test.tsx new file mode 100644 index 0000000000000..2cf626d182eeb --- /dev/null +++ b/src/plugins/discover/public/application/components/sidebar/discover_field_details.test.tsx @@ -0,0 +1,103 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import React from 'react'; +import { findTestSubject } from '@elastic/eui/lib/test'; +// @ts-ignore +import stubbedLogstashFields from 'fixtures/logstash_fields'; +import { mountWithIntl } from 'test_utils/enzyme_helpers'; +import { DiscoverFieldDetails } from './discover_field_details'; +import { coreMock } from '../../../../../../core/public/mocks'; +import { IndexPatternField } from '../../../../../data/public'; +import { getStubIndexPattern } from '../../../../../data/public/test_utils'; + +const indexPattern = getStubIndexPattern( + 'logstash-*', + (cfg: any) => cfg, + 'time', + stubbedLogstashFields(), + coreMock.createSetup() +); + +describe('discover sidebar field details', function () { + const defaultProps = { + indexPattern, + details: { buckets: [], error: '', exists: 1, total: true, columns: [] }, + onAddFilter: jest.fn(), + }; + + function mountComponent(field: IndexPatternField) { + const compProps = { ...defaultProps, field }; + return mountWithIntl(); + } + + it('should enable the visualize link for a number field', function () { + const visualizableField = new IndexPatternField( + { + name: 'bytes', + type: 'number', + esTypes: ['long'], + count: 10, + scripted: false, + searchable: true, + aggregatable: true, + readFromDocValues: true, + }, + 'bytes' + ); + const comp = mountComponent(visualizableField); + expect(findTestSubject(comp, 'fieldVisualize-bytes')).toBeTruthy(); + }); + + it('should disable the visualize link for an _id field', function () { + const conflictField = new IndexPatternField( + { + name: '_id', + type: 'string', + esTypes: ['_id'], + count: 0, + scripted: false, + searchable: true, + aggregatable: true, + readFromDocValues: true, + }, + 'test' + ); + const comp = mountComponent(conflictField); + expect(findTestSubject(comp, 'fieldVisualize-_id')).toEqual({}); + }); + + it('should disable the visualize link for an unknown field', function () { + const unknownField = new IndexPatternField( + { + name: 'test', + type: 'unknown', + esTypes: ['double'], + count: 0, + scripted: false, + searchable: true, + aggregatable: true, + readFromDocValues: true, + }, + 'test' + ); + const comp = mountComponent(unknownField); + expect(findTestSubject(comp, 'fieldVisualize-test')).toEqual({}); + }); +}); diff --git a/src/plugins/embeddable/public/lib/panel/_embeddable_panel.scss b/src/plugins/embeddable/public/lib/panel/_embeddable_panel.scss index cdc0f9f0e0451..6b8654f6c3528 100644 --- a/src/plugins/embeddable/public/lib/panel/_embeddable_panel.scss +++ b/src/plugins/embeddable/public/lib/panel/_embeddable_panel.scss @@ -64,12 +64,12 @@ .embPanel__titleText { @include euiTextTruncate; + font-weight: $euiFontWeightBold; } .embPanel__placeholderTitleText { - @include euiTextTruncate; - font-weight: $euiFontWeightRegular; color: $euiColorMediumShade; + font-weight: $euiFontWeightRegular; } } diff --git a/src/plugins/embeddable/public/lib/panel/embeddable_panel.test.tsx b/src/plugins/embeddable/public/lib/panel/embeddable_panel.test.tsx index fcf79c1d6b211..c717e4370231e 100644 --- a/src/plugins/embeddable/public/lib/panel/embeddable_panel.test.tsx +++ b/src/plugins/embeddable/public/lib/panel/embeddable_panel.test.tsx @@ -19,7 +19,7 @@ import React from 'react'; import { mount } from 'enzyme'; -import { nextTick } from 'test_utils/enzyme_helpers'; +import { mountWithIntl, nextTick } from 'test_utils/enzyme_helpers'; import { findTestSubject } from '@elastic/eui/lib/test'; import { I18nProvider } from '@kbn/i18n/react'; @@ -343,6 +343,88 @@ test('HelloWorldContainer in edit mode shows edit mode actions', async () => { // expect(action.length).toBe(1); }); +test('Panel title customize link does not exist in view mode', async () => { + const inspector = inspectorPluginMock.createStartContract(); + + const container = new HelloWorldContainer( + { id: '123', panels: {}, viewMode: ViewMode.VIEW, hidePanelTitles: false }, + { getEmbeddableFactory } as any + ); + + const embeddable = await container.addNewEmbeddable< + ContactCardEmbeddableInput, + ContactCardEmbeddableOutput, + ContactCardEmbeddable + >(CONTACT_CARD_EMBEDDABLE, { + firstName: 'Vayon', + lastName: 'Poole', + }); + + const component = mountWithIntl( + Promise.resolve([])} + getAllEmbeddableFactories={start.getEmbeddableFactories} + getEmbeddableFactory={start.getEmbeddableFactory} + notifications={{} as any} + overlays={{} as any} + application={applicationMock} + inspector={inspector} + SavedObjectFinder={() => null} + /> + ); + + const titleLink = findTestSubject(component, 'embeddablePanelTitleLink'); + expect(titleLink.length).toBe(0); +}); + +test('Runs customize panel action on title click when in edit mode', async () => { + const inspector = inspectorPluginMock.createStartContract(); + + const container = new HelloWorldContainer( + { id: '123', panels: {}, viewMode: ViewMode.EDIT, hidePanelTitles: false }, + { getEmbeddableFactory } as any + ); + + const embeddable = await container.addNewEmbeddable< + ContactCardEmbeddableInput, + ContactCardEmbeddableOutput, + ContactCardEmbeddable + >(CONTACT_CARD_EMBEDDABLE, { + firstName: 'Vayon', + lastName: 'Poole', + }); + + const component = mountWithIntl( + Promise.resolve([])} + getAllEmbeddableFactories={start.getEmbeddableFactories} + getEmbeddableFactory={start.getEmbeddableFactory} + notifications={{} as any} + overlays={{} as any} + application={applicationMock} + inspector={inspector} + SavedObjectFinder={() => null} + /> + ); + + const titleExecute = jest.fn(); + component.setState((s: any) => ({ + ...s, + universalActions: { + ...s.universalActions, + customizePanelTitle: { execute: titleExecute, isCompatible: jest.fn() }, + }, + })); + + const titleLink = findTestSubject(component, 'embeddablePanelTitleLink'); + expect(titleLink.length).toBe(1); + titleLink.simulate('click'); + await nextTick(); + expect(titleExecute).toHaveBeenCalledTimes(1); +}); + test('Updates when hidePanelTitles is toggled', async () => { const inspector = inspectorPluginMock.createStartContract(); diff --git a/src/plugins/embeddable/public/lib/panel/embeddable_panel.tsx b/src/plugins/embeddable/public/lib/panel/embeddable_panel.tsx index 137f8c24b1fae..1cd48e85469fd 100644 --- a/src/plugins/embeddable/public/lib/panel/embeddable_panel.tsx +++ b/src/plugins/embeddable/public/lib/panel/embeddable_panel.tsx @@ -76,6 +76,7 @@ interface Props { interface State { panels: EuiContextMenuPanelDescriptor[]; + universalActions: PanelUniversalActions; focusedPanelIndex?: string; viewMode: ViewMode; hidePanelTitle: boolean; @@ -86,6 +87,14 @@ interface State { error?: EmbeddableError; } +interface PanelUniversalActions { + customizePanelTitle: CustomizePanelTitleAction; + addPanel: AddPanelAction; + inspectPanel: InspectPanelAction; + removePanel: RemovePanelAction; + editPanel: EditPanelAction; +} + export class EmbeddablePanel extends React.Component { private embeddableRoot: React.RefObject; private parentSubscription?: Subscription; @@ -102,6 +111,7 @@ export class EmbeddablePanel extends React.Component { Boolean(embeddable.getInput()?.hidePanelTitles); this.state = { + universalActions: this.getUniversalActions(), panels: [], viewMode, hidePanelTitle, @@ -229,6 +239,7 @@ export class EmbeddablePanel extends React.Component { getActionContextMenuPanel={this.getActionContextMenuPanel} hidePanelTitle={this.state.hidePanelTitle} isViewMode={viewOnlyMode} + customizeTitle={this.state.universalActions.customizePanelTitle} closeContextMenu={this.state.closeContextMenu} title={title} badges={this.state.badges} @@ -267,17 +278,7 @@ export class EmbeddablePanel extends React.Component { } }; - private getActionContextMenuPanel = async () => { - let regularActions = await this.props.getActions(CONTEXT_MENU_TRIGGER, { - embeddable: this.props.embeddable, - }); - - const { disabledActions } = this.props.embeddable.getInput(); - if (disabledActions) { - const removeDisabledActions = removeById(disabledActions); - regularActions = regularActions.filter(removeDisabledActions); - } - + private getUniversalActions = (): PanelUniversalActions => { const createGetUserData = (overlays: OverlayStart) => async function getUserData(context: { embeddable: IEmbeddable }) { return new Promise<{ title: string | undefined; hideTitle?: boolean }>((resolve) => { @@ -299,27 +300,41 @@ export class EmbeddablePanel extends React.Component { }); }; - // These actions are exposed on the context menu for every embeddable, they bypass the trigger + // Universal actions are exposed on the context menu for every embeddable, they bypass the trigger // registry. - const extraActions: Array> = [ - new CustomizePanelTitleAction(createGetUserData(this.props.overlays)), - new AddPanelAction( + return { + customizePanelTitle: new CustomizePanelTitleAction(createGetUserData(this.props.overlays)), + addPanel: new AddPanelAction( this.props.getEmbeddableFactory, this.props.getAllEmbeddableFactories, this.props.overlays, this.props.notifications, this.props.SavedObjectFinder ), - new InspectPanelAction(this.props.inspector), - new RemovePanelAction(), - new EditPanelAction( + inspectPanel: new InspectPanelAction(this.props.inspector), + removePanel: new RemovePanelAction(), + editPanel: new EditPanelAction( this.props.getEmbeddableFactory, this.props.application, this.props.stateTransfer ), - ]; + }; + }; - const sortedActions = [...regularActions, ...extraActions].sort(sortByOrderField); + private getActionContextMenuPanel = async () => { + let regularActions = await this.props.getActions(CONTEXT_MENU_TRIGGER, { + embeddable: this.props.embeddable, + }); + + const { disabledActions } = this.props.embeddable.getInput(); + if (disabledActions) { + const removeDisabledActions = removeById(disabledActions); + regularActions = regularActions.filter(removeDisabledActions); + } + + const sortedActions = [...regularActions, ...Object.values(this.state.universalActions)].sort( + sortByOrderField + ); return await buildContextMenuForActions({ actions: sortedActions.map((action) => ({ diff --git a/src/plugins/embeddable/public/lib/panel/panel_header/panel_header.tsx b/src/plugins/embeddable/public/lib/panel/panel_header/panel_header.tsx index 9bcef051a9359..44f5c3df2709d 100644 --- a/src/plugins/embeddable/public/lib/panel/panel_header/panel_header.tsx +++ b/src/plugins/embeddable/public/lib/panel/panel_header/panel_header.tsx @@ -24,6 +24,7 @@ import { EuiToolTip, EuiScreenReaderOnly, EuiNotificationBadge, + EuiLink, } from '@elastic/eui'; import classNames from 'classnames'; import React from 'react'; @@ -32,6 +33,7 @@ import { PanelOptionsMenu } from './panel_options_menu'; import { IEmbeddable } from '../../embeddables'; import { EmbeddableContext, panelBadgeTrigger, panelNotificationTrigger } from '../../triggers'; import { uiToReactComponent } from '../../../../../kibana_react/public'; +import { CustomizePanelTitleAction } from '.'; export interface PanelHeaderProps { title?: string; @@ -44,6 +46,7 @@ export interface PanelHeaderProps { embeddable: IEmbeddable; headerId?: string; showPlaceholderTitle?: boolean; + customizeTitle: CustomizePanelTitleAction; } function renderBadges(badges: Array>, embeddable: IEmbeddable) { @@ -129,6 +132,7 @@ export function PanelHeader({ notifications, embeddable, headerId, + customizeTitle, }: PanelHeaderProps) { const description = getViewDescription(embeddable); const showTitle = !hidePanelTitle && (!isViewMode || title); @@ -172,11 +176,35 @@ export function PanelHeader({ } const renderTitle = () => { - const titleComponent = showTitle ? ( - - {title || placeholderTitle} - - ) : undefined; + let titleComponent; + if (showTitle) { + titleComponent = isViewMode ? ( + + {title || placeholderTitle} + + ) : ( + customizeTitle.execute({ embeddable })} + > + {title || placeholderTitle} + + ); + } return description ? ( { test('creates default ExecutionContext', () => { const execution = createExecution(); expect(execution.context).toMatchObject({ - getInitialInput: expect.any(Function), + getSearchContext: expect.any(Function), variables: expect.any(Object), types: expect.any(Object), }); @@ -143,6 +145,7 @@ describe('Execution', () => { const execution = new Execution({ executor, expression, + params: {}, }); expect(execution.expression).toBe(expression); }); @@ -153,6 +156,7 @@ describe('Execution', () => { const execution = new Execution({ ast: parseExpression(expression), executor, + params: {}, }); expect(execution.expression).toBe(expression); }); @@ -619,7 +623,7 @@ describe('Execution', () => { const execution = new Execution({ executor, ast: parseExpression('add val=1 | throws | add val=3'), - debug: true, + params: { debug: true }, }); execution.start(0); await execution.result; @@ -637,7 +641,7 @@ describe('Execution', () => { const execution = new Execution({ executor, ast: parseExpression('add val=1 | throws | add val=3'), - debug: true, + params: { debug: true }, }); execution.start(0); await execution.result; @@ -658,7 +662,7 @@ describe('Execution', () => { const execution = new Execution({ executor, ast: parseExpression('add val=1 | throws | add val=3'), - debug: true, + params: { debug: true }, }); execution.start(0); await execution.result; diff --git a/src/plugins/expressions/common/execution/execution.ts b/src/plugins/expressions/common/execution/execution.ts index 69140453f486d..2bcf441b14203 100644 --- a/src/plugins/expressions/common/execution/execution.ts +++ b/src/plugins/expressions/common/execution/execution.ts @@ -19,7 +19,7 @@ import { i18n } from '@kbn/i18n'; import { keys, last, mapValues, reduce, zipObject } from 'lodash'; -import { Executor, ExpressionExecOptions } from '../executor'; +import { Executor } from '../executor'; import { createExecutionContainer, ExecutionContainer } from './container'; import { createError } from '../util'; import { Defer, now } from '../../../kibana_utils/common'; @@ -39,6 +39,7 @@ import { getType, ExpressionValue } from '../expression_types'; import { ArgumentType, ExpressionFunction } from '../expression_functions'; import { getByAlias } from '../util/get_by_alias'; import { ExecutionContract } from './execution_contract'; +import { ExpressionExecutionParams } from '../service'; const createAbortErrorValue = () => createError({ @@ -46,20 +47,11 @@ const createAbortErrorValue = () => name: 'AbortError', }); -export interface ExecutionParams< - ExtraContext extends Record = Record -> { +export interface ExecutionParams { executor: Executor; ast?: ExpressionAstExpression; expression?: string; - context?: ExtraContext; - - /** - * Whether to execute expression in *debug mode*. In *debug mode* inputs and - * outputs as well as all resolved arguments and time it took to execute each - * function are saved and are available for introspection. - */ - debug?: boolean; + params: ExpressionExecutionParams; } const createDefaultInspectorAdapters = (): DefaultInspectorAdapters => ({ @@ -68,11 +60,10 @@ const createDefaultInspectorAdapters = (): DefaultInspectorAdapters => ({ }); export class Execution< - ExtraContext extends Record = Record, Input = unknown, Output = unknown, - InspectorAdapters extends Adapters = ExtraContext['inspectorAdapters'] extends object - ? ExtraContext['inspectorAdapters'] + InspectorAdapters extends Adapters = ExpressionExecutionParams['inspectorAdapters'] extends object + ? ExpressionExecutionParams['inspectorAdapters'] : DefaultInspectorAdapters > { /** @@ -92,7 +83,7 @@ export class Execution< * Execution context - object that allows to do side-effects. Context is passed * to every function. */ - public readonly context: ExecutionContext & ExtraContext; + public readonly context: ExecutionContext; /** * AbortController to cancel this Execution. @@ -126,11 +117,10 @@ export class Execution< * can return to other plugins for their consumption. */ public readonly contract: ExecutionContract< - ExtraContext, Input, Output, InspectorAdapters - > = new ExecutionContract(this); + > = new ExecutionContract(this); public readonly expression: string; @@ -142,17 +132,17 @@ export class Execution< return this.context.inspectorAdapters; } - constructor(public readonly params: ExecutionParams) { - const { executor } = params; + constructor(public readonly execution: ExecutionParams) { + const { executor } = execution; - if (!params.ast && !params.expression) { + if (!execution.ast && !execution.expression) { throw new TypeError('Execution params should contain at least .ast or .expression key.'); - } else if (params.ast && params.expression) { + } else if (execution.ast && execution.expression) { throw new TypeError('Execution params cannot contain both .ast and .expression key.'); } - this.expression = params.expression || formatExpression(params.ast!); - const ast = params.ast || parseExpression(this.expression); + this.expression = execution.expression || formatExpression(execution.ast!); + const ast = execution.ast || parseExpression(this.expression); this.state = createExecutionContainer({ ...executor.state.get(), @@ -161,14 +151,13 @@ export class Execution< }); this.context = { - getInitialInput: () => this.input, - variables: {}, + getSearchContext: () => this.execution.params.searchContext || {}, + getSearchSessionId: () => execution.params.searchSessionId, + variables: execution.params.variables || {}, types: executor.getTypes(), abortSignal: this.abortController.signal, - ...(params.context || ({} as ExtraContext)), - inspectorAdapters: (params.context && params.context.inspectorAdapters - ? params.context.inspectorAdapters - : createDefaultInspectorAdapters()) as InspectorAdapters, + inspectorAdapters: execution.params.inspectorAdapters || createDefaultInspectorAdapters(), + ...(execution.params as any).extraContext, }; } @@ -249,10 +238,10 @@ export class Execution< // actually have a `then` function which would be treated as a `Promise`. const { resolvedArgs } = await this.race(this.resolveArgs(fn, input, fnArgs)); args = resolvedArgs; - timeStart = this.params.debug ? now() : 0; + timeStart = this.execution.params.debug ? now() : 0; const output = await this.race(this.invokeFunction(fn, input, resolvedArgs)); - if (this.params.debug) { + if (this.execution.params.debug) { const timeEnd: number = now(); (link as ExpressionAstFunction).debug = { success: true, @@ -267,11 +256,11 @@ export class Execution< if (getType(output) === 'error') return output; input = output; } catch (rawError) { - const timeEnd: number = this.params.debug ? now() : 0; + const timeEnd: number = this.execution.params.debug ? now() : 0; const error = createError(rawError) as ExpressionValueError; error.error.message = `[${fnName}] > ${error.error.message}`; - if (this.params.debug) { + if (this.execution.params.debug) { (link as ExpressionAstFunction).debug = { success: false, fn: fn.name, @@ -404,9 +393,7 @@ export class Execution< const resolveArgFns = mapValues(argAstsWithDefaults, (asts, argName) => { return asts.map((item: ExpressionAstExpression) => { return async (subInput = input) => { - const output = await this.interpret(item, subInput, { - debug: this.params.debug, - }); + const output = await this.interpret(item, subInput); if (isExpressionValueError(output)) throw output.error; const casted = this.cast(output, argDefs[argName as any].types); return casted; @@ -438,17 +425,12 @@ export class Execution< return { resolvedArgs }; } - public async interpret( - ast: ExpressionAstNode, - input: T, - options?: ExpressionExecOptions - ): Promise { + public async interpret(ast: ExpressionAstNode, input: T): Promise { switch (getType(ast)) { case 'expression': - const execution = this.params.executor.createExecution( + const execution = this.execution.executor.createExecution( ast as ExpressionAstExpression, - this.context, - options + this.execution.params ); execution.start(input); return await execution.result; diff --git a/src/plugins/expressions/common/execution/execution_contract.test.ts b/src/plugins/expressions/common/execution/execution_contract.test.ts index c33f8a1a0f36e..856b22470d782 100644 --- a/src/plugins/expressions/common/execution/execution_contract.test.ts +++ b/src/plugins/expressions/common/execution/execution_contract.test.ts @@ -30,7 +30,7 @@ const createExecution = ( const execution = new Execution({ executor, ast: parseExpression(expression), - context, + params: { ...context }, }); return execution; }; diff --git a/src/plugins/expressions/common/execution/execution_contract.ts b/src/plugins/expressions/common/execution/execution_contract.ts index 79bb4c58ab48d..f05f1ded82799 100644 --- a/src/plugins/expressions/common/execution/execution_contract.ts +++ b/src/plugins/expressions/common/execution/execution_contract.ts @@ -25,21 +25,14 @@ import { ExpressionAstExpression } from '../ast'; * `ExecutionContract` is a wrapper around `Execution` class. It provides the * same functionality but does not expose Expressions plugin internals. */ -export class ExecutionContract< - ExtraContext extends Record = Record, - Input = unknown, - Output = unknown, - InspectorAdapters = unknown -> { +export class ExecutionContract { public get isPending(): boolean { const state = this.execution.state.get().state; const finished = state === 'error' || state === 'result'; return !finished; } - constructor( - protected readonly execution: Execution - ) {} + constructor(protected readonly execution: Execution) {} /** * Cancel the execution of the expression. This will set abort signal diff --git a/src/plugins/expressions/common/execution/types.ts b/src/plugins/expressions/common/execution/types.ts index 7c26e586fb790..50475c3bd94ae 100644 --- a/src/plugins/expressions/common/execution/types.ts +++ b/src/plugins/expressions/common/execution/types.ts @@ -26,11 +26,11 @@ import { SavedObject, SavedObjectAttributes } from '../../../../core/public'; * `ExecutionContext` is an object available to all functions during a single execution; * it provides various methods to perform side-effects. */ -export interface ExecutionContext { +export interface ExecutionContext { /** - * Get initial input with which execution started. + * Get search context of the expression. */ - getInitialInput: () => Input; + getSearchContext: () => ExecutionContextSearch; /** * Context variables that can be consumed using `var` and `var_set` functions. @@ -55,7 +55,7 @@ export interface ExecutionContext string | undefined; /** * Allows to fetch saved objects from ElasticSearch. In browser `getSavedObject` diff --git a/src/plugins/expressions/common/executor/executor.ts b/src/plugins/expressions/common/executor/executor.ts index fd7f5808f0340..85b5589b593af 100644 --- a/src/plugins/expressions/common/executor/executor.ts +++ b/src/plugins/expressions/common/executor/executor.ts @@ -33,6 +33,7 @@ import { functionSpecs } from '../expression_functions/specs'; import { getByAlias } from '../util'; import { SavedObjectReference } from '../../../../core/types'; import { PersistableState } from '../../../kibana_utils/common'; +import { ExpressionExecutionParams } from '../service'; export interface ExpressionExecOptions { /** @@ -166,43 +167,34 @@ export class Executor = Record = Record - >( + public async run( ast: string | ExpressionAstExpression, input: Input, - context?: ExtraContext, - options?: ExpressionExecOptions + params: ExpressionExecutionParams = {} ) { - const execution = this.createExecution(ast, context, options); + const execution = this.createExecution(ast, params); execution.start(input); return (await execution.result) as Output; } - public createExecution< - ExtraContext extends Record = Record, - Input = unknown, - Output = unknown - >( + public createExecution( ast: string | ExpressionAstExpression, - context: ExtraContext = {} as ExtraContext, - { debug }: ExpressionExecOptions = {} as ExpressionExecOptions - ): Execution { - const params: ExecutionParams = { + params: ExpressionExecutionParams = {} + ): Execution { + const executionParams: ExecutionParams = { executor: this, - context: { - ...this.context, - ...context, - } as Context & ExtraContext, - debug, + params: { + ...params, + // for canvas we are passing this in, + // canvas should be refactored to not pass any extra context in + extraContext: this.context, + } as any, }; - if (typeof ast === 'string') params.expression = ast; - else params.ast = ast; + if (typeof ast === 'string') executionParams.expression = ast; + else executionParams.ast = ast; - const execution = new Execution(params); + const execution = new Execution(executionParams); return execution; } diff --git a/src/plugins/expressions/common/expression_functions/specs/kibana.ts b/src/plugins/expressions/common/expression_functions/specs/kibana.ts index 2144a8aba2d19..3ec4c23eab28d 100644 --- a/src/plugins/expressions/common/expression_functions/specs/kibana.ts +++ b/src/plugins/expressions/common/expression_functions/specs/kibana.ts @@ -44,15 +44,15 @@ export const kibana: ExpressionFunctionKibana = { args: {}, - fn(input, _, { search = {} }) { + fn(input, _, { getSearchContext }) { const output: ExpressionValueSearchContext = { // TODO: This spread is left here for legacy reasons, possibly Lens uses it. // TODO: But it shouldn't be need. ...input, type: 'kibana_context', - query: [...toArray(search.query), ...toArray((input || {}).query)], - filters: [...(search.filters || []), ...((input || {}).filters || [])], - timeRange: search.timeRange || (input ? input.timeRange : undefined), + query: [...toArray(getSearchContext().query), ...toArray((input || {}).query)], + filters: [...(getSearchContext().filters || []), ...((input || {}).filters || [])], + timeRange: getSearchContext().timeRange || (input ? input.timeRange : undefined), }; return output; diff --git a/src/plugins/expressions/common/expression_functions/specs/tests/kibana.test.ts b/src/plugins/expressions/common/expression_functions/specs/tests/kibana.test.ts index e5bd53f63c91d..e5c4b92de4fdb 100644 --- a/src/plugins/expressions/common/expression_functions/specs/tests/kibana.test.ts +++ b/src/plugins/expressions/common/expression_functions/specs/tests/kibana.test.ts @@ -46,8 +46,8 @@ describe('interpreter/functions#kibana', () => { timeRange: { from: '2', to: '3' }, }; context = { - search, - getInitialInput: () => input, + getSearchContext: () => search, + getSearchSessionId: () => undefined, types: {}, variables: {}, abortSignal: {} as any, diff --git a/src/plugins/expressions/common/expression_functions/specs/tests/theme.test.ts b/src/plugins/expressions/common/expression_functions/specs/tests/theme.test.ts index 263409f0caca2..88511d4fd571e 100644 --- a/src/plugins/expressions/common/expression_functions/specs/tests/theme.test.ts +++ b/src/plugins/expressions/common/expression_functions/specs/tests/theme.test.ts @@ -37,7 +37,8 @@ describe('expression_functions', () => { }; context = { - getInitialInput: () => {}, + getSearchContext: () => ({} as any), + getSearchSessionId: () => undefined, types: {}, variables: { theme: themeProps }, abortSignal: {} as any, diff --git a/src/plugins/expressions/common/expression_functions/specs/tests/var.test.ts b/src/plugins/expressions/common/expression_functions/specs/tests/var.test.ts index ccf49ec918d3d..762f34e3f5566 100644 --- a/src/plugins/expressions/common/expression_functions/specs/tests/var.test.ts +++ b/src/plugins/expressions/common/expression_functions/specs/tests/var.test.ts @@ -31,7 +31,8 @@ describe('expression_functions', () => { beforeEach(() => { input = { timeRange: { from: '0', to: '1' } }; context = { - getInitialInput: () => input, + getSearchContext: () => input, + getSearchSessionId: () => undefined, types: {}, variables: { test: 1 }, abortSignal: {} as any, diff --git a/src/plugins/expressions/common/expression_functions/specs/tests/var_set.test.ts b/src/plugins/expressions/common/expression_functions/specs/tests/var_set.test.ts index b1ae44e6f899e..365ae5b89baea 100644 --- a/src/plugins/expressions/common/expression_functions/specs/tests/var_set.test.ts +++ b/src/plugins/expressions/common/expression_functions/specs/tests/var_set.test.ts @@ -32,7 +32,8 @@ describe('expression_functions', () => { beforeEach(() => { input = { timeRange: { from: '0', to: '1' } }; context = { - getInitialInput: () => input, + getSearchContext: () => input, + getSearchSessionId: () => undefined, types: {}, variables: { test: 1 }, abortSignal: {} as any, diff --git a/src/plugins/expressions/common/mocks.ts b/src/plugins/expressions/common/mocks.ts index 502d88ac955ae..52f96953885cf 100644 --- a/src/plugins/expressions/common/mocks.ts +++ b/src/plugins/expressions/common/mocks.ts @@ -23,7 +23,8 @@ export const createMockExecutionContext = extraContext: ExtraContext = {} as ExtraContext ): ExecutionContext & ExtraContext => { const executionContext: ExecutionContext = { - getInitialInput: jest.fn(), + getSearchContext: jest.fn(), + getSearchSessionId: jest.fn(), variables: {}, types: {}, abortSignal: { @@ -37,7 +38,6 @@ export const createMockExecutionContext = requests: {} as any, data: {} as any, }, - search: {}, }; return { diff --git a/src/plugins/expressions/common/service/expressions_services.ts b/src/plugins/expressions/common/service/expressions_services.ts index 3d0fb968e8a3a..abbba433ab3ca 100644 --- a/src/plugins/expressions/common/service/expressions_services.ts +++ b/src/plugins/expressions/common/service/expressions_services.ts @@ -17,7 +17,7 @@ * under the License. */ -import { Executor, ExpressionExecOptions } from '../executor'; +import { Executor } from '../executor'; import { AnyExpressionRenderDefinition, ExpressionRendererRegistry } from '../expression_renderers'; import { ExpressionAstExpression } from '../ast'; import { ExecutionContract } from '../execution/execution_contract'; @@ -25,6 +25,8 @@ import { AnyExpressionTypeDefinition } from '../expression_types'; import { AnyExpressionFunctionDefinition } from '../expression_functions'; import { SavedObjectReference } from '../../../../core/types'; import { PersistableState } from '../../../kibana_utils/common'; +import { Adapters } from '../../../inspector/common/adapters'; +import { ExecutionContextSearch } from '../execution'; /** * The public contract that `ExpressionsService` provides to other plugins @@ -45,6 +47,23 @@ export type ExpressionsServiceSetup = Pick< | 'fork' >; +export interface ExpressionExecutionParams { + searchContext?: ExecutionContextSearch; + + variables?: Record; + + /** + * Whether to execute expression in *debug mode*. In *debug mode* inputs and + * outputs as well as all resolved arguments and time it took to execute each + * function are saved and are available for introspection. + */ + debug?: boolean; + + searchSessionId?: string; + + inspectorAdapters?: Adapters; +} + /** * The public contract that `ExpressionsService` provides to other plugins * in Kibana Platform in *start* life-cycle. @@ -98,11 +117,10 @@ export interface ExpressionsServiceStart { * expressions.run('...', null, { elasticsearchClient }); * ``` */ - run: = Record>( + run: ( ast: string | ExpressionAstExpression, input: Input, - context?: ExtraContext, - options?: ExpressionExecOptions + params?: ExpressionExecutionParams ) => Promise; /** @@ -110,17 +128,12 @@ export interface ExpressionsServiceStart { * instance that tracks the progress of the execution and can be used to * interact with the execution. */ - execute: < - Input = unknown, - Output = unknown, - ExtraContext extends Record = Record - >( + execute: ( ast: string | ExpressionAstExpression, // This any is for legacy reasons. input: Input, - context?: ExtraContext, - options?: ExpressionExecOptions - ) => ExecutionContract; + params?: ExpressionExecutionParams + ) => ExecutionContract; /** * Create a new instance of `ExpressionsService`. The new instance inherits @@ -214,8 +227,8 @@ export class ExpressionsService implements PersistableState AnyExpressionRenderDefinition) ): void => this.renderers.register(definition); - public readonly run: ExpressionsServiceStart['run'] = (ast, input, context, options) => - this.executor.run(ast, input, context, options); + public readonly run: ExpressionsServiceStart['run'] = (ast, input, params) => + this.executor.run(ast, input, params); public readonly getFunction: ExpressionsServiceStart['getFunction'] = (name) => this.executor.getFunction(name); @@ -246,8 +259,8 @@ export class ExpressionsService implements PersistableState => this.executor.getTypes(); - public readonly execute: ExpressionsServiceStart['execute'] = ((ast, input, context, options) => { - const execution = this.executor.createExecution(ast, context, options); + public readonly execute: ExpressionsServiceStart['execute'] = ((ast, input, params) => { + const execution = this.executor.createExecution(ast, params); execution.start(input); return execution.contract; }) as ExpressionsServiceStart['execute']; diff --git a/src/plugins/expressions/public/loader.ts b/src/plugins/expressions/public/loader.ts index aef4b73f86e34..91c482621de36 100644 --- a/src/plugins/expressions/public/loader.ts +++ b/src/plugins/expressions/public/loader.ts @@ -145,18 +145,13 @@ export class ExpressionLoader { this.execution.cancel(); } this.setParams(params); - this.execution = getExpressionsService().execute( - expression, - params.context, - { - search: params.searchContext, - variables: params.variables || {}, - inspectorAdapters: params.inspectorAdapters, - }, - { - debug: params.debug, - } - ); + this.execution = getExpressionsService().execute(expression, params.context, { + searchContext: params.searchContext, + variables: params.variables || {}, + inspectorAdapters: params.inspectorAdapters, + searchSessionId: params.searchSessionId, + debug: params.debug, + }); const prevDataHandler = this.execution; const data = await prevDataHandler.getData(); @@ -188,6 +183,9 @@ export class ExpressionLoader { if (params.variables && this.params) { this.params.variables = params.variables; } + if (params.searchSessionId && this.params) { + this.params.searchSessionId = params.searchSessionId; + } this.params.debug = Boolean(params.debug); this.params.inspectorAdapters = (params.inspectorAdapters || diff --git a/src/plugins/expressions/public/public.api.md b/src/plugins/expressions/public/public.api.md index 4739b9434bdaa..fe95cf5eb0cda 100644 --- a/src/plugins/expressions/public/public.api.md +++ b/src/plugins/expressions/public/public.api.md @@ -90,33 +90,32 @@ export type DatatableColumnType = '_source' | 'attachment' | 'boolean' | 'date' export type DatatableRow = Record; // Warning: (ae-forgotten-export) The symbol "Adapters" needs to be exported by the entry point index.d.ts +// Warning: (ae-forgotten-export) The symbol "ExpressionExecutionParams" needs to be exported by the entry point index.d.ts // Warning: (ae-forgotten-export) The symbol "DefaultInspectorAdapters" needs to be exported by the entry point index.d.ts // Warning: (ae-missing-release-tag) "Execution" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export class Execution = Record, Input = unknown, Output = unknown, InspectorAdapters extends Adapters = ExtraContext['inspectorAdapters'] extends object ? ExtraContext['inspectorAdapters'] : DefaultInspectorAdapters> { - constructor(params: ExecutionParams); +export class Execution { + constructor(execution: ExecutionParams); cancel(): void; // (undocumented) cast(value: any, toTypeNames?: string[]): any; - readonly context: ExecutionContext & ExtraContext; - readonly contract: ExecutionContract; + readonly context: ExecutionContext; + readonly contract: ExecutionContract; + // (undocumented) + readonly execution: ExecutionParams; // (undocumented) readonly expression: string; input: Input; // (undocumented) get inspectorAdapters(): InspectorAdapters; - // Warning: (ae-forgotten-export) The symbol "ExpressionExecOptions" needs to be exported by the entry point index.d.ts - // // (undocumented) - interpret(ast: ExpressionAstNode, input: T, options?: ExpressionExecOptions): Promise; + interpret(ast: ExpressionAstNode, input: T): Promise; // (undocumented) invokeChain(chainArr: ExpressionAstFunction[], input: unknown): Promise; // (undocumented) invokeFunction(fn: ExpressionFunction, input: unknown, args: Record): Promise; // (undocumented) - readonly params: ExecutionParams; - // (undocumented) resolveArgs(fnDef: ExpressionFunction, input: unknown, argAsts: any): Promise; // (undocumented) get result(): Promise; @@ -134,15 +133,15 @@ export type ExecutionContainer = StateContainer { +export interface ExecutionContext { abortSignal: AbortSignal; - getInitialInput: () => Input; // Warning: (ae-forgotten-export) The symbol "SavedObjectAttributes" needs to be exported by the entry point index.d.ts // Warning: (ae-forgotten-export) The symbol "SavedObject" needs to be exported by the entry point index.d.ts getSavedObject?: (type: string, id: string) => Promise>; - inspectorAdapters: InspectorAdapters; // Warning: (ae-forgotten-export) The symbol "ExecutionContextSearch" needs to be exported by the entry point index.d.ts - search?: ExecutionContextSearch; + getSearchContext: () => ExecutionContextSearch; + getSearchSessionId: () => string | undefined; + inspectorAdapters: InspectorAdapters; types: Record; variables: Record; } @@ -150,11 +149,11 @@ export interface ExecutionContext = Record, Input = unknown, Output = unknown, InspectorAdapters = unknown> { - constructor(execution: Execution); +export class ExecutionContract { + constructor(execution: Execution); cancel: () => void; // (undocumented) - protected readonly execution: Execution; + protected readonly execution: Execution; getAst: () => ExpressionAstExpression; getData: () => Promise; getExpression: () => string; @@ -166,16 +165,15 @@ export class ExecutionContract = Re // Warning: (ae-missing-release-tag) "ExecutionParams" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export interface ExecutionParams = Record> { +export interface ExecutionParams { // (undocumented) ast?: ExpressionAstExpression; // (undocumented) - context?: ExtraContext; - debug?: boolean; - // (undocumented) executor: Executor; // (undocumented) expression?: string; + // (undocumented) + params: ExpressionExecutionParams; } // Warning: (ae-missing-release-tag) "ExecutionState" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -198,7 +196,7 @@ export class Executor = Record; // (undocumented) - createExecution = Record, Input = unknown, Output = unknown>(ast: string | ExpressionAstExpression, context?: ExtraContext, { debug }?: ExpressionExecOptions): Execution; + createExecution(ast: string | ExpressionAstExpression, params?: ExpressionExecutionParams): Execution; // (undocumented) static createWithDefaults = Record>(state?: ExecutorState): Executor; // (undocumented) @@ -228,7 +226,7 @@ export class Executor = Record AnyExpressionFunctionDefinition)): void; // (undocumented) registerType(typeDefinition: AnyExpressionTypeDefinition | (() => AnyExpressionTypeDefinition)): void; - run = Record>(ast: string | ExpressionAstExpression, input: Input, context?: ExtraContext, options?: ExpressionExecOptions): Promise; + run(ast: string | ExpressionAstExpression, input: Input, params?: ExpressionExecutionParams): Promise; // (undocumented) readonly state: ExecutorContainer; // (undocumented) @@ -613,12 +611,12 @@ export type ExpressionsServiceSetup = Pick = Record>(ast: string | ExpressionAstExpression, input: Input, context?: ExtraContext, options?: ExpressionExecOptions) => ExecutionContract; + execute: (ast: string | ExpressionAstExpression, input: Input, params?: ExpressionExecutionParams) => ExecutionContract; fork: () => ExpressionsService; getFunction: (name: string) => ReturnType; getRenderer: (name: string) => ReturnType; getType: (name: string) => ReturnType; - run: = Record>(ast: string | ExpressionAstExpression, input: Input, context?: ExtraContext, options?: ExpressionExecOptions) => Promise; + run: (ast: string | ExpressionAstExpression, input: Input, params?: ExpressionExecutionParams) => Promise; } // Warning: (ae-missing-release-tag) "ExpressionsSetup" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -894,6 +892,8 @@ export interface IExpressionLoaderParams { // (undocumented) searchContext?: ExecutionContextSearch; // (undocumented) + searchSessionId?: string; + // (undocumented) uiState?: unknown; // (undocumented) variables?: Record; diff --git a/src/plugins/expressions/public/types/index.ts b/src/plugins/expressions/public/types/index.ts index 054c5ac3dc467..1643b5734ef1a 100644 --- a/src/plugins/expressions/public/types/index.ts +++ b/src/plugins/expressions/public/types/index.ts @@ -53,6 +53,7 @@ export interface IExpressionLoaderParams { uiState?: unknown; inspectorAdapters?: Adapters; onRenderError?: RenderErrorHandlerFnType; + searchSessionId?: string; } export interface ExpressionRenderError extends Error { diff --git a/src/plugins/expressions/server/server.api.md b/src/plugins/expressions/server/server.api.md index fcdfd5ef3246c..d6925a027358c 100644 --- a/src/plugins/expressions/server/server.api.md +++ b/src/plugins/expressions/server/server.api.md @@ -87,34 +87,33 @@ export type DatatableColumnType = '_source' | 'attachment' | 'boolean' | 'date' export type DatatableRow = Record; // Warning: (ae-forgotten-export) The symbol "Adapters" needs to be exported by the entry point index.d.ts +// Warning: (ae-forgotten-export) The symbol "ExpressionExecutionParams" needs to be exported by the entry point index.d.ts // Warning: (ae-forgotten-export) The symbol "DefaultInspectorAdapters" needs to be exported by the entry point index.d.ts // Warning: (ae-missing-release-tag) "Execution" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export class Execution = Record, Input = unknown, Output = unknown, InspectorAdapters extends Adapters = ExtraContext['inspectorAdapters'] extends object ? ExtraContext['inspectorAdapters'] : DefaultInspectorAdapters> { - constructor(params: ExecutionParams); +export class Execution { + constructor(execution: ExecutionParams); cancel(): void; // (undocumented) cast(value: any, toTypeNames?: string[]): any; - readonly context: ExecutionContext & ExtraContext; + readonly context: ExecutionContext; // Warning: (ae-forgotten-export) The symbol "ExecutionContract" needs to be exported by the entry point index.d.ts - readonly contract: ExecutionContract; + readonly contract: ExecutionContract; + // (undocumented) + readonly execution: ExecutionParams; // (undocumented) readonly expression: string; input: Input; // (undocumented) get inspectorAdapters(): InspectorAdapters; - // Warning: (ae-forgotten-export) The symbol "ExpressionExecOptions" needs to be exported by the entry point index.d.ts - // // (undocumented) - interpret(ast: ExpressionAstNode, input: T, options?: ExpressionExecOptions): Promise; + interpret(ast: ExpressionAstNode, input: T): Promise; // (undocumented) invokeChain(chainArr: ExpressionAstFunction[], input: unknown): Promise; // (undocumented) invokeFunction(fn: ExpressionFunction, input: unknown, args: Record): Promise; // (undocumented) - readonly params: ExecutionParams; - // (undocumented) resolveArgs(fnDef: ExpressionFunction, input: unknown, argAsts: any): Promise; // (undocumented) get result(): Promise; @@ -132,15 +131,15 @@ export type ExecutionContainer = StateContainer { +export interface ExecutionContext { abortSignal: AbortSignal; - getInitialInput: () => Input; // Warning: (ae-forgotten-export) The symbol "SavedObjectAttributes" needs to be exported by the entry point index.d.ts // Warning: (ae-forgotten-export) The symbol "SavedObject" needs to be exported by the entry point index.d.ts getSavedObject?: (type: string, id: string) => Promise>; - inspectorAdapters: InspectorAdapters; // Warning: (ae-forgotten-export) The symbol "ExecutionContextSearch" needs to be exported by the entry point index.d.ts - search?: ExecutionContextSearch; + getSearchContext: () => ExecutionContextSearch; + getSearchSessionId: () => string | undefined; + inspectorAdapters: InspectorAdapters; types: Record; variables: Record; } @@ -148,16 +147,15 @@ export interface ExecutionContext = Record> { +export interface ExecutionParams { // (undocumented) ast?: ExpressionAstExpression; // (undocumented) - context?: ExtraContext; - debug?: boolean; - // (undocumented) executor: Executor; // (undocumented) expression?: string; + // (undocumented) + params: ExpressionExecutionParams; } // Warning: (ae-missing-release-tag) "ExecutionState" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -180,7 +178,7 @@ export class Executor = Record; // (undocumented) - createExecution = Record, Input = unknown, Output = unknown>(ast: string | ExpressionAstExpression, context?: ExtraContext, { debug }?: ExpressionExecOptions): Execution; + createExecution(ast: string | ExpressionAstExpression, params?: ExpressionExecutionParams): Execution; // (undocumented) static createWithDefaults = Record>(state?: ExecutorState): Executor; // (undocumented) @@ -210,7 +208,7 @@ export class Executor = Record AnyExpressionFunctionDefinition)): void; // (undocumented) registerType(typeDefinition: AnyExpressionTypeDefinition | (() => AnyExpressionTypeDefinition)): void; - run = Record>(ast: string | ExpressionAstExpression, input: Input, context?: ExtraContext, options?: ExpressionExecOptions): Promise; + run(ast: string | ExpressionAstExpression, input: Input, params?: ExpressionExecutionParams): Promise; // (undocumented) readonly state: ExecutorContainer; // (undocumented) diff --git a/src/plugins/home/public/application/components/solutions_section/solutions_section.tsx b/src/plugins/home/public/application/components/solutions_section/solutions_section.tsx index 13b70383147eb..bbc7f28235d6a 100644 --- a/src/plugins/home/public/application/components/solutions_section/solutions_section.tsx +++ b/src/plugins/home/public/application/components/solutions_section/solutions_section.tsx @@ -25,8 +25,8 @@ import { SolutionPanel } from './solution_panel'; import { FeatureCatalogueEntry, FeatureCatalogueSolution } from '../../../'; const sortByOrder = ( - { order: orderA = 0 }: FeatureCatalogueSolution, - { order: orderB = 0 }: FeatureCatalogueSolution + { order: orderA = 0 }: FeatureCatalogueSolution | FeatureCatalogueEntry, + { order: orderB = 0 }: FeatureCatalogueSolution | FeatureCatalogueEntry ) => orderA - orderB; interface Props { @@ -38,7 +38,9 @@ interface Props { export const SolutionsSection: FC = ({ addBasePath, solutions, directories }) => { // Separate Kibana from other solutions const kibana = solutions.find(({ id }) => id === 'kibana'); - const kibanaApps = directories.filter(({ solutionId }) => solutionId === 'kibana'); + const kibanaApps = directories + .filter(({ solutionId }) => solutionId === 'kibana') + .sort(sortByOrder); solutions = solutions.sort(sortByOrder).filter(({ id }) => id !== 'kibana'); return ( diff --git a/src/plugins/kibana_overview/public/plugin.ts b/src/plugins/kibana_overview/public/plugin.ts index 663058ad16ec8..1139002dc19b6 100644 --- a/src/plugins/kibana_overview/public/plugin.ts +++ b/src/plugins/kibana_overview/public/plugin.ts @@ -109,7 +109,7 @@ export class KibanaOverviewPlugin defaultMessage: 'Search and find insights.', }), i18n.translate('kibanaOverview.kibana.appDescription3', { - defaultMessage: 'Design pixel-perfect reports.', + defaultMessage: 'Design pixel-perfect presentations.', }), i18n.translate('kibanaOverview.kibana.appDescription4', { defaultMessage: 'Plot geographic data.', diff --git a/src/plugins/kibana_usage_collection/server/collectors/csp/csp_collector.ts b/src/plugins/kibana_usage_collection/server/collectors/csp/csp_collector.ts index c45a83588ee44..8cd60e5477d02 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/csp/csp_collector.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/csp/csp_collector.ts @@ -17,7 +17,7 @@ * under the License. */ -import { UsageCollectionSetup, CollectorOptions } from 'src/plugins/usage_collection/server'; +import { UsageCollectionSetup, UsageCollectorOptions } from 'src/plugins/usage_collection/server'; import { HttpServiceSetup, CspConfig } from '../../../../../core/server'; interface Usage { @@ -26,7 +26,7 @@ interface Usage { rulesChangedFromDefault: boolean; } -export function createCspCollector(http: HttpServiceSetup): CollectorOptions { +export function createCspCollector(http: HttpServiceSetup): UsageCollectorOptions { return { type: 'csp', isReady: () => true, diff --git a/src/plugins/telemetry/schema/legacy_plugins.json b/src/plugins/telemetry/schema/legacy_plugins.json new file mode 100644 index 0000000000000..1a7c0ccb15082 --- /dev/null +++ b/src/plugins/telemetry/schema/legacy_plugins.json @@ -0,0 +1,21 @@ +{ + "properties": { + "localization": { + "properties": { + "locale": { + "type": "keyword" + }, + "integrities": { + "properties": { + "DYNAMIC_KEY": { + "type": "text" + } + } + }, + "labelsCount": { + "type": "long" + } + } + } + } +} diff --git a/src/plugins/usage_collection/server/collector/collector.test.ts b/src/plugins/usage_collection/server/collector/collector.test.ts index 375fe4f7686c0..875414fbeec48 100644 --- a/src/plugins/usage_collection/server/collector/collector.test.ts +++ b/src/plugins/usage_collection/server/collector/collector.test.ts @@ -122,6 +122,7 @@ describe('collector', () => { type: 'my_test_collector', isReady: () => false, fetch: () => fetchOutput, + schema: { testPass: { type: 'long' } }, }); expect(collector.formatForBulkUpload(fetchOutput)).toStrictEqual({ type: 'kibana_stats', diff --git a/src/plugins/usage_collection/server/collector/collector.ts b/src/plugins/usage_collection/server/collector/collector.ts index 11a709c037783..951418d448cbd 100644 --- a/src/plugins/usage_collection/server/collector/collector.ts +++ b/src/plugins/usage_collection/server/collector/collector.ts @@ -71,17 +71,27 @@ export interface CollectorFetchContext { } export interface CollectorOptions { + /** + * Unique string identifier for the collector + */ type: string; init?: Function; + /** + * Method to return `true`/`false` to confirm if the collector is ready for the `fetch` method to be called. + */ + isReady: () => Promise | boolean; + /** + * Schema definition of the output of the `fetch` method. + */ schema?: MakeSchemaFrom; fetch: (collectorFetchContext: CollectorFetchContext) => Promise | T; /* * A hook for allowing the fetched data payload to be organized into a typed * data model for internal bulk upload. See defaultFormatterForBulkUpload for * a generic example. + * @deprecated Used only by the Legacy Monitoring collection (to be removed in 8.0) */ formatForBulkUpload?: CollectorFormatForBulkUpload; - isReady: () => Promise | boolean; } export class Collector { diff --git a/src/plugins/usage_collection/server/collector/collector_set.test.ts b/src/plugins/usage_collection/server/collector/collector_set.test.ts index 45a3437777c5f..359a2d214f991 100644 --- a/src/plugins/usage_collection/server/collector/collector_set.test.ts +++ b/src/plugins/usage_collection/server/collector/collector_set.test.ts @@ -20,7 +20,7 @@ import { noop } from 'lodash'; import { Collector } from './collector'; import { CollectorSet } from './collector_set'; -import { UsageCollector } from './usage_collector'; +import { UsageCollector, UsageCollectorOptions } from './usage_collector'; import { elasticsearchServiceMock, loggingSystemMock, @@ -73,8 +73,9 @@ describe('CollectorSet', () => { // Even for Collector vs. UsageCollector new UsageCollector(logger, { type: 'test_duplicated', - fetch: () => 2, + fetch: () => ({ prop: 2 }), isReady: () => false, + schema: { prop: { type: 'long' } }, }) ) ).toThrowError(`Usage collector's type "test_duplicated" is duplicated.`); @@ -252,7 +253,12 @@ describe('CollectorSet', () => { }); describe('isUsageCollector', () => { - const collectorOptions = { type: 'MY_TEST_COLLECTOR', fetch: () => {}, isReady: () => true }; + const collectorOptions: UsageCollectorOptions = { + type: 'MY_TEST_COLLECTOR', + fetch: () => ({ test: 1 }), + isReady: () => true, + schema: { test: { type: 'long' } }, + }; it('returns true only for UsageCollector instances', () => { const collectors = new CollectorSet({ logger }); diff --git a/src/plugins/usage_collection/server/collector/collector_set.ts b/src/plugins/usage_collection/server/collector/collector_set.ts index 4e64cbc1bf30f..c52830cda6513 100644 --- a/src/plugins/usage_collection/server/collector/collector_set.ts +++ b/src/plugins/usage_collection/server/collector/collector_set.ts @@ -26,7 +26,7 @@ import { SavedObjectsClientContract, } from 'kibana/server'; import { Collector, CollectorOptions } from './collector'; -import { UsageCollector } from './usage_collector'; +import { UsageCollector, UsageCollectorOptions } from './usage_collector'; interface CollectorSetConfig { logger: Logger; @@ -45,10 +45,22 @@ export class CollectorSet { this.maximumWaitTimeForAllCollectorsInS = maximumWaitTimeForAllCollectorsInS || 60; } - public makeStatsCollector = (options: CollectorOptions) => { + public makeStatsCollector = < + T, + U, + O extends CollectorOptions = CollectorOptions // Used to allow extra properties (the Collector constructor extends the class with the additional options provided) + >( + options: O + ) => { return new Collector(this.logger, options); }; - public makeUsageCollector = (options: CollectorOptions) => { + public makeUsageCollector = < + T, + U = T, + O extends UsageCollectorOptions = UsageCollectorOptions + >( + options: O + ) => { return new UsageCollector(this.logger, options); }; diff --git a/src/plugins/usage_collection/server/collector/index.ts b/src/plugins/usage_collection/server/collector/index.ts index c294ba77d3cdb..da85f9ab181c9 100644 --- a/src/plugins/usage_collection/server/collector/index.ts +++ b/src/plugins/usage_collection/server/collector/index.ts @@ -26,4 +26,4 @@ export { CollectorOptions, CollectorFetchContext, } from './collector'; -export { UsageCollector } from './usage_collector'; +export { UsageCollector, UsageCollectorOptions } from './usage_collector'; diff --git a/src/plugins/usage_collection/server/collector/usage_collector.ts b/src/plugins/usage_collection/server/collector/usage_collector.ts index bf861a94fccff..5bfc36537e0b0 100644 --- a/src/plugins/usage_collection/server/collector/usage_collector.ts +++ b/src/plugins/usage_collection/server/collector/usage_collector.ts @@ -17,13 +17,22 @@ * under the License. */ +import { Logger } from 'src/core/server'; import { KIBANA_STATS_TYPE } from '../../common/constants'; -import { Collector } from './collector'; +import { Collector, CollectorOptions } from './collector'; + +// Enforce the `schema` property for UsageCollectors +export type UsageCollectorOptions = CollectorOptions & + Required, 'schema'>>; export class UsageCollector extends Collector< T, U > { + constructor(protected readonly log: Logger, collectorOptions: UsageCollectorOptions) { + super(log, collectorOptions); + } + protected defaultFormatterForBulkUpload(result: T) { return { type: KIBANA_STATS_TYPE, diff --git a/src/plugins/usage_collection/server/index.ts b/src/plugins/usage_collection/server/index.ts index 80e34b1502cda..f7a08fdb5e9dd 100644 --- a/src/plugins/usage_collection/server/index.ts +++ b/src/plugins/usage_collection/server/index.ts @@ -25,6 +25,7 @@ export { MakeSchemaFrom, SchemaField, CollectorOptions, + UsageCollectorOptions, Collector, CollectorFetchContext, } from './collector'; diff --git a/src/plugins/vis_default_editor/public/components/agg_params_helper.test.ts b/src/plugins/vis_default_editor/public/components/agg_params_helper.test.ts index 834ad8b70ad0d..a56155db02f6b 100644 --- a/src/plugins/vis_default_editor/public/components/agg_params_helper.test.ts +++ b/src/plugins/vis_default_editor/public/components/agg_params_helper.test.ts @@ -33,6 +33,7 @@ import { FieldParamEditor, OrderByParamEditor } from './controls'; import { EditorConfig } from './utils'; import { Schema } from '../schemas'; import { EditorVisState } from './sidebar/state/reducers'; +import { groupAndSortBy } from '../utils'; jest.mock('../utils', () => ({ groupAndSortBy: jest.fn(() => ['indexedFields']), @@ -169,6 +170,9 @@ describe('DefaultEditorAggParams helpers', () => { ], advanced: [], }); + + // Should be grouped using displayName as label + expect(groupAndSortBy).toHaveBeenCalledWith(expect.anything(), 'type', 'displayName', 'name'); }); }); diff --git a/src/plugins/vis_default_editor/public/components/agg_params_helper.ts b/src/plugins/vis_default_editor/public/components/agg_params_helper.ts index b13ca32601aa9..271fc75a0853e 100644 --- a/src/plugins/vis_default_editor/public/components/agg_params_helper.ts +++ b/src/plugins/vis_default_editor/public/components/agg_params_helper.ts @@ -93,7 +93,7 @@ function getAggParamsToRender({ } } fields = filterAggTypeFields(availableFields, agg); - indexedFields = groupAndSortBy(fields, 'type', 'name'); + indexedFields = groupAndSortBy(fields, 'type', 'displayName', 'name'); if (fields && !indexedFields.length && index > 0) { // don't draw the rest of the options if there are no indexed fields and it's an extra param (index > 0). diff --git a/src/plugins/vis_default_editor/public/components/controls/field.tsx b/src/plugins/vis_default_editor/public/components/controls/field.tsx index bfc4f881f8458..41d6db25da5e2 100644 --- a/src/plugins/vis_default_editor/public/components/controls/field.tsx +++ b/src/plugins/vis_default_editor/public/components/controls/field.tsx @@ -52,7 +52,7 @@ function FieldParamEditor({ }: FieldParamEditorProps) { const [isDirty, setIsDirty] = useState(false); const selectedOptions: ComboBoxGroupedOptions = value - ? [{ label: value.displayName || value.name, target: value }] + ? [{ label: value.displayName, target: value, key: value.name }] : []; const onChange = (options: EuiComboBoxOptionOption[]) => { diff --git a/src/plugins/vis_default_editor/public/utils.ts b/src/plugins/vis_default_editor/public/utils.ts index d0a9c067e9da2..11b7c07acc2c2 100644 --- a/src/plugins/vis_default_editor/public/utils.ts +++ b/src/plugins/vis_default_editor/public/utils.ts @@ -18,10 +18,12 @@ */ interface ComboBoxOption { + key?: string; label: string; target: T; } interface ComboBoxGroupedOption { + key?: string; label: string; options: Array>; } @@ -40,15 +42,22 @@ export type ComboBoxGroupedOptions = Array>; * @returns An array of grouped and sorted alphabetically `objects` that are compatible with EuiComboBox options. */ export function groupAndSortBy< - T extends Record, + T extends Record, TGroupBy extends string = 'type', - TLabelName extends string = 'title' ->(objects: T[], groupBy: TGroupBy, labelName: TLabelName): ComboBoxGroupedOptions { + TLabelName extends string = 'title', + TKeyName extends string = never +>( + objects: T[], + groupBy: TGroupBy, + labelName: TLabelName, + keyName?: TKeyName +): ComboBoxGroupedOptions { const groupedOptions = objects.reduce((array, obj) => { const group = array.find((element) => element.label === obj[groupBy]); const option = { label: obj[labelName], target: obj, + ...(keyName ? { key: obj[keyName] } : {}), }; if (group && group.options) { diff --git a/src/plugins/vis_type_timeseries/server/lib/get_fields.ts b/src/plugins/vis_type_timeseries/server/lib/get_fields.ts index 682d0a071e50d..b52188129f77f 100644 --- a/src/plugins/vis_type_timeseries/server/lib/get_fields.ts +++ b/src/plugins/vis_type_timeseries/server/lib/get_fields.ts @@ -45,7 +45,7 @@ export async function getFields( payload: {}, pre: { indexPatternsService: new IndexPatternsFetcher( - requestContext.core.elasticsearch.legacy.client.callAsCurrentUser + requestContext.core.elasticsearch.client.asCurrentUser ), }, getUiSettingsService: () => requestContext.core.uiSettings.client, diff --git a/src/plugins/vis_type_timeseries/server/lib/search_strategies/strategies/abstract_search_strategy.test.js b/src/plugins/vis_type_timeseries/server/lib/search_strategies/strategies/abstract_search_strategy.test.js index ceae784cf74a6..613f33a47f1f4 100644 --- a/src/plugins/vis_type_timeseries/server/lib/search_strategies/strategies/abstract_search_strategy.test.js +++ b/src/plugins/vis_type_timeseries/server/lib/search_strategies/strategies/abstract_search_strategy.test.js @@ -50,7 +50,7 @@ describe('AbstractSearchStrategy', () => { expect(fields).toBe(mockedFields); expect(req.pre.indexPatternsService.getFieldsForWildcard).toHaveBeenCalledWith({ pattern: indexPattern, - fieldCapsOptions: { allowNoIndices: true }, + fieldCapsOptions: { allow_no_indices: true }, }); }); diff --git a/src/plugins/vis_type_timeseries/server/lib/search_strategies/strategies/abstract_search_strategy.ts b/src/plugins/vis_type_timeseries/server/lib/search_strategies/strategies/abstract_search_strategy.ts index 7b62ad310a354..8b16048f0dce0 100644 --- a/src/plugins/vis_type_timeseries/server/lib/search_strategies/strategies/abstract_search_strategy.ts +++ b/src/plugins/vis_type_timeseries/server/lib/search_strategies/strategies/abstract_search_strategy.ts @@ -86,7 +86,7 @@ export class AbstractSearchStrategy { return await indexPatternsService!.getFieldsForWildcard({ pattern: indexPattern, - fieldCapsOptions: { allowNoIndices: true }, + fieldCapsOptions: { allow_no_indices: true }, }); } diff --git a/src/plugins/vis_type_vega/public/vega_fn.ts b/src/plugins/vis_type_vega/public/vega_fn.ts index c109bb3c6e90c..c88b78948133c 100644 --- a/src/plugins/vis_type_vega/public/vega_fn.ts +++ b/src/plugins/vis_type_vega/public/vega_fn.ts @@ -53,7 +53,7 @@ export const createVegaFn = ( Input, Arguments, Output, - ExecutionContext + ExecutionContext > => ({ name: 'vega', type: 'render', diff --git a/test/functional/apps/discover/_discover_histogram.ts b/test/functional/apps/discover/_discover_histogram.ts index e06783174e83b..672becca614c9 100644 --- a/test/functional/apps/discover/_discover_histogram.ts +++ b/test/functional/apps/discover/_discover_histogram.ts @@ -31,7 +31,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { 'dateFormat:tz': 'Europe/Berlin', }; - describe('discover histogram', function describeIndexTests() { + // FLAKY: https://github.com/elastic/kibana/issues/81576 + describe.skip('discover histogram', function describeIndexTests() { before(async () => { await esArchiver.loadIfNeeded('logstash_functional'); await esArchiver.load('long_window_logstash'); diff --git a/test/functional/apps/discover/_doc_navigation.js b/test/functional/apps/discover/_doc_navigation.js index 87a150c7d6961..31aef96918ffa 100644 --- a/test/functional/apps/discover/_doc_navigation.js +++ b/test/functional/apps/discover/_doc_navigation.js @@ -28,7 +28,8 @@ export default function ({ getService, getPageObjects }) { const esArchiver = getService('esArchiver'); const retry = getService('retry'); - describe('doc link in discover', function contextSize() { + // FLAKY: https://github.com/elastic/kibana/issues/78373 + describe.skip('doc link in discover', function contextSize() { beforeEach(async function () { log.debug('load kibana index with default index pattern'); await esArchiver.loadIfNeeded('discover'); diff --git a/test/functional/page_objects/common_page.ts b/test/functional/page_objects/common_page.ts index cc229ef0c2e08..4a14d43aec249 100644 --- a/test/functional/page_objects/common_page.ts +++ b/test/functional/page_objects/common_page.ts @@ -117,11 +117,12 @@ export function CommonPageProvider({ getService, getPageObjects }: FtrProviderCo } else { log.debug(`navigateToUrl ${appUrl}`); await browser.get(appUrl, insertTimestamp); - // accept alert if it pops up - const alert = await browser.getAlert(); - await alert?.accept(); } + // accept alert if it pops up + const alert = await browser.getAlert(); + await alert?.accept(); + const currentUrl = shouldLoginIfPrompted ? await this.loginIfPrompted(appUrl, insertTimestamp) : await browser.getCurrentUrl(); diff --git a/test/functional/page_objects/time_picker.ts b/test/functional/page_objects/time_picker.ts index 3ac6c83e61f14..272fc5a7914be 100644 --- a/test/functional/page_objects/time_picker.ts +++ b/test/functional/page_objects/time_picker.ts @@ -35,12 +35,18 @@ export type CommonlyUsed = export function TimePickerProvider({ getService, getPageObjects }: FtrProviderContext) { const log = getService('log'); - const retry = getService('retry'); const find = getService('find'); const browser = getService('browser'); const testSubjects = getService('testSubjects'); const { header } = getPageObjects(['header']); const kibanaServer = getService('kibanaServer'); + const MenuToggle = getService('MenuToggle'); + + const quickSelectTimeMenuToggle = new MenuToggle({ + name: 'QuickSelectTime Menu', + menuTestSubject: 'superDatePickerQuickMenu', + toggleButtonTestSubject: 'superDatePickerToggleQuickMenuButton', + }); class TimePicker { defaultStartTime = 'Sep 19, 2015 @ 06:31:44.000'; @@ -158,34 +164,8 @@ export function TimePickerProvider({ getService, getPageObjects }: FtrProviderCo return await find.existsByCssSelector('.euiDatePickerRange--readOnly'); } - public async isQuickSelectMenuOpen() { - return await testSubjects.exists('superDatePickerQuickMenu'); - } - - public async openQuickSelectTimeMenu() { - log.debug('openQuickSelectTimeMenu'); - const isMenuOpen = await this.isQuickSelectMenuOpen(); - if (!isMenuOpen) { - log.debug('opening quick select menu'); - await retry.try(async () => { - await testSubjects.click('superDatePickerToggleQuickMenuButton'); - }); - } - } - - public async closeQuickSelectTimeMenu() { - log.debug('closeQuickSelectTimeMenu'); - const isMenuOpen = await this.isQuickSelectMenuOpen(); - if (isMenuOpen) { - log.debug('closing quick select menu'); - await retry.try(async () => { - await testSubjects.click('superDatePickerToggleQuickMenuButton'); - }); - } - } - public async getRefreshConfig(keepQuickSelectOpen = false) { - await this.openQuickSelectTimeMenu(); + await quickSelectTimeMenuToggle.open(); const interval = await testSubjects.getAttribute( 'superDatePickerRefreshIntervalInput', 'value' @@ -207,7 +187,7 @@ export function TimePickerProvider({ getService, getPageObjects }: FtrProviderCo 'superDatePickerToggleRefreshButton' ); if (!keepQuickSelectOpen) { - await this.closeQuickSelectTimeMenu(); + await quickSelectTimeMenuToggle.close(); } return { @@ -270,26 +250,26 @@ export function TimePickerProvider({ getService, getPageObjects }: FtrProviderCo } public async startAutoRefresh(intervalS = 3) { - await this.openQuickSelectTimeMenu(); + await quickSelectTimeMenuToggle.open(); await this.inputValue('superDatePickerRefreshIntervalInput', intervalS.toString()); const refreshConfig = await this.getRefreshConfig(true); if (refreshConfig.isPaused) { log.debug('start auto refresh'); await testSubjects.click('superDatePickerToggleRefreshButton'); } - await this.closeQuickSelectTimeMenu(); + await quickSelectTimeMenuToggle.close(); } public async pauseAutoRefresh() { log.debug('pauseAutoRefresh'); const refreshConfig = await this.getRefreshConfig(true); + if (!refreshConfig.isPaused) { log.debug('pause auto refresh'); await testSubjects.click('superDatePickerToggleRefreshButton'); - await this.closeQuickSelectTimeMenu(); } - await this.closeQuickSelectTimeMenu(); + await quickSelectTimeMenuToggle.close(); } public async resumeAutoRefresh() { @@ -300,7 +280,7 @@ export function TimePickerProvider({ getService, getPageObjects }: FtrProviderCo await testSubjects.click('superDatePickerToggleRefreshButton'); } - await this.closeQuickSelectTimeMenu(); + await quickSelectTimeMenuToggle.close(); } public async setHistoricalDataRange() { diff --git a/test/functional/services/common/test_subjects.ts b/test/functional/services/common/test_subjects.ts index 204edcb215107..e464eaf943627 100644 --- a/test/functional/services/common/test_subjects.ts +++ b/test/functional/services/common/test_subjects.ts @@ -170,11 +170,27 @@ export function TestSubjectsProvider({ getService }: FtrProviderContext) { public async getAttribute( selector: string, attribute: string, - timeout?: number + options?: + | number + | { + findTimeout?: number; + tryTimeout?: number; + } ): Promise { - return await retry.try(async () => { - log.debug(`TestSubjects.getAttribute(${selector}, ${attribute})`); - const element = await this.find(selector, timeout); + const findTimeout = + (typeof options === 'number' ? options : options?.findTimeout) ?? + config.get('timeouts.find'); + + const tryTimeout = + (typeof options !== 'number' ? options?.tryTimeout : undefined) ?? + config.get('timeouts.try'); + + log.debug( + `TestSubjects.getAttribute(${selector}, ${attribute}, tryTimeout=${tryTimeout}, findTimeout=${findTimeout})` + ); + + return await retry.tryForTime(tryTimeout, async () => { + const element = await this.find(selector, findTimeout); return await element.getAttribute(attribute); }); } diff --git a/test/functional/services/index.ts b/test/functional/services/index.ts index 057ae0bd13b6e..2c71fd8ef8f55 100644 --- a/test/functional/services/index.ts +++ b/test/functional/services/index.ts @@ -57,6 +57,7 @@ import { import { ListingTableProvider } from './listing_table'; import { SavedQueryManagementComponentProvider } from './saved_query_management_component'; import { KibanaSupertestProvider } from './supertest'; +import { MenuToggleProvider } from './menu_toggle'; export const services = { ...commonServiceProviders, @@ -93,4 +94,5 @@ export const services = { elasticChart: ElasticChartProvider, supertest: KibanaSupertestProvider, managementMenu: ManagementMenuProvider, + MenuToggle: MenuToggleProvider, }; diff --git a/test/functional/services/menu_toggle.ts b/test/functional/services/menu_toggle.ts new file mode 100644 index 0000000000000..073aa846f47a8 --- /dev/null +++ b/test/functional/services/menu_toggle.ts @@ -0,0 +1,77 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { FtrProviderContext } from '../ftr_provider_context'; + +export function MenuToggleProvider({ getService }: FtrProviderContext) { + const log = getService('log'); + const retry = getService('retry'); + const testSubjects = getService('testSubjects'); + + interface Options { + name: string; + menuTestSubject: string; + toggleButtonTestSubject: string; + } + + return class MenuToggle { + private readonly name: string; + private readonly menuTestSubject: string; + private readonly toggleButtonTestSubject: string; + + constructor(options: Options) { + this.name = options.name; + this.menuTestSubject = options.menuTestSubject; + this.toggleButtonTestSubject = options.toggleButtonTestSubject; + } + + async open() { + await this.setState(true); + } + + async close() { + await this.setState(false); + } + + private async setState(expectedState: boolean) { + log.debug( + `setting menu open state [name=${this.name}] [state=${expectedState ? 'open' : 'closed'}]` + ); + + await retry.try(async () => { + // if the menu is clearly in the expected state already, bail out quickly if so + const isOpen = await testSubjects.exists(this.menuTestSubject, { timeout: 1000 }); + if (isOpen === expectedState) { + return; + } + + // toggle the view state by clicking the button + await testSubjects.click(this.toggleButtonTestSubject); + + if (expectedState === true) { + // wait for up to 10 seconds for the menu to show up, otherwise fail and retry + await testSubjects.existOrFail(this.menuTestSubject, { timeout: 10000 }); + } else { + // wait for the form to hide, otherwise fail and retry + await testSubjects.waitForDeleted(this.menuTestSubject); + } + }); + } + }; +} diff --git a/test/interpreter_functional/plugins/kbn_tp_run_pipeline/public/app/components/main.tsx b/test/interpreter_functional/plugins/kbn_tp_run_pipeline/public/app/components/main.tsx index b4f9634b23d29..8c62fa246dd59 100644 --- a/test/interpreter_functional/plugins/kbn_tp_run_pipeline/public/app/components/main.tsx +++ b/test/interpreter_functional/plugins/kbn_tp_run_pipeline/public/app/components/main.tsx @@ -63,7 +63,7 @@ class Main extends React.Component<{}, State> { return getExpressions() .execute(expression, context || { type: 'null' }, { inspectorAdapters: adapters, - search: initialContext as any, + searchContext: initialContext as any, }) .getData(); }; diff --git a/x-pack/package.json b/x-pack/package.json index d91e11134f9c8..77dc2e662dd28 100644 --- a/x-pack/package.json +++ b/x-pack/package.json @@ -152,7 +152,7 @@ "copy-to-clipboard": "^3.0.8", "copy-webpack-plugin": "^6.0.2", "cronstrue": "^1.51.0", - "cypress": "^5.0.0", + "cypress": "5.4.0", "cypress-multi-reporters": "^1.2.3", "cypress-promise": "^1.1.0", "d3": "3.5.17", diff --git a/x-pack/plugins/actions/server/builtin_action_types/webhook.test.ts b/x-pack/plugins/actions/server/builtin_action_types/webhook.test.ts index 23ce527d4ae0d..74feb8ee57d48 100644 --- a/x-pack/plugins/actions/server/builtin_action_types/webhook.test.ts +++ b/x-pack/plugins/actions/server/builtin_action_types/webhook.test.ts @@ -90,8 +90,9 @@ describe('config validation', () => { }; test('config validation passes when only required fields are provided', () => { - const config: Record = { + const config: Record = { url: 'http://mylisteningserver:9200/endpoint', + hasAuth: true, }; expect(validateConfig(actionType, config)).toEqual({ ...defaultValues, @@ -101,9 +102,10 @@ describe('config validation', () => { test('config validation passes when valid methods are provided', () => { ['post', 'put'].forEach((method) => { - const config: Record = { + const config: Record = { url: 'http://mylisteningserver:9200/endpoint', method, + hasAuth: true, }; expect(validateConfig(actionType, config)).toEqual({ ...defaultValues, @@ -127,8 +129,9 @@ describe('config validation', () => { }); test('config validation passes when a url is specified', () => { - const config: Record = { + const config: Record = { url: 'http://mylisteningserver:9200/endpoint', + hasAuth: true, }; expect(validateConfig(actionType, config)).toEqual({ ...defaultValues, @@ -155,6 +158,7 @@ describe('config validation', () => { headers: { 'Content-Type': 'application/json', }, + hasAuth: true, }; expect(validateConfig(actionType, config)).toEqual({ ...defaultValues, @@ -184,6 +188,7 @@ describe('config validation', () => { headers: { 'Content-Type': 'application/json', }, + hasAuth: true, }; expect(validateConfig(actionType, config)).toEqual({ @@ -263,6 +268,7 @@ describe('execute()', () => { headers: { aheader: 'a value', }, + hasAuth: true, }; await actionType.executor({ actionId: 'some-id', @@ -320,6 +326,7 @@ describe('execute()', () => { headers: { aheader: 'a value', }, + hasAuth: false, }; const secrets: ActionTypeSecretsType = { user: null, password: null }; await actionType.executor({ diff --git a/x-pack/plugins/actions/server/builtin_action_types/webhook.ts b/x-pack/plugins/actions/server/builtin_action_types/webhook.ts index d0ec31721685e..dc9de86d3d951 100644 --- a/x-pack/plugins/actions/server/builtin_action_types/webhook.ts +++ b/x-pack/plugins/actions/server/builtin_action_types/webhook.ts @@ -42,6 +42,7 @@ const configSchemaProps = { defaultValue: WebhookMethods.POST, }), headers: nullableType(HeadersSchema), + hasAuth: schema.boolean({ defaultValue: true }), }; const ConfigSchema = schema.object(configSchemaProps); export type ActionTypeConfigType = TypeOf; @@ -128,12 +129,12 @@ export async function executor( execOptions: WebhookActionTypeExecutorOptions ): Promise> { const actionId = execOptions.actionId; - const { method, url, headers = {} } = execOptions.config; + const { method, url, headers = {}, hasAuth } = execOptions.config; const { body: data } = execOptions.params; const secrets: ActionTypeSecretsType = execOptions.secrets; const basicAuth = - isString(secrets.user) && isString(secrets.password) + hasAuth && isString(secrets.user) && isString(secrets.password) ? { auth: { username: secrets.user, password: secrets.password } } : {}; diff --git a/x-pack/plugins/actions/server/saved_objects/migrations.test.ts b/x-pack/plugins/actions/server/saved_objects/migrations.test.ts index 1fa5889e77cb0..f1bd1ba2aeb60 100644 --- a/x-pack/plugins/actions/server/saved_objects/migrations.test.ts +++ b/x-pack/plugins/actions/server/saved_objects/migrations.test.ts @@ -24,7 +24,11 @@ describe('7.10.0', () => { test('add hasAuth config property for .email actions', () => { const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; const action = getMockDataForEmail({}); - expect(migration710(action, context)).toMatchObject({ + const migratedAction = migration710(action, context); + expect(migratedAction.attributes.config).toEqual({ + hasAuth: true, + }); + expect(migratedAction).toEqual({ ...action, attributes: { ...action.attributes, @@ -38,7 +42,11 @@ describe('7.10.0', () => { test('rename cases configuration object', () => { const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; const action = getMockData({}); - expect(migration710(action, context)).toMatchObject({ + const migratedAction = migration710(action, context); + expect(migratedAction.attributes.config).toEqual({ + incidentConfiguration: { mapping: [] }, + }); + expect(migratedAction).toEqual({ ...action, attributes: { ...action.attributes, @@ -50,6 +58,63 @@ describe('7.10.0', () => { }); }); +describe('7.11.0', () => { + beforeEach(() => { + jest.resetAllMocks(); + encryptedSavedObjectsSetup.createMigration.mockImplementation( + (shouldMigrateWhenPredicate, migration) => migration + ); + }); + + test('add hasAuth = true for .webhook actions with user and password', () => { + const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; + const action = getMockDataForWebhook({}, true); + expect(migration711(action, context)).toMatchObject({ + ...action, + attributes: { + ...action.attributes, + config: { + hasAuth: true, + }, + }, + }); + }); + + test('add hasAuth = false for .webhook actions without user and password', () => { + const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; + const action = getMockDataForWebhook({}, false); + expect(migration711(action, context)).toMatchObject({ + ...action, + attributes: { + ...action.attributes, + config: { + hasAuth: false, + }, + }, + }); + }); +}); + +function getMockDataForWebhook( + overwrites: Record = {}, + hasUserAndPassword: boolean +): SavedObjectUnsanitizedDoc { + const secrets = hasUserAndPassword + ? { user: 'test', password: '123' } + : { user: '', password: '' }; + return { + attributes: { + name: 'abc', + actionTypeId: '.webhook', + config: {}, + secrets, + ...overwrites, + }, + id: uuid.v4(), + type: 'action', + }; +} + function getMockDataForEmail( overwrites: Record = {} ): SavedObjectUnsanitizedDoc { diff --git a/x-pack/plugins/actions/server/saved_objects/migrations.ts b/x-pack/plugins/actions/server/saved_objects/migrations.ts index 993beef8d9b2b..1e2290b14ec1b 100644 --- a/x-pack/plugins/actions/server/saved_objects/migrations.ts +++ b/x-pack/plugins/actions/server/saved_objects/migrations.ts @@ -25,8 +25,18 @@ export function getMigrations( pipeMigrations(renameCasesConfigurationObject, addHasAuthConfigurationObject) ); + const migrationWebhookConnectorHasAuth = encryptedSavedObjects.createMigration< + RawAction, + RawAction + >( + (doc): doc is SavedObjectUnsanitizedDoc => + doc.attributes.actionTypeId === '.webhook', + pipeMigrations(addHasAuthConfigurationObject) + ); + return { '7.10.0': executeMigrationWithErrorHandling(migrationActions, '7.10.0'), + '7.11.0': executeMigrationWithErrorHandling(migrationWebhookConnectorHasAuth, '7.11.0'), }; } @@ -70,6 +80,9 @@ function renameCasesConfigurationObject( const addHasAuthConfigurationObject = ( doc: SavedObjectUnsanitizedDoc ): SavedObjectUnsanitizedDoc => { + if (doc.attributes.actionTypeId !== '.email' && doc.attributes.actionTypeId !== '.webhook') { + return doc; + } const hasAuth = !!doc.attributes.secrets.user || !!doc.attributes.secrets.password; return { ...doc, diff --git a/x-pack/plugins/alerts/server/task_runner/task_runner.test.ts b/x-pack/plugins/alerts/server/task_runner/task_runner.test.ts index d9af3d0ae6d5b..8e345d6ff66a8 100644 --- a/x-pack/plugins/alerts/server/task_runner/task_runner.test.ts +++ b/x-pack/plugins/alerts/server/task_runner/task_runner.test.ts @@ -621,9 +621,7 @@ describe('Task Runner', () => { expect(await taskRunner.run()).toMatchInlineSnapshot(` Object { "runAt": 1970-01-01T00:00:10.000Z, - "state": Object { - "previousStartedAt": 1970-01-01T00:00:00.000Z, - }, + "state": Object {}, } `); expect(taskRunnerFactoryInitializerParams.logger.error).toHaveBeenCalledWith( @@ -727,9 +725,7 @@ describe('Task Runner', () => { expect(runnerResult).toMatchInlineSnapshot(` Object { "runAt": 1970-01-01T00:00:10.000Z, - "state": Object { - "previousStartedAt": 1970-01-01T00:00:00.000Z, - }, + "state": Object {}, } `); @@ -781,9 +777,7 @@ describe('Task Runner', () => { expect(runnerResult).toMatchInlineSnapshot(` Object { "runAt": 1970-01-01T00:05:00.000Z, - "state": Object { - "previousStartedAt": 1970-01-01T00:00:00.000Z, - }, + "state": Object {}, } `); }); @@ -814,9 +808,7 @@ describe('Task Runner', () => { expect(runnerResult).toMatchInlineSnapshot(` Object { "runAt": 1970-01-01T00:05:00.000Z, - "state": Object { - "previousStartedAt": 1970-01-01T00:00:00.000Z, - }, + "state": Object {}, } `); }); @@ -846,13 +838,48 @@ describe('Task Runner', () => { expect(runnerResult).toMatchInlineSnapshot(` Object { "runAt": 1970-01-01T00:05:00.000Z, - "state": Object { - "previousStartedAt": 1970-01-01T00:00:00.000Z, - }, + "state": Object {}, } `); }); + test(`doesn't change previousStartedAt when it fails to run`, async () => { + const originalAlertSate = { + previousStartedAt: '1970-01-05T00:00:00.000Z', + }; + + alertType.executor.mockImplementation( + ({ services: executorServices }: AlertExecutorOptions) => { + throw new Error('OMG'); + } + ); + + const taskRunner = new TaskRunner( + alertType, + { + ...mockedTaskInstance, + state: originalAlertSate, + }, + taskRunnerFactoryInitializerParams + ); + + alertsClient.get.mockResolvedValueOnce(mockedAlertTypeSavedObject); + encryptedSavedObjectsClient.getDecryptedAsInternalUser.mockResolvedValueOnce({ + id: '1', + type: 'alert', + attributes: { + apiKey: Buffer.from('123:abc').toString('base64'), + }, + references: [], + }); + + const runnerResult = await taskRunner.run(); + + expect(runnerResult.state.previousStartedAt).toEqual( + new Date(originalAlertSate.previousStartedAt) + ); + }); + test('avoids rescheduling a failed Alert Task Runner when it throws due to failing to fetch the alert', async () => { alertsClient.get.mockImplementation(() => { throw SavedObjectsErrorHelpers.createGenericNotFoundError('task', '1'); @@ -878,9 +905,7 @@ describe('Task Runner', () => { expect(runnerResult).toMatchInlineSnapshot(` Object { "runAt": undefined, - "state": Object { - "previousStartedAt": 1970-01-01T00:00:00.000Z, - }, + "state": Object {}, } `); }); diff --git a/x-pack/plugins/alerts/server/task_runner/task_runner.ts b/x-pack/plugins/alerts/server/task_runner/task_runner.ts index 1ccf14a3a5334..7da16717aef9f 100644 --- a/x-pack/plugins/alerts/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerts/server/task_runner/task_runner.ts @@ -322,7 +322,7 @@ export class TaskRunner { async run(): Promise { const { params: { alertId, spaceId }, - startedAt: previousStartedAt, + startedAt, state: originalState, } = this.taskInstance; @@ -360,7 +360,7 @@ export class TaskRunner { (stateUpdates: AlertTaskState) => { return { ...stateUpdates, - previousStartedAt, + previousStartedAt: startedAt, }; }, (err: Error) => { @@ -370,10 +370,7 @@ export class TaskRunner { } else { this.logger.error(message); } - return { - ...originalState, - previousStartedAt, - }; + return originalState; } ), runAt: resolveErr(runAt, (err) => { diff --git a/x-pack/plugins/apm/public/components/app/RumDashboard/URLFilter/URLSearch/SelectableUrlList.tsx b/x-pack/plugins/apm/public/components/app/RumDashboard/URLFilter/URLSearch/SelectableUrlList.tsx index d9d3d23299371..7bd9b2c87814b 100644 --- a/x-pack/plugins/apm/public/components/app/RumDashboard/URLFilter/URLSearch/SelectableUrlList.tsx +++ b/x-pack/plugins/apm/public/components/app/RumDashboard/URLFilter/URLSearch/SelectableUrlList.tsx @@ -10,6 +10,7 @@ import React, { useRef, useState, KeyboardEvent, + useEffect, } from 'react'; import { EuiFlexGroup, @@ -67,6 +68,7 @@ interface Props { searchValue: string; onClose: () => void; popoverIsOpen: boolean; + initialValue?: string; setPopoverIsOpen: React.Dispatch>; } @@ -80,6 +82,7 @@ export function SelectableUrlList({ onClose, popoverIsOpen, setPopoverIsOpen, + initialValue, }: Props) { const [darkMode] = useUiSetting$('theme:darkMode'); @@ -92,6 +95,9 @@ export function SelectableUrlList({ if (evt.key.toLowerCase() === 'enter') { onTermChange(); setPopoverIsOpen(false); + if (searchRef) { + searchRef.blur(); + } } }; @@ -126,6 +132,16 @@ export function SelectableUrlList({ } }; + useEffect(() => { + if (searchRef && initialValue) { + searchRef.value = initialValue; + } + + // only want to call it at initial render to set value + // coming from initial value/url + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [searchRef]); + const loadingMessage = ( @@ -165,12 +181,12 @@ export function SelectableUrlList({ renderOption={selectableRenderOptions} singleSelection={false} searchProps={{ - placeholder: I18LABELS.searchByUrl, isClearable: true, onFocus: searchOnFocus, onBlur: searchOnBlur, onInput: onSearchInput, inputRef: setSearchRef, + placeholder: I18LABELS.searchByUrl, }} listProps={{ rowHeight: 68, @@ -197,7 +213,7 @@ export function SelectableUrlList({ {searchValue}, icon: ( diff --git a/x-pack/plugins/apm/public/components/app/RumDashboard/URLFilter/URLSearch/__tests__/SelectableUrlList.test.tsx b/x-pack/plugins/apm/public/components/app/RumDashboard/URLFilter/URLSearch/__tests__/SelectableUrlList.test.tsx new file mode 100644 index 0000000000000..abafdf089748b --- /dev/null +++ b/x-pack/plugins/apm/public/components/app/RumDashboard/URLFilter/URLSearch/__tests__/SelectableUrlList.test.tsx @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import React from 'react'; +import { createMemoryHistory } from 'history'; +import * as fetcherHook from '../../../../../../hooks/useFetcher'; +import { SelectableUrlList } from '../SelectableUrlList'; +import { render } from '../../../utils/test_helper'; + +describe('SelectableUrlList', () => { + it('it uses search term value from url', () => { + jest.spyOn(fetcherHook, 'useFetcher').mockReturnValue({ + data: {}, + status: fetcherHook.FETCH_STATUS.SUCCESS, + refetch: jest.fn(), + }); + + const customHistory = createMemoryHistory({ + initialEntries: ['/?searchTerm=blog'], + }); + + const { getByDisplayValue } = render( + , + { customHistory } + ); + expect(getByDisplayValue('blog')).toBeInTheDocument(); + }); +}); diff --git a/x-pack/plugins/apm/public/components/app/RumDashboard/URLFilter/URLSearch/index.tsx b/x-pack/plugins/apm/public/components/app/RumDashboard/URLFilter/URLSearch/index.tsx index 661f4406990f6..02334be5f722e 100644 --- a/x-pack/plugins/apm/public/components/app/RumDashboard/URLFilter/URLSearch/index.tsx +++ b/x-pack/plugins/apm/public/components/app/RumDashboard/URLFilter/URLSearch/index.tsx @@ -30,9 +30,9 @@ export function URLSearch({ onChange: onFilterChange }: Props) { const [popoverIsOpen, setPopoverIsOpen] = useState(false); - const [searchValue, setSearchValue] = useState(''); + const [searchValue, setSearchValue] = useState(searchTerm ?? ''); - const [debouncedValue, setDebouncedValue] = useState(''); + const [debouncedValue, setDebouncedValue] = useState(searchTerm ?? ''); useDebounce( () => { @@ -44,12 +44,16 @@ export function URLSearch({ onChange: onFilterChange }: Props) { const updateSearchTerm = useCallback( (searchTermN: string) => { + const newQuery = { + ...toQuery(history.location.search), + searchTerm: searchTermN || undefined, + }; + if (!searchTermN) { + delete newQuery.searchTerm; + } const newLocation = { ...history.location, - search: fromQuery({ - ...toQuery(history.location.search), - searchTerm: searchTermN, - }), + search: fromQuery(newQuery), }; history.push(newLocation); }, @@ -133,6 +137,7 @@ export function URLSearch({ onChange: onFilterChange }: Props) {

{I18LABELS.url}

true, + get$: (key: string) => of(true), + }, +} as unknown) as CoreStart; + +export const render = ( + component: React.ReactNode, + options: { customHistory: MemoryHistory } +) => { + const history = options?.customHistory ?? createMemoryHistory(); + + history.location.key = 'TestKeyForTesting'; + + return testLibRender( + + + + {component} + + + + ); +}; diff --git a/x-pack/plugins/apm/server/lib/index_pattern/get_dynamic_index_pattern.ts b/x-pack/plugins/apm/server/lib/index_pattern/get_dynamic_index_pattern.ts index 49030dc8cacc5..cf1f4852002ec 100644 --- a/x-pack/plugins/apm/server/lib/index_pattern/get_dynamic_index_pattern.ts +++ b/x-pack/plugins/apm/server/lib/index_pattern/get_dynamic_index_pattern.ts @@ -5,7 +5,6 @@ */ import LRU from 'lru-cache'; -import { LegacyAPICaller } from '../../../../../../src/core/server'; import { IndexPatternsFetcher, FieldDescriptor, @@ -45,8 +44,7 @@ export const getDynamicIndexPattern = async ({ } const indexPatternsFetcher = new IndexPatternsFetcher( - (...rest: Parameters) => - context.core.elasticsearch.legacy.client.callAsCurrentUser(...rest) + context.core.elasticsearch.client.asCurrentUser ); // Since `getDynamicIndexPattern` is called in setup_request (and thus by every endpoint) diff --git a/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts b/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts index 8b3163e44915a..a869c1e1186fa 100644 --- a/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts +++ b/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts @@ -145,6 +145,15 @@ export interface AggregationOptionsByType { >; keyed?: boolean; } & AggregationSourceOptions; + range: { + field: string; + ranges: Array< + | { key?: string; from: string | number } + | { key?: string; to: string | number } + | { key?: string; from: string | number; to: string | number } + >; + keyed?: boolean; + }; auto_date_histogram: { buckets: number; } & AggregationSourceOptions; @@ -324,6 +333,18 @@ interface AggregationResponsePart< ? Record : { buckets: DateRangeBucket[] }; }; + range: { + buckets: TAggregationOptionsMap extends { range: { keyed: true } } + ? Record< + string, + DateRangeBucket & + SubAggregationResponseOf + > + : Array< + DateRangeBucket & + SubAggregationResponseOf + >; + }; auto_date_histogram: { buckets: Array< DateHistogramBucket & diff --git a/x-pack/plugins/canvas/public/feature_catalogue_entry.ts b/x-pack/plugins/canvas/public/feature_catalogue_entry.ts index ccc642177780d..2b6d2f105454b 100644 --- a/x-pack/plugins/canvas/public/feature_catalogue_entry.ts +++ b/x-pack/plugins/canvas/public/feature_catalogue_entry.ts @@ -11,7 +11,7 @@ export const featureCatalogueEntry = { id: 'canvas', title: 'Canvas', subtitle: i18n.translate('xpack.canvas.featureCatalogue.canvasSubtitle', { - defaultMessage: 'Design pixel-perfect reports.', + defaultMessage: 'Design pixel-perfect presentations.', }), description: i18n.translate('xpack.canvas.appDescription', { defaultMessage: 'Showcase your data in a pixel-perfect way.', diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/constants.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/constants.ts index 374a2420f5ba7..decf1e2158744 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/constants.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/constants.ts @@ -11,6 +11,16 @@ export enum ApiTokenTypes { Search = 'search', } +export const CREATE_MESSAGE = i18n.translate('xpack.enterpriseSearch.appSearch.tokens.created', { + defaultMessage: 'Successfully created key.', +}); +export const UPDATE_MESSAGE = i18n.translate('xpack.enterpriseSearch.appSearch.tokens.update', { + defaultMessage: 'Successfully updated API Key.', +}); +export const DELETE_MESSAGE = i18n.translate('xpack.enterpriseSearch.appSearch.tokens.deleted', { + defaultMessage: 'Successfully deleted key.', +}); + export const SEARCH_DISPLAY = i18n.translate( 'xpack.enterpriseSearch.appSearch.tokens.permissions.display.search', { @@ -81,3 +91,7 @@ export const TOKEN_TYPE_INFO = [ { value: ApiTokenTypes.Private, text: TOKEN_TYPE_DISPLAY_NAMES[ApiTokenTypes.Private] }, { value: ApiTokenTypes.Admin, text: TOKEN_TYPE_DISPLAY_NAMES[ApiTokenTypes.Admin] }, ]; + +export const FLYOUT_ARIA_LABEL_ID = 'credentialsFlyoutTitle'; + +export const DOCS_HREF = 'https://www.elastic.co/guide/en/app-search/current/authentication.html'; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials.test.tsx index a265b2c998d39..a9a0dab044351 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials.test.tsx @@ -14,6 +14,7 @@ import { Credentials } from './credentials'; import { EuiCopy, EuiLoadingContent, EuiPageContentBody } from '@elastic/eui'; import { externalUrl } from '../../../shared/enterprise_search_url'; +import { CredentialsFlyout } from './credentials_flyout'; describe('Credentials', () => { // Kea mocks @@ -71,4 +72,16 @@ describe('Credentials', () => { button.props().onClick(); expect(actions.showCredentialsForm).toHaveBeenCalledTimes(1); }); + + it('will render CredentialsFlyout if shouldShowCredentialsForm is true', () => { + setMockValues({ shouldShowCredentialsForm: true }); + const wrapper = shallow(); + expect(wrapper.find(CredentialsFlyout)).toHaveLength(1); + }); + + it('will NOT render CredentialsFlyout if shouldShowCredentialsForm is false', () => { + setMockValues({ shouldShowCredentialsForm: false }); + const wrapper = shallow(); + expect(wrapper.find(CredentialsFlyout)).toHaveLength(0); + }); }); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials.tsx index b9a482ae462d5..c8eae8cc13f5f 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials.tsx @@ -24,16 +24,19 @@ import { import { i18n } from '@kbn/i18n'; import { SetAppSearchChrome as SetPageChrome } from '../../../shared/kibana_chrome'; +import { FlashMessages } from '../../../shared/flash_messages'; + import { CredentialsLogic } from './credentials_logic'; import { externalUrl } from '../../../shared/enterprise_search_url/external_url'; import { CredentialsList } from './credentials_list'; +import { CredentialsFlyout } from './credentials_flyout'; export const Credentials: React.FC = () => { const { initializeCredentialsData, resetCredentials, showCredentialsForm } = useActions( CredentialsLogic ); - const { dataLoading } = useValues(CredentialsLogic); + const { dataLoading, shouldShowCredentialsForm } = useValues(CredentialsLogic); useEffect(() => { initializeCredentialsData(); @@ -63,6 +66,7 @@ export const Credentials: React.FC = () => { + {shouldShowCredentialsForm && }

@@ -120,7 +124,8 @@ export const Credentials: React.FC = () => { )} - + + {!!dataLoading ? : } diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/body.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/body.test.tsx new file mode 100644 index 0000000000000..e9217da163636 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/body.test.tsx @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { setMockValues, setMockActions } from '../../../../__mocks__/kea.mock'; + +import React from 'react'; +import { shallow } from 'enzyme'; +import { EuiFlyoutBody, EuiForm } from '@elastic/eui'; + +import { ApiTokenTypes } from '../constants'; +import { defaultApiToken } from '../credentials_logic'; + +import { + FormKeyName, + FormKeyType, + FormKeyReadWriteAccess, + FormKeyEngineAccess, + FormKeyUpdateWarning, +} from './form_components'; +import { CredentialsFlyoutBody } from './body'; + +describe('CredentialsFlyoutBody', () => { + const values = { + activeApiToken: defaultApiToken, + activeApiTokenExists: false, + }; + const actions = { + onApiTokenChange: jest.fn(), + }; + + beforeEach(() => { + jest.clearAllMocks(); + setMockValues(values); + setMockActions(actions); + }); + + it('renders', () => { + const wrapper = shallow(); + + expect(wrapper.find(EuiFlyoutBody)).toHaveLength(1); + expect(wrapper.find(EuiForm)).toHaveLength(1); + }); + + it('shows the expected form components on default private key creation', () => { + const wrapper = shallow(); + + expect(wrapper.find(FormKeyName)).toHaveLength(1); + expect(wrapper.find(FormKeyType)).toHaveLength(1); + expect(wrapper.find(FormKeyReadWriteAccess)).toHaveLength(1); + expect(wrapper.find(FormKeyEngineAccess)).toHaveLength(1); + expect(wrapper.find(FormKeyUpdateWarning)).toHaveLength(0); + }); + + it('does not show read-write access options for search keys', () => { + setMockValues({ + ...values, + activeApiToken: { + ...defaultApiToken, + type: ApiTokenTypes.Search, + }, + }); + const wrapper = shallow(); + + expect(wrapper.find(FormKeyReadWriteAccess)).toHaveLength(0); + expect(wrapper.find(FormKeyEngineAccess)).toHaveLength(1); + }); + + it('does not show read-write or engine access options for admin keys', () => { + setMockValues({ + ...values, + activeApiToken: { + ...defaultApiToken, + type: ApiTokenTypes.Admin, + }, + }); + const wrapper = shallow(); + + expect(wrapper.find(FormKeyReadWriteAccess)).toHaveLength(0); + expect(wrapper.find(FormKeyEngineAccess)).toHaveLength(0); + }); + + it('shows a warning if updating an existing key', () => { + setMockValues({ ...values, activeApiTokenExists: true }); + const wrapper = shallow(); + + expect(wrapper.find(FormKeyUpdateWarning)).toHaveLength(1); + }); + + it('calls onApiTokenChange on form submit', () => { + const wrapper = shallow(); + + const preventDefault = jest.fn(); + wrapper.find(EuiForm).simulate('submit', { preventDefault }); + + expect(preventDefault).toHaveBeenCalled(); + expect(actions.onApiTokenChange).toHaveBeenCalled(); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/body.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/body.tsx new file mode 100644 index 0000000000000..0395c77cf9d89 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/body.tsx @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { useValues, useActions } from 'kea'; +import { EuiFlyoutBody, EuiForm } from '@elastic/eui'; + +import { FlashMessages } from '../../../../shared/flash_messages'; +import { CredentialsLogic } from '../credentials_logic'; +import { ApiTokenTypes } from '../constants'; + +import { + FormKeyName, + FormKeyType, + FormKeyReadWriteAccess, + FormKeyEngineAccess, + FormKeyUpdateWarning, +} from './form_components'; + +export const CredentialsFlyoutBody: React.FC = () => { + const { onApiTokenChange } = useActions(CredentialsLogic); + const { activeApiToken, activeApiTokenExists } = useValues(CredentialsLogic); + + return ( + + + { + e.preventDefault(); + onApiTokenChange(); + }} + component="form" + > + + + {activeApiToken.type === ApiTokenTypes.Private && } + {activeApiToken.type !== ApiTokenTypes.Admin && } + + {activeApiTokenExists && } + + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/footer.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/footer.test.tsx new file mode 100644 index 0000000000000..c31546472b036 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/footer.test.tsx @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { setMockValues, setMockActions } from '../../../../__mocks__/kea.mock'; + +import React from 'react'; +import { shallow } from 'enzyme'; +import { EuiFlyoutFooter, EuiButtonEmpty } from '@elastic/eui'; + +import { CredentialsFlyoutFooter } from './footer'; + +describe('CredentialsFlyoutFooter', () => { + const values = { + activeApiTokenExists: false, + }; + const actions = { + hideCredentialsForm: jest.fn(), + onApiTokenChange: jest.fn(), + }; + + beforeEach(() => { + jest.clearAllMocks(); + setMockValues(values); + setMockActions(actions); + }); + + it('renders', () => { + const wrapper = shallow(); + expect(wrapper.find(EuiFlyoutFooter)).toHaveLength(1); + }); + + it('closes the flyout', () => { + const wrapper = shallow(); + const button = wrapper.find(EuiButtonEmpty); + button.simulate('click'); + expect(button.prop('children')).toEqual('Close'); + expect(actions.hideCredentialsForm).toHaveBeenCalled(); + }); + + it('renders action button text for new tokens', () => { + const wrapper = shallow(); + const button = wrapper.find('[data-test-subj="APIKeyActionButton"]'); + + expect(button.prop('children')).toEqual('Save'); + }); + + it('renders action button text for existing tokens', () => { + setMockValues({ activeApiTokenExists: true }); + const wrapper = shallow(); + const button = wrapper.find('[data-test-subj="APIKeyActionButton"]'); + + expect(button.prop('children')).toEqual('Update'); + }); + + it('calls onApiTokenChange on action button press', () => { + const wrapper = shallow(); + const button = wrapper.find('[data-test-subj="APIKeyActionButton"]'); + button.simulate('click'); + + expect(actions.onApiTokenChange).toHaveBeenCalled(); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/footer.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/footer.tsx new file mode 100644 index 0000000000000..e59a75a578ba4 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/footer.tsx @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { useValues, useActions } from 'kea'; +import { + EuiFlyoutFooter, + EuiFlexGroup, + EuiFlexItem, + EuiButtonEmpty, + EuiButton, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; + +import { CredentialsLogic } from '../credentials_logic'; + +export const CredentialsFlyoutFooter: React.FC = () => { + const { hideCredentialsForm, onApiTokenChange } = useActions(CredentialsLogic); + const { activeApiTokenExists } = useValues(CredentialsLogic); + + return ( + + + + + {i18n.translate('xpack.enterpriseSearch.appSearch.credentials.flyout.closeText', { + defaultMessage: 'Close', + })} + + + + + {activeApiTokenExists + ? i18n.translate('xpack.enterpriseSearch.appSearch.credentials.flyout.updateText', { + defaultMessage: 'Update', + }) + : i18n.translate('xpack.enterpriseSearch.appSearch.credentials.flyout.saveText', { + defaultMessage: 'Save', + })} + + + + + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/index.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/index.ts new file mode 100644 index 0000000000000..ad39717ff8979 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/index.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { FormKeyName } from './key_name'; +export { FormKeyType } from './key_type'; +export { FormKeyReadWriteAccess } from './key_read_write_access'; +export { FormKeyEngineAccess } from './key_engine_access'; +export { FormKeyUpdateWarning } from './key_update_warning'; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_engine_access.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_engine_access.test.tsx new file mode 100644 index 0000000000000..b4b092f17a6aa --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_engine_access.test.tsx @@ -0,0 +1,135 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { setMockValues, setMockActions } from '../../../../../__mocks__/kea.mock'; + +import React from 'react'; +import { shallow } from 'enzyme'; +import { EuiRadio, EuiCheckbox } from '@elastic/eui'; + +import { FormKeyEngineAccess, EngineSelection } from './key_engine_access'; + +describe('FormKeyEngineAccess', () => { + const values = { + myRole: { canAccessAllEngines: true }, + fullEngineAccessChecked: true, + }; + const actions = { + setAccessAllEngines: jest.fn(), + }; + + beforeEach(() => { + jest.clearAllMocks(); + setMockValues(values); + setMockActions(actions); + }); + + it('renders', () => { + const wrapper = shallow(); + + expect(wrapper.find(EuiRadio)).toHaveLength(2); + expect(wrapper.find(EngineSelection)).toHaveLength(0); + }); + + it('hides the full access radio option if the user does not have access to all engines', () => { + setMockValues({ + ...values, + myRole: { canAccessAllEngines: false }, + }); + const wrapper = shallow(); + + expect(wrapper.find('#all_engines').prop('hidden')).toEqual(true); + }); + + it('controls the checked values for access radios', () => { + setMockValues({ + ...values, + fullEngineAccessChecked: true, + }); + const wrapper = shallow(); + + expect(wrapper.find('#all_engines').prop('checked')).toEqual(true); + expect(wrapper.find('#all_engines').prop('value')).toEqual('true'); + expect(wrapper.find('#specific_engines').prop('checked')).toEqual(false); + expect(wrapper.find('#specific_engines').prop('value')).toEqual('false'); + + setMockValues({ + ...values, + fullEngineAccessChecked: false, + }); + wrapper.setProps({}); // Re-render + + expect(wrapper.find('#all_engines').prop('checked')).toEqual(false); + expect(wrapper.find('#all_engines').prop('value')).toEqual('false'); + expect(wrapper.find('#specific_engines').prop('checked')).toEqual(true); + expect(wrapper.find('#specific_engines').prop('value')).toEqual('true'); + }); + + it('calls setAccessAllEngines when the radios are changed', () => { + const wrapper = shallow(); + + wrapper.find('#all_engines').simulate('change'); + expect(actions.setAccessAllEngines).toHaveBeenCalledWith(true); + + wrapper.find('#specific_engines').simulate('change'); + expect(actions.setAccessAllEngines).toHaveBeenCalledWith(false); + }); + + it('displays the engine selection panel if the limited access radio is selected', () => { + setMockValues({ + ...values, + fullEngineAccessChecked: false, + }); + const wrapper = shallow(); + + expect(wrapper.find(EngineSelection)).toHaveLength(1); + }); +}); + +describe('EngineSelection', () => { + const values = { + activeApiToken: { engines: [] }, + engines: [{ name: 'engine1' }, { name: 'engine2' }, { name: 'engine3' }], + }; + const actions = { + onEngineSelect: jest.fn(), + }; + + beforeEach(() => { + jest.clearAllMocks(); + setMockValues(values); + setMockActions(actions); + }); + + it('renders', () => { + const wrapper = shallow(); + + expect(wrapper.find('h4').text()).toEqual('Select Engines'); + expect(wrapper.find(EuiCheckbox)).toHaveLength(3); + expect(wrapper.find(EuiCheckbox).first().prop('label')).toEqual('engine1'); + }); + + it('controls the engines checked state', () => { + setMockValues({ + ...values, + activeApiToken: { engines: ['engine3'] }, + }); + const wrapper = shallow(); + + expect(wrapper.find(EuiCheckbox).first().prop('checked')).toEqual(false); + expect(wrapper.find(EuiCheckbox).last().prop('checked')).toEqual(true); + }); + + it('calls onEngineSelect when the checkboxes are changed', () => { + const wrapper = shallow(); + + wrapper.find(EuiCheckbox).first().simulate('change'); + expect(actions.onEngineSelect).toHaveBeenCalledWith('engine1'); + + wrapper.find(EuiCheckbox).last().simulate('change'); + expect(actions.onEngineSelect).toHaveBeenCalledWith('engine3'); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_engine_access.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_engine_access.tsx new file mode 100644 index 0000000000000..88e345d0f9966 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_engine_access.tsx @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { useValues, useActions } from 'kea'; +import { + EuiFormRow, + EuiRadio, + EuiCheckbox, + EuiText, + EuiTitle, + EuiSpacer, + EuiPanel, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; + +import { AppLogic } from '../../../../app_logic'; +import { CredentialsLogic } from '../../credentials_logic'; + +export const FormKeyEngineAccess: React.FC = () => { + const { myRole } = useValues(AppLogic); + const { setAccessAllEngines } = useActions(CredentialsLogic); + const { fullEngineAccessChecked } = useValues(CredentialsLogic); + + return ( + <> + + + <> + + +

+ {i18n.translate( + 'xpack.enterpriseSearch.appSearch.credentials.formEngineAccess.fullAccess.label', + { defaultMessage: 'Full Engine Access' } + )} +

+ + + {i18n.translate( + 'xpack.enterpriseSearch.appSearch.credentials.formEngineAccess.fullAccess.helpText', + { defaultMessage: 'Access to all current and future Engines.' } + )} + + + } + hidden={!myRole.canAccessAllEngines} + checked={fullEngineAccessChecked} + value={fullEngineAccessChecked.toString()} + onChange={() => setAccessAllEngines(true)} + /> + + + +

+ {i18n.translate( + 'xpack.enterpriseSearch.appSearch.credentials.formEngineAccess.limitedAccess.label', + { defaultMessage: 'Limited Engine Access' } + )} +

+ + + {i18n.translate( + 'xpack.enterpriseSearch.appSearch.credentials.formEngineAccess.limitedAccess.helpText', + { defaultMessage: 'Limit key access to specific Engines.' } + )} + + + } + checked={!fullEngineAccessChecked} + value={(!fullEngineAccessChecked).toString()} + onChange={() => setAccessAllEngines(false)} + /> + + + {!fullEngineAccessChecked && } + + ); +}; + +export const EngineSelection: React.FC = () => { + const { onEngineSelect } = useActions(CredentialsLogic); + const { activeApiToken, engines } = useValues(CredentialsLogic); + + return ( + <> + + + +

+ {i18n.translate( + 'xpack.enterpriseSearch.appSearch.credentials.formEngineAccess.engineAccess.label', + { defaultMessage: 'Select Engines' } + )} +

+ + + {i18n.translate( + 'xpack.enterpriseSearch.appSearch.credentials.formEngineAccess.engineAccess.helpText', + { defaultMessage: 'Engines which the key can access:' } + )} + + + {engines.map((engine) => ( + onEngineSelect(engine.name)} + /> + ))} + + + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_name.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_name.test.tsx new file mode 100644 index 0000000000000..87f0f843dfa67 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_name.test.tsx @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { setMockValues, setMockActions } from '../../../../../__mocks__/kea.mock'; + +import React from 'react'; +import { shallow } from 'enzyme'; +import { EuiFieldText, EuiFormRow } from '@elastic/eui'; + +import { FormKeyName } from './'; + +describe('FormKeyName', () => { + const values = { + activeApiToken: { name: '' }, + activeApiTokenRawName: '', + activeApiTokenExists: false, + }; + const actions = { + setNameInputBlurred: jest.fn(), + setTokenName: jest.fn(), + }; + + beforeEach(() => { + jest.clearAllMocks(); + setMockValues(values); + setMockActions(actions); + }); + + it('renders', () => { + const wrapper = shallow(); + + expect(wrapper.find(EuiFieldText)).toHaveLength(1); + expect(wrapper.find(EuiFieldText).prop('placeholder')).toEqual('i.e., my-engine-key'); + expect(wrapper.find(EuiFieldText).prop('value')).toEqual(''); + expect(wrapper.find(EuiFieldText).prop('disabled')).toEqual(false); + expect(wrapper.find(EuiFormRow).prop('helpText')).toEqual(''); + }); + + it('shows help text if the raw name does not match the expected name', () => { + setMockValues({ + ...values, + activeApiToken: { name: 'my-engine-key' }, + activeApiTokenRawName: 'my engine key!!', + }); + const wrapper = shallow(); + + expect(wrapper.find(EuiFormRow).prop('helpText')).toEqual( + 'Your key will be named: my-engine-key' + ); + }); + + it('controls the input value', () => { + setMockValues({ + ...values, + activeApiTokenRawName: 'test', + }); + const wrapper = shallow(); + + expect(wrapper.find(EuiFieldText).prop('value')).toEqual('test'); + }); + + it('disables the input if editing an existing key', () => { + setMockValues({ + ...values, + activeApiTokenExists: true, + }); + const wrapper = shallow(); + + expect(wrapper.find(EuiFieldText).prop('disabled')).toEqual(true); + }); + + it('calls setTokenName when the input value is changed', () => { + const wrapper = shallow(); + wrapper.find(EuiFieldText).simulate('change', { target: { value: 'changed' } }); + + expect(actions.setTokenName).toHaveBeenCalledWith('changed'); + }); + + it('calls setNameInputBlurred when the user stops focusing the input', () => { + const wrapper = shallow(); + wrapper.find(EuiFieldText).simulate('blur'); + + expect(actions.setNameInputBlurred).toHaveBeenCalledWith(true); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_name.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_name.tsx new file mode 100644 index 0000000000000..fb8de2b244ecc --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_name.tsx @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { useValues, useActions } from 'kea'; +import { EuiFormRow, EuiFieldText } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; + +import { CredentialsLogic } from '../../credentials_logic'; + +export const FormKeyName: React.FC = () => { + const { setNameInputBlurred, setTokenName } = useActions(CredentialsLogic); + const { + activeApiToken: { name }, + activeApiTokenRawName: rawName, + activeApiTokenExists, + } = useValues(CredentialsLogic); + + return ( + + setTokenName(e.target.value)} + onBlur={() => setNameInputBlurred(true)} + autoComplete="off" + maxLength={64} + disabled={activeApiTokenExists} + required + fullWidth + autoFocus + /> + + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_read_write_access.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_read_write_access.test.tsx new file mode 100644 index 0000000000000..2f1be1b07cbe1 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_read_write_access.test.tsx @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { setMockValues, setMockActions } from '../../../../../__mocks__/kea.mock'; + +import React from 'react'; +import { shallow } from 'enzyme'; +import { EuiCheckbox } from '@elastic/eui'; + +import { FormKeyReadWriteAccess } from './'; + +describe('FormKeyReadWriteAccess', () => { + const values = { + activeApiToken: { read: false, write: false }, + }; + const actions = { + setTokenReadWrite: jest.fn(), + }; + + beforeEach(() => { + jest.clearAllMocks(); + setMockValues(values); + setMockActions(actions); + }); + + it('renders', () => { + const wrapper = shallow(); + + expect(wrapper.find('h3').text()).toEqual('Read and Write Access Levels'); + expect(wrapper.find(EuiCheckbox)).toHaveLength(2); + }); + + it('controls the checked state for the read checkbox', () => { + setMockValues({ + ...values, + activeApiToken: { read: true, write: false }, + }); + const wrapper = shallow(); + + expect(wrapper.find('#read').prop('checked')).toEqual(true); + expect(wrapper.find('#write').prop('checked')).toEqual(false); + }); + + it('controls the checked state for the write checkbox', () => { + setMockValues({ + ...values, + activeApiToken: { read: false, write: true }, + }); + const wrapper = shallow(); + + expect(wrapper.find('#read').prop('checked')).toEqual(false); + expect(wrapper.find('#write').prop('checked')).toEqual(true); + }); + + it('calls setTokenReadWrite when the checkboxes are changed', () => { + const wrapper = shallow(); + + wrapper.find('#read').simulate('change', { target: { name: 'read', checked: true } }); + expect(actions.setTokenReadWrite).toHaveBeenCalledWith({ name: 'read', checked: true }); + + wrapper.find('#write').simulate('change', { target: { name: 'write', checked: false } }); + expect(actions.setTokenReadWrite).toHaveBeenCalledWith({ name: 'write', checked: false }); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_read_write_access.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_read_write_access.tsx new file mode 100644 index 0000000000000..a02b00b6ad377 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_read_write_access.tsx @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { useValues, useActions } from 'kea'; +import { EuiCheckbox, EuiText, EuiTitle, EuiSpacer, EuiPanel } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; + +import { CredentialsLogic } from '../../credentials_logic'; +import { ITokenReadWrite } from '../../types'; + +export const FormKeyReadWriteAccess: React.FC = () => { + const { setTokenReadWrite } = useActions(CredentialsLogic); + const { activeApiToken } = useValues(CredentialsLogic); + + return ( + <> + + + +

+ {i18n.translate('xpack.enterpriseSearch.appSearch.credentials.formReadWrite.label', { + defaultMessage: 'Read and Write Access Levels', + })} +

+ + + {i18n.translate('xpack.enterpriseSearch.appSearch.credentials.formReadWrite.helpText', { + defaultMessage: 'Only applies to Private API Keys.', + })} + + + setTokenReadWrite(e.target as ITokenReadWrite)} + label={i18n.translate( + 'xpack.enterpriseSearch.appSearch.credentials.formReadWrite.readLabel', + { defaultMessage: 'Read Access' } + )} + /> + setTokenReadWrite(e.target as ITokenReadWrite)} + label={i18n.translate( + 'xpack.enterpriseSearch.appSearch.credentials.formReadWrite.writeLabel', + { defaultMessage: 'Write Access' } + )} + /> + + + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_type.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_type.test.tsx new file mode 100644 index 0000000000000..d07a705b2d90b --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_type.test.tsx @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { setMockValues, setMockActions } from '../../../../../__mocks__/kea.mock'; + +import React from 'react'; +import { shallow } from 'enzyme'; +import { EuiSelect } from '@elastic/eui'; + +import { ApiTokenTypes, TOKEN_TYPE_INFO } from '../../constants'; +import { FormKeyType } from './'; + +describe('FormKeyType', () => { + const values = { + myRole: { credentialTypes: ['search', 'private', 'admin'] }, + activeApiToken: { type: ApiTokenTypes.Private }, + activeApiTokenExists: false, + }; + const actions = { + setTokenType: jest.fn(), + }; + + beforeEach(() => { + jest.clearAllMocks(); + setMockValues(values); + setMockActions(actions); + }); + + it('renders', () => { + const wrapper = shallow(); + + expect(wrapper.find(EuiSelect)).toHaveLength(1); + expect(wrapper.find(EuiSelect).prop('placeholder')).toEqual('Select a key type'); + expect(wrapper.find(EuiSelect).prop('options')).toEqual(TOKEN_TYPE_INFO); + expect(wrapper.find(EuiSelect).prop('value')).toEqual(ApiTokenTypes.Private); + expect(wrapper.find(EuiSelect).prop('disabled')).toEqual(false); + }); + + it('only shows the type options that the user has access to', () => { + setMockValues({ + ...values, + myRole: { credentialTypes: ['search'] }, + }); + const wrapper = shallow(); + + expect(wrapper.find(EuiSelect).prop('options')).toEqual([ + expect.objectContaining({ value: ApiTokenTypes.Search }), + ]); + }); + + it('controls the select value', () => { + setMockValues({ + ...values, + activeApiToken: { type: ApiTokenTypes.Search }, + }); + const wrapper = shallow(); + + expect(wrapper.find(EuiSelect).prop('value')).toEqual(ApiTokenTypes.Search); + }); + + it('disables the select if editing an existing key', () => { + setMockValues({ + ...values, + activeApiTokenExists: true, + }); + const wrapper = shallow(); + + expect(wrapper.find(EuiSelect).prop('disabled')).toEqual(true); + }); + + it('calls setTokenType when the select value is changed', () => { + const wrapper = shallow(); + wrapper.find(EuiSelect).simulate('change', { target: { value: ApiTokenTypes.Admin } }); + + expect(actions.setTokenType).toHaveBeenCalledWith(ApiTokenTypes.Admin); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_type.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_type.tsx new file mode 100644 index 0000000000000..7268c12614e8b --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_type.tsx @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { useValues, useActions } from 'kea'; +import { EuiFormRow, EuiSelect, EuiText, EuiLink } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; + +import { AppLogic } from '../../../../app_logic'; +import { CredentialsLogic } from '../../credentials_logic'; +import { TOKEN_TYPE_DESCRIPTION, TOKEN_TYPE_INFO, DOCS_HREF } from '../../constants'; + +export const FormKeyType: React.FC = () => { + const { myRole } = useValues(AppLogic); + const { setTokenType } = useActions(CredentialsLogic); + const { activeApiToken, activeApiTokenExists } = useValues(CredentialsLogic); + + const tokenDescription = TOKEN_TYPE_DESCRIPTION[activeApiToken.type]; + const tokenOptions = TOKEN_TYPE_INFO.filter((typeInfo) => + myRole?.credentialTypes?.includes(typeInfo.value) + ); + + return ( + +

+ {tokenDescription}{' '} + + {i18n.translate('xpack.enterpriseSearch.appSearch.credentials.documentationLink1', { + defaultMessage: 'Visit the documentation', + })} + {' '} + {i18n.translate('xpack.enterpriseSearch.appSearch.credentials.documentationLink2', { + defaultMessage: 'to learn more about keys.', + })} +

+ + } + > + setTokenType(e.target.value)} + placeholder={i18n.translate( + 'xpack.enterpriseSearch.appSearch.credentials.formType.placeholder', + { defaultMessage: 'Select a key type' } + )} + disabled={activeApiTokenExists} + required + fullWidth + /> + + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_update_warning.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_update_warning.test.tsx new file mode 100644 index 0000000000000..c0ff892c220c7 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_update_warning.test.tsx @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { shallow } from 'enzyme'; +import { EuiCallOut } from '@elastic/eui'; + +import { FormKeyUpdateWarning } from './'; + +describe('FormKeyUpdateWarning', () => { + it('renders', () => { + const wrapper = shallow(); + expect(wrapper.find(EuiCallOut)).toHaveLength(1); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_update_warning.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_update_warning.tsx new file mode 100644 index 0000000000000..7e7aaa583325d --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_update_warning.tsx @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { EuiSpacer, EuiCallOut } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; + +export const FormKeyUpdateWarning: React.FC = () => ( + <> + + +

+ {i18n.translate('xpack.enterpriseSearch.appSearch.credentials.updateWarning', { + defaultMessage: + 'Existing API keys may be shared between users. Changing permissions for this key will affect all users who have access to this key.', + })} +

+ + +); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/header.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/header.test.tsx new file mode 100644 index 0000000000000..a8d9505136faa --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/header.test.tsx @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { setMockValues } from '../../../../__mocks__/kea.mock'; + +import React from 'react'; +import { shallow } from 'enzyme'; +import { EuiFlyoutHeader } from '@elastic/eui'; + +import { ApiTokenTypes } from '../constants'; +import { IApiToken } from '../types'; + +import { CredentialsFlyoutHeader } from './header'; + +describe('CredentialsFlyoutHeader', () => { + const apiToken: IApiToken = { + name: '', + type: ApiTokenTypes.Private, + read: true, + write: true, + access_all_engines: true, + }; + const values = { + activeApiToken: apiToken, + }; + + beforeEach(() => { + jest.clearAllMocks(); + setMockValues(values); + }); + + it('renders', () => { + const wrapper = shallow(); + + expect(wrapper.find(EuiFlyoutHeader)).toHaveLength(1); + expect(wrapper.find('h2').prop('id')).toEqual('credentialsFlyoutTitle'); + expect(wrapper.find('h2').prop('children')).toEqual('Create a new key'); + }); + + it('changes the title text if editing an existing token', () => { + setMockValues({ + activeApiToken: { + ...apiToken, + id: 'some-id', + name: 'search-key', + }, + }); + const wrapper = shallow(); + + expect(wrapper.find('h2').prop('children')).toEqual('Update search-key'); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/header.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/header.tsx new file mode 100644 index 0000000000000..f208cd1c5918f --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/header.tsx @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { useValues } from 'kea'; +import { EuiFlyoutHeader, EuiTitle } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; + +import { CredentialsLogic } from '../credentials_logic'; +import { FLYOUT_ARIA_LABEL_ID } from '../constants'; + +export const CredentialsFlyoutHeader: React.FC = () => { + const { activeApiToken } = useValues(CredentialsLogic); + + return ( + + +

+ {activeApiToken.id + ? i18n.translate('xpack.enterpriseSearch.appSearch.credentials.flyout.updateTitle', { + defaultMessage: 'Update {tokenName}', + values: { tokenName: activeApiToken.name }, + }) + : i18n.translate('xpack.enterpriseSearch.appSearch.credentials.flyout.createTitle', { + defaultMessage: 'Create a new key', + })} +

+ + + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/index.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/index.test.tsx new file mode 100644 index 0000000000000..16b669c530012 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/index.test.tsx @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { setMockActions } from '../../../../__mocks__/kea.mock'; + +import React from 'react'; +import { shallow } from 'enzyme'; +import { EuiFlyout } from '@elastic/eui'; + +import { CredentialsFlyout } from './'; + +describe('CredentialsFlyout', () => { + const actions = { + hideCredentialsForm: jest.fn(), + }; + + beforeEach(() => { + jest.clearAllMocks(); + setMockActions(actions); + }); + + it('renders', () => { + const wrapper = shallow(); + const flyout = wrapper.find(EuiFlyout); + + expect(flyout).toHaveLength(1); + expect(flyout.prop('aria-labelledby')).toEqual('credentialsFlyoutTitle'); + expect(flyout.prop('onClose')).toEqual(actions.hideCredentialsForm); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/index.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/index.tsx new file mode 100644 index 0000000000000..602a5250716c3 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/index.tsx @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { useActions } from 'kea'; +import { EuiPortal, EuiFlyout } from '@elastic/eui'; + +import { CredentialsLogic } from '../credentials_logic'; +import { FLYOUT_ARIA_LABEL_ID } from '../constants'; +import { CredentialsFlyoutHeader } from './header'; +import { CredentialsFlyoutBody } from './body'; +import { CredentialsFlyoutFooter } from './footer'; + +export const CredentialsFlyout: React.FC = () => { + const { hideCredentialsForm } = useActions(CredentialsLogic); + + return ( + + + + + + + + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_logic.test.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_logic.test.ts index 11b1253332cb2..de79862b540ba 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_logic.test.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_logic.test.ts @@ -6,17 +6,33 @@ import { resetContext } from 'kea'; -import { CredentialsLogic } from './credentials_logic'; -import { ApiTokenTypes } from './constants'; - +import { mockHttpValues } from '../../../__mocks__'; jest.mock('../../../shared/http', () => ({ - HttpLogic: { values: { http: { get: jest.fn(), delete: jest.fn() } } }, + HttpLogic: { values: mockHttpValues }, })); -import { HttpLogic } from '../../../shared/http'; +const { http } = mockHttpValues; + jest.mock('../../../shared/flash_messages', () => ({ + FlashMessagesLogic: { actions: { clearFlashMessages: jest.fn() } }, + setSuccessMessage: jest.fn(), flashAPIErrors: jest.fn(), })); -import { flashAPIErrors } from '../../../shared/flash_messages'; +import { + FlashMessagesLogic, + setSuccessMessage, + flashAPIErrors, +} from '../../../shared/flash_messages'; + +jest.mock('../../app_logic', () => ({ + AppLogic: { + selectors: { myRole: jest.fn(() => ({})) }, + values: { myRole: jest.fn(() => ({})) }, + }, +})); +import { AppLogic } from '../../app_logic'; + +import { ApiTokenTypes } from './constants'; +import { CredentialsLogic } from './credentials_logic'; describe('CredentialsLogic', () => { const DEFAULT_VALUES = { @@ -38,6 +54,7 @@ describe('CredentialsLogic', () => { meta: {}, nameInputBlurred: false, shouldShowCredentialsForm: false, + fullEngineAccessChecked: false, }; const mount = (defaults?: object) => { @@ -952,6 +969,13 @@ describe('CredentialsLogic', () => { }); }); }); + + describe('listener side-effects', () => { + it('should clear flashMessages whenever the credentials form flyout is opened', () => { + CredentialsLogic.actions.showCredentialsForm(); + expect(FlashMessagesLogic.actions.clearFlashMessages).toHaveBeenCalled(); + }); + }); }); describe('hideCredentialsForm', () => { @@ -1068,10 +1092,10 @@ describe('CredentialsLogic', () => { mount(); jest.spyOn(CredentialsLogic.actions, 'setCredentialsData').mockImplementationOnce(() => {}); const promise = Promise.resolve({ meta, results }); - (HttpLogic.values.http.get as jest.Mock).mockReturnValue(promise); + http.get.mockReturnValue(promise); CredentialsLogic.actions.fetchCredentials(2); - expect(HttpLogic.values.http.get).toHaveBeenCalledWith('/api/app_search/credentials', { + expect(http.get).toHaveBeenCalledWith('/api/app_search/credentials', { query: { 'page[current]': 2, }, @@ -1083,7 +1107,7 @@ describe('CredentialsLogic', () => { it('handles errors', async () => { mount(); const promise = Promise.reject('An error occured'); - (HttpLogic.values.http.get as jest.Mock).mockReturnValue(promise); + http.get.mockReturnValue(promise); CredentialsLogic.actions.fetchCredentials(); try { @@ -1101,12 +1125,10 @@ describe('CredentialsLogic', () => { .spyOn(CredentialsLogic.actions, 'setCredentialsDetails') .mockImplementationOnce(() => {}); const promise = Promise.resolve(credentialsDetails); - (HttpLogic.values.http.get as jest.Mock).mockReturnValue(promise); + http.get.mockReturnValue(promise); CredentialsLogic.actions.fetchDetails(); - expect(HttpLogic.values.http.get).toHaveBeenCalledWith( - '/api/app_search/credentials/details' - ); + expect(http.get).toHaveBeenCalledWith('/api/app_search/credentials/details'); await promise; expect(CredentialsLogic.actions.setCredentialsDetails).toHaveBeenCalledWith( credentialsDetails @@ -1116,7 +1138,7 @@ describe('CredentialsLogic', () => { it('handles errors', async () => { mount(); const promise = Promise.reject('An error occured'); - (HttpLogic.values.http.get as jest.Mock).mockReturnValue(promise); + http.get.mockReturnValue(promise); CredentialsLogic.actions.fetchDetails(); try { @@ -1134,20 +1156,19 @@ describe('CredentialsLogic', () => { mount(); jest.spyOn(CredentialsLogic.actions, 'onApiKeyDelete').mockImplementationOnce(() => {}); const promise = Promise.resolve(); - (HttpLogic.values.http.delete as jest.Mock).mockReturnValue(promise); + http.delete.mockReturnValue(promise); CredentialsLogic.actions.deleteApiKey(tokenName); - expect(HttpLogic.values.http.delete).toHaveBeenCalledWith( - `/api/app_search/credentials/${tokenName}` - ); + expect(http.delete).toHaveBeenCalledWith(`/api/app_search/credentials/${tokenName}`); await promise; expect(CredentialsLogic.actions.onApiKeyDelete).toHaveBeenCalledWith(tokenName); + expect(setSuccessMessage).toHaveBeenCalled(); }); it('handles errors', async () => { mount(); const promise = Promise.reject('An error occured'); - (HttpLogic.values.http.delete as jest.Mock).mockReturnValue(promise); + http.delete.mockReturnValue(promise); CredentialsLogic.actions.deleteApiKey(tokenName); try { @@ -1157,9 +1178,189 @@ describe('CredentialsLogic', () => { } }); }); + + describe('onApiTokenChange', () => { + it('calls a POST API endpoint that creates a new token if the active token does not exist yet', async () => { + const createdToken = { + name: 'new-key', + type: ApiTokenTypes.Admin, + }; + mount({ + activeApiToken: createdToken, + }); + jest.spyOn(CredentialsLogic.actions, 'onApiTokenCreateSuccess'); + const promise = Promise.resolve(createdToken); + http.post.mockReturnValue(promise); + + CredentialsLogic.actions.onApiTokenChange(); + expect(http.post).toHaveBeenCalledWith('/api/app_search/credentials', { + body: JSON.stringify(createdToken), + }); + await promise; + expect(CredentialsLogic.actions.onApiTokenCreateSuccess).toHaveBeenCalledWith(createdToken); + expect(setSuccessMessage).toHaveBeenCalled(); + }); + + it('calls a PUT endpoint that updates the active token if it already exists', async () => { + const updatedToken = { + name: 'test-key', + type: ApiTokenTypes.Private, + read: true, + write: false, + access_all_engines: false, + engines: ['engine1'], + }; + mount({ + activeApiToken: { + ...updatedToken, + id: 'some-id', + }, + }); + jest.spyOn(CredentialsLogic.actions, 'onApiTokenUpdateSuccess'); + const promise = Promise.resolve(updatedToken); + http.put.mockReturnValue(promise); + + CredentialsLogic.actions.onApiTokenChange(); + expect(http.put).toHaveBeenCalledWith('/api/app_search/credentials/test-key', { + body: JSON.stringify(updatedToken), + }); + await promise; + expect(CredentialsLogic.actions.onApiTokenUpdateSuccess).toHaveBeenCalledWith(updatedToken); + expect(setSuccessMessage).toHaveBeenCalled(); + }); + + it('handles errors', async () => { + mount(); + const promise = Promise.reject('An error occured'); + http.post.mockReturnValue(promise); + + CredentialsLogic.actions.onApiTokenChange(); + try { + await promise; + } catch { + expect(flashAPIErrors).toHaveBeenCalledWith('An error occured'); + } + }); + + describe('token type data', () => { + it('does not send extra read/write/engine access data for admin tokens', () => { + const correctAdminToken = { + name: 'bogus-admin', + type: ApiTokenTypes.Admin, + }; + const extraData = { + read: true, + write: true, + access_all_engines: true, + }; + mount({ activeApiToken: { ...correctAdminToken, ...extraData } }); + + CredentialsLogic.actions.onApiTokenChange(); + expect(http.post).toHaveBeenCalledWith('/api/app_search/credentials', { + body: JSON.stringify(correctAdminToken), + }); + }); + + it('does not send extra read/write access data for search tokens', () => { + const correctSearchToken = { + name: 'bogus-search', + type: ApiTokenTypes.Search, + access_all_engines: false, + engines: ['some-engine'], + }; + const extraData = { + read: true, + write: false, + }; + mount({ activeApiToken: { ...correctSearchToken, ...extraData } }); + + CredentialsLogic.actions.onApiTokenChange(); + expect(http.post).toHaveBeenCalledWith('/api/app_search/credentials', { + body: JSON.stringify(correctSearchToken), + }); + }); + + // Private tokens send all data per the PUT test above. + // If that ever changes, we should capture that in another test here. + }); + }); + + describe('onEngineSelect', () => { + it('calls addEngineName if the engine is not selected', () => { + mount({ + activeApiToken: { + ...DEFAULT_VALUES.activeApiToken, + engines: [], + }, + }); + jest.spyOn(CredentialsLogic.actions, 'addEngineName'); + + CredentialsLogic.actions.onEngineSelect('engine1'); + expect(CredentialsLogic.actions.addEngineName).toHaveBeenCalledWith('engine1'); + expect(CredentialsLogic.values.activeApiToken.engines).toEqual(['engine1']); + }); + + it('calls removeEngineName if the engine is already selected', () => { + mount({ + activeApiToken: { + ...DEFAULT_VALUES.activeApiToken, + engines: ['engine1', 'engine2'], + }, + }); + jest.spyOn(CredentialsLogic.actions, 'removeEngineName'); + + CredentialsLogic.actions.onEngineSelect('engine1'); + expect(CredentialsLogic.actions.removeEngineName).toHaveBeenCalledWith('engine1'); + expect(CredentialsLogic.values.activeApiToken.engines).toEqual(['engine2']); + }); + }); }); describe('selectors', () => { + describe('fullEngineAccessChecked', () => { + it('should be true if active token is set to access all engines and the user can access all engines', () => { + (AppLogic.selectors.myRole as jest.Mock).mockReturnValueOnce({ + canAccessAllEngines: true, + }); + mount({ + activeApiToken: { + ...DEFAULT_VALUES.activeApiToken, + access_all_engines: true, + }, + }); + + expect(CredentialsLogic.values.fullEngineAccessChecked).toEqual(true); + }); + + it('should be false if the token is not set to access all engines', () => { + (AppLogic.selectors.myRole as jest.Mock).mockReturnValueOnce({ + canAccessAllEngines: true, + }); + mount({ + activeApiToken: { + ...DEFAULT_VALUES.activeApiToken, + access_all_engines: false, + }, + }); + + expect(CredentialsLogic.values.fullEngineAccessChecked).toEqual(false); + }); + + it('should be false if the user cannot acess all engines', () => { + (AppLogic.selectors.myRole as jest.Mock).mockReturnValueOnce({ + canAccessAllEngines: false, + }); + mount({ + activeApiToken: { + ...DEFAULT_VALUES.activeApiToken, + access_all_engines: true, + }, + }); + + expect(CredentialsLogic.values.fullEngineAccessChecked).toEqual(false); + }); + }); + describe('activeApiTokenExists', () => { it('should be false if the token has no id', () => { mount({ diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_logic.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_logic.ts index c6f929c45eb23..7b8b864b3a317 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_logic.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_logic.ts @@ -7,15 +7,21 @@ import { kea, MakeLogicType } from 'kea'; import { formatApiName } from '../../utils/format_api_name'; -import { ApiTokenTypes } from './constants'; +import { ApiTokenTypes, CREATE_MESSAGE, UPDATE_MESSAGE, DELETE_MESSAGE } from './constants'; import { HttpLogic } from '../../../shared/http'; +import { + FlashMessagesLogic, + setSuccessMessage, + flashAPIErrors, +} from '../../../shared/flash_messages'; +import { AppLogic } from '../../app_logic'; + import { IMeta } from '../../../../../common/types'; -import { flashAPIErrors } from '../../../shared/flash_messages'; import { IEngine } from '../../types'; import { IApiToken, ICredentialsDetails, ITokenReadWrite } from './types'; -const defaultApiToken: IApiToken = { +export const defaultApiToken: IApiToken = { name: '', type: ApiTokenTypes.Private, read: true, @@ -23,9 +29,7 @@ const defaultApiToken: IApiToken = { access_all_engines: true, }; -// TODO CREATE_MESSAGE, UPDATE_MESSAGE, and DELETE_MESSAGE from ent-search - -export interface ICredentialsLogicActions { +interface ICredentialsLogicActions { addEngineName(engineName: string): string; onApiKeyDelete(tokenName: string): string; onApiTokenCreateSuccess(apiToken: IApiToken): IApiToken; @@ -46,9 +50,11 @@ export interface ICredentialsLogicActions { fetchCredentials(page?: number): number; fetchDetails(): { value: boolean }; deleteApiKey(tokenName: string): string; + onApiTokenChange(): void; + onEngineSelect(engineName: string): string; } -export interface ICredentialsLogicValues { +interface ICredentialsLogicValues { activeApiToken: IApiToken; activeApiTokenExists: boolean; activeApiTokenRawName: string; @@ -79,10 +85,7 @@ export const CredentialsLogic = kea< setCredentialsData: (meta, apiTokens) => ({ meta, apiTokens }), setCredentialsDetails: (details) => details, setNameInputBlurred: (nameInputBlurred) => nameInputBlurred, - setTokenReadWrite: ({ name, checked }) => ({ - name, - checked, - }), + setTokenReadWrite: ({ name, checked }) => ({ name, checked }), setTokenName: (name) => name, setTokenType: (tokenType) => tokenType, showCredentialsForm: (apiToken = { ...defaultApiToken }) => apiToken, @@ -92,6 +95,8 @@ export const CredentialsLogic = kea< fetchCredentials: (page) => page, fetchDetails: true, deleteApiKey: (tokenName) => tokenName, + onApiTokenChange: () => null, + onEngineSelect: (engineName) => engineName, }), reducers: () => ({ apiTokens: [ @@ -204,7 +209,11 @@ export const CredentialsLogic = kea< ], }), selectors: ({ selectors }) => ({ - // TODO fullEngineAccessChecked from ent-search + fullEngineAccessChecked: [ + () => [AppLogic.selectors.myRole, selectors.activeApiToken], + (myRole, activeApiToken) => + !!(myRole.canAccessAllEngines && activeApiToken.access_all_engines), + ], dataLoading: [ () => [selectors.isCredentialsDetailsComplete, selectors.isCredentialsDataComplete], (isCredentialsDetailsComplete, isCredentialsDataComplete) => { @@ -217,6 +226,9 @@ export const CredentialsLogic = kea< ], }), listeners: ({ actions, values }) => ({ + showCredentialsForm: () => { + FlashMessagesLogic.actions.clearFlashMessages(); + }, initializeCredentialsData: () => { actions.fetchCredentials(); actions.fetchDetails(); @@ -247,11 +259,50 @@ export const CredentialsLogic = kea< await http.delete(`/api/app_search/credentials/${tokenName}`); actions.onApiKeyDelete(tokenName); + setSuccessMessage(DELETE_MESSAGE); } catch (e) { flashAPIErrors(e); } }, - // TODO onApiTokenChange from ent-search - // TODO onEngineSelect from ent-search + onApiTokenChange: async () => { + const { id, name, engines, type, read, write } = values.activeApiToken; + + const data: IApiToken = { + name, + type, + }; + if (type === ApiTokenTypes.Private) { + data.read = read; + data.write = write; + } + if (type !== ApiTokenTypes.Admin) { + data.access_all_engines = values.fullEngineAccessChecked; + data.engines = engines; + } + + try { + const { http } = HttpLogic.values; + const body = JSON.stringify(data); + + if (id) { + const response = await http.put(`/api/app_search/credentials/${name}`, { body }); + actions.onApiTokenUpdateSuccess(response); + setSuccessMessage(UPDATE_MESSAGE); + } else { + const response = await http.post('/api/app_search/credentials', { body }); + actions.onApiTokenCreateSuccess(response); + setSuccessMessage(CREATE_MESSAGE); + } + } catch (e) { + flashAPIErrors(e); + } + }, + onEngineSelect: (engineName: string) => { + if (values.activeApiToken?.engines?.includes(engineName)) { + actions.removeEngineName(engineName); + } else { + actions.addEngineName(engineName); + } + }, }), }); diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/product_selector/product_selector.test.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/product_selector/product_selector.test.tsx index f1f16d1a6f7a4..f2bdc1a8c75b5 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/product_selector/product_selector.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/product_selector/product_selector.test.tsx @@ -4,28 +4,30 @@ * you may not use this file except in compliance with the Elastic License. */ -import '../../../__mocks__/kea.mock'; +import { setMockValues } from '../../../__mocks__/kea.mock'; import React from 'react'; -import { useValues } from 'kea'; import { shallow } from 'enzyme'; import { EuiPage } from '@elastic/eui'; -import { ProductSelector } from './'; +import { SetupGuideCta } from '../setup_guide'; import { ProductCard } from '../product_card'; +import { ProductSelector } from './'; + describe('ProductSelector', () => { - it('renders the overview page and product cards with no host set', () => { - (useValues as jest.Mock).mockImplementationOnce(() => ({ config: { host: '' } })); + it('renders the overview page, product cards, & setup guide CTAs with no host set', () => { + setMockValues({ config: { host: '' } }); const wrapper = shallow(); expect(wrapper.find(EuiPage).hasClass('enterpriseSearchOverview')).toBe(true); expect(wrapper.find(ProductCard)).toHaveLength(2); + expect(wrapper.find(SetupGuideCta)).toHaveLength(1); }); describe('access checks when host is set', () => { beforeEach(() => { - (useValues as jest.Mock).mockImplementationOnce(() => ({ config: { host: 'localhost' } })); + setMockValues({ config: { host: 'localhost' } }); }); it('does not render the App Search card if the user does not have access to AS', () => { diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/product_selector/product_selector.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/product_selector/product_selector.tsx index 6d76b741d7a97..235ececd8b6fc 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/product_selector/product_selector.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/product_selector/product_selector.tsx @@ -3,11 +3,6 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ import React from 'react'; import { useValues } from 'kea'; @@ -30,6 +25,7 @@ import { SetEnterpriseSearchChrome as SetPageChrome } from '../../../shared/kiba import { SendEnterpriseSearchTelemetry as SendTelemetry } from '../../../shared/telemetry'; import { ProductCard } from '../product_card'; +import { SetupGuideCta } from '../setup_guide'; import AppSearchImage from '../../assets/app_search.png'; import WorkplaceSearchImage from '../../assets/workplace_search.png'; @@ -66,9 +62,13 @@ export const ProductSelector: React.FC = ({ access }) =>

- {i18n.translate('xpack.enterpriseSearch.overview.subheading', { - defaultMessage: 'Select a product to get started', - })} + {config.host + ? i18n.translate('xpack.enterpriseSearch.overview.subheading', { + defaultMessage: 'Select a product to get started.', + }) + : i18n.translate('xpack.enterpriseSearch.overview.setupHeading', { + defaultMessage: 'Choose a product to set up and get started.', + })}

@@ -87,6 +87,7 @@ export const ProductSelector: React.FC = ({ access }) => )} + {!config.host && } diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/setup_guide/index.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/setup_guide/index.ts index c367424d375f9..89f7da4547569 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/setup_guide/index.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/setup_guide/index.ts @@ -5,3 +5,4 @@ */ export { SetupGuide } from './setup_guide'; +export { SetupGuideCta } from './setup_guide_cta'; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/setup_guide/setup_guide_cta.scss b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/setup_guide/setup_guide_cta.scss new file mode 100644 index 0000000000000..103ef8eccb558 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/setup_guide/setup_guide_cta.scss @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +.enterpriseSearchSetupCta { + margin: $euiSize auto $euiSizeXL; + + // Clickable EuiPanel override - line panel up with product cards + &.euiPanel--isClickable { + width: calc(100% - #{$euiSize}); + } + + &__text { + max-width: $euiSize * 40; + } + + &__image { + display: block; + max-width: 100%; + width: $euiSize * 10; + margin: 0 auto; + + @include euiBreakpoint('xs', 's') { + width: $euiSize * 15; + } + } +} diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/setup_guide/setup_guide_cta.test.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/setup_guide/setup_guide_cta.test.tsx new file mode 100644 index 0000000000000..f235beef3b337 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/setup_guide/setup_guide_cta.test.tsx @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { shallow } from 'enzyme'; + +import { SetupGuideCta } from './'; + +describe('SetupGuideCta', () => { + it('renders', () => { + const wrapper = shallow(); + + expect(wrapper.find('.enterpriseSearchSetupCta')).toHaveLength(1); + expect(wrapper.find('img')).toHaveLength(1); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/setup_guide/setup_guide_cta.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/setup_guide/setup_guide_cta.tsx new file mode 100644 index 0000000000000..2a0e2ffc34f3f --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/setup_guide/setup_guide_cta.tsx @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { i18n } from '@kbn/i18n'; +import { EuiFlexGroup, EuiFlexItem, EuiTitle, EuiText } from '@elastic/eui'; +import { EuiPanel } from '../../../shared/react_router_helpers'; + +import CtaImage from './assets/getting_started.png'; +import './setup_guide_cta.scss'; + +export const SetupGuideCta: React.FC = () => ( + + + + +

+ {i18n.translate('xpack.enterpriseSearch.overview.setupCta.title', { + defaultMessage: 'Enterprise-grade functionality for teams big and small', + })} +

+ + + {i18n.translate('xpack.enterpriseSearch.overview.setupCta.description', { + defaultMessage: + 'Add search to your app or internal organization with Elastic App Search and Workplace Search. Watch the video to see what you can do when search is made easy.', + })} + + + + + + + +); diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search/index.test.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/index.test.tsx index 803d2c8462b1b..0e929c9191e0f 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search/index.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/index.test.tsx @@ -6,10 +6,8 @@ import React from 'react'; import { shallow } from 'enzyme'; -import { EuiPage } from '@elastic/eui'; -import '../__mocks__/kea.mock'; -import { useValues } from 'kea'; +import { setMockValues } from '../__mocks__/kea.mock'; import { EnterpriseSearch } from './'; import { SetupGuide } from './components/setup_guide'; @@ -18,7 +16,7 @@ import { ProductSelector } from './components/product_selector'; describe('EnterpriseSearch', () => { it('renders the Setup Guide and Product Selector', () => { - (useValues as jest.Mock).mockReturnValue({ + setMockValues({ errorConnecting: false, config: { host: 'localhost' }, }); @@ -28,15 +26,23 @@ describe('EnterpriseSearch', () => { expect(wrapper.find(ProductSelector)).toHaveLength(1); }); - it('renders the error connecting prompt when host is not configured', () => { - (useValues as jest.Mock).mockReturnValueOnce({ + it('renders the error connecting prompt only if host is configured', () => { + setMockValues({ errorConnecting: true, - config: { host: '' }, + config: { host: 'localhost' }, }); const wrapper = shallow(); expect(wrapper.find(ErrorConnecting)).toHaveLength(1); - expect(wrapper.find(EuiPage)).toHaveLength(0); expect(wrapper.find(ProductSelector)).toHaveLength(0); + + setMockValues({ + errorConnecting: true, + config: { host: '' }, + }); + wrapper.setProps({}); // Re-render + + expect(wrapper.find(ErrorConnecting)).toHaveLength(0); + expect(wrapper.find(ProductSelector)).toHaveLength(1); }); }); diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search/index.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/index.tsx index 7b97c6c9e58b6..048baabe6a1dd 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search/index.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/index.tsx @@ -25,7 +25,7 @@ export const EnterpriseSearch: React.FC = ({ access = {} }) => const { errorConnecting } = useValues(HttpLogic); const { config } = useValues(KibanaLogic); - const showErrorConnecting = config.host && errorConnecting; + const showErrorConnecting = !!(config.host && errorConnecting); return ( diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/react_router_helpers/eui_link.test.tsx b/x-pack/plugins/enterprise_search/public/applications/shared/react_router_helpers/eui_link.test.tsx index 82fbb8940d460..3a4585b6d9a71 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/react_router_helpers/eui_link.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/shared/react_router_helpers/eui_link.test.tsx @@ -8,18 +8,18 @@ import '../../__mocks__/kea.mock'; import React from 'react'; import { shallow, mount } from 'enzyme'; -import { EuiLink, EuiButton } from '@elastic/eui'; +import { EuiLink, EuiButton, EuiPanel } from '@elastic/eui'; import { mockKibanaValues, mockHistory } from '../../__mocks__'; -import { EuiReactRouterLink, EuiReactRouterButton } from './eui_link'; +import { EuiReactRouterLink, EuiReactRouterButton, EuiReactRouterPanel } from './eui_link'; describe('EUI & React Router Component Helpers', () => { beforeEach(() => { jest.clearAllMocks(); }); - it('renders', () => { + it('renders an EuiLink', () => { const wrapper = shallow(); expect(wrapper.find(EuiLink)).toHaveLength(1); @@ -31,6 +31,13 @@ describe('EUI & React Router Component Helpers', () => { expect(wrapper.find(EuiButton)).toHaveLength(1); }); + it('renders an EuiPanel', () => { + const wrapper = shallow(); + + expect(wrapper.find(EuiPanel)).toHaveLength(1); + expect(wrapper.find(EuiPanel).prop('paddingSize')).toEqual('l'); + }); + it('passes down all ...rest props', () => { const wrapper = shallow(); const link = wrapper.find(EuiLink); diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/react_router_helpers/eui_link.tsx b/x-pack/plugins/enterprise_search/public/applications/shared/react_router_helpers/eui_link.tsx index f9f6ec54e8832..78546911813ec 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/react_router_helpers/eui_link.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/shared/react_router_helpers/eui_link.tsx @@ -6,14 +6,15 @@ import React from 'react'; import { useValues } from 'kea'; -import { EuiLink, EuiButton, EuiButtonProps, EuiLinkAnchorProps } from '@elastic/eui'; +import { EuiLink, EuiButton, EuiButtonProps, EuiLinkAnchorProps, EuiPanel } from '@elastic/eui'; +import { EuiPanelProps } from '@elastic/eui/src/components/panel/panel'; import { KibanaLogic } from '../kibana'; import { HttpLogic } from '../http'; import { letBrowserHandleEvent, createHref } from './'; /** - * Generates either an EuiLink or EuiButton with a React-Router-ified link + * Generates EUI components with React-Router-ified links * * Based off of EUI's recommendations for handling React Router: * https://github.com/elastic/eui/blob/master/wiki/react-router.md#react-router-51 @@ -54,9 +55,11 @@ export const EuiReactRouterHelper: React.FC = ({ return React.cloneElement(children as React.ReactElement, reactRouterProps); }; -type TEuiReactRouterLinkProps = EuiLinkAnchorProps & IEuiReactRouterProps; -type TEuiReactRouterButtonProps = EuiButtonProps & IEuiReactRouterProps; +/** + * Component helpers + */ +type TEuiReactRouterLinkProps = EuiLinkAnchorProps & IEuiReactRouterProps; export const EuiReactRouterLink: React.FC = ({ to, onClick, @@ -68,6 +71,7 @@ export const EuiReactRouterLink: React.FC = ({ ); +type TEuiReactRouterButtonProps = EuiButtonProps & IEuiReactRouterProps; export const EuiReactRouterButton: React.FC = ({ to, onClick, @@ -78,3 +82,15 @@ export const EuiReactRouterButton: React.FC = ({ ); + +type TEuiReactRouterPanelProps = EuiPanelProps & IEuiReactRouterProps; +export const EuiReactRouterPanel: React.FC = ({ + to, + onClick, + shouldNotCreateHref, + ...rest +}) => ( + + + +); diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/react_router_helpers/index.ts b/x-pack/plugins/enterprise_search/public/applications/shared/react_router_helpers/index.ts index 6915d3222c45c..36fb0560d7323 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/react_router_helpers/index.ts +++ b/x-pack/plugins/enterprise_search/public/applications/shared/react_router_helpers/index.ts @@ -6,5 +6,8 @@ export { letBrowserHandleEvent } from './link_events'; export { createHref, ICreateHrefOptions } from './create_href'; -export { EuiReactRouterLink as EuiLink } from './eui_link'; -export { EuiReactRouterButton as EuiButton } from './eui_link'; +export { + EuiReactRouterLink as EuiLink, + EuiReactRouterButton as EuiButton, + EuiReactRouterPanel as EuiPanel, +} from './eui_link'; diff --git a/x-pack/plugins/enterprise_search/server/routes/app_search/credentials.test.ts b/x-pack/plugins/enterprise_search/server/routes/app_search/credentials.test.ts index 6b5f4a05b3aa6..357b49de93412 100644 --- a/x-pack/plugins/enterprise_search/server/routes/app_search/credentials.test.ts +++ b/x-pack/plugins/enterprise_search/server/routes/app_search/credentials.test.ts @@ -41,6 +41,115 @@ describe('credentials routes', () => { }); }); + describe('POST /api/app_search/credentials', () => { + let mockRouter: MockRouter; + + beforeEach(() => { + jest.clearAllMocks(); + mockRouter = new MockRouter({ method: 'post', payload: 'body' }); + + registerCredentialsRoutes({ + ...mockDependencies, + router: mockRouter.router, + }); + }); + + it('creates a request handler', () => { + expect(mockRequestHandler.createRequest).toHaveBeenCalledWith({ + path: '/as/credentials/collection', + }); + }); + + describe('validates', () => { + describe('admin keys', () => { + it('correctly', () => { + const request = { + body: { + name: 'admin-key', + type: 'admin', + }, + }; + mockRouter.shouldValidate(request); + }); + + it('throws on unnecessary properties', () => { + const request = { + body: { + name: 'admin-key', + type: 'admin', + read: true, + access_all_engines: true, + }, + }; + mockRouter.shouldThrow(request); + }); + }); + + describe('private keys', () => { + it('correctly', () => { + const request = { + body: { + name: 'private-key', + type: 'private', + read: true, + write: false, + access_all_engines: false, + engines: ['engine1', 'engine2'], + }, + }; + mockRouter.shouldValidate(request); + }); + + it('throws on missing keys', () => { + const request = { + body: { + name: 'private-key', + type: 'private', + }, + }; + mockRouter.shouldThrow(request); + }); + }); + + describe('search keys', () => { + it('correctly', () => { + const request = { + body: { + name: 'search-key', + type: 'search', + access_all_engines: true, + }, + }; + mockRouter.shouldValidate(request); + }); + + it('throws on missing keys', () => { + const request = { + body: { + name: 'search-key', + type: 'search', + }, + }; + mockRouter.shouldThrow(request); + }); + + it('throws on extra keys', () => { + const request = { + body: { + name: 'search-key', + type: 'search', + read: true, + write: false, + access_all_engines: false, + engines: ['engine1', 'engine2'], + }, + }; + mockRouter.shouldThrow(request); + }); + }); + }); + }); + describe('GET /api/app_search/credentials/details', () => { let mockRouter: MockRouter; @@ -61,6 +170,123 @@ describe('credentials routes', () => { }); }); + describe('PUT /api/app_search/credentials/{name}', () => { + let mockRouter: MockRouter; + + beforeEach(() => { + jest.clearAllMocks(); + mockRouter = new MockRouter({ method: 'put', payload: 'body' }); + + registerCredentialsRoutes({ + ...mockDependencies, + router: mockRouter.router, + }); + }); + + it('creates a request to enterprise search', () => { + const mockRequest = { + params: { + name: 'abc123', + }, + }; + + mockRouter.callRoute(mockRequest); + + expect(mockRequestHandler.createRequest).toHaveBeenCalledWith({ + path: '/as/credentials/abc123', + }); + }); + + describe('validates', () => { + describe('admin keys', () => { + it('correctly', () => { + const request = { + body: { + name: 'admin-key', + type: 'admin', + }, + }; + mockRouter.shouldValidate(request); + }); + + it('throws on unnecessary properties', () => { + const request = { + body: { + name: 'admin-key', + type: 'admin', + read: true, + access_all_engines: true, + }, + }; + mockRouter.shouldThrow(request); + }); + }); + + describe('private keys', () => { + it('correctly', () => { + const request = { + body: { + name: 'private-key', + type: 'private', + read: true, + write: false, + access_all_engines: false, + engines: ['engine1', 'engine2'], + }, + }; + mockRouter.shouldValidate(request); + }); + + it('throws on missing keys', () => { + const request = { + body: { + name: 'private-key', + type: 'private', + }, + }; + mockRouter.shouldThrow(request); + }); + }); + + describe('search keys', () => { + it('correctly', () => { + const request = { + body: { + name: 'search-key', + type: 'search', + access_all_engines: true, + }, + }; + mockRouter.shouldValidate(request); + }); + + it('throws on missing keys', () => { + const request = { + body: { + name: 'search-key', + type: 'search', + }, + }; + mockRouter.shouldThrow(request); + }); + + it('throws on extra keys', () => { + const request = { + body: { + name: 'search-key', + type: 'search', + read: true, + write: false, + access_all_engines: false, + engines: ['engine1', 'engine2'], + }, + }; + mockRouter.shouldThrow(request); + }); + }); + }); + }); + describe('DELETE /api/app_search/credentials/{name}', () => { let mockRouter: MockRouter; diff --git a/x-pack/plugins/enterprise_search/server/routes/app_search/credentials.ts b/x-pack/plugins/enterprise_search/server/routes/app_search/credentials.ts index 0f2c1133192c5..85d213c82dd05 100644 --- a/x-pack/plugins/enterprise_search/server/routes/app_search/credentials.ts +++ b/x-pack/plugins/enterprise_search/server/routes/app_search/credentials.ts @@ -8,10 +8,32 @@ import { schema } from '@kbn/config-schema'; import { IRouteDependencies } from '../../plugin'; +const tokenSchema = schema.oneOf([ + schema.object({ + name: schema.string(), + type: schema.literal('admin'), + }), + schema.object({ + name: schema.string(), + type: schema.literal('private'), + read: schema.boolean(), + write: schema.boolean(), + access_all_engines: schema.boolean(), + engines: schema.maybe(schema.arrayOf(schema.string())), + }), + schema.object({ + name: schema.string(), + type: schema.literal('search'), + access_all_engines: schema.boolean(), + engines: schema.maybe(schema.arrayOf(schema.string())), + }), +]); + export function registerCredentialsRoutes({ router, enterpriseSearchRequestHandler, }: IRouteDependencies) { + // Credentials API router.get( { path: '/api/app_search/credentials', @@ -25,6 +47,19 @@ export function registerCredentialsRoutes({ path: '/as/credentials/collection', }) ); + router.post( + { + path: '/api/app_search/credentials', + validate: { + body: tokenSchema, + }, + }, + enterpriseSearchRequestHandler.createRequest({ + path: '/as/credentials/collection', + }) + ); + + // TODO: It would be great to remove this someday router.get( { path: '/api/app_search/credentials/details', @@ -34,6 +69,24 @@ export function registerCredentialsRoutes({ path: '/as/credentials/details', }) ); + + // Single credential API + router.put( + { + path: '/api/app_search/credentials/{name}', + validate: { + params: schema.object({ + name: schema.string(), + }), + body: tokenSchema, + }, + }, + async (context, request, response) => { + return enterpriseSearchRequestHandler.createRequest({ + path: `/as/credentials/${request.params.name}`, + })(context, request, response); + } + ); router.delete( { path: '/api/app_search/credentials/{name}', diff --git a/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/constants.ts b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/constants.ts index bd845b0a7d9a7..0a96146339a58 100644 --- a/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/constants.ts +++ b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/constants.ts @@ -14,17 +14,52 @@ export const DEFAULT_POLICY: PolicyFromES = { version: 1, modified_date: Date.now().toString(), policy: { - name: '', + name: 'my_policy', + phases: { + hot: { + min_age: '0ms', + actions: { + rollover: { + max_age: '30d', + max_size: '50gb', + }, + }, + }, + }, + }, + name: 'my_policy', +}; + +export const POLICY_WITH_INCLUDE_EXCLUDE: PolicyFromES = { + version: 1, + modified_date: Date.now().toString(), + policy: { + name: 'my_policy', phases: { hot: { min_age: '123ms', actions: { - rollover: {}, + rollover: { + max_age: '30d', + max_size: '50gb', + }, + }, + }, + warm: { + actions: { + allocate: { + include: { + abc: '123', + }, + exclude: { + def: '456', + }, + }, }, }, }, }, - name: '', + name: 'my_policy', }; export const DELETE_PHASE_POLICY: PolicyFromES = { @@ -60,3 +95,58 @@ export const DELETE_PHASE_POLICY: PolicyFromES = { }, name: POLICY_NAME, }; + +export const POLICY_WITH_NODE_ATTR_AND_OFF_ALLOCATION: PolicyFromES = { + version: 1, + modified_date: Date.now().toString(), + policy: { + phases: { + hot: { + min_age: '0ms', + actions: { + rollover: { + max_size: '50gb', + }, + }, + }, + warm: { + actions: { + allocate: { + require: {}, + include: { test: '123' }, + exclude: {}, + }, + }, + }, + cold: { + actions: { + migrate: { enabled: false }, + }, + }, + }, + name: POLICY_NAME, + }, + name: POLICY_NAME, +}; + +export const POLICY_WITH_NODE_ROLE_ALLOCATION: PolicyFromES = { + version: 1, + modified_date: Date.now().toString(), + policy: { + phases: { + hot: { + min_age: '0ms', + actions: { + rollover: { + max_size: '50gb', + }, + }, + }, + warm: { + actions: {}, + }, + }, + name: POLICY_NAME, + }, + name: POLICY_NAME, +}; diff --git a/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/edit_policy.helpers.tsx b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/edit_policy.helpers.tsx index 0cfccba761309..1716f124b0c83 100644 --- a/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/edit_policy.helpers.tsx +++ b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/edit_policy.helpers.tsx @@ -9,11 +9,16 @@ import { act } from 'react-dom/test-utils'; import { registerTestBed, TestBedConfig } from '../../../../../test_utils'; +import { EditPolicy } from '../../../public/application/sections/edit_policy'; +import { DataTierAllocationType } from '../../../public/application/sections/edit_policy/types'; + +import { Phases as PolicyPhases } from '../../../common/types'; + +type Phases = keyof PolicyPhases; + import { POLICY_NAME } from './constants'; import { TestSubjects } from '../helpers'; -import { EditPolicy } from '../../../public/application/sections/edit_policy'; - jest.mock('@elastic/eui', () => { const original = jest.requireActual('@elastic/eui'); @@ -52,7 +57,23 @@ export type EditPolicyTestBed = SetupReturn extends Promise ? U : Setup export const setup = async () => { const testBed = await initTestBed(); - const { find, component } = testBed; + const { find, component, form } = testBed; + + const createFormToggleAction = (dataTestSubject: string) => async (checked: boolean) => { + await act(async () => { + form.toggleEuiSwitch(dataTestSubject, checked); + }); + component.update(); + }; + + function createFormSetValueAction(dataTestSubject: string) { + return async (value: V) => { + await act(async () => { + form.setInputValue(dataTestSubject, value); + }); + component.update(); + }; + } const setWaitForSnapshotPolicy = async (snapshotPolicyName: string) => { act(() => { @@ -68,12 +89,7 @@ export const setup = async () => { component.update(); }; - const toggleRollover = async (checked: boolean) => { - await act(async () => { - find('rolloverSwitch').simulate('click', { target: { checked } }); - }); - component.update(); - }; + const toggleRollover = createFormToggleAction('rolloverSwitch'); const setMaxSize = async (value: string, units?: string) => { await act(async () => { @@ -87,12 +103,7 @@ export const setup = async () => { component.update(); }; - const setMaxDocs = async (value: string) => { - await act(async () => { - find('hot-selectedMaxDocuments').simulate('change', { target: { value } }); - }); - component.update(); - }; + const setMaxDocs = createFormSetValueAction('hot-selectedMaxDocuments'); const setMaxAge = async (value: string, units?: string) => { await act(async () => { @@ -104,32 +115,56 @@ export const setup = async () => { component.update(); }; - const toggleForceMerge = (phase: string) => async (checked: boolean) => { - await act(async () => { - find(`${phase}-forceMergeSwitch`).simulate('click', { target: { checked } }); + const toggleForceMerge = (phase: Phases) => createFormToggleAction(`${phase}-forceMergeSwitch`); + + const setForcemergeSegmentsCount = (phase: Phases) => + createFormSetValueAction(`${phase}-selectedForceMergeSegments`); + + const setBestCompression = (phase: Phases) => createFormToggleAction(`${phase}-bestCompression`); + + const setIndexPriority = (phase: Phases) => + createFormSetValueAction(`${phase}-phaseIndexPriority`); + + const enable = (phase: Phases) => createFormToggleAction(`enablePhaseSwitch-${phase}`); + + const warmPhaseOnRollover = createFormToggleAction(`warm-warmPhaseOnRollover`); + + const setMinAgeValue = (phase: Phases) => createFormSetValueAction(`${phase}-selectedMinimumAge`); + + const setMinAgeUnits = (phase: Phases) => + createFormSetValueAction(`${phase}-selectedMinimumAgeUnits`); + + const setDataAllocation = (phase: Phases) => async (value: DataTierAllocationType) => { + act(() => { + find(`${phase}-dataTierAllocationControls.dataTierSelect`).simulate('click'); }); component.update(); - }; - - const setForcemergeSegmentsCount = (phase: string) => async (value: string) => { await act(async () => { - find(`${phase}-selectedForceMergeSegments`).simulate('change', { target: { value } }); + switch (value) { + case 'node_roles': + find(`${phase}-dataTierAllocationControls.defaultDataAllocationOption`).simulate('click'); + break; + case 'node_attrs': + find(`${phase}-dataTierAllocationControls.customDataAllocationOption`).simulate('click'); + break; + default: + find(`${phase}-dataTierAllocationControls.noneDataAllocationOption`).simulate('click'); + } }); component.update(); }; - const setBestCompression = (phase: string) => async (checked: boolean) => { - await act(async () => { - find(`${phase}-bestCompression`).simulate('click', { target: { checked } }); - }); - component.update(); + const setSelectedNodeAttribute = (phase: string) => + createFormSetValueAction(`${phase}-selectedNodeAttrs`); + + const setReplicas = async (value: string) => { + await createFormToggleAction('warm-setReplicasSwitch')(true); + await createFormSetValueAction('warm-selectedReplicaCount')(value); }; - const setIndexPriority = (phase: string) => async (value: string) => { - await act(async () => { - find(`${phase}-phaseIndexPriority`).simulate('change', { target: { value } }); - }); - component.update(); + const setShrink = async (value: string) => { + await createFormToggleAction('shrinkSwitch')(true); + await createFormSetValueAction('warm-selectedPrimaryShardCount')(value); }; return { @@ -147,6 +182,20 @@ export const setup = async () => { setBestCompression: setBestCompression('hot'), setIndexPriority: setIndexPriority('hot'), }, + warm: { + enable: enable('warm'), + warmPhaseOnRollover, + setMinAgeValue: setMinAgeValue('warm'), + setMinAgeUnits: setMinAgeUnits('warm'), + setDataAllocation: setDataAllocation('warm'), + setSelectedNodeAttribute: setSelectedNodeAttribute('warm'), + setReplicas, + setShrink, + toggleForceMerge: toggleForceMerge('warm'), + setForcemergeSegments: setForcemergeSegmentsCount('warm'), + setBestCompression: setBestCompression('warm'), + setIndexPriority: setIndexPriority('warm'), + }, }, }; }; diff --git a/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/edit_policy.test.ts b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/edit_policy.test.ts index 3cbc2d982566e..fccffde3f793f 100644 --- a/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/edit_policy.test.ts +++ b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/edit_policy.test.ts @@ -15,6 +15,9 @@ import { NEW_SNAPSHOT_POLICY_NAME, SNAPSHOT_POLICY_NAME, DEFAULT_POLICY, + POLICY_WITH_INCLUDE_EXCLUDE, + POLICY_WITH_NODE_ATTR_AND_OFF_ALLOCATION, + POLICY_WITH_NODE_ROLE_ALLOCATION, } from './constants'; window.scrollTo = jest.fn(); @@ -53,7 +56,8 @@ describe('', () => { await actions.savePolicy(); const latestRequest = server.requests[server.requests.length - 1]; - expect(JSON.parse(JSON.parse(latestRequest.requestBody).body)).toMatchInlineSnapshot(` + const entirePolicy = JSON.parse(JSON.parse(latestRequest.requestBody).body); + expect(entirePolicy).toMatchInlineSnapshot(` Object { "name": "my_policy", "phases": Object { @@ -81,21 +85,184 @@ describe('', () => { test('disabling rollover', async () => { const { actions } = testBed; - await actions.hot.toggleRollover(false); + await actions.hot.toggleRollover(true); await actions.savePolicy(); const latestRequest = server.requests[server.requests.length - 1]; - expect(JSON.parse(JSON.parse(latestRequest.requestBody).body)).toMatchInlineSnapshot(` + const policy = JSON.parse(JSON.parse(latestRequest.requestBody).body); + const hotActions = policy.phases.hot.actions; + const rolloverAction = hotActions.rollover; + expect(rolloverAction).toBe(undefined); + expect(hotActions).toMatchInlineSnapshot(` + Object { + "set_priority": Object { + "priority": 100, + }, + } + `); + }); + }); + }); + + describe('warm phase', () => { + describe('serialization', () => { + beforeEach(async () => { + httpRequestsMockHelpers.setLoadPolicies([DEFAULT_POLICY]); + httpRequestsMockHelpers.setListNodes({ + nodesByRoles: {}, + nodesByAttributes: { test: ['123'] }, + isUsingDeprecatedDataRoleConfig: false, + }); + httpRequestsMockHelpers.setLoadSnapshotPolicies([]); + + await act(async () => { + testBed = await setup(); + }); + + const { component } = testBed; + component.update(); + }); + + test('default values', async () => { + const { actions } = testBed; + await actions.warm.enable(true); + await actions.savePolicy(); + const latestRequest = server.requests[server.requests.length - 1]; + const warmPhase = JSON.parse(JSON.parse(latestRequest.requestBody).body).phases.warm; + expect(warmPhase).toMatchInlineSnapshot(` + Object { + "actions": Object { + "set_priority": Object { + "priority": 50, + }, + }, + "min_age": "0ms", + } + `); + }); + + test('setting all values', async () => { + const { actions } = testBed; + await actions.warm.enable(true); + await actions.warm.setMinAgeValue('123'); + await actions.warm.setMinAgeUnits('d'); + await actions.warm.setDataAllocation('node_attrs'); + await actions.warm.setSelectedNodeAttribute('test:123'); + await actions.warm.setReplicas('123'); + await actions.warm.setShrink('123'); + await actions.warm.toggleForceMerge(true); + await actions.warm.setForcemergeSegments('123'); + await actions.warm.setBestCompression(true); + await actions.warm.setIndexPriority('123'); + await actions.savePolicy(); + const latestRequest = server.requests[server.requests.length - 1]; + const entirePolicy = JSON.parse(JSON.parse(latestRequest.requestBody).body); + // Check shape of entire policy + expect(entirePolicy).toMatchInlineSnapshot(` Object { "name": "my_policy", "phases": Object { "hot": Object { "actions": Object { + "rollover": Object { + "max_age": "30d", + "max_size": "50gb", + }, "set_priority": Object { "priority": 100, }, }, "min_age": "0ms", }, + "warm": Object { + "actions": Object { + "allocate": Object { + "number_of_replicas": 123, + "require": Object { + "test": "123", + }, + }, + "forcemerge": Object { + "index_codec": "best_compression", + "max_num_segments": 123, + }, + "set_priority": Object { + "priority": 123, + }, + "shrink": Object { + "number_of_shards": 123, + }, + }, + "min_age": "123d", + }, + }, + } + `); + }); + + test('default allocation with replicas set', async () => { + const { actions } = testBed; + await actions.warm.enable(true); + await actions.warm.setReplicas('123'); + await actions.savePolicy(); + const latestRequest = server.requests[server.requests.length - 1]; + const warmPhaseActions = JSON.parse(JSON.parse(latestRequest.requestBody).body).phases.warm + .actions; + expect(warmPhaseActions).toMatchInlineSnapshot(` + Object { + "allocate": Object { + "number_of_replicas": 123, + }, + "set_priority": Object { + "priority": 50, + }, + } + `); + }); + + test('setting warm phase on rollover to "true"', async () => { + const { actions } = testBed; + await actions.warm.enable(true); + await actions.warm.warmPhaseOnRollover(true); + await actions.savePolicy(); + const latestRequest = server.requests[server.requests.length - 1]; + const warmPhaseMinAge = JSON.parse(JSON.parse(latestRequest.requestBody).body).phases.warm + .min_age; + expect(warmPhaseMinAge).toBe(undefined); + }); + }); + + describe('policy with include and exclude', () => { + beforeEach(async () => { + httpRequestsMockHelpers.setLoadPolicies([POLICY_WITH_INCLUDE_EXCLUDE]); + httpRequestsMockHelpers.setListNodes({ + nodesByRoles: {}, + nodesByAttributes: { test: ['123'] }, + isUsingDeprecatedDataRoleConfig: false, + }); + httpRequestsMockHelpers.setLoadSnapshotPolicies([]); + + await act(async () => { + testBed = await setup(); + }); + + const { component } = testBed; + component.update(); + }); + + test('preserves include, exclude allocation settings', async () => { + const { actions } = testBed; + await actions.warm.setDataAllocation('node_attrs'); + await actions.savePolicy(); + const latestRequest = server.requests[server.requests.length - 1]; + const warmPhaseAllocate = JSON.parse(JSON.parse(latestRequest.requestBody).body).phases.warm + .actions.allocate; + expect(warmPhaseAllocate).toMatchInlineSnapshot(` + Object { + "exclude": Object { + "def": "456", + }, + "include": Object { + "abc": "123", }, } `); @@ -216,4 +383,57 @@ describe('', () => { expect(testBed.find('policiesErrorCallout').exists()).toBeTruthy(); }); }); + + describe('data allocation', () => { + describe('node roles', () => { + beforeEach(async () => { + httpRequestsMockHelpers.setLoadPolicies([POLICY_WITH_NODE_ROLE_ALLOCATION]); + httpRequestsMockHelpers.setListNodes({ + isUsingDeprecatedDataRoleConfig: false, + nodesByAttributes: { test: ['123'] }, + nodesByRoles: { data: ['123'] }, + }); + + await act(async () => { + testBed = await setup(); + }); + + const { component } = testBed; + component.update(); + }); + test('showing "default" type', () => { + const { find } = testBed; + expect(find('warm-dataTierAllocationControls.dataTierSelect').text()).toContain( + 'recommended' + ); + expect(find('warm-dataTierAllocationControls.dataTierSelect').text()).not.toContain( + 'Custom' + ); + expect(find('warm-dataTierAllocationControls.dataTierSelect').text()).not.toContain('Off'); + }); + }); + describe('node attr and none', () => { + beforeEach(async () => { + httpRequestsMockHelpers.setLoadPolicies([POLICY_WITH_NODE_ATTR_AND_OFF_ALLOCATION]); + httpRequestsMockHelpers.setListNodes({ + isUsingDeprecatedDataRoleConfig: false, + nodesByAttributes: { test: ['123'] }, + nodesByRoles: { data: ['123'] }, + }); + + await act(async () => { + testBed = await setup(); + }); + + const { component } = testBed; + component.update(); + }); + + test('showing "custom" and "off" types', () => { + const { find } = testBed; + expect(find('warm-dataTierAllocationControls.dataTierSelect').text()).toContain('Custom'); + expect(find('cold-dataTierAllocationControls.dataTierSelect').text()).toContain('Off'); + }); + }); + }); }); diff --git a/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/helpers/http_requests.ts b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/helpers/http_requests.ts index 04f58f93939ca..c7a493ce80d96 100644 --- a/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/helpers/http_requests.ts +++ b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/helpers/http_requests.ts @@ -6,6 +6,7 @@ import { fakeServer, SinonFakeServer } from 'sinon'; import { API_BASE_PATH } from '../../../common/constants'; +import { ListNodesRouteResponse } from '../../../common/types'; export const init = () => { const server = fakeServer.create(); @@ -38,8 +39,17 @@ const registerHttpRequestMockHelpers = (server: SinonFakeServer) => { ]); }; + const setListNodes = (body: ListNodesRouteResponse) => { + server.respondWith('GET', `${API_BASE_PATH}/nodes/list`, [ + 200, + { 'Content-Type': 'application/json' }, + JSON.stringify(body), + ]); + }; + return { setLoadPolicies, setLoadSnapshotPolicies, + setListNodes, }; }; diff --git a/x-pack/plugins/index_lifecycle_management/__jest__/components/edit_policy.test.tsx b/x-pack/plugins/index_lifecycle_management/__jest__/components/edit_policy.test.tsx index c7664f2d837b9..4ba6cee7b027f 100644 --- a/x-pack/plugins/index_lifecycle_management/__jest__/components/edit_policy.test.tsx +++ b/x-pack/plugins/index_lifecycle_management/__jest__/components/edit_policy.test.tsx @@ -30,9 +30,7 @@ import { init as initUiMetric } from '../../public/application/services/ui_metri import { init as initNotification } from '../../public/application/services/notification'; import { PolicyFromES } from '../../common/types'; import { - positiveNumbersAboveZeroErrorMessage, positiveNumberRequiredMessage, - numberRequiredMessage, policyNameRequiredMessage, policyNameStartsWithUnderscoreErrorMessage, policyNameContainsCommaErrorMessage, @@ -40,6 +38,8 @@ import { policyNameMustBeDifferentErrorMessage, policyNameAlreadyUsedErrorMessage, } from '../../public/application/services/policies/policy_validation'; + +import { i18nTexts } from '../../public/application/sections/edit_policy/i18n_texts'; import { editPolicyHelpers } from './helpers'; // @ts-ignore @@ -89,13 +89,13 @@ const activatePhase = async (rendered: ReactWrapper, phase: string) => { }); rendered.update(); }; -const openNodeAttributesSection = (rendered: ReactWrapper, phase: string) => { +const openNodeAttributesSection = async (rendered: ReactWrapper, phase: string) => { const getControls = () => findTestSubject(rendered, `${phase}-dataTierAllocationControls`); - act(() => { + await act(async () => { findTestSubject(getControls(), 'dataTierSelect').simulate('click'); }); rendered.update(); - act(() => { + await act(async () => { findTestSubject(getControls(), 'customDataAllocationOption').simulate('click'); }); rendered.update(); @@ -119,19 +119,29 @@ const noRollover = async (rendered: ReactWrapper) => { }); rendered.update(); }; -const getNodeAttributeSelect = (rendered: ReactWrapper, phase: string) => { +const getNodeAttributeSelectLegacy = (rendered: ReactWrapper, phase: string) => { return rendered.find(`select#${phase}-selectedNodeAttrs`); }; +const getNodeAttributeSelect = (rendered: ReactWrapper, phase: string) => { + return findTestSubject(rendered, `${phase}-selectedNodeAttrs`); +}; const setPolicyName = (rendered: ReactWrapper, policyName: string) => { const policyNameField = findTestSubject(rendered, 'policyNameField'); policyNameField.simulate('change', { target: { value: policyName } }); rendered.update(); }; -const setPhaseAfter = (rendered: ReactWrapper, phase: string, after: string | number) => { +const setPhaseAfterLegacy = (rendered: ReactWrapper, phase: string, after: string | number) => { const afterInput = rendered.find(`input#${phase}-selectedMinimumAge`); afterInput.simulate('change', { target: { value: after } }); rendered.update(); }; +const setPhaseAfter = async (rendered: ReactWrapper, phase: string, after: string | number) => { + const afterInput = findTestSubject(rendered, `${phase}-selectedMinimumAge`); + await act(async () => { + afterInput.simulate('change', { target: { value: after } }); + }); + rendered.update(); +}; const setPhaseIndexPriorityLegacy = ( rendered: ReactWrapper, phase: string, @@ -172,10 +182,11 @@ describe('edit policy', () => { * any validation errors. This helper advances timers and can trigger component * state changes. */ - const waitForFormLibValidation = () => { + const waitForFormLibValidation = (rendered: ReactWrapper) => { act(() => { jest.advanceTimersByTime(1000); }); + rendered.update(); }; beforeEach(() => { @@ -288,13 +299,12 @@ describe('edit policy', () => { await act(async () => { maxSizeInput.simulate('change', { target: { value: '' } }); }); - waitForFormLibValidation(); + waitForFormLibValidation(rendered); const maxAgeInput = findTestSubject(rendered, 'hot-selectedMaxAge'); await act(async () => { maxAgeInput.simulate('change', { target: { value: '' } }); }); - waitForFormLibValidation(); - rendered.update(); + waitForFormLibValidation(rendered); await save(rendered); expect(findTestSubject(rendered, 'rolloverSettingsRequired').exists()).toBeTruthy(); }); @@ -305,9 +315,9 @@ describe('edit policy', () => { await act(async () => { maxSizeInput.simulate('change', { target: { value: '-1' } }); }); - waitForFormLibValidation(); + waitForFormLibValidation(rendered); rendered.update(); - expectedErrorMessages(rendered, [positiveNumbersAboveZeroErrorMessage]); + expectedErrorMessages(rendered, [i18nTexts.editPolicy.errors.numberGreatThan0Required]); }); test('should show number above 0 required error when trying to save with 0 for max size', async () => { const rendered = mountWithIntl(component); @@ -316,9 +326,8 @@ describe('edit policy', () => { await act(async () => { maxSizeInput.simulate('change', { target: { value: '-1' } }); }); - waitForFormLibValidation(); - rendered.update(); - expectedErrorMessages(rendered, [positiveNumbersAboveZeroErrorMessage]); + waitForFormLibValidation(rendered); + expectedErrorMessages(rendered, [i18nTexts.editPolicy.errors.numberGreatThan0Required]); }); test('should show number above 0 required error when trying to save with -1 for max age', async () => { const rendered = mountWithIntl(component); @@ -327,9 +336,8 @@ describe('edit policy', () => { await act(async () => { maxAgeInput.simulate('change', { target: { value: '-1' } }); }); - waitForFormLibValidation(); - rendered.update(); - expectedErrorMessages(rendered, [positiveNumbersAboveZeroErrorMessage]); + waitForFormLibValidation(rendered); + expectedErrorMessages(rendered, [i18nTexts.editPolicy.errors.numberGreatThan0Required]); }); test('should show number above 0 required error when trying to save with 0 for max age', async () => { const rendered = mountWithIntl(component); @@ -338,9 +346,8 @@ describe('edit policy', () => { await act(async () => { maxAgeInput.simulate('change', { target: { value: '0' } }); }); - waitForFormLibValidation(); - rendered.update(); - expectedErrorMessages(rendered, [positiveNumbersAboveZeroErrorMessage]); + waitForFormLibValidation(rendered); + expectedErrorMessages(rendered, [i18nTexts.editPolicy.errors.numberGreatThan0Required]); }); test('should show forcemerge input when rollover enabled', () => { const rendered = mountWithIntl(component); @@ -351,8 +358,7 @@ describe('edit policy', () => { const rendered = mountWithIntl(component); setPolicyName(rendered, 'mypolicy'); await noRollover(rendered); - waitForFormLibValidation(); - rendered.update(); + waitForFormLibValidation(rendered); expect(findTestSubject(rendered, 'hot-forceMergeSwitch').exists()).toBeFalsy(); }); test('should show positive number required above zero error when trying to save hot phase with 0 for force merge', async () => { @@ -366,9 +372,8 @@ describe('edit policy', () => { await act(async () => { forcemergeInput.simulate('change', { target: { value: '0' } }); }); - waitForFormLibValidation(); - rendered.update(); - expectedErrorMessages(rendered, [positiveNumbersAboveZeroErrorMessage]); + waitForFormLibValidation(rendered); + expectedErrorMessages(rendered, [i18nTexts.editPolicy.errors.numberGreatThan0Required]); }); test('should show positive number above 0 required error when trying to save hot phase with -1 for force merge', async () => { const rendered = mountWithIntl(component); @@ -379,19 +384,17 @@ describe('edit policy', () => { await act(async () => { forcemergeInput.simulate('change', { target: { value: '-1' } }); }); - waitForFormLibValidation(); - rendered.update(); + waitForFormLibValidation(rendered); await save(rendered); - expectedErrorMessages(rendered, [positiveNumbersAboveZeroErrorMessage]); + expectedErrorMessages(rendered, [i18nTexts.editPolicy.errors.numberGreatThan0Required]); }); test('should show positive number required error when trying to save with -1 for index priority', async () => { const rendered = mountWithIntl(component); await noRollover(rendered); setPolicyName(rendered, 'mypolicy'); await setPhaseIndexPriority(rendered, 'hot', '-1'); - waitForFormLibValidation(); - rendered.update(); - expectedErrorMessages(rendered, [positiveNumbersAboveZeroErrorMessage]); + waitForFormLibValidation(rendered); + expectedErrorMessages(rendered, [i18nTexts.editPolicy.errors.numberGreatThan0Required]); }); }); describe('warm phase', () => { @@ -408,17 +411,17 @@ describe('edit policy', () => { await noRollover(rendered); setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'warm'); - setPhaseAfter(rendered, 'warm', ''); - await save(rendered); - expectedErrorMessages(rendered, [numberRequiredMessage]); + await setPhaseAfter(rendered, 'warm', ''); + waitForFormLibValidation(rendered); + expectedErrorMessages(rendered, [i18nTexts.editPolicy.errors.nonNegativeNumberRequired]); }); test('should allow 0 for phase timing', async () => { const rendered = mountWithIntl(component); await noRollover(rendered); setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'warm'); - setPhaseAfter(rendered, 'warm', '0'); - await save(rendered); + await setPhaseAfter(rendered, 'warm', '0'); + waitForFormLibValidation(rendered); expectedErrorMessages(rendered, []); }); test('should show positive number required error when trying to save warm phase with -1 for after', async () => { @@ -426,75 +429,87 @@ describe('edit policy', () => { await noRollover(rendered); setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'warm'); - setPhaseAfter(rendered, 'warm', '-1'); - await save(rendered); - expectedErrorMessages(rendered, [positiveNumberRequiredMessage]); + await setPhaseAfter(rendered, 'warm', '-1'); + waitForFormLibValidation(rendered); + expectedErrorMessages(rendered, [i18nTexts.editPolicy.errors.nonNegativeNumberRequired]); }); test('should show positive number required error when trying to save warm phase with -1 for index priority', async () => { const rendered = mountWithIntl(component); await noRollover(rendered); setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'warm'); - setPhaseAfter(rendered, 'warm', '1'); - setPhaseIndexPriorityLegacy(rendered, 'warm', '-1'); - await save(rendered); - expectedErrorMessages(rendered, [positiveNumberRequiredMessage]); + await setPhaseAfter(rendered, 'warm', '1'); + await setPhaseAfter(rendered, 'warm', '-1'); + waitForFormLibValidation(rendered); + expectedErrorMessages(rendered, [i18nTexts.editPolicy.errors.nonNegativeNumberRequired]); }); test('should show positive number required above zero error when trying to save warm phase with 0 for shrink', async () => { const rendered = mountWithIntl(component); await noRollover(rendered); setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'warm'); - findTestSubject(rendered, 'shrinkSwitch').simulate('click'); - rendered.update(); - setPhaseAfter(rendered, 'warm', '1'); - const shrinkInput = rendered.find('input#warm-selectedPrimaryShardCount'); - shrinkInput.simulate('change', { target: { value: '0' } }); + act(() => { + findTestSubject(rendered, 'shrinkSwitch').simulate('click'); + }); rendered.update(); - await save(rendered); - expectedErrorMessages(rendered, [positiveNumbersAboveZeroErrorMessage]); + await setPhaseAfter(rendered, 'warm', '1'); + const shrinkInput = findTestSubject(rendered, 'warm-selectedPrimaryShardCount'); + await act(async () => { + shrinkInput.simulate('change', { target: { value: '0' } }); + }); + waitForFormLibValidation(rendered); + expectedErrorMessages(rendered, [i18nTexts.editPolicy.errors.numberGreatThan0Required]); }); test('should show positive number above 0 required error when trying to save warm phase with -1 for shrink', async () => { const rendered = mountWithIntl(component); await noRollover(rendered); setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'warm'); - setPhaseAfter(rendered, 'warm', '1'); - findTestSubject(rendered, 'shrinkSwitch').simulate('click'); - rendered.update(); - const shrinkInput = rendered.find('input#warm-selectedPrimaryShardCount'); - shrinkInput.simulate('change', { target: { value: '-1' } }); + await setPhaseAfter(rendered, 'warm', '1'); + act(() => { + findTestSubject(rendered, 'shrinkSwitch').simulate('click'); + }); rendered.update(); - await save(rendered); - expectedErrorMessages(rendered, [positiveNumbersAboveZeroErrorMessage]); + const shrinkInput = findTestSubject(rendered, 'warm-selectedPrimaryShardCount'); + await act(async () => { + shrinkInput.simulate('change', { target: { value: '-1' } }); + }); + waitForFormLibValidation(rendered); + expectedErrorMessages(rendered, [i18nTexts.editPolicy.errors.numberGreatThan0Required]); }); test('should show positive number required above zero error when trying to save warm phase with 0 for force merge', async () => { const rendered = mountWithIntl(component); await noRollover(rendered); setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'warm'); - setPhaseAfter(rendered, 'warm', '1'); - findTestSubject(rendered, 'warm-forceMergeSwitch').simulate('click'); + await setPhaseAfter(rendered, 'warm', '1'); + act(() => { + findTestSubject(rendered, 'warm-forceMergeSwitch').simulate('click'); + }); rendered.update(); const forcemergeInput = findTestSubject(rendered, 'warm-selectedForceMergeSegments'); - forcemergeInput.simulate('change', { target: { value: '0' } }); - rendered.update(); - await save(rendered); - expectedErrorMessages(rendered, [positiveNumbersAboveZeroErrorMessage]); + await act(async () => { + forcemergeInput.simulate('change', { target: { value: '0' } }); + }); + waitForFormLibValidation(rendered); + expectedErrorMessages(rendered, [i18nTexts.editPolicy.errors.numberGreatThan0Required]); }); test('should show positive number above 0 required error when trying to save warm phase with -1 for force merge', async () => { const rendered = mountWithIntl(component); await noRollover(rendered); setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'warm'); - setPhaseAfter(rendered, 'warm', '1'); - findTestSubject(rendered, 'warm-forceMergeSwitch').simulate('click'); + await setPhaseAfter(rendered, 'warm', '1'); + await act(async () => { + findTestSubject(rendered, 'warm-forceMergeSwitch').simulate('click'); + }); rendered.update(); const forcemergeInput = findTestSubject(rendered, 'warm-selectedForceMergeSegments'); - forcemergeInput.simulate('change', { target: { value: '-1' } }); - rendered.update(); - await save(rendered); - expectedErrorMessages(rendered, [positiveNumbersAboveZeroErrorMessage]); + await act(async () => { + forcemergeInput.simulate('change', { target: { value: '-1' } }); + }); + waitForFormLibValidation(rendered); + expectedErrorMessages(rendered, [i18nTexts.editPolicy.errors.numberGreatThan0Required]); }); test('should show spinner for node attributes input when loading', async () => { server.respondImmediately = false; @@ -504,7 +519,7 @@ describe('edit policy', () => { await activatePhase(rendered, 'warm'); expect(rendered.find('.euiLoadingSpinner').exists()).toBeTruthy(); expect(rendered.find('.euiCallOut--warning').exists()).toBeFalsy(); - expect(getNodeAttributeSelect(rendered, 'warm').exists()).toBeFalsy(); + expect(getNodeAttributeSelectLegacy(rendered, 'warm').exists()).toBeFalsy(); }); test('should show warning instead of node attributes input when none exist', async () => { http.setupNodeListResponse({ @@ -517,9 +532,9 @@ describe('edit policy', () => { setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'warm'); expect(rendered.find('.euiLoadingSpinner').exists()).toBeFalsy(); - openNodeAttributesSection(rendered, 'warm'); + await openNodeAttributesSection(rendered, 'warm'); expect(findTestSubject(rendered, 'noNodeAttributesWarning').exists()).toBeTruthy(); - expect(getNodeAttributeSelect(rendered, 'warm').exists()).toBeFalsy(); + expect(getNodeAttributeSelectLegacy(rendered, 'warm').exists()).toBeFalsy(); }); test('should show node attributes input when attributes exist', async () => { const rendered = mountWithIntl(component); @@ -527,7 +542,7 @@ describe('edit policy', () => { setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'warm'); expect(rendered.find('.euiLoadingSpinner').exists()).toBeFalsy(); - openNodeAttributesSection(rendered, 'warm'); + await openNodeAttributesSection(rendered, 'warm'); expect(findTestSubject(rendered, 'noNodeAttributesWarning').exists()).toBeFalsy(); const nodeAttributesSelect = getNodeAttributeSelect(rendered, 'warm'); expect(nodeAttributesSelect.exists()).toBeTruthy(); @@ -539,13 +554,15 @@ describe('edit policy', () => { setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'warm'); expect(rendered.find('.euiLoadingSpinner').exists()).toBeFalsy(); - openNodeAttributesSection(rendered, 'warm'); + await openNodeAttributesSection(rendered, 'warm'); expect(findTestSubject(rendered, 'noNodeAttributesWarning').exists()).toBeFalsy(); const nodeAttributesSelect = getNodeAttributeSelect(rendered, 'warm'); expect(nodeAttributesSelect.exists()).toBeTruthy(); expect(findTestSubject(rendered, 'warm-viewNodeDetailsFlyoutButton').exists()).toBeFalsy(); expect(nodeAttributesSelect.find('option').length).toBe(2); - nodeAttributesSelect.simulate('change', { target: { value: 'attribute:true' } }); + await act(async () => { + nodeAttributesSelect.simulate('change', { target: { value: 'attribute:true' } }); + }); rendered.update(); const flyoutButton = findTestSubject(rendered, 'warm-viewNodeDetailsFlyoutButton'); expect(flyoutButton.exists()).toBeTruthy(); @@ -608,7 +625,7 @@ describe('edit policy', () => { await noRollover(rendered); setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'cold'); - setPhaseAfter(rendered, 'cold', '0'); + setPhaseAfterLegacy(rendered, 'cold', '0'); await save(rendered); expectedErrorMessages(rendered, []); }); @@ -617,7 +634,7 @@ describe('edit policy', () => { await noRollover(rendered); setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'cold'); - setPhaseAfter(rendered, 'cold', '-1'); + setPhaseAfterLegacy(rendered, 'cold', '-1'); await save(rendered); expectedErrorMessages(rendered, [positiveNumberRequiredMessage]); }); @@ -629,7 +646,7 @@ describe('edit policy', () => { await activatePhase(rendered, 'cold'); expect(rendered.find('.euiLoadingSpinner').exists()).toBeTruthy(); expect(rendered.find('.euiCallOut--warning').exists()).toBeFalsy(); - expect(getNodeAttributeSelect(rendered, 'cold').exists()).toBeFalsy(); + expect(getNodeAttributeSelectLegacy(rendered, 'cold').exists()).toBeFalsy(); }); test('should show warning instead of node attributes input when none exist', async () => { http.setupNodeListResponse({ @@ -642,9 +659,9 @@ describe('edit policy', () => { setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'cold'); expect(rendered.find('.euiLoadingSpinner').exists()).toBeFalsy(); - openNodeAttributesSection(rendered, 'cold'); + await openNodeAttributesSection(rendered, 'cold'); expect(findTestSubject(rendered, 'noNodeAttributesWarning').exists()).toBeTruthy(); - expect(getNodeAttributeSelect(rendered, 'cold').exists()).toBeFalsy(); + expect(getNodeAttributeSelectLegacy(rendered, 'cold').exists()).toBeFalsy(); }); test('should show node attributes input when attributes exist', async () => { const rendered = mountWithIntl(component); @@ -652,9 +669,9 @@ describe('edit policy', () => { setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'cold'); expect(rendered.find('.euiLoadingSpinner').exists()).toBeFalsy(); - openNodeAttributesSection(rendered, 'cold'); + await openNodeAttributesSection(rendered, 'cold'); expect(findTestSubject(rendered, 'noNodeAttributesWarning').exists()).toBeFalsy(); - const nodeAttributesSelect = getNodeAttributeSelect(rendered, 'cold'); + const nodeAttributesSelect = getNodeAttributeSelectLegacy(rendered, 'cold'); expect(nodeAttributesSelect.exists()).toBeTruthy(); expect(nodeAttributesSelect.find('option').length).toBe(2); }); @@ -664,9 +681,9 @@ describe('edit policy', () => { setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'cold'); expect(rendered.find('.euiLoadingSpinner').exists()).toBeFalsy(); - openNodeAttributesSection(rendered, 'cold'); + await openNodeAttributesSection(rendered, 'cold'); expect(findTestSubject(rendered, 'noNodeAttributesWarning').exists()).toBeFalsy(); - const nodeAttributesSelect = getNodeAttributeSelect(rendered, 'cold'); + const nodeAttributesSelect = getNodeAttributeSelectLegacy(rendered, 'cold'); expect(nodeAttributesSelect.exists()).toBeTruthy(); expect(findTestSubject(rendered, 'cold-viewNodeDetailsFlyoutButton').exists()).toBeFalsy(); expect(nodeAttributesSelect.find('option').length).toBe(2); @@ -685,7 +702,7 @@ describe('edit policy', () => { await noRollover(rendered); setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'cold'); - setPhaseAfter(rendered, 'cold', '1'); + setPhaseAfterLegacy(rendered, 'cold', '1'); setPhaseIndexPriorityLegacy(rendered, 'cold', '-1'); await save(rendered); expectedErrorMessages(rendered, [positiveNumberRequiredMessage]); @@ -736,7 +753,7 @@ describe('edit policy', () => { await noRollover(rendered); setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'delete'); - setPhaseAfter(rendered, 'delete', '0'); + setPhaseAfterLegacy(rendered, 'delete', '0'); await save(rendered); expectedErrorMessages(rendered, []); }); @@ -745,7 +762,7 @@ describe('edit policy', () => { await noRollover(rendered); setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'delete'); - setPhaseAfter(rendered, 'delete', '-1'); + setPhaseAfterLegacy(rendered, 'delete', '-1'); await save(rendered); expectedErrorMessages(rendered, [positiveNumberRequiredMessage]); }); @@ -796,7 +813,8 @@ describe('edit policy', () => { test('should hide data tier option on cloud using legacy node role configuration', async () => { http.setupNodeListResponse({ nodesByAttributes: { test: ['123'] }, - nodesByRoles: { data: ['test'], data_hot: ['test'], data_warm: ['test'] }, + // On cloud, if using legacy config there will not be any "data_*" roles set. + nodesByRoles: { data: ['test'] }, isUsingDeprecatedDataRoleConfig: true, }); const rendered = mountWithIntl(component); diff --git a/x-pack/plugins/index_lifecycle_management/common/constants/data_tiers.ts b/x-pack/plugins/index_lifecycle_management/common/constants/data_tiers.ts index 8a1acf72949e6..f6b9506f9068c 100644 --- a/x-pack/plugins/index_lifecycle_management/common/constants/data_tiers.ts +++ b/x-pack/plugins/index_lifecycle_management/common/constants/data_tiers.ts @@ -6,13 +6,13 @@ // Order of node roles matters here, the warm phase prefers allocating data // to the data_warm role. -import { NodeDataRole, PhaseWithAllocation } from '../types'; +import { DataTierRole, PhaseWithAllocation } from '../types'; -const WARM_PHASE_NODE_PREFERENCE: NodeDataRole[] = ['data_warm', 'data_hot']; +const WARM_PHASE_NODE_PREFERENCE: DataTierRole[] = ['data_warm', 'data_hot']; -const COLD_PHASE_NODE_PREFERENCE: NodeDataRole[] = ['data_cold', 'data_warm', 'data_hot']; +const COLD_PHASE_NODE_PREFERENCE: DataTierRole[] = ['data_cold', 'data_warm', 'data_hot']; -export const phaseToNodePreferenceMap: Record = Object.freeze({ +export const phaseToNodePreferenceMap: Record = Object.freeze({ warm: WARM_PHASE_NODE_PREFERENCE, cold: COLD_PHASE_NODE_PREFERENCE, }); diff --git a/x-pack/plugins/index_lifecycle_management/common/types/api.ts b/x-pack/plugins/index_lifecycle_management/common/types/api.ts index ccdd7fcb11778..b7ca16ac46dde 100644 --- a/x-pack/plugins/index_lifecycle_management/common/types/api.ts +++ b/x-pack/plugins/index_lifecycle_management/common/types/api.ts @@ -4,11 +4,11 @@ * you may not use this file except in compliance with the Elastic License. */ -import { NodeDataRoleWithCatchAll } from '.'; +import { AnyDataRole } from '.'; export interface ListNodesRouteResponse { nodesByAttributes: { [attributePair: string]: string[] }; - nodesByRoles: { [role in NodeDataRoleWithCatchAll]?: string[] }; + nodesByRoles: { [role in AnyDataRole]?: string[] }; /** * A flag to indicate whether a node is using `settings.node.data` which is the now deprecated way cloud configured diff --git a/x-pack/plugins/index_lifecycle_management/common/types/index.ts b/x-pack/plugins/index_lifecycle_management/common/types/index.ts index 1f41370e48f18..737e5a551aae8 100644 --- a/x-pack/plugins/index_lifecycle_management/common/types/index.ts +++ b/x-pack/plugins/index_lifecycle_management/common/types/index.ts @@ -9,8 +9,17 @@ export * from './api'; export * from './policies'; /** - * These roles reflect how nodes are stratified into different data tiers. The "data" role - * is a catch-all that can be used to store data in any phase. + * These roles reflect how nodes are stratified into different data tiers. */ -export type NodeDataRole = 'data_hot' | 'data_warm' | 'data_cold'; -export type NodeDataRoleWithCatchAll = 'data' | NodeDataRole; +export type DataTierRole = 'data_hot' | 'data_warm' | 'data_cold'; + +/** + * The "data_content" role can store all data the ES stack uses for feature + * functionality like security-related indices. + */ +export type DataRole = 'data_content' | DataTierRole; + +/** + * The "data" role can store data allocated to any tier. + */ +export type AnyDataRole = 'data' | DataRole; diff --git a/x-pack/plugins/index_lifecycle_management/common/types/policies.ts b/x-pack/plugins/index_lifecycle_management/common/types/policies.ts index 152c5e4e9e0d7..813fcd9c253f1 100644 --- a/x-pack/plugins/index_lifecycle_management/common/types/policies.ts +++ b/x-pack/plugins/index_lifecycle_management/common/types/policies.ts @@ -35,6 +35,19 @@ export interface SerializedPhase { }; } +export interface MigrateAction { + /** + * If enabled is ever set it will probably only be set to `false` because the default value + * for this is `true`. Rather leave unspecified for true when serialising. + */ + enabled: boolean; +} + +export interface SerializedActionWithAllocation { + allocate?: AllocateAction; + migrate?: MigrateAction; +} + export interface SerializedHotPhase extends SerializedPhase { actions: { rollover?: { @@ -59,7 +72,7 @@ export interface SerializedWarmPhase extends SerializedPhase { set_priority?: { priority: number | null; }; - migrate?: { enabled: boolean }; + migrate?: MigrateAction; }; } @@ -70,7 +83,7 @@ export interface SerializedColdPhase extends SerializedPhase { set_priority?: { priority: number | null; }; - migrate?: { enabled: boolean }; + migrate?: MigrateAction; }; } @@ -87,18 +100,11 @@ export interface SerializedDeletePhase extends SerializedPhase { export interface AllocateAction { number_of_replicas?: number; - include: {}; - exclude: {}; + include?: {}; + exclude?: {}; require?: { [attribute: string]: string; }; - migrate?: { - /** - * If enabled is ever set it will only be set to `false` because the default value - * for this is `true`. Rather leave unspecified for true. - */ - enabled: false; - }; } export interface ForcemergeAction { @@ -110,7 +116,6 @@ export interface ForcemergeAction { export interface LegacyPolicy { name: string; phases: { - warm: WarmPhase; cold: ColdPhase; delete: DeletePhase; }; @@ -154,17 +159,6 @@ export interface PhaseWithForcemergeAction { bestCompressionEnabled: boolean; } -export interface WarmPhase - extends CommonPhaseSettings, - PhaseWithMinAge, - PhaseWithAllocationAction, - PhaseWithIndexPriority, - PhaseWithForcemergeAction { - warmPhaseOnRollover: boolean; - shrinkEnabled: boolean; - selectedPrimaryShardCount: string; -} - export interface ColdPhase extends CommonPhaseSettings, PhaseWithMinAge, diff --git a/x-pack/plugins/index_lifecycle_management/public/application/constants/policy.ts b/x-pack/plugins/index_lifecycle_management/public/application/constants/policy.ts index c919331082ec3..136b68727672a 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/constants/policy.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/constants/policy.ts @@ -4,16 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -import { - SerializedPhase, - ColdPhase, - DeletePhase, - WarmPhase, - SerializedPolicy, -} from '../../../common/types'; +import { SerializedPhase, ColdPhase, DeletePhase, SerializedPolicy } from '../../../common/types'; export const defaultSetPriority: string = '100'; +export const defaultPhaseIndexPriority: string = '50'; + export const defaultPolicy: SerializedPolicy = { name: '', phases: { @@ -28,22 +24,6 @@ export const defaultPolicy: SerializedPolicy = { }, }; -export const defaultNewWarmPhase: WarmPhase = { - phaseEnabled: false, - forceMergeEnabled: false, - selectedForceMergeSegments: '', - bestCompressionEnabled: false, - selectedMinimumAge: '0', - selectedMinimumAgeUnits: 'd', - selectedNodeAttrs: '', - shrinkEnabled: false, - selectedPrimaryShardCount: '', - selectedReplicaCount: '', - warmPhaseOnRollover: true, - phaseIndexPriority: '50', - dataTierAllocationType: 'default', -}; - export const defaultNewColdPhase: ColdPhase = { phaseEnabled: false, selectedMinimumAge: '0', diff --git a/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/determine_allocation_type.ts b/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/determine_allocation_type.ts index 4067ad97fc43b..20ac439e9964f 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/determine_allocation_type.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/determine_allocation_type.ts @@ -4,31 +4,63 @@ * you may not use this file except in compliance with the Elastic License. */ -import { DataTierAllocationType, AllocateAction } from '../../../../common/types'; +import { DataTierAllocationType, AllocateAction, MigrateAction } from '../../../../common/types'; /** * Determine what deserialized state the policy config represents. * * See {@DataTierAllocationType} for more information. */ -export const determineDataTierAllocationType = ( - allocateAction?: AllocateAction +export const determineDataTierAllocationTypeLegacy = ( + actions: { + allocate?: AllocateAction; + migrate?: MigrateAction; + } = {} ): DataTierAllocationType => { - if (!allocateAction) { - return 'default'; - } + const { allocate, migrate } = actions; - if (allocateAction.migrate?.enabled === false) { + if (migrate?.enabled === false) { return 'none'; } + if (!allocate) { + return 'default'; + } + if ( - (allocateAction.require && Object.keys(allocateAction.require).length) || - (allocateAction.include && Object.keys(allocateAction.include).length) || - (allocateAction.exclude && Object.keys(allocateAction.exclude).length) + (allocate.require && Object.keys(allocate.require).length) || + (allocate.include && Object.keys(allocate.include).length) || + (allocate.exclude && Object.keys(allocate.exclude).length) ) { return 'custom'; } return 'default'; }; + +export const determineDataTierAllocationType = ( + actions: { + allocate?: AllocateAction; + migrate?: MigrateAction; + } = {} +) => { + const { allocate, migrate } = actions; + + if (migrate?.enabled === false) { + return 'none'; + } + + if (!allocate) { + return 'node_roles'; + } + + if ( + (allocate.require && Object.keys(allocate.require).length) || + (allocate.include && Object.keys(allocate.include).length) || + (allocate.exclude && Object.keys(allocate.exclude).length) + ) { + return 'node_attrs'; + } + + return 'node_roles'; +}; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/get_available_node_roles_for_phase.ts b/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/get_available_node_roles_for_phase.ts index 6daae57330886..179de0b1d8c8e 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/get_available_node_roles_for_phase.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/get_available_node_roles_for_phase.ts @@ -5,14 +5,14 @@ */ import { - NodeDataRole, + DataTierRole, ListNodesRouteResponse, PhaseWithAllocation, } from '../../../../common/types'; import { phaseToNodePreferenceMap } from '../../../../common/constants'; -export type AllocationNodeRole = NodeDataRole | 'none'; +export type AllocationNodeRole = DataTierRole | 'none'; /** * Given a phase and current cluster node roles, determine which nodes the phase diff --git a/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/index.ts b/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/index.ts index 87f2cbc08ecc0..ac1aae270628d 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/index.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/index.ts @@ -7,3 +7,5 @@ export * from './determine_allocation_type'; export * from './get_available_node_roles_for_phase'; + +export * from './is_node_role_first_preference'; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/is_node_role_first_preference.ts b/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/is_node_role_first_preference.ts index 872efa740b131..fff640a5eb957 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/is_node_role_first_preference.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/is_node_role_first_preference.ts @@ -4,9 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -import { NodeDataRole, PhaseWithAllocation } from '../../../../common/types'; +import { DataTierRole, PhaseWithAllocation } from '../../../../common/types'; import { phaseToNodePreferenceMap } from '../../../../common/constants'; -export const isNodeRoleFirstPreference = (phase: PhaseWithAllocation, nodeRole: NodeDataRole) => { +export const isNodeRoleFirstPreference = (phase: PhaseWithAllocation, nodeRole: DataTierRole) => { return phaseToNodePreferenceMap[phase][0] === nodeRole; }; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/data_tier_allocation/default_allocation_notice.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/data_tier_allocation/default_allocation_notice.tsx index 42f9e8494a0b3..3d0052c69607b 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/data_tier_allocation/default_allocation_notice.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/data_tier_allocation/default_allocation_notice.tsx @@ -8,11 +8,11 @@ import { i18n } from '@kbn/i18n'; import React, { FunctionComponent } from 'react'; import { EuiCallOut } from '@elastic/eui'; -import { PhaseWithAllocation, NodeDataRole } from '../../../../../../common/types'; +import { PhaseWithAllocation, DataTierRole } from '../../../../../../common/types'; import { AllocationNodeRole } from '../../../../lib'; -const i18nTextsNodeRoleToDataTier: Record = { +const i18nTextsNodeRoleToDataTier: Record = { data_hot: i18n.translate('xpack.indexLifecycleMgmt.editPolicy.dataTierHotLabel', { defaultMessage: 'hot', }), @@ -31,7 +31,7 @@ const i18nTexts = { 'xpack.indexLifecycleMgmt.warmPhase.dataTier.defaultAllocationNotice.warm.title', { defaultMessage: 'No nodes assigned to the warm tier' } ), - body: (nodeRole: NodeDataRole) => + body: (nodeRole: DataTierRole) => i18n.translate('xpack.indexLifecycleMgmt.warmPhase.dataTier.defaultAllocationNotice.warm', { defaultMessage: 'This policy will move data in the warm phase to {tier} tier nodes instead.', @@ -43,7 +43,7 @@ const i18nTexts = { 'xpack.indexLifecycleMgmt.warmPhase.dataTier.defaultAllocationNotice.cold.title', { defaultMessage: 'No nodes assigned to the cold tier' } ), - body: (nodeRole: NodeDataRole) => + body: (nodeRole: DataTierRole) => i18n.translate('xpack.indexLifecycleMgmt.warmPhase.dataTier.defaultAllocationNotice.cold', { defaultMessage: 'This policy will move data in the cold phase to {tier} tier nodes instead.', diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/forcemerge.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/forcemerge_legacy.tsx similarity index 100% rename from x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/forcemerge.tsx rename to x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/forcemerge_legacy.tsx diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/index.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/index.ts index 04d9a6ef3cf67..2b774b00b98a9 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/index.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/index.ts @@ -7,11 +7,11 @@ export { ActiveBadge } from './active_badge'; export { ErrableFormRow } from './form_errors'; export { LearnMoreLink } from './learn_more_link'; -export { MinAgeInput } from './min_age_input'; +export { MinAgeInput } from './min_age_input_legacy'; export { OptionalLabel } from './optional_label'; export { PhaseErrorMessage } from './phase_error_message'; export { PolicyJsonFlyout } from './policy_json_flyout'; -export { SetPriorityInput } from './set_priority_input'; +export { SetPriorityInput } from './set_priority_input_legacy'; export { SnapshotPolicies } from './snapshot_policies'; export { DataTierAllocation, @@ -21,6 +21,6 @@ export { DefaultAllocationNotice, } from './data_tier_allocation'; export { DescribedFormField } from './described_form_field'; -export { Forcemerge } from './forcemerge'; +export { Forcemerge } from './forcemerge_legacy'; export * from './phases'; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/min_age_input.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/min_age_input_legacy.tsx similarity index 100% rename from x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/min_age_input.tsx rename to x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/min_age_input_legacy.tsx diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/cold_phase.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/cold_phase.tsx index 7a22fb5bc1f0c..da6c358aa67c1 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/cold_phase.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/cold_phase.tsx @@ -28,7 +28,7 @@ import { DescribedFormField, } from '../'; -import { DataTierAllocationField, useRolloverPath } from './shared'; +import { DataTierAllocationFieldLegacy, useRolloverPath } from './shared'; const i18nTexts = { freezeLabel: i18n.translate('xpack.indexLifecycleMgmt.coldPhase.freezeIndexLabel', { @@ -124,7 +124,7 @@ export const ColdPhase: FunctionComponent = ({ {phaseData.phaseEnabled ? ( {/* Data tier allocation section */} - void }> = ({ - setWarmPhaseOnRollover, -}) => { +export const HotPhase: FunctionComponent = () => { const form = useFormContext(); const [formData] = useFormData({ watch: useRolloverPath, @@ -55,10 +51,6 @@ export const HotPhase: FunctionComponent<{ setWarmPhaseOnRollover: (v: boolean) const isShowingErrors = form.isValid === false; const [showEmptyRolloverFieldsError, setShowEmptyRolloverFieldsError] = useState(false); - useEffect(() => { - setWarmPhaseOnRollover(isRolloverEnabled ?? false); - }, [setWarmPhaseOnRollover, isRolloverEnabled]); - return ( <> -
{i18nTexts.rollOverConfigurationCallout.body}
+
{i18nTexts.editPolicy.errors.rollOverConfigurationCallout.body}
 diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/hot_phase/i18n_texts.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/hot_phase/i18n_texts.ts deleted file mode 100644 index 6423b12b86dd2..0000000000000 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/hot_phase/i18n_texts.ts +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { i18n } from '@kbn/i18n'; - -export const i18nTexts = { - maximumAgeRequiredMessage: i18n.translate( - 'xpack.indexLifecycleMgmt.editPolicy.maximumAgeMissingError', - { - defaultMessage: 'A maximum age is required.', - } - ), - maximumSizeRequiredMessage: i18n.translate( - 'xpack.indexLifecycleMgmt.editPolicy.maximumIndexSizeMissingError', - { - defaultMessage: 'A maximum index size is required.', - } - ), - maximumDocumentsRequiredMessage: i18n.translate( - 'xpack.indexLifecycleMgmt.editPolicy.maximumDocumentsMissingError', - { - defaultMessage: 'Maximum documents is required.', - } - ), - rollOverConfigurationCallout: { - title: i18n.translate('xpack.indexLifecycleMgmt.editPolicy.rolloverConfigurationError.title', { - defaultMessage: 'Invalid rollover configuration', - }), - body: i18n.translate('xpack.indexLifecycleMgmt.editPolicy.rolloverConfigurationError.body', { - defaultMessage: - 'A value for one of maximum size, maximum documents, or maximum age is required.', - }), - }, -}; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/cloud_data_tier_callout.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/cloud_data_tier_callout.tsx new file mode 100644 index 0000000000000..2dff55ac10de1 --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/cloud_data_tier_callout.tsx @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import React, { FunctionComponent } from 'react'; +import { EuiCallOut } from '@elastic/eui'; + +const i18nTexts = { + title: i18n.translate('xpack.indexLifecycleMgmt.editPolicy.cloudDataTierCallout.title', { + defaultMessage: 'Create a cold tier', + }), + body: i18n.translate('xpack.indexLifecycleMgmt.editPolicy.cloudDataTierCallout.body', { + defaultMessage: 'Edit your Elastic Cloud deployment to set up a cold tier.', + }), +}; + +export const CloudDataTierCallout: FunctionComponent = () => { + return ( + + {i18nTexts.body} + + ); +}; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/data_tier_allocation.scss b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/data_tier_allocation.scss new file mode 100644 index 0000000000000..62ec3f303e1e8 --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/data_tier_allocation.scss @@ -0,0 +1,9 @@ +.indexLifecycleManagement__phase__dataTierAllocation { + &__controlSection { + background-color: $euiColorLightestShade; + padding-top: $euiSizeM; + padding-left: $euiSizeM; + padding-right: $euiSizeM; + padding-bottom: $euiSizeM; + } +} diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/data_tier_allocation.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/data_tier_allocation.tsx new file mode 100644 index 0000000000000..56af4bbac7a26 --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/data_tier_allocation.tsx @@ -0,0 +1,191 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FunctionComponent } from 'react'; +import { get } from 'lodash'; +import { i18n } from '@kbn/i18n'; +import { EuiText, EuiSpacer, EuiSuperSelectOption } from '@elastic/eui'; + +import { UseField, SuperSelectField, useFormData } from '../../../../../../../../shared_imports'; +import { PhaseWithAllocation } from '../../../../../../../../../common/types'; + +import { DataTierAllocationType } from '../../../../../types'; + +import { NodeAllocation } from './node_allocation'; +import { SharedProps } from './types'; + +import './data_tier_allocation.scss'; + +type SelectOptions = EuiSuperSelectOption; + +const i18nTexts = { + allocationOptions: { + warm: { + default: { + input: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.common.dataTierAllocation.warm.defaultOption.input', + { defaultMessage: 'Use warm nodes (recommended)' } + ), + helpText: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.common.dataTierAllocation.warm.defaultOption.helpText', + { defaultMessage: 'Move data to nodes in the warm tier.' } + ), + }, + none: { + inputDisplay: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.common.dataTierAllocation.warm.noneOption.input', + { defaultMessage: 'Off' } + ), + helpText: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.common.dataTierAllocation.warm.noneOption.helpText', + { defaultMessage: 'Do not move data in the warm phase.' } + ), + }, + custom: { + inputDisplay: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.common.dataTierAllocation.warm.customOption.input', + { defaultMessage: 'Custom' } + ), + helpText: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.common.dataTierAllocation.warm.customOption.helpText', + { defaultMessage: 'Move data based on node attributes.' } + ), + }, + }, + cold: { + default: { + input: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.common.dataTierAllocation.cold.defaultOption.input', + { defaultMessage: 'Use cold nodes (recommended)' } + ), + helpText: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.common.dataTierAllocation.cold.defaultOption.helpText', + { defaultMessage: 'Move data to nodes in the cold tier.' } + ), + }, + none: { + inputDisplay: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.common.dataTierAllocation.cold.noneOption.input', + { defaultMessage: 'Off' } + ), + helpText: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.common.dataTierAllocation.cold.noneOption.helpText', + { defaultMessage: 'Do not move data in the cold phase.' } + ), + }, + custom: { + inputDisplay: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.common.dataTierAllocation.cold.customOption.input', + { defaultMessage: 'Custom' } + ), + helpText: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.common.dataTierAllocation.cold.customOption.helpText', + { defaultMessage: 'Move data based on node attributes.' } + ), + }, + }, + }, +}; + +const getSelectOptions = (phase: PhaseWithAllocation, disableDataTierOption: boolean) => + [ + disableDataTierOption + ? undefined + : { + 'data-test-subj': 'defaultDataAllocationOption', + value: 'node_roles', + inputDisplay: i18nTexts.allocationOptions[phase].default.input, + dropdownDisplay: ( + <> + {i18nTexts.allocationOptions[phase].default.input} + +

+ {i18nTexts.allocationOptions[phase].default.helpText} +

+ + + ), + }, + { + 'data-test-subj': 'customDataAllocationOption', + value: 'node_attrs', + inputDisplay: i18nTexts.allocationOptions[phase].custom.inputDisplay, + dropdownDisplay: ( + <> + {i18nTexts.allocationOptions[phase].custom.inputDisplay} + +

+ {i18nTexts.allocationOptions[phase].custom.helpText} +

+ + + ), + }, + { + 'data-test-subj': 'noneDataAllocationOption', + value: 'none', + inputDisplay: i18nTexts.allocationOptions[phase].none.inputDisplay, + dropdownDisplay: ( + <> + {i18nTexts.allocationOptions[phase].none.inputDisplay} + +

+ {i18nTexts.allocationOptions[phase].none.helpText} +

+ + + ), + }, + ].filter(Boolean) as SelectOptions[]; + +export const DataTierAllocation: FunctionComponent = (props) => { + const { phase, hasNodeAttributes, disableDataTierOption } = props; + + const dataTierAllocationTypePath = `_meta.${phase}.dataTierAllocationType`; + + const [formData] = useFormData({ + watch: dataTierAllocationTypePath, + }); + + const dataTierAllocationType = get(formData, dataTierAllocationTypePath); + + return ( +
+ + {(field) => { + /** + * We reset the value to "custom" if we deserialized to "default". + * + * It would be better if we had all the information we needed before deserializing and + * were able to handle this at the deserialization step instead of patching further down + * the component tree - this should be a future refactor. + */ + if (disableDataTierOption && field.value === 'node_roles') { + field.setValue('node_attrs'); + } + return ( + + ); + }} + + {dataTierAllocationType === 'node_attrs' && hasNodeAttributes && ( + <> + +
+ +
+ + )} +
+ ); +}; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/default_allocation_notice.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/default_allocation_notice.tsx new file mode 100644 index 0000000000000..f1f8fef62f1f8 --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/default_allocation_notice.tsx @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import React, { FunctionComponent } from 'react'; +import { EuiCallOut } from '@elastic/eui'; + +import { PhaseWithAllocation, DataTierRole } from '../../../../../../../../../common/types'; + +import { AllocationNodeRole } from '../../../../../../../lib'; + +const i18nTextsNodeRoleToDataTier: Record = { + data_hot: i18n.translate('xpack.indexLifecycleMgmt.editPolicy.dataTierHotLabel', { + defaultMessage: 'hot', + }), + data_warm: i18n.translate('xpack.indexLifecycleMgmt.editPolicy.dataTierWarmLabel', { + defaultMessage: 'warm', + }), + data_cold: i18n.translate('xpack.indexLifecycleMgmt.editPolicy.dataTierColdLabel', { + defaultMessage: 'cold', + }), +}; + +const i18nTexts = { + notice: { + warm: { + title: i18n.translate( + 'xpack.indexLifecycleMgmt.warmPhase.dataTier.defaultAllocationNotice.warm.title', + { defaultMessage: 'No nodes assigned to the warm tier' } + ), + body: (nodeRole: DataTierRole) => + i18n.translate('xpack.indexLifecycleMgmt.warmPhase.dataTier.defaultAllocationNotice.warm', { + defaultMessage: + 'This policy will move data in the warm phase to {tier} tier nodes instead.', + values: { tier: i18nTextsNodeRoleToDataTier[nodeRole] }, + }), + }, + cold: { + title: i18n.translate( + 'xpack.indexLifecycleMgmt.warmPhase.dataTier.defaultAllocationNotice.cold.title', + { defaultMessage: 'No nodes assigned to the cold tier' } + ), + body: (nodeRole: DataTierRole) => + i18n.translate('xpack.indexLifecycleMgmt.warmPhase.dataTier.defaultAllocationNotice.cold', { + defaultMessage: + 'This policy will move data in the cold phase to {tier} tier nodes instead.', + values: { tier: i18nTextsNodeRoleToDataTier[nodeRole] }, + }), + }, + }, + warning: { + warm: { + title: i18n.translate( + 'xpack.indexLifecycleMgmt.warmPhase.dataTier.defaultAllocationNotAvailableTitle', + { defaultMessage: 'No nodes assigned to the warm tier' } + ), + body: i18n.translate( + 'xpack.indexLifecycleMgmt.warmPhase.dataTier.defaultAllocationNotAvailableBody', + { + defaultMessage: + 'Assign at least one node to the warm or hot tier to use role-based allocation. The policy will fail to complete allocation if there are no available nodes.', + } + ), + }, + cold: { + title: i18n.translate( + 'xpack.indexLifecycleMgmt.coldPhase.dataTier.defaultAllocationNotAvailableTitle', + { defaultMessage: 'No nodes assigned to the cold tier' } + ), + body: i18n.translate( + 'xpack.indexLifecycleMgmt.coldPhase.dataTier.defaultAllocationNotAvailableBody', + { + defaultMessage: + 'Assign at least one node to the cold, warm, or hot tier to use role-based allocation. The policy will fail to complete allocation if there are no available nodes.', + } + ), + }, + }, +}; + +interface Props { + phase: PhaseWithAllocation; + targetNodeRole: AllocationNodeRole; +} + +export const DefaultAllocationNotice: FunctionComponent = ({ phase, targetNodeRole }) => { + const content = + targetNodeRole === 'none' ? ( + + {i18nTexts.warning[phase].body} + + ) : ( + + {i18nTexts.notice[phase].body(targetNodeRole)} + + ); + + return content; +}; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/index.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/index.ts new file mode 100644 index 0000000000000..0782782e77fad --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { NodesDataProvider } from './node_data_provider'; + +export { NodeAllocation } from './node_allocation'; + +export { NodeAttrsDetails } from './node_attrs_details'; + +export { DataTierAllocation } from './data_tier_allocation'; + +export { DefaultAllocationNotice } from './default_allocation_notice'; + +export { NoNodeAttributesWarning } from './no_node_attributes_warning'; + +export { CloudDataTierCallout } from './cloud_data_tier_callout'; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/no_node_attributes_warning.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/no_node_attributes_warning.tsx new file mode 100644 index 0000000000000..338e5367a1d0d --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/no_node_attributes_warning.tsx @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FunctionComponent } from 'react'; +import { EuiCallOut } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; + +import { PhaseWithAllocation } from '../../../../../../../../../common/types'; + +const i18nTexts = { + title: i18n.translate('xpack.indexLifecycleMgmt.editPolicy.nodeAttributesMissingLabel', { + defaultMessage: 'No custom node attributes configured', + }), + warm: { + body: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.warm.nodeAttributesMissingDescription', + { + defaultMessage: + 'Define custom node attributes in elasticsearch.yml to use attribute-based allocation. Warm nodes will be used instead.', + } + ), + }, + cold: { + body: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.cold.nodeAttributesMissingDescription', + { + defaultMessage: + 'Define custom node attributes in elasticsearch.yml to use attribute-based allocation. Cold nodes will be used instead.', + } + ), + }, +}; + +export const NoNodeAttributesWarning: FunctionComponent<{ phase: PhaseWithAllocation }> = ({ + phase, +}) => { + return ( + + {i18nTexts[phase].body} + + ); +}; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/node_allocation.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/node_allocation.tsx new file mode 100644 index 0000000000000..407bb9ea92e85 --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/node_allocation.tsx @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { useState, FunctionComponent } from 'react'; +import { get } from 'lodash'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { i18n } from '@kbn/i18n'; +import { EuiButtonEmpty, EuiText, EuiSpacer } from '@elastic/eui'; + +import { PhaseWithAllocationAction } from '../../../../../../../../../common/types'; + +import { UseField, SelectField, useFormData } from '../../../../../../../../shared_imports'; + +import { propertyof } from '../../../../../../../services/policies/policy_validation'; + +import { LearnMoreLink } from '../../../../learn_more_link'; + +import { NodeAttrsDetails } from './node_attrs_details'; + +import { SharedProps } from './types'; + +const learnMoreLink = ( + + } + docPath="modules-cluster.html#cluster-shard-allocation-settings" + /> +); + +const i18nTexts = { + doNotModifyAllocationOption: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.nodeAllocation.doNotModifyAllocationOption', + { defaultMessage: 'Do not modify allocation configuration' } + ), +}; + +export const NodeAllocation: FunctionComponent = ({ phase, nodes }) => { + const allocationNodeAttributePath = `_meta.${phase}.allocationNodeAttribute`; + + const [formData] = useFormData({ + watch: [allocationNodeAttributePath], + }); + + const selectedAllocationNodeAttribute = get(formData, allocationNodeAttributePath); + + const [selectedNodeAttrsForDetails, setSelectedNodeAttrsForDetails] = useState( + null + ); + + const nodeOptions = Object.keys(nodes).map((attrs) => ({ + text: `${attrs} (${nodes[attrs].length})`, + value: attrs, + })); + + nodeOptions.sort((a, b) => a.value.localeCompare(b.value)); + + // check that this string is a valid property + const nodeAttrsProperty = propertyof('selectedNodeAttrs'); + + return ( + <> + +

+ +

+ + + + {/* + TODO: this field component must be revisited to support setting multiple require values and to support + setting `include and exclude values on ILM policies. See https://github.com/elastic/kibana/issues/77344 + */} + setSelectedNodeAttrsForDetails(selectedAllocationNodeAttribute)} + > + + + ) : undefined, + euiFieldProps: { + 'data-test-subj': `${phase}-${nodeAttrsProperty}`, + options: [{ text: i18nTexts.doNotModifyAllocationOption, value: '' }].concat( + nodeOptions + ), + hasNoInitialSelection: false, + }, + }} + /> + {selectedNodeAttrsForDetails ? ( + setSelectedNodeAttrsForDetails(null)} + /> + ) : null} + + ); +}; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/node_attrs_details.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/node_attrs_details.tsx new file mode 100644 index 0000000000000..61de977b1cb12 --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/node_attrs_details.tsx @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; + +import { + EuiFlyoutBody, + EuiFlyout, + EuiTitle, + EuiInMemoryTable, + EuiSpacer, + EuiPortal, + EuiLoadingContent, + EuiCallOut, + EuiButton, +} from '@elastic/eui'; + +import { useLoadNodeDetails } from '../../../../../../../services/api'; + +interface Props { + close: () => void; + selectedNodeAttrs: string; +} + +export const NodeAttrsDetails: React.FunctionComponent = ({ close, selectedNodeAttrs }) => { + const { data, isLoading, error, resendRequest } = useLoadNodeDetails(selectedNodeAttrs); + let content; + if (isLoading) { + content = ; + } else if (error) { + const { statusCode, message } = error; + content = ( + + } + color="danger" + > +

+ {message} ({statusCode}) +

+ + + + + ); + } else { + content = ( + + ); + } + return ( + + + + +

+ +

+ + + {content} + + + + ); +}; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/node_data_provider.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/node_data_provider.tsx new file mode 100644 index 0000000000000..bb01a42e378fa --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/node_data_provider.tsx @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { EuiButton, EuiCallOut, EuiLoadingSpinner, EuiSpacer } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; + +import { ListNodesRouteResponse } from '../../../../../../../../../common/types'; +import { useLoadNodes } from '../../../../../../../services/api'; + +interface Props { + children: (data: ListNodesRouteResponse) => JSX.Element; +} + +export const NodesDataProvider = ({ children }: Props): JSX.Element => { + const { isLoading, data, error, resendRequest } = useLoadNodes(); + + if (isLoading) { + return ( + <> + + + + ); + } + + const renderError = () => { + if (error) { + const { statusCode, message } = error; + return ( + <> + + } + color="danger" + > +

+ {message} ({statusCode}) +

+ + + + + + + + ); + } + return null; + }; + + return ( + <> + {renderError()} + {/* `data` will always be defined because we use an initial value when loading */} + {children(data!)} + + ); +}; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/types.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/types.ts new file mode 100644 index 0000000000000..7bd620757a8ac --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/types.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + ListNodesRouteResponse, + PhaseWithAllocation, +} from '../../../../../../../../../common/types'; + +export interface SharedProps { + phase: PhaseWithAllocation; + nodes: ListNodesRouteResponse['nodesByAttributes']; + hasNodeAttributes: boolean; + /** + * When on Cloud we want to disable the data tier allocation option when we detect that we are not + * using node roles in our Node config yet. See {@link ListNodesRouteResponse} for information about how this is + * detected. + */ + disableDataTierOption: boolean; +} diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/data_tier_allocation_field.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/data_tier_allocation_field.tsx new file mode 100644 index 0000000000000..73814537ff276 --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/data_tier_allocation_field.tsx @@ -0,0 +1,137 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { get } from 'lodash'; +import React, { FunctionComponent } from 'react'; +import { i18n } from '@kbn/i18n'; + +import { EuiDescribedFormGroup, EuiFormRow, EuiSpacer } from '@elastic/eui'; + +import { useKibana, useFormData } from '../../../../../../../shared_imports'; + +import { PhaseWithAllocation } from '../../../../../../../../common/types'; + +import { getAvailableNodeRoleForPhase } from '../../../../../../lib/data_tiers'; + +import { isNodeRoleFirstPreference } from '../../../../../../lib'; + +import { DataTierAllocationType } from '../../../../types'; + +import { + DataTierAllocation, + DefaultAllocationNotice, + NoNodeAttributesWarning, + NodesDataProvider, + CloudDataTierCallout, +} from './components'; + +const i18nTexts = { + title: i18n.translate('xpack.indexLifecycleMgmt.common.dataTier.title', { + defaultMessage: 'Data allocation', + }), +}; + +interface Props { + phase: PhaseWithAllocation; + description: React.ReactNode; +} + +/** + * Top-level layout control for the data tier allocation field. + */ +export const DataTierAllocationField: FunctionComponent = ({ phase, description }) => { + const { + services: { cloud }, + } = useKibana(); + + const dataTierAllocationTypePath = `_meta.${phase}.dataTierAllocationType`; + const [formData] = useFormData({ watch: dataTierAllocationTypePath }); + const allocationType: DataTierAllocationType = get(formData, dataTierAllocationTypePath); + + return ( + + {({ nodesByRoles, nodesByAttributes, isUsingDeprecatedDataRoleConfig }) => { + const hasDataNodeRoles = Object.keys(nodesByRoles).some((nodeRole) => + // match any of the "data_" roles, including data_content. + nodeRole.trim().startsWith('data_') + ); + const hasNodeAttrs = Boolean(Object.keys(nodesByAttributes ?? {}).length); + const isCloudEnabled = cloud?.isCloudEnabled ?? false; + + const renderNotice = () => { + switch (allocationType) { + case 'node_roles': + if (isCloudEnabled && phase === 'cold') { + const isUsingNodeRolesAllocation = + !isUsingDeprecatedDataRoleConfig && hasDataNodeRoles; + const hasNoNodesWithNodeRole = !nodesByRoles.data_cold?.length; + + if (isUsingNodeRolesAllocation && hasNoNodesWithNodeRole) { + // Tell cloud users they can deploy nodes on cloud. + return ( + <> + + + + ); + } + } + + const allocationNodeRole = getAvailableNodeRoleForPhase(phase, nodesByRoles); + if ( + allocationNodeRole === 'none' || + !isNodeRoleFirstPreference(phase, allocationNodeRole) + ) { + return ( + <> + + + + ); + } + break; + case 'node_attrs': + if (!hasNodeAttrs) { + return ( + <> + + + + ); + } + break; + default: + return null; + } + }; + + return ( + {i18nTexts.title}

} + description={description} + fullWidth + > + + <> + + + {/* Data tier related warnings and call-to-action notices */} + {renderNotice()} + + + + ); + }} + + ); +}; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/index.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/index.ts new file mode 100644 index 0000000000000..f9e939058adb9 --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { DataTierAllocationField } from './data_tier_allocation_field'; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_legacy_field.tsx similarity index 85% rename from x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field.tsx rename to x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_legacy_field.tsx index df59efcbfd299..d64df468620e6 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_legacy_field.tsx @@ -11,8 +11,7 @@ import { EuiDescribedFormGroup, EuiFormRow, EuiSpacer } from '@elastic/eui'; import { useKibana } from '../../../../../../shared_imports'; import { PhaseWithAllocationAction, PhaseWithAllocation } from '../../../../../../../common/types'; import { PhaseValidationErrors } from '../../../../../services/policies/policy_validation'; -import { getAvailableNodeRoleForPhase } from '../../../../../lib/data_tiers'; -import { isNodeRoleFirstPreference } from '../../../../../lib/data_tiers/is_node_role_first_preference'; +import { getAvailableNodeRoleForPhase, isNodeRoleFirstPreference } from '../../../../../lib'; import { DataTierAllocation, @@ -40,7 +39,7 @@ interface Props { /** * Top-level layout control for the data tier allocation field. */ -export const DataTierAllocationField: FunctionComponent = ({ +export const DataTierAllocationFieldLegacy: FunctionComponent = ({ description, phase, phaseData, @@ -55,6 +54,10 @@ export const DataTierAllocationField: FunctionComponent = ({ return ( {({ nodesByRoles, nodesByAttributes, isUsingDeprecatedDataRoleConfig }) => { + const hasDataNodeRoles = Object.keys(nodesByRoles).some((nodeRole) => + // match any of the "data_" roles, including data_content. + nodeRole.trim().startsWith('data_') + ); const hasNodeAttrs = Boolean(Object.keys(nodesByAttributes ?? {}).length); const renderNotice = () => { @@ -62,7 +65,8 @@ export const DataTierAllocationField: FunctionComponent = ({ case 'default': const isCloudEnabled = cloud?.isCloudEnabled ?? false; if (isCloudEnabled && phase === 'cold') { - const isUsingNodeRolesAllocation = !isUsingDeprecatedDataRoleConfig; + const isUsingNodeRolesAllocation = + !isUsingDeprecatedDataRoleConfig && hasDataNodeRoles; const hasNoNodesWithNodeRole = !nodesByRoles.data_cold?.length; if (isUsingNodeRolesAllocation && hasNoNodesWithNodeRole) { @@ -120,9 +124,9 @@ export const DataTierAllocationField: FunctionComponent = ({ phaseData={phaseData} isShowingErrors={isShowingErrors} nodes={nodesByAttributes} - disableDataTierOption={ - !!(isUsingDeprecatedDataRoleConfig && cloud?.isCloudEnabled) - } + disableDataTierOption={Boolean( + cloud?.isCloudEnabled && !hasDataNodeRoles && isUsingDeprecatedDataRoleConfig + )} /> {/* Data tier related warnings and call-to-action notices */} diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/forcemerge_field.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/forcemerge_field.tsx index c6f02fd219130..b410bd0e6b3b0 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/forcemerge_field.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/forcemerge_field.tsx @@ -3,33 +3,32 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { get } from 'lodash'; -import { FormattedMessage } from '@kbn/i18n/react'; -import { EuiDescribedFormGroup, EuiSpacer, EuiTextColor } from '@elastic/eui'; -import React from 'react'; -import { Phases } from '../../../../../../../common/types'; +import React, { useMemo } from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { EuiSpacer, EuiTextColor } from '@elastic/eui'; -import { UseField, ToggleField, NumericField, useFormData } from '../../../../../../shared_imports'; +import { UseField, ToggleField, NumericField } from '../../../../../../shared_imports'; import { i18nTexts } from '../../../i18n_texts'; -import { LearnMoreLink } from '../../'; +import { useEditPolicyContext } from '../../../edit_policy_context'; + +import { LearnMoreLink, DescribedFormField } from '../../'; interface Props { - phase: keyof Phases & string; + phase: 'hot' | 'warm'; } -const forceMergeEnabledPath = '_meta.hot.forceMergeEnabled'; - export const Forcemerge: React.FunctionComponent = ({ phase }) => { - const [formData] = useFormData({ - watch: forceMergeEnabledPath, - }); - const forceMergeEnabled = get(formData, forceMergeEnabledPath); + const { originalPolicy } = useEditPolicyContext(); + + const initialToggleValue = useMemo(() => { + return Boolean(originalPolicy.phases[phase]?.actions?.forcemerge); + }, [originalPolicy, phase]); return ( - = ({ phase }) => { } titleSize="xs" fullWidth + switchProps={{ + 'aria-label': i18nTexts.editPolicy.forceMergeEnabledFieldLabel, + 'data-test-subj': `${phase}-forceMergeSwitch`, + 'aria-controls': 'forcemergeContent', + label: i18nTexts.editPolicy.forceMergeEnabledFieldLabel, + initialValue: initialToggleValue, + }} > -
- {forceMergeEnabled && ( - <> - - - - )} + +
- + ); }; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/index.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/index.ts index 3b94d36a977d1..0cae3eea6316b 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/index.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/index.ts @@ -6,8 +6,12 @@ export { useRolloverPath } from '../../../constants'; +export { DataTierAllocationFieldLegacy } from './data_tier_allocation_legacy_field'; + export { DataTierAllocationField } from './data_tier_allocation_field'; export { Forcemerge } from './forcemerge_field'; export { SetPriorityInput } from './set_priority_input'; + +export { MinAgeInputField } from './min_age_input_field'; diff --git a/x-pack/plugins/painless_lab/public/lib/index.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/min_age_input_field/index.ts similarity index 79% rename from x-pack/plugins/painless_lab/public/lib/index.ts rename to x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/min_age_input_field/index.ts index 2421307b7c107..0228a524f8129 100644 --- a/x-pack/plugins/painless_lab/public/lib/index.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/min_age_input_field/index.ts @@ -4,4 +4,4 @@ * you may not use this file except in compliance with the Elastic License. */ -export { monacoPainlessLang } from './monaco_painless_lang'; +export { MinAgeInputField } from './min_age_input_field'; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/min_age_input_field/min_age_input_field.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/min_age_input_field/min_age_input_field.tsx new file mode 100644 index 0000000000000..f37c387354418 --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/min_age_input_field/min_age_input_field.tsx @@ -0,0 +1,212 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import React, { FunctionComponent } from 'react'; +import { get } from 'lodash'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { i18n } from '@kbn/i18n'; + +import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; + +import { + useFormData, + UseField, + NumericField, + SelectField, +} from '../../../../../../../shared_imports'; + +import { LearnMoreLink } from '../../../learn_more_link'; +import { useRolloverPath } from '../../../../constants'; + +import { getUnitsAriaLabelForPhase, getTimingLabelForPhase } from './util'; + +type PhaseWithMinAgeAction = 'warm' | 'cold' | 'delete'; + +interface Props { + phase: PhaseWithMinAgeAction; +} + +export const MinAgeInputField: FunctionComponent = ({ phase }): React.ReactElement => { + const [formData] = useFormData({ watch: useRolloverPath }); + const rolloverEnabled = get(formData, useRolloverPath); + + let daysOptionLabel; + let hoursOptionLabel; + let minutesOptionLabel; + let secondsOptionLabel; + let millisecondsOptionLabel; + let microsecondsOptionLabel; + let nanosecondsOptionLabel; + + if (rolloverEnabled) { + daysOptionLabel = i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.rolloverDaysOptionLabel', + { + defaultMessage: 'days from rollover', + } + ); + + hoursOptionLabel = i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.rolloverHoursOptionLabel', + { + defaultMessage: 'hours from rollover', + } + ); + minutesOptionLabel = i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.rolloverMinutesOptionLabel', + { + defaultMessage: 'minutes from rollover', + } + ); + + secondsOptionLabel = i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.rolloverSecondsOptionLabel', + { + defaultMessage: 'seconds from rollover', + } + ); + millisecondsOptionLabel = i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.rolloverMilliSecondsOptionLabel', + { + defaultMessage: 'milliseconds from rollover', + } + ); + + microsecondsOptionLabel = i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.rolloverMicroSecondsOptionLabel', + { + defaultMessage: 'microseconds from rollover', + } + ); + + nanosecondsOptionLabel = i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.rolloverNanoSecondsOptionLabel', + { + defaultMessage: 'nanoseconds from rollover', + } + ); + } else { + daysOptionLabel = i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.creationDaysOptionLabel', + { + defaultMessage: 'days from index creation', + } + ); + + hoursOptionLabel = i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.creationHoursOptionLabel', + { + defaultMessage: 'hours from index creation', + } + ); + + minutesOptionLabel = i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.creationMinutesOptionLabel', + { + defaultMessage: 'minutes from index creation', + } + ); + + secondsOptionLabel = i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.creationSecondsOptionLabel', + { + defaultMessage: 'seconds from index creation', + } + ); + + millisecondsOptionLabel = i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.creationMilliSecondsOptionLabel', + { + defaultMessage: 'milliseconds from index creation', + } + ); + + microsecondsOptionLabel = i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.creationMicroSecondsOptionLabel', + { + defaultMessage: 'microseconds from index creation', + } + ); + + nanosecondsOptionLabel = i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.creationNanoSecondsOptionLabel', + { + defaultMessage: 'nanoseconds from index creation', + } + ); + } + + return ( + + + + } + /> + ), + euiFieldProps: { + 'data-test-subj': `${phase}-selectedMinimumAge`, + min: 0, + }, + }} + /> + + + + + + ); +}; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/min_age_input_field/util.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/min_age_input_field/util.ts new file mode 100644 index 0000000000000..181894badba7b --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/min_age_input_field/util.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import { Phases } from '../../../../../../../../common/types'; + +type PhaseWithMinAgeAction = 'warm' | 'cold' | 'delete'; + +export function getUnitsAriaLabelForPhase(phase: keyof Phases) { + // NOTE: Hot phase isn't necessary, because indices begin in the hot phase. + switch (phase) { + case 'warm': + return i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.phaseWarm.minimumAgeUnitsAriaLabel', + { + defaultMessage: 'Units for timing of warm phase', + } + ); + + case 'cold': + return i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.phaseCold.minimumAgeUnitsAriaLabel', + { + defaultMessage: 'Units for timing of cold phase', + } + ); + + case 'delete': + return i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.phaseDelete.minimumAgeUnitsAriaLabel', + { + defaultMessage: 'Units for timing of delete phase', + } + ); + } +} +export function getTimingLabelForPhase(phase: PhaseWithMinAgeAction) { + // NOTE: Hot phase isn't necessary, because indices begin in the hot phase. + switch (phase) { + case 'warm': + return i18n.translate('xpack.indexLifecycleMgmt.editPolicy.phaseWarm.minimumAgeLabel', { + defaultMessage: 'Timing for warm phase', + }); + + case 'cold': + return i18n.translate('xpack.indexLifecycleMgmt.editPolicy.phaseCold.minimumAgeLabel', { + defaultMessage: 'Timing for cold phase', + }); + + case 'delete': + return i18n.translate('xpack.indexLifecycleMgmt.editPolicy.phaseDelete.minimumAgeLabel', { + defaultMessage: 'Timing for delete phase', + }); + } +} diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/warm_phase.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/warm_phase.tsx deleted file mode 100644 index d0aaec367104e..0000000000000 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/warm_phase.tsx +++ /dev/null @@ -1,316 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import React, { Fragment, FunctionComponent } from 'react'; -import { get } from 'lodash'; -import { FormattedMessage } from '@kbn/i18n/react'; -import { i18n } from '@kbn/i18n'; -import { - EuiTextColor, - EuiFlexGroup, - EuiFlexItem, - EuiSpacer, - EuiFormRow, - EuiFieldNumber, - EuiSwitch, - EuiDescribedFormGroup, -} from '@elastic/eui'; - -import { useFormData } from '../../../../../shared_imports'; -import { Phases, WarmPhase as WarmPhaseInterface } from '../../../../../../common/types'; -import { PhaseValidationErrors } from '../../../../services/policies/policy_validation'; - -import { useRolloverPath } from './shared'; - -import { - LearnMoreLink, - ActiveBadge, - PhaseErrorMessage, - OptionalLabel, - ErrableFormRow, - SetPriorityInput, - MinAgeInput, - DescribedFormField, - Forcemerge, -} from '../'; - -import { DataTierAllocationField } from './shared'; - -const i18nTexts = { - shrinkLabel: i18n.translate('xpack.indexLifecycleMgmt.warmPhase.shrinkIndexLabel', { - defaultMessage: 'Shrink index', - }), - moveToWarmPhaseOnRolloverLabel: i18n.translate( - 'xpack.indexLifecycleMgmt.warmPhase.moveToWarmPhaseOnRolloverLabel', - { - defaultMessage: 'Move to warm phase on rollover', - } - ), - dataTierAllocation: { - description: i18n.translate('xpack.indexLifecycleMgmt.warmPhase.dataTier.description', { - defaultMessage: 'Move data to nodes optimized for less-frequent, read-only access.', - }), - }, -}; - -const warmProperty: keyof Phases = 'warm'; -const phaseProperty = (propertyName: keyof WarmPhaseInterface) => propertyName; - -interface Props { - setPhaseData: ( - key: keyof WarmPhaseInterface & string, - value: boolean | string | undefined - ) => void; - phaseData: WarmPhaseInterface; - isShowingErrors: boolean; - errors?: PhaseValidationErrors; -} -export const WarmPhase: FunctionComponent = ({ - setPhaseData, - phaseData, - errors, - isShowingErrors, -}) => { - const [formData] = useFormData({ - watch: useRolloverPath, - }); - - const hotPhaseRolloverEnabled = get(formData, useRolloverPath); - - return ( -
- <> - -

- -

{' '} - {phaseData.phaseEnabled && !isShowingErrors ? : null} - -
- } - titleSize="s" - description={ - -

- -

- - } - id={`${warmProperty}-${phaseProperty('phaseEnabled')}`} - checked={phaseData.phaseEnabled} - onChange={(e) => { - setPhaseData(phaseProperty('phaseEnabled'), e.target.checked); - }} - aria-controls="warmPhaseContent" - /> - - } - fullWidth - > - - {phaseData.phaseEnabled ? ( - - {hotPhaseRolloverEnabled ? ( - - { - setPhaseData(phaseProperty('warmPhaseOnRollover'), e.target.checked); - }} - /> - - ) : null} - {!phaseData.warmPhaseOnRollover || !hotPhaseRolloverEnabled ? ( - - - - errors={errors} - phaseData={phaseData} - phase={warmProperty} - isShowingErrors={isShowingErrors} - setPhaseData={setPhaseData} - rolloverEnabled={hotPhaseRolloverEnabled} - /> - - ) : null} - - ) : null} - - - - {phaseData.phaseEnabled ? ( - - {/* Data tier allocation section */} - - - - {i18n.translate('xpack.indexLifecycleMgmt.warmPhase.replicasTitle', { - defaultMessage: 'Replicas', - })} - - } - description={i18n.translate( - 'xpack.indexLifecycleMgmt.warmPhase.numberOfReplicasDescription', - { - defaultMessage: - 'Set the number of replicas. Remains the same as the previous phase by default.', - } - )} - switchProps={{ - label: i18n.translate( - 'xpack.indexLifecycleMgmt.editPolicy.warmPhase.numberOfReplicas.switchLabel', - { defaultMessage: 'Set replicas' } - ), - initialValue: Boolean(phaseData.selectedReplicaCount), - onChange: (v) => { - if (!v) { - setPhaseData('selectedReplicaCount', ''); - } - }, - }} - fullWidth - > - - - - - } - isShowingErrors={isShowingErrors} - errors={errors?.selectedReplicaCount} - > - { - setPhaseData('selectedReplicaCount', e.target.value); - }} - min={0} - /> - - - - - - } - description={ - - {' '} - - - } - fullWidth - titleSize="xs" - > - - { - setPhaseData(phaseProperty('shrinkEnabled'), e.target.checked); - }} - label={i18nTexts.shrinkLabel} - aria-label={i18nTexts.shrinkLabel} - aria-controls="shrinkContent" - /> - -
- {phaseData.shrinkEnabled ? ( - - - - - - { - setPhaseData( - phaseProperty('selectedPrimaryShardCount'), - e.target.value - ); - }} - min={1} - /> - - - - - - ) : null} -
- - - - - errors={errors} - phaseData={phaseData} - phase={warmProperty} - isShowingErrors={isShowingErrors} - setPhaseData={setPhaseData} - /> - - ) : null} - -
- ); -}; diff --git a/x-pack/plugins/painless_lab/public/services/index.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/warm_phase/index.ts similarity index 81% rename from x-pack/plugins/painless_lab/public/services/index.ts rename to x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/warm_phase/index.ts index 20bec9de24550..d0686270ba559 100644 --- a/x-pack/plugins/painless_lab/public/services/index.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/warm_phase/index.ts @@ -4,4 +4,4 @@ * you may not use this file except in compliance with the Elastic License. */ -export { LanguageService } from './language_service'; +export { WarmPhase } from './warm_phase'; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/warm_phase/warm_phase.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/warm_phase/warm_phase.tsx new file mode 100644 index 0000000000000..7b1a4f44b5de6 --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/warm_phase/warm_phase.tsx @@ -0,0 +1,233 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { Fragment, FunctionComponent } from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { i18n } from '@kbn/i18n'; +import { get } from 'lodash'; + +import { + EuiTextColor, + EuiFlexGroup, + EuiFlexItem, + EuiSpacer, + EuiDescribedFormGroup, +} from '@elastic/eui'; + +import { + useFormData, + UseField, + ToggleField, + useFormContext, + NumericField, +} from '../../../../../../shared_imports'; + +import { Phases } from '../../../../../../../common/types'; + +import { useRolloverPath, MinAgeInputField, Forcemerge, SetPriorityInput } from '../shared'; + +import { useEditPolicyContext } from '../../../edit_policy_context'; + +import { LearnMoreLink, ActiveBadge, PhaseErrorMessage, DescribedFormField } from '../../'; + +import { DataTierAllocationField } from '../shared'; + +const i18nTexts = { + shrinkLabel: i18n.translate('xpack.indexLifecycleMgmt.warmPhase.shrinkIndexLabel', { + defaultMessage: 'Shrink index', + }), + dataTierAllocation: { + description: i18n.translate('xpack.indexLifecycleMgmt.warmPhase.dataTier.description', { + defaultMessage: 'Move data to nodes optimized for less-frequent, read-only access.', + }), + }, +}; + +const warmProperty: keyof Phases = 'warm'; + +export const WarmPhase: FunctionComponent = () => { + const { originalPolicy } = useEditPolicyContext(); + const form = useFormContext(); + const [formData] = useFormData({ + watch: [useRolloverPath, '_meta.warm.enabled', '_meta.warm.warmPhaseOnRollover'], + }); + + const enabled = get(formData, '_meta.warm.enabled'); + const hotPhaseRolloverEnabled = get(formData, useRolloverPath); + const warmPhaseOnRollover = get(formData, '_meta.warm.warmPhaseOnRollover'); + const isShowingErrors = form.isValid === false; + + return ( +
+ <> + +

+ +

{' '} + {enabled && !isShowingErrors ? : null} + +
+ } + titleSize="s" + description={ + +

+ +

+ + + } + fullWidth + > + + {enabled && ( + + {hotPhaseRolloverEnabled && ( + + )} + {(!warmPhaseOnRollover || !hotPhaseRolloverEnabled) && ( + <> + + + + )} + + )} + + + + {enabled && ( + + {/* Data tier allocation section */} + + + + {i18n.translate('xpack.indexLifecycleMgmt.warmPhase.replicasTitle', { + defaultMessage: 'Replicas', + })} + + } + description={i18n.translate( + 'xpack.indexLifecycleMgmt.warmPhase.numberOfReplicasDescription', + { + defaultMessage: + 'Set the number of replicas. Remains the same as the previous phase by default.', + } + )} + switchProps={{ + 'data-test-subj': 'warm-setReplicasSwitch', + label: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.warmPhase.numberOfReplicas.switchLabel', + { defaultMessage: 'Set replicas' } + ), + initialValue: Boolean( + originalPolicy.phases.warm?.actions?.allocate?.number_of_replicas + ), + }} + fullWidth + > + + + + + + } + description={ + + {' '} + + + } + titleSize="xs" + switchProps={{ + 'aria-controls': 'shrinkContent', + 'data-test-subj': 'shrinkSwitch', + label: i18nTexts.shrinkLabel, + 'aria-label': i18nTexts.shrinkLabel, + initialValue: Boolean(originalPolicy.phases.warm?.actions?.shrink), + }} + fullWidth + > +
+ + + + + + + +
+ + + + + + + )} + + + ); +}; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/policy_json_flyout.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/policy_json_flyout.tsx index e9ce193118b35..9ed24355ce7b3 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/policy_json_flyout.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/policy_json_flyout.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { useEffect, useState } from 'react'; +import React, { useCallback, useEffect, useState } from 'react'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; @@ -25,6 +25,7 @@ import { import { SerializedPolicy } from '../../../../../common/types'; import { useFormContext, useFormData } from '../../../../shared_imports'; +import { FormInternal } from '../types'; interface Props { legacyPolicy: SerializedPolicy; @@ -44,27 +45,29 @@ export const PolicyJsonFlyout: React.FunctionComponent = ({ */ const [policy, setPolicy] = useState(undefined); - const form = useFormContext(); - const [formData, getFormData] = useFormData(); + const { validate: validateForm } = useFormContext(); + const [, getFormData] = useFormData(); + + const updatePolicy = useCallback(async () => { + setPolicy(undefined); + if (await validateForm()) { + const p = getFormData() as SerializedPolicy; + setPolicy({ + ...legacyPolicy, + phases: { + ...legacyPolicy.phases, + hot: p.phases.hot, + warm: p.phases.warm, + }, + }); + } else { + setPolicy(null); + } + }, [setPolicy, getFormData, legacyPolicy, validateForm]); useEffect(() => { - (async function checkPolicy() { - setPolicy(undefined); - if (await form.validate()) { - const p = getFormData() as SerializedPolicy; - setPolicy({ - ...legacyPolicy, - phases: { - ...legacyPolicy.phases, - hot: p.phases.hot, - }, - }); - } else { - setPolicy(null); - } - })(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [form, legacyPolicy, formData]); + updatePolicy(); + }, [updatePolicy]); let content: React.ReactNode; switch (policy) { diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/set_priority_input.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/set_priority_input_legacy.tsx similarity index 100% rename from x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/set_priority_input.tsx rename to x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/set_priority_input_legacy.tsx diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/toggleable_field.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/toggleable_field.tsx index ff4301808db33..d188a172d746b 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/toggleable_field.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/toggleable_field.tsx @@ -9,7 +9,7 @@ import { EuiSpacer, EuiSwitch, EuiSwitchProps } from '@elastic/eui'; export interface Props extends Omit { initialValue: boolean; - onChange: (nextValue: boolean) => void; + onChange?: (nextValue: boolean) => void; } export const ToggleableField: FunctionComponent = ({ @@ -28,7 +28,9 @@ export const ToggleableField: FunctionComponent = ({ onChange={(e) => { const nextValue = e.target.checked; setIsContentVisible(nextValue); - onChange(nextValue); + if (onChange) { + onChange(nextValue); + } }} /> diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/deserializer.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/deserializer.ts index bb24eea64ec8c..760c6ad713ea0 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/deserializer.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/deserializer.ts @@ -10,22 +10,32 @@ import { SerializedPolicy } from '../../../../common/types'; import { splitSizeAndUnits } from '../../services/policies/policy_serialization'; +import { determineDataTierAllocationType } from '../../lib'; + import { FormInternal } from './types'; -export const deserializer = (policy: SerializedPolicy): FormInternal => - produce( +export const deserializer = (policy: SerializedPolicy): FormInternal => { + const _meta: FormInternal['_meta'] = { + hot: { + useRollover: Boolean(policy.phases.hot?.actions?.rollover), + forceMergeEnabled: Boolean(policy.phases.hot?.actions?.forcemerge), + bestCompression: policy.phases.hot?.actions?.forcemerge?.index_codec === 'best_compression', + }, + warm: { + enabled: Boolean(policy.phases.warm), + warmPhaseOnRollover: Boolean(policy.phases.warm?.min_age === '0ms'), + forceMergeEnabled: Boolean(policy.phases.warm?.actions?.forcemerge), + bestCompression: policy.phases.warm?.actions?.forcemerge?.index_codec === 'best_compression', + dataTierAllocationType: determineDataTierAllocationType(policy.phases.warm?.actions), + }, + }; + + return produce( { ...policy, - _meta: { - hot: { - useRollover: Boolean(policy.phases.hot?.actions?.rollover), - forceMergeEnabled: Boolean(policy.phases.hot?.actions?.forcemerge), - bestCompression: - policy.phases.hot?.actions?.forcemerge?.index_codec === 'best_compression', - }, - }, + _meta, }, - (draft) => { + (draft: FormInternal) => { if (draft.phases.hot?.actions?.rollover) { if (draft.phases.hot.actions.rollover.max_size) { const maxSize = splitSizeAndUnits(draft.phases.hot.actions.rollover.max_size); @@ -39,5 +49,20 @@ export const deserializer = (policy: SerializedPolicy): FormInternal => draft._meta.hot.maxAgeUnit = maxAge.units; } } + + if (draft.phases.warm) { + if (draft.phases.warm.actions?.allocate?.require) { + Object.entries(draft.phases.warm.actions.allocate.require).forEach((entry) => { + draft._meta.warm.allocationNodeAttribute = entry.join(':'); + }); + } + + if (draft.phases.warm.min_age) { + const minAge = splitSizeAndUnits(draft.phases.warm.min_age); + draft.phases.warm.min_age = minAge.size; + draft._meta.warm.minAgeUnit = minAge.units; + } + } } ); +}; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/edit_policy.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/edit_policy.tsx index 8f8b0447f378a..eecdfb4871a67 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/edit_policy.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/edit_policy.tsx @@ -67,6 +67,8 @@ import { schema } from './form_schema'; import { deserializer } from './deserializer'; import { createSerializer } from './serializer'; +import { EditPolicyContextProvider } from './edit_policy_context'; + export interface Props { policies: PolicyFromES[]; policyName: string; @@ -89,6 +91,7 @@ const mergeAllSerializedPolicies = ( phases: { ...legacySerializedPolicy.phases, hot: serializedPolicy.phases.hot, + warm: serializedPolicy.phases.warm, }, }; }; @@ -113,9 +116,11 @@ export const EditPolicy: React.FunctionComponent = ({ return createSerializer(existingPolicy?.policy); }, [existingPolicy?.policy]); + const originalPolicy = existingPolicy?.policy ?? defaultPolicy; + const { form } = useForm({ schema, - defaultValue: existingPolicy?.policy ?? defaultPolicy, + defaultValue: originalPolicy, deserializer, serializer, }); @@ -132,22 +137,6 @@ export const EditPolicy: React.FunctionComponent = ({ history.push('/policies'); }; - const setWarmPhaseOnRollover = useCallback( - (value: boolean) => { - setPolicy((p) => ({ - ...p, - phases: { - ...p.phases, - warm: { - ...p.phases.warm, - warmPhaseOnRollover: value, - }, - }, - })); - }, - [setPolicy] - ); - const submit = async () => { setIsShowingErrors(true); const { data: formLibPolicy, isValid: newIsValid } = await form.submit(); @@ -206,10 +195,6 @@ export const EditPolicy: React.FunctionComponent = ({ [setPolicy] ); - const setWarmPhaseData = useCallback( - (key: string, value: any) => setPhaseData('warm', key, value), - [setPhaseData] - ); const setColdPhaseData = useCallback( (key: string, value: any) => setPhaseData('cold', key, value), [setPhaseData] @@ -220,234 +205,236 @@ export const EditPolicy: React.FunctionComponent = ({ ); return ( - - - - -

- {isNewPolicy - ? i18n.translate('xpack.indexLifecycleMgmt.editPolicy.createPolicyMessage', { - defaultMessage: 'Create an index lifecycle policy', - }) - : i18n.translate('xpack.indexLifecycleMgmt.editPolicy.editPolicyMessage', { - defaultMessage: 'Edit index lifecycle policy {originalPolicyName}', - values: { originalPolicyName }, - })} -

- - -
-
- - -

- + +

+ {isNewPolicy + ? i18n.translate('xpack.indexLifecycleMgmt.editPolicy.createPolicyMessage', { + defaultMessage: 'Create an index lifecycle policy', + }) + : i18n.translate('xpack.indexLifecycleMgmt.editPolicy.editPolicyMessage', { + defaultMessage: 'Edit index lifecycle policy {originalPolicyName}', + values: { originalPolicyName }, + })} +

+ + +
+ + + +

+ {' '} - - } - /> -

- + />{' '} + + } + /> +

+ - + - {isNewPolicy ? null : ( - - -

- + {isNewPolicy ? null : ( + + +

+ + + + .{' '} - - .{' '} - +

+ + + + + { + setSaveAsNew(e.target.checked); + }} + label={ + + + + } /> -

- - - - - { - setSaveAsNew(e.target.checked); - }} - label={ - + + + )} + + {saveAsNew || isNewPolicy ? ( + + - } - /> - - - )} - - {saveAsNew || isNewPolicy ? ( - - +
+ } + titleSize="s" + fullWidth + > + - -
- } - titleSize="s" - fullWidth - > - + { + setPolicy({ ...policy, name: e.target.value }); + }} /> - } - > - { - setPolicy({ ...policy, name: e.target.value }); - }} - /> - - - ) : null} - - - - - - - - 0} - setPhaseData={setWarmPhaseData} - phaseData={policy.phases.warm} - /> - - - - 0} - setPhaseData={setColdPhaseData} - phaseData={policy.phases.cold} - /> - - - - 0 - } - getUrlForApp={getUrlForApp} - setPhaseData={setDeletePhaseData} - phaseData={policy.phases.delete} - /> - - - - - - - - - {saveAsNew ? ( - - ) : ( + + + ) : null} + + + + + + + + + + + + 0 + } + setPhaseData={setColdPhaseData} + phaseData={policy.phases.cold} + /> + + + + 0 + } + getUrlForApp={getUrlForApp} + setPhaseData={setDeletePhaseData} + phaseData={policy.phases.delete} + /> + + + + + + + + + {saveAsNew ? ( + + ) : ( + + )} + + + + + - )} - - - - - + + + + + + + + {isShowingPolicyJsonFlyout ? ( - - - - - - - - {isShowingPolicyJsonFlyout ? ( - - ) : ( - - )} - - - - - {isShowingPolicyJsonFlyout ? ( - setIsShowingPolicyJsonFlyout(false)} - /> - ) : null} - - - - - + ) : ( + + )} + + + + + {isShowingPolicyJsonFlyout ? ( + setIsShowingPolicyJsonFlyout(false)} + /> + ) : null} + + + + + + ); }; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/edit_policy_context.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/edit_policy_context.tsx new file mode 100644 index 0000000000000..4748c26d6cec1 --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/edit_policy_context.tsx @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { createContext, ReactChild, useContext } from 'react'; +import { SerializedPolicy } from '../../../../common/types'; + +interface EditPolicyContextValue { + originalPolicy: SerializedPolicy; +} + +const EditPolicyContext = createContext(null as any); + +export const EditPolicyContextProvider = ({ + value, + children, +}: { + value: EditPolicyContextValue; + children: ReactChild; +}) => { + return {children}; +}; + +export const useEditPolicyContext = () => { + const ctx = useContext(EditPolicyContext); + if (!ctx) { + throw new Error('useEditPolicyContext can only be called inside of EditPolicyContext!'); + } + return ctx; +}; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/form_schema.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/form_schema.ts index 806164c8b0da1..a80382e87539c 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/form_schema.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/form_schema.ts @@ -7,13 +7,19 @@ import { i18n } from '@kbn/i18n'; import { FormSchema, fieldValidators } from '../../../shared_imports'; -import { defaultSetPriority } from '../../constants'; +import { defaultSetPriority, defaultPhaseIndexPriority } from '../../constants'; import { FormInternal } from './types'; + import { ifExistsNumberGreaterThanZero, rolloverThresholdsValidator } from './form_validations'; + import { i18nTexts } from './i18n_texts'; -const { emptyField } = fieldValidators; +const { emptyField, numberGreaterThanField } = fieldValidators; + +const serializers = { + stringToNumber: (v: string): any => (v ? parseInt(v, 10) : undefined), +}; export const schema: FormSchema = { _meta: { @@ -30,21 +36,38 @@ export const schema: FormSchema = { maxAgeUnit: { defaultValue: 'd', }, - forceMergeEnabled: { - label: i18nTexts.editPolicy.forceMergeEnabledFieldLabel, - }, bestCompression: { - label: i18n.translate('xpack.indexLifecycleMgmt.forcemerge.bestCompressionLabel', { - defaultMessage: 'Compress stored fields', - }), - helpText: i18n.translate( - 'xpack.indexLifecycleMgmt.editPolicy.forceMerge.bestCompressionText', - { - defaultMessage: - 'Use higher compression for stored fields at the cost of slower performance.', - } + label: i18nTexts.editPolicy.bestCompressionFieldLabel, + helpText: i18nTexts.editPolicy.bestCompressionFieldHelpText, + }, + }, + warm: { + enabled: { + defaultValue: false, + label: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.warmPhase.activateWarmPhaseSwitchLabel', + { defaultMessage: 'Activate warm phase' } ), }, + warmPhaseOnRollover: { + defaultValue: true, + label: i18n.translate('xpack.indexLifecycleMgmt.warmPhase.moveToWarmPhaseOnRolloverLabel', { + defaultMessage: 'Move to warm phase on rollover', + }), + }, + minAgeUnit: { + defaultValue: 'ms', + }, + bestCompression: { + label: i18nTexts.editPolicy.bestCompressionFieldLabel, + helpText: i18nTexts.editPolicy.bestCompressionFieldHelpText, + }, + dataTierAllocationType: { + label: i18nTexts.editPolicy.allocationTypeOptionsFieldLabel, + }, + allocationNodeAttribute: { + label: i18nTexts.editPolicy.allocationNodeAttributeFieldLabel, + }, }, }, phases: { @@ -76,7 +99,7 @@ export const schema: FormSchema = { validator: ifExistsNumberGreaterThanZero, }, ], - serializer: (v: string): any => (v ? parseInt(v, 10) : undefined), + serializer: serializers.stringToNumber, }, max_size: { label: i18n.translate('xpack.indexLifecycleMgmt.hotPhase.maximumIndexSizeLabel', { @@ -94,9 +117,7 @@ export const schema: FormSchema = { }, forcemerge: { max_num_segments: { - label: i18n.translate('xpack.indexLifecycleMgmt.forceMerge.numberOfSegmentsLabel', { - defaultMessage: 'Number of segments', - }), + label: i18nTexts.editPolicy.maxNumSegmentsFieldLabel, validations: [ { validator: emptyField( @@ -110,17 +131,94 @@ export const schema: FormSchema = { validator: ifExistsNumberGreaterThanZero, }, ], - serializer: (v: string): any => (v ? parseInt(v, 10) : undefined), + serializer: serializers.stringToNumber, }, }, set_priority: { priority: { defaultValue: defaultSetPriority as any, - label: i18n.translate('xpack.indexLifecycleMgmt.editPolicy.indexPriorityLabel', { - defaultMessage: 'Index priority (optional)', + label: i18nTexts.editPolicy.setPriorityFieldLabel, + validations: [{ validator: ifExistsNumberGreaterThanZero }], + serializer: serializers.stringToNumber, + }, + }, + }, + }, + warm: { + min_age: { + defaultValue: '0', + validations: [ + { + validator: (arg) => + numberGreaterThanField({ + than: 0, + allowEquality: true, + message: i18nTexts.editPolicy.errors.nonNegativeNumberRequired, + })({ + ...arg, + value: arg.value === '' ? -Infinity : parseInt(arg.value, 10), + }), + }, + ], + }, + actions: { + allocate: { + number_of_replicas: { + label: i18n.translate('xpack.indexLifecycleMgmt.warmPhase.numberOfReplicasLabel', { + defaultMessage: 'Number of replicas (optional)', + }), + validations: [ + { + validator: ifExistsNumberGreaterThanZero, + }, + ], + serializer: serializers.stringToNumber, + }, + }, + shrink: { + number_of_shards: { + label: i18n.translate('xpack.indexLifecycleMgmt.warmPhase.numberOfPrimaryShardsLabel', { + defaultMessage: 'Number of primary shards', }), + validations: [ + { + validator: emptyField(i18nTexts.editPolicy.errors.numberRequired), + }, + { + validator: numberGreaterThanField({ + message: i18nTexts.editPolicy.errors.numberGreatThan0Required, + than: 0, + }), + }, + ], + serializer: serializers.stringToNumber, + }, + }, + forcemerge: { + max_num_segments: { + label: i18nTexts.editPolicy.maxNumSegmentsFieldLabel, + validations: [ + { + validator: emptyField( + i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.forcemerge.numberOfSegmentsRequiredError', + { defaultMessage: 'A value for number of segments is required.' } + ) + ), + }, + { + validator: ifExistsNumberGreaterThanZero, + }, + ], + serializer: serializers.stringToNumber, + }, + }, + set_priority: { + priority: { + defaultValue: defaultPhaseIndexPriority as any, + label: i18nTexts.editPolicy.setPriorityFieldLabel, validations: [{ validator: ifExistsNumberGreaterThanZero }], - serializer: (v: string): any => (v ? parseInt(v, 10) : undefined), + serializer: serializers.stringToNumber, }, }, }, diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/form_validations.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/form_validations.ts index b937ea2043138..37ca4e9def340 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/form_validations.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/form_validations.ts @@ -4,27 +4,19 @@ * you may not use this file except in compliance with the Elastic License. */ -import { i18n } from '@kbn/i18n'; import { fieldValidators, ValidationFunc } from '../../../shared_imports'; -import { i18nTexts } from './components/phases/hot_phase/i18n_texts'; - import { ROLLOVER_FORM_PATHS } from './constants'; -const { numberGreaterThanField } = fieldValidators; +import { i18nTexts } from './i18n_texts'; -export const positiveNumberRequiredMessage = i18n.translate( - 'xpack.indexLifecycleMgmt.editPolicy.numberAboveZeroRequiredError', - { - defaultMessage: 'Only numbers above 0 are allowed.', - } -); +const { numberGreaterThanField } = fieldValidators; export const ifExistsNumberGreaterThanZero: ValidationFunc = (arg) => { if (arg.value) { return numberGreaterThanField({ than: 0, - message: positiveNumberRequiredMessage, + message: i18nTexts.editPolicy.errors.numberGreatThan0Required, })({ ...arg, value: parseInt(arg.value, 10), @@ -54,16 +46,22 @@ export const rolloverThresholdsValidator: ValidationFunc = ({ form }) => { ) ) { fields[ROLLOVER_FORM_PATHS.maxAge].setErrors([ - { validationType: ROLLOVER_EMPTY_VALIDATION, message: i18nTexts.maximumAgeRequiredMessage }, + { + validationType: ROLLOVER_EMPTY_VALIDATION, + message: i18nTexts.editPolicy.errors.maximumAgeRequiredMessage, + }, ]); fields[ROLLOVER_FORM_PATHS.maxDocs].setErrors([ { validationType: ROLLOVER_EMPTY_VALIDATION, - message: i18nTexts.maximumDocumentsRequiredMessage, + message: i18nTexts.editPolicy.errors.maximumDocumentsRequiredMessage, }, ]); fields[ROLLOVER_FORM_PATHS.maxSize].setErrors([ - { validationType: ROLLOVER_EMPTY_VALIDATION, message: i18nTexts.maximumSizeRequiredMessage }, + { + validationType: ROLLOVER_EMPTY_VALIDATION, + message: i18nTexts.editPolicy.errors.maximumSizeRequiredMessage, + }, ]); } else { fields[ROLLOVER_FORM_PATHS.maxAge].clearErrors(ROLLOVER_EMPTY_VALIDATION); diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/i18n_texts.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/i18n_texts.ts index 31bb10b402d27..1fba69b7634ae 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/i18n_texts.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/i18n_texts.ts @@ -11,5 +11,93 @@ export const i18nTexts = { forceMergeEnabledFieldLabel: i18n.translate('xpack.indexLifecycleMgmt.forcemerge.enableLabel', { defaultMessage: 'Force merge data', }), + maxNumSegmentsFieldLabel: i18n.translate( + 'xpack.indexLifecycleMgmt.forceMerge.numberOfSegmentsLabel', + { + defaultMessage: 'Number of segments', + } + ), + setPriorityFieldLabel: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.indexPriorityLabel', + { + defaultMessage: 'Index priority (optional)', + } + ), + bestCompressionFieldLabel: i18n.translate( + 'xpack.indexLifecycleMgmt.forcemerge.bestCompressionLabel', + { + defaultMessage: 'Compress stored fields', + } + ), + bestCompressionFieldHelpText: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.forceMerge.bestCompressionText', + { + defaultMessage: + 'Use higher compression for stored fields at the cost of slower performance.', + } + ), + allocationTypeOptionsFieldLabel: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.dataTierAllocation.allocationFieldLabel', + { defaultMessage: 'Data tier options' } + ), + allocationNodeAttributeFieldLabel: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.dataTierAllocation.nodeAllocationFieldLabel', + { + defaultMessage: 'Select a node attribute', + } + ), + errors: { + numberRequired: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.errors.numberRequiredErrorMessage', + { + defaultMessage: 'A number is required.', + } + ), + numberGreatThan0Required: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.errors.numberAboveZeroRequiredError', + { + defaultMessage: 'Only numbers above 0 are allowed.', + } + ), + maximumAgeRequiredMessage: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.errors.maximumAgeMissingError', + { + defaultMessage: 'A maximum age is required.', + } + ), + maximumSizeRequiredMessage: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.errors.maximumIndexSizeMissingError', + { + defaultMessage: 'A maximum index size is required.', + } + ), + maximumDocumentsRequiredMessage: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.errors.maximumDocumentsMissingError', + { + defaultMessage: 'Maximum documents is required.', + } + ), + rollOverConfigurationCallout: { + title: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.errors.rolloverConfigurationError.title', + { + defaultMessage: 'Invalid rollover configuration', + } + ), + body: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.errors.rolloverConfigurationError.body', + { + defaultMessage: + 'A value for one of maximum size, maximum documents, or maximum age is required.', + } + ), + }, + nonNegativeNumberRequired: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.errors.nonNegativeNumberRequiredError', + { + defaultMessage: 'Only non-negative numbers are allowed.', + } + ), + }, }, }; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/serializer.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/serializer.ts index e0e1ad44f1557..90e81528f5afe 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/serializer.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/serializer.ts @@ -4,40 +4,130 @@ * you may not use this file except in compliance with the Elastic License. */ -import { SerializedPolicy } from '../../../../common/types'; +import { isEmpty } from 'lodash'; -import { FormInternal } from './types'; +import { SerializedPolicy, SerializedActionWithAllocation } from '../../../../common/types'; + +import { FormInternal, DataAllocationMetaFields } from './types'; +import { isNumber } from '../../services/policies/policy_serialization'; + +const serializeAllocateAction = ( + { dataTierAllocationType, allocationNodeAttribute }: DataAllocationMetaFields, + newActions: SerializedActionWithAllocation = {}, + originalActions: SerializedActionWithAllocation = {} +): SerializedActionWithAllocation => { + const { allocate, migrate, ...rest } = newActions; + // First copy over all non-allocate and migrate actions. + const actions: SerializedActionWithAllocation = { allocate, migrate, ...rest }; + + switch (dataTierAllocationType) { + case 'node_attrs': + if (allocationNodeAttribute) { + const [name, value] = allocationNodeAttribute.split(':'); + actions.allocate = { + // copy over any other allocate details like "number_of_replicas" + ...actions.allocate, + require: { + [name]: value, + }, + }; + } + + // copy over the original include and exclude values until we can set them in the form. + if (!isEmpty(originalActions?.allocate?.include)) { + actions.allocate = { + ...actions.allocate, + include: { ...originalActions?.allocate?.include }, + }; + } + + if (!isEmpty(originalActions?.allocate?.exclude)) { + actions.allocate = { + ...actions.allocate, + exclude: { ...originalActions?.allocate?.exclude }, + }; + } + break; + case 'none': + actions.migrate = { enabled: false }; + break; + default: + } + return actions; +}; export const createSerializer = (originalPolicy?: SerializedPolicy) => ( data: FormInternal ): SerializedPolicy => { - const { _meta, ...rest } = data; + const { _meta, ...policy } = data; - if (!rest.phases || !rest.phases.hot) { - rest.phases = { hot: { actions: {} } }; + if (!policy.phases || !policy.phases.hot) { + policy.phases = { hot: { actions: {} } }; } - if (rest.phases.hot) { - rest.phases.hot.min_age = originalPolicy?.phases.hot?.min_age ?? '0ms'; + /** + * HOT PHASE SERIALIZATION + */ + if (policy.phases.hot) { + policy.phases.hot.min_age = originalPolicy?.phases.hot?.min_age ?? '0ms'; } - if (rest.phases.hot?.actions) { - if (rest.phases.hot.actions?.rollover && _meta.hot.useRollover) { - if (rest.phases.hot.actions.rollover.max_age) { - rest.phases.hot.actions.rollover.max_age = `${rest.phases.hot.actions.rollover.max_age}${_meta.hot.maxAgeUnit}`; + if (policy.phases.hot?.actions) { + if (policy.phases.hot.actions?.rollover && _meta.hot.useRollover) { + if (policy.phases.hot.actions.rollover.max_age) { + policy.phases.hot.actions.rollover.max_age = `${policy.phases.hot.actions.rollover.max_age}${_meta.hot.maxAgeUnit}`; } - if (rest.phases.hot.actions.rollover.max_size) { - rest.phases.hot.actions.rollover.max_size = `${rest.phases.hot.actions.rollover.max_size}${_meta.hot.maxStorageSizeUnit}`; + if (policy.phases.hot.actions.rollover.max_size) { + policy.phases.hot.actions.rollover.max_size = `${policy.phases.hot.actions.rollover.max_size}${_meta.hot.maxStorageSizeUnit}`; } - if (_meta.hot.bestCompression && rest.phases.hot.actions?.forcemerge) { - rest.phases.hot.actions.forcemerge.index_codec = 'best_compression'; + if (_meta.hot.bestCompression && policy.phases.hot.actions?.forcemerge) { + policy.phases.hot.actions.forcemerge.index_codec = 'best_compression'; } } else { - delete rest.phases.hot.actions?.rollover; + delete policy.phases.hot.actions?.rollover; + } + } + + /** + * WARM PHASE SERIALIZATION + */ + if (policy.phases.warm) { + // If warm phase on rollover is enabled, delete min age field + // An index lifecycle switches to warm phase when rollover occurs, so you cannot specify a warm phase time + // They are mutually exclusive + if (_meta.hot.useRollover && _meta.warm.warmPhaseOnRollover) { + delete policy.phases.warm.min_age; + } else if ( + (!_meta.hot.useRollover || !_meta.warm.warmPhaseOnRollover) && + policy.phases.warm.min_age + ) { + policy.phases.warm.min_age = `${policy.phases.warm.min_age}${_meta.warm.minAgeUnit}`; + } + + policy.phases.warm.actions = serializeAllocateAction( + _meta.warm, + policy.phases.warm.actions, + originalPolicy?.phases.warm?.actions + ); + + if ( + policy.phases.warm.actions.allocate && + !policy.phases.warm.actions.allocate.require && + !isNumber(policy.phases.warm.actions.allocate.number_of_replicas) && + isEmpty(policy.phases.warm.actions.allocate.include) && + isEmpty(policy.phases.warm.actions.allocate.exclude) + ) { + // remove allocate action if it does not define require or number of nodes + // and both include and exclude are empty objects (ES will fail to parse if we don't) + delete policy.phases.warm.actions.allocate; + } + + if (_meta.warm.bestCompression && policy.phases.warm.actions?.forcemerge) { + policy.phases.warm.actions.forcemerge.index_codec = 'best_compression'; } } - return rest; + return policy; }; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/types.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/types.ts index dba56eb8ecbf3..6fcfbd050c69d 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/types.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/types.ts @@ -6,6 +6,29 @@ import { SerializedPolicy } from '../../../../common/types'; +export type DataTierAllocationType = 'node_roles' | 'node_attrs' | 'none'; + +export interface DataAllocationMetaFields { + dataTierAllocationType: DataTierAllocationType; + allocationNodeAttribute?: string; +} + +interface HotPhaseMetaFields { + useRollover: boolean; + forceMergeEnabled: boolean; + bestCompression: boolean; + maxStorageSizeUnit?: string; + maxAgeUnit?: string; +} + +interface WarmPhaseMetaFields extends DataAllocationMetaFields { + enabled: boolean; + forceMergeEnabled: boolean; + bestCompression: boolean; + warmPhaseOnRollover: boolean; + minAgeUnit?: string; +} + /** * Describes the shape of data after deserialization. */ @@ -15,12 +38,7 @@ export interface FormInternal extends SerializedPolicy { * certain form fields which affects what is ultimately serialized. */ _meta: { - hot: { - useRollover: boolean; - forceMergeEnabled: boolean; - bestCompression: boolean; - maxStorageSizeUnit?: string; - maxAgeUnit?: string; - }; + hot: HotPhaseMetaFields; + warm: WarmPhaseMetaFields; }; } diff --git a/x-pack/plugins/index_lifecycle_management/public/application/services/policies/cold_phase.ts b/x-pack/plugins/index_lifecycle_management/public/application/services/policies/cold_phase.ts index 9f5f603fbc564..faf3954f93fd8 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/services/policies/cold_phase.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/services/policies/cold_phase.ts @@ -13,7 +13,7 @@ import { PhaseValidationErrors, positiveNumberRequiredMessage, } from './policy_validation'; -import { determineDataTierAllocationType } from '../../lib'; +import { determineDataTierAllocationTypeLegacy } from '../../lib'; import { serializePhaseWithAllocation } from './shared'; export const coldPhaseInitialization: ColdPhase = { @@ -35,10 +35,8 @@ export const coldPhaseFromES = (phaseSerialized?: SerializedColdPhase): ColdPhas phase.phaseEnabled = true; - if (phaseSerialized.actions.allocate) { - phase.dataTierAllocationType = determineDataTierAllocationType( - phaseSerialized.actions.allocate - ); + if (phaseSerialized.actions) { + phase.dataTierAllocationType = determineDataTierAllocationTypeLegacy(phaseSerialized.actions); } if (phaseSerialized.min_age) { diff --git a/x-pack/plugins/index_lifecycle_management/public/application/services/policies/policy_serialization.test.ts b/x-pack/plugins/index_lifecycle_management/public/application/services/policies/policy_serialization.test.ts index 5c7f04986827b..0be6ab3521736 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/services/policies/policy_serialization.test.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/services/policies/policy_serialization.test.ts @@ -7,7 +7,7 @@ // eslint-disable-next-line no-restricted-imports import cloneDeep from 'lodash/cloneDeep'; import { deserializePolicy, legacySerializePolicy } from './policy_serialization'; -import { defaultNewColdPhase, defaultNewDeletePhase, defaultNewWarmPhase } from '../../constants'; +import { defaultNewColdPhase, defaultNewDeletePhase } from '../../constants'; import { DataTierAllocationType } from '../../../../common/types'; import { coldPhaseInitialization } from './cold_phase'; @@ -18,13 +18,6 @@ describe('Policy serialization', () => { { name: 'test', phases: { - warm: { - ...defaultNewWarmPhase, - dataTierAllocationType: 'default', - // These selected attrs should be ignored - selectedNodeAttrs: 'another:thing', - phaseEnabled: true, - }, cold: { ...defaultNewColdPhase, dataTierAllocationType: 'default', @@ -38,9 +31,6 @@ describe('Policy serialization', () => { name: 'test', phases: { hot: { actions: {} }, - warm: { - actions: { allocate: { include: {}, exclude: {}, require: { something: 'here' } } }, - }, cold: { actions: { allocate: { include: {}, exclude: {}, require: { something: 'here' } } }, }, @@ -50,13 +40,6 @@ describe('Policy serialization', () => { ).toEqual({ name: 'test', phases: { - warm: { - actions: { - set_priority: { - priority: 50, - }, - }, - }, cold: { actions: { set_priority: { @@ -75,12 +58,6 @@ describe('Policy serialization', () => { { name: 'test', phases: { - warm: { - ...defaultNewWarmPhase, - dataTierAllocationType: 'custom', - selectedNodeAttrs: 'another:thing', - phaseEnabled: true, - }, cold: { ...defaultNewColdPhase, dataTierAllocationType: 'custom', @@ -94,15 +71,6 @@ describe('Policy serialization', () => { name: 'test', phases: { hot: { actions: {} }, - warm: { - actions: { - allocate: { - include: { keep: 'this' }, - exclude: { keep: 'this' }, - require: { something: 'here' }, - }, - }, - }, cold: { actions: { allocate: { @@ -118,20 +86,6 @@ describe('Policy serialization', () => { ).toEqual({ name: 'test', phases: { - warm: { - actions: { - allocate: { - include: { keep: 'this' }, - exclude: { keep: 'this' }, - require: { - another: 'thing', - }, - }, - set_priority: { - priority: 50, - }, - }, - }, cold: { actions: { allocate: { @@ -157,12 +111,6 @@ describe('Policy serialization', () => { { name: 'test', phases: { - warm: { - ...defaultNewWarmPhase, - dataTierAllocationType: 'custom', - selectedNodeAttrs: '', - phaseEnabled: true, - }, cold: { ...defaultNewColdPhase, dataTierAllocationType: 'custom', @@ -176,9 +124,6 @@ describe('Policy serialization', () => { name: 'test', phases: { hot: { actions: {} }, - warm: { - actions: { allocate: { include: {}, exclude: {}, require: { something: 'here' } } }, - }, cold: { actions: { allocate: { include: {}, exclude: {}, require: { something: 'here' } } }, }, @@ -189,14 +134,6 @@ describe('Policy serialization', () => { // There should be no allocation action in any phases... name: 'test', phases: { - warm: { - actions: { - allocate: { include: {}, exclude: {}, require: { something: 'here' } }, - set_priority: { - priority: 50, - }, - }, - }, cold: { actions: { allocate: { include: {}, exclude: {}, require: { something: 'here' } }, @@ -216,12 +153,6 @@ describe('Policy serialization', () => { { name: 'test', phases: { - warm: { - ...defaultNewWarmPhase, - dataTierAllocationType: 'none', - selectedNodeAttrs: 'ignore:this', - phaseEnabled: true, - }, cold: { ...defaultNewColdPhase, dataTierAllocationType: 'none', @@ -235,9 +166,6 @@ describe('Policy serialization', () => { name: 'test', phases: { hot: { actions: {} }, - warm: { - actions: { allocate: { include: {}, exclude: {}, require: { something: 'here' } } }, - }, cold: { actions: { allocate: { include: {}, exclude: {}, require: { something: 'here' } } }, }, @@ -248,16 +176,6 @@ describe('Policy serialization', () => { // There should be no allocation action in any phases... name: 'test', phases: { - warm: { - actions: { - migrate: { - enabled: false, - }, - set_priority: { - priority: 50, - }, - }, - }, cold: { actions: { migrate: { @@ -277,9 +195,6 @@ describe('Policy serialization', () => { const originalPolicy = { name: 'test', phases: { - warm: { - actions: { allocate: { include: {}, exclude: {}, require: { something: 'here' } } }, - }, cold: { actions: { allocate: { include: {}, exclude: {}, require: { something: 'here' } } }, }, @@ -291,12 +206,6 @@ describe('Policy serialization', () => { const deserializedPolicy = { name: 'test', phases: { - warm: { - ...defaultNewWarmPhase, - dataTierAllocationType: 'none' as DataTierAllocationType, - selectedNodeAttrs: 'ignore:this', - phaseEnabled: true, - }, cold: { ...defaultNewColdPhase, dataTierAllocationType: 'none' as DataTierAllocationType, @@ -308,10 +217,6 @@ describe('Policy serialization', () => { }, }; - legacySerializePolicy(deserializedPolicy, originalPolicy); - deserializedPolicy.phases.warm.dataTierAllocationType = 'custom'; - legacySerializePolicy(deserializedPolicy, originalPolicy); - deserializedPolicy.phases.warm.dataTierAllocationType = 'default'; legacySerializePolicy(deserializedPolicy, originalPolicy); expect(originalPolicy).toEqual(originalClone); }); @@ -322,13 +227,6 @@ describe('Policy serialization', () => { { name: 'test', phases: { - warm: { - ...defaultNewWarmPhase, - phaseEnabled: true, - forceMergeEnabled: true, - selectedForceMergeSegments: '1', - bestCompressionEnabled: true, - }, cold: { ...defaultNewColdPhase, }, @@ -344,19 +242,7 @@ describe('Policy serialization', () => { ) ).toEqual({ name: 'test', - phases: { - warm: { - actions: { - forcemerge: { - max_num_segments: 1, - index_codec: 'best_compression', - }, - set_priority: { - priority: 50, - }, - }, - }, - }, + phases: {}, }); }); @@ -384,31 +270,12 @@ describe('Policy serialization', () => { }, }, }, - warm: { - actions: { - forcemerge: { - max_num_segments: 1, - index_codec: 'best_compression', - }, - set_priority: { - priority: 50, - }, - }, - }, }, }, }) ).toEqual({ name: 'test', phases: { - warm: { - ...defaultNewWarmPhase, - warmPhaseOnRollover: false, - phaseEnabled: true, - forceMergeEnabled: true, - selectedForceMergeSegments: '1', - bestCompressionEnabled: true, - }, cold: { ...coldPhaseInitialization, }, @@ -423,13 +290,6 @@ describe('Policy serialization', () => { { name: 'test', phases: { - warm: { - ...defaultNewWarmPhase, - phaseEnabled: true, - forceMergeEnabled: true, - selectedForceMergeSegments: '1', - bestCompressionEnabled: false, - }, cold: { ...defaultNewColdPhase, }, @@ -438,32 +298,12 @@ describe('Policy serialization', () => { }, { name: 'test', - phases: { - warm: { - actions: { - forcemerge: { - max_num_segments: 1, - index_codec: 'best_compression', - }, - }, - }, - }, + phases: {}, } ) ).toEqual({ name: 'test', - phases: { - warm: { - actions: { - forcemerge: { - max_num_segments: 1, - }, - set_priority: { - priority: 50, - }, - }, - }, - }, + phases: {}, }); }); }); diff --git a/x-pack/plugins/index_lifecycle_management/public/application/services/policies/policy_serialization.ts b/x-pack/plugins/index_lifecycle_management/public/application/services/policies/policy_serialization.ts index 0dce7efce4623..32c7e698b0920 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/services/policies/policy_serialization.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/services/policies/policy_serialization.ts @@ -9,11 +9,9 @@ import { LegacyPolicy, PolicyFromES, SerializedPolicy } from '../../../../common import { defaultNewColdPhase, defaultNewDeletePhase, - defaultNewWarmPhase, serializedPhaseInitialization, } from '../../constants'; -import { warmPhaseFromES, warmPhaseToES } from './warm_phase'; import { coldPhaseFromES, coldPhaseToES } from './cold_phase'; import { deletePhaseFromES, deletePhaseToES } from './delete_phase'; @@ -48,7 +46,6 @@ export const initializeNewPolicy = (newPolicyName: string = ''): LegacyPolicy => return { name: newPolicyName, phases: { - warm: { ...defaultNewWarmPhase }, cold: { ...defaultNewColdPhase }, delete: { ...defaultNewDeletePhase }, }, @@ -64,7 +61,6 @@ export const deserializePolicy = (policy: PolicyFromES): LegacyPolicy => { return { name, phases: { - warm: warmPhaseFromES(phases.warm), cold: coldPhaseFromES(phases.cold), delete: deletePhaseFromES(phases.delete), }, @@ -82,9 +78,6 @@ export const legacySerializePolicy = ( name: policy.name, phases: {}, } as SerializedPolicy; - if (policy.phases.warm.phaseEnabled) { - serializedPolicy.phases.warm = warmPhaseToES(policy.phases.warm, originalEsPolicy.phases.warm); - } if (policy.phases.cold.phaseEnabled) { serializedPolicy.phases.cold = coldPhaseToES(policy.phases.cold, originalEsPolicy.phases.cold); diff --git a/x-pack/plugins/index_lifecycle_management/public/application/services/policies/policy_validation.ts b/x-pack/plugins/index_lifecycle_management/public/application/services/policies/policy_validation.ts index eeceb97c409f5..a113cb68a2349 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/services/policies/policy_validation.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/services/policies/policy_validation.ts @@ -5,14 +5,7 @@ */ import { i18n } from '@kbn/i18n'; -import { - ColdPhase, - DeletePhase, - LegacyPolicy, - PolicyFromES, - WarmPhase, -} from '../../../../common/types'; -import { validateWarmPhase } from './warm_phase'; +import { ColdPhase, DeletePhase, LegacyPolicy, PolicyFromES } from '../../../../common/types'; import { validateColdPhase } from './cold_phase'; import { validateDeletePhase } from './delete_phase'; @@ -89,7 +82,6 @@ export type PhaseValidationErrors = { }; export interface ValidationErrors { - warm: PhaseValidationErrors; cold: PhaseValidationErrors; delete: PhaseValidationErrors; policyName: string[]; @@ -128,19 +120,16 @@ export const validatePolicy = ( } } - const warmPhaseErrors = validateWarmPhase(policy.phases.warm); const coldPhaseErrors = validateColdPhase(policy.phases.cold); const deletePhaseErrors = validateDeletePhase(policy.phases.delete); const isValid = policyNameErrors.length === 0 && - Object.keys(warmPhaseErrors).length === 0 && Object.keys(coldPhaseErrors).length === 0 && Object.keys(deletePhaseErrors).length === 0; return [ isValid, { policyName: [...policyNameErrors], - warm: warmPhaseErrors, cold: coldPhaseErrors, delete: deletePhaseErrors, }, @@ -156,9 +145,6 @@ export const findFirstError = (errors?: ValidationErrors): string | undefined => return propertyof('policyName'); } - if (Object.keys(errors.warm).length > 0) { - return `${propertyof('warm')}.${Object.keys(errors.warm)[0]}`; - } if (Object.keys(errors.cold).length > 0) { return `${propertyof('cold')}.${Object.keys(errors.cold)[0]}`; } diff --git a/x-pack/plugins/index_lifecycle_management/public/application/services/policies/warm_phase.ts b/x-pack/plugins/index_lifecycle_management/public/application/services/policies/warm_phase.ts deleted file mode 100644 index 436e5a222f86d..0000000000000 --- a/x-pack/plugins/index_lifecycle_management/public/application/services/policies/warm_phase.ts +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { isEmpty } from 'lodash'; -import { AllocateAction, WarmPhase, SerializedWarmPhase } from '../../../../common/types'; -import { serializedPhaseInitialization } from '../../constants'; -import { isNumber, splitSizeAndUnits } from './policy_serialization'; - -import { - numberRequiredMessage, - PhaseValidationErrors, - positiveNumberRequiredMessage, - positiveNumbersAboveZeroErrorMessage, -} from './policy_validation'; - -import { determineDataTierAllocationType } from '../../lib'; -import { serializePhaseWithAllocation } from './shared'; - -const warmPhaseInitialization: WarmPhase = { - phaseEnabled: false, - warmPhaseOnRollover: false, - selectedMinimumAge: '0', - selectedMinimumAgeUnits: 'd', - selectedNodeAttrs: '', - selectedReplicaCount: '', - shrinkEnabled: false, - selectedPrimaryShardCount: '', - forceMergeEnabled: false, - selectedForceMergeSegments: '', - bestCompressionEnabled: false, - phaseIndexPriority: '', - dataTierAllocationType: 'default', -}; - -export const warmPhaseFromES = (phaseSerialized?: SerializedWarmPhase): WarmPhase => { - const phase: WarmPhase = { ...warmPhaseInitialization }; - - if (phaseSerialized === undefined || phaseSerialized === null) { - return phase; - } - - phase.phaseEnabled = true; - - if (phaseSerialized.actions.allocate) { - phase.dataTierAllocationType = determineDataTierAllocationType( - phaseSerialized.actions.allocate - ); - } - - if (phaseSerialized.min_age) { - if (phaseSerialized.min_age === '0ms') { - phase.warmPhaseOnRollover = true; - } else { - const { size: minAge, units: minAgeUnits } = splitSizeAndUnits(phaseSerialized.min_age); - phase.selectedMinimumAge = minAge; - phase.selectedMinimumAgeUnits = minAgeUnits; - } - } - if (phaseSerialized.actions) { - const actions = phaseSerialized.actions; - if (actions.allocate) { - const allocate = actions.allocate; - if (allocate.require) { - Object.entries(allocate.require).forEach((entry) => { - phase.selectedNodeAttrs = entry.join(':'); - }); - if (allocate.number_of_replicas) { - phase.selectedReplicaCount = allocate.number_of_replicas.toString(); - } - } - } - - if (actions.forcemerge) { - const forcemerge = actions.forcemerge; - phase.forceMergeEnabled = true; - phase.selectedForceMergeSegments = forcemerge.max_num_segments.toString(); - // only accepted value for index_codec - phase.bestCompressionEnabled = forcemerge.index_codec === 'best_compression'; - } - - if (actions.shrink) { - phase.shrinkEnabled = true; - phase.selectedPrimaryShardCount = actions.shrink.number_of_shards - ? actions.shrink.number_of_shards.toString() - : ''; - } - - if (actions.set_priority) { - phase.phaseIndexPriority = actions.set_priority.priority - ? actions.set_priority.priority.toString() - : ''; - } - } - return phase; -}; - -export const warmPhaseToES = ( - phase: WarmPhase, - originalEsPhase?: SerializedWarmPhase -): SerializedWarmPhase => { - if (!originalEsPhase) { - originalEsPhase = { ...serializedPhaseInitialization }; - } - - const esPhase = { ...originalEsPhase }; - - if (isNumber(phase.selectedMinimumAge)) { - esPhase.min_age = `${phase.selectedMinimumAge}${phase.selectedMinimumAgeUnits}`; - } - - // If warm phase on rollover is enabled, delete min age field - // An index lifecycle switches to warm phase when rollover occurs, so you cannot specify a warm phase time - // They are mutually exclusive - if (phase.warmPhaseOnRollover) { - delete esPhase.min_age; - } - - esPhase.actions = serializePhaseWithAllocation(phase, esPhase.actions); - - if (isNumber(phase.selectedReplicaCount)) { - esPhase.actions.allocate = esPhase.actions.allocate || ({} as AllocateAction); - esPhase.actions.allocate.number_of_replicas = parseInt(phase.selectedReplicaCount, 10); - } else { - if (esPhase.actions.allocate) { - delete esPhase.actions.allocate.number_of_replicas; - } - } - - if ( - esPhase.actions.allocate && - !esPhase.actions.allocate.require && - !isNumber(esPhase.actions.allocate.number_of_replicas) && - isEmpty(esPhase.actions.allocate.include) && - isEmpty(esPhase.actions.allocate.exclude) - ) { - // remove allocate action if it does not define require or number of nodes - // and both include and exclude are empty objects (ES will fail to parse if we don't) - delete esPhase.actions.allocate; - } - - if (phase.forceMergeEnabled) { - esPhase.actions.forcemerge = { - max_num_segments: parseInt(phase.selectedForceMergeSegments, 10), - }; - if (phase.bestCompressionEnabled) { - // only accepted value for index_codec - esPhase.actions.forcemerge.index_codec = 'best_compression'; - } - } else { - delete esPhase.actions.forcemerge; - } - - if (phase.shrinkEnabled && isNumber(phase.selectedPrimaryShardCount)) { - esPhase.actions.shrink = { - number_of_shards: parseInt(phase.selectedPrimaryShardCount, 10), - }; - } else { - delete esPhase.actions.shrink; - } - - if (isNumber(phase.phaseIndexPriority)) { - esPhase.actions.set_priority = { - priority: parseInt(phase.phaseIndexPriority, 10), - }; - } else { - delete esPhase.actions.set_priority; - } - - return esPhase; -}; - -export const validateWarmPhase = (phase: WarmPhase): PhaseValidationErrors => { - if (!phase.phaseEnabled) { - return {}; - } - - const phaseErrors = {} as PhaseValidationErrors; - - // index priority is optional, but if it's set, it needs to be a positive number - if (phase.phaseIndexPriority) { - if (!isNumber(phase.phaseIndexPriority)) { - phaseErrors.phaseIndexPriority = [numberRequiredMessage]; - } else if (parseInt(phase.phaseIndexPriority, 10) < 0) { - phaseErrors.phaseIndexPriority = [positiveNumberRequiredMessage]; - } - } - - // if warm phase on rollover is disabled, min age needs to be a positive number - if (!phase.warmPhaseOnRollover) { - if (!isNumber(phase.selectedMinimumAge)) { - phaseErrors.selectedMinimumAge = [numberRequiredMessage]; - } else if (parseInt(phase.selectedMinimumAge, 10) < 0) { - phaseErrors.selectedMinimumAge = [positiveNumberRequiredMessage]; - } - } - - // if forcemerge is enabled, force merge segments needs to be a number above zero - if (phase.forceMergeEnabled) { - if (!isNumber(phase.selectedForceMergeSegments)) { - phaseErrors.selectedForceMergeSegments = [numberRequiredMessage]; - } else if (parseInt(phase.selectedForceMergeSegments, 10) < 1) { - phaseErrors.selectedForceMergeSegments = [positiveNumbersAboveZeroErrorMessage]; - } - } - - // if shrink is enabled, primary shard count needs to be a number above zero - if (phase.shrinkEnabled) { - if (!isNumber(phase.selectedPrimaryShardCount)) { - phaseErrors.selectedPrimaryShardCount = [numberRequiredMessage]; - } else if (parseInt(phase.selectedPrimaryShardCount, 10) < 1) { - phaseErrors.selectedPrimaryShardCount = [positiveNumbersAboveZeroErrorMessage]; - } - } - - // replica count is optional, but if it's set, it needs to be a positive number - if (phase.selectedReplicaCount) { - if (!isNumber(phase.selectedReplicaCount)) { - phaseErrors.selectedReplicaCount = [numberRequiredMessage]; - } else if (parseInt(phase.selectedReplicaCount, 10) < 0) { - phaseErrors.selectedReplicaCount = [numberRequiredMessage]; - } - } - - return { - ...phaseErrors, - }; -}; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/services/ui_metric.test.ts b/x-pack/plugins/index_lifecycle_management/public/application/services/ui_metric.test.ts index 81eb1c8cad135..c77e3d22f0e37 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/services/ui_metric.test.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/services/ui_metric.test.ts @@ -9,8 +9,8 @@ import { UIM_CONFIG_WARM_PHASE, UIM_CONFIG_SET_PRIORITY, UIM_CONFIG_FREEZE_INDEX, - defaultNewWarmPhase, defaultNewColdPhase, + defaultPhaseIndexPriority, } from '../constants/'; import { getUiMetricsForPhases } from './ui_metric'; @@ -38,7 +38,7 @@ describe('getUiMetricsForPhases', () => { min_age: '0ms', actions: { set_priority: { - priority: parseInt(defaultNewWarmPhase.phaseIndexPriority, 10), + priority: parseInt(defaultPhaseIndexPriority, 10), }, }, }, @@ -53,7 +53,7 @@ describe('getUiMetricsForPhases', () => { min_age: '0ms', actions: { set_priority: { - priority: parseInt(defaultNewWarmPhase.phaseIndexPriority, 10) + 1, + priority: parseInt(defaultPhaseIndexPriority, 10) + 1, }, }, }, diff --git a/x-pack/plugins/index_lifecycle_management/public/application/services/ui_metric.ts b/x-pack/plugins/index_lifecycle_management/public/application/services/ui_metric.ts index ea5c5619da589..305b35b23e4d8 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/services/ui_metric.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/services/ui_metric.ts @@ -14,8 +14,8 @@ import { UIM_CONFIG_SET_PRIORITY, UIM_CONFIG_WARM_PHASE, defaultNewColdPhase, - defaultNewWarmPhase, defaultSetPriority, + defaultPhaseIndexPriority, } from '../constants'; import { Phases } from '../../../common/types'; @@ -50,8 +50,7 @@ export function getUiMetricsForPhases(phases: Phases): string[] { const isWarmPhasePriorityChanged = phases.warm && phases.warm.actions.set_priority && - phases.warm.actions.set_priority.priority !== - parseInt(defaultNewWarmPhase.phaseIndexPriority, 10); + phases.warm.actions.set_priority.priority !== parseInt(defaultPhaseIndexPriority, 10); const isColdPhasePriorityChanged = phases.cold && diff --git a/x-pack/plugins/index_lifecycle_management/public/shared_imports.ts b/x-pack/plugins/index_lifecycle_management/public/shared_imports.ts index dc3e1b1d1b62d..023aeba57aa7a 100644 --- a/x-pack/plugins/index_lifecycle_management/public/shared_imports.ts +++ b/x-pack/plugins/index_lifecycle_management/public/shared_imports.ts @@ -26,6 +26,7 @@ export { ToggleField, NumericField, SelectField, + SuperSelectField, } from '../../../../src/plugins/es_ui_shared/static/forms/components'; export { KibanaContextProvider } from '../../../../src/plugins/kibana_react/public'; diff --git a/x-pack/plugins/index_lifecycle_management/server/routes/api/nodes/register_list_route.ts b/x-pack/plugins/index_lifecycle_management/server/routes/api/nodes/register_list_route.ts index 53955d93c1e09..0603ebf6eebe0 100644 --- a/x-pack/plugins/index_lifecycle_management/server/routes/api/nodes/register_list_route.ts +++ b/x-pack/plugins/index_lifecycle_management/server/routes/api/nodes/register_list_route.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { ListNodesRouteResponse, NodeDataRole } from '../../../../common/types'; +import { ListNodesRouteResponse, DataTierRole } from '../../../../common/types'; import { RouteDependencies } from '../../../types'; import { addBasePath } from '../../../services'; @@ -39,10 +39,10 @@ export function convertSettingsIntoLists( } } - const dataRoles = nodeSettings.roles.filter((r) => r.startsWith('data')) as NodeDataRole[]; + const dataRoles = nodeSettings.roles.filter((r) => r.startsWith('data')) as DataTierRole[]; for (const role of dataRoles) { - accum.nodesByRoles[role as NodeDataRole] = accum.nodesByRoles[role] ?? []; - accum.nodesByRoles[role as NodeDataRole]!.push(nodeId); + accum.nodesByRoles[role as DataTierRole] = accum.nodesByRoles[role] ?? []; + accum.nodesByRoles[role as DataTierRole]!.push(nodeId); } // If we detect a single node using legacy "data:true" setting we know we are not using data roles for diff --git a/x-pack/plugins/index_management/public/application/sections/home/index_list/index_actions_context_menu/index_actions_context_menu.js b/x-pack/plugins/index_management/public/application/sections/home/index_list/index_actions_context_menu/index_actions_context_menu.js index 5c249ee474b00..6e96ef56d683f 100644 --- a/x-pack/plugins/index_management/public/application/sections/home/index_list/index_actions_context_menu/index_actions_context_menu.js +++ b/x-pack/plugins/index_management/public/application/sections/home/index_list/index_actions_context_menu/index_actions_context_menu.js @@ -361,9 +361,10 @@ export class IndexActionsContextMenu extends Component {

diff --git a/x-pack/plugins/infra/server/lib/adapters/framework/kibana_framework_adapter.ts b/x-pack/plugins/infra/server/lib/adapters/framework/kibana_framework_adapter.ts index 2dcab5b49dcdb..2d84e36f3a3ac 100644 --- a/x-pack/plugins/infra/server/lib/adapters/framework/kibana_framework_adapter.ts +++ b/x-pack/plugins/infra/server/lib/adapters/framework/kibana_framework_adapter.ts @@ -26,7 +26,6 @@ import { RequestHandlerContext, KibanaResponseFactory, RouteMethod, - LegacyAPICaller, } from '../../../../../../../src/core/server'; import { RequestHandler } from '../../../../../../../src/core/server'; import { InfraConfig } from '../../../plugin'; @@ -218,11 +217,7 @@ export class KibanaFramework { } public getIndexPatternsService(requestContext: RequestHandlerContext): IndexPatternsFetcher { - return new IndexPatternsFetcher((...rest: Parameters) => { - rest[1] = rest[1] || {}; - rest[1].allowNoIndices = true; - return requestContext.core.elasticsearch.legacy.client.callAsCurrentUser(...rest); - }); + return new IndexPatternsFetcher(requestContext.core.elasticsearch.client.asCurrentUser, true); } public getSpaceId(request: KibanaRequest): string { diff --git a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_metric_to_metrics_api_metric.ts b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_metric_to_metrics_api_metric.ts index 93948a8b8797e..c4f2c5f7981d4 100644 --- a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_metric_to_metrics_api_metric.ts +++ b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_metric_to_metrics_api_metric.ts @@ -9,7 +9,7 @@ import { MetricsAPIMetric, MetricsExplorerMetric } from '../../../../common/http export const convertMetricToMetricsAPIMetric = ( metric: MetricsExplorerMetric, index: number -): MetricsAPIMetric => { +): MetricsAPIMetric | undefined => { const id = `metric_${index}`; if (metric.aggregation === 'rate' && metric.field) { return { @@ -44,19 +44,21 @@ export const convertMetricToMetricsAPIMetric = ( }; } - return { - id, - aggregations: { - [id]: { - bucket_script: { - buckets_path: { count: '_count' }, - script: { - source: 'count * 1', - lang: 'expression', + if (metric.aggregation === 'count') { + return { + id, + aggregations: { + [id]: { + bucket_script: { + buckets_path: { count: '_count' }, + script: { + source: 'count * 1', + lang: 'expression', + }, + gap_policy: 'skip', }, - gap_policy: 'skip', }, }, - }, - }; + }; + } }; diff --git a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_request_to_metrics_api_options.test.ts b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_request_to_metrics_api_options.test.ts index 4c423aee347e9..887f464b1a564 100644 --- a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_request_to_metrics_api_options.test.ts +++ b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_request_to_metrics_api_options.test.ts @@ -120,4 +120,16 @@ describe('convertRequestToMetricsAPIOptions', () => { metrics: [], }); }); + + it('should work with empty field', () => { + expect( + convertRequestToMetricsAPIOptions({ + ...BASE_REQUEST, + metrics: [{ aggregation: 'avg' }], + }) + ).toEqual({ + ...BASE_METRICS_UI_OPTIONS, + metrics: [], + }); + }); }); diff --git a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_request_to_metrics_api_options.ts b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_request_to_metrics_api_options.ts index 2dd00c4aed59c..4bb6d8f55a11a 100644 --- a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_request_to_metrics_api_options.ts +++ b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_request_to_metrics_api_options.ts @@ -15,7 +15,9 @@ import { convertMetricToMetricsAPIMetric } from './convert_metric_to_metrics_api export const convertRequestToMetricsAPIOptions = ( options: MetricsExplorerRequestBody ): MetricsAPIRequest => { - const metrics = options.metrics.map(convertMetricToMetricsAPIMetric); + const metrics = options.metrics + .map(convertMetricToMetricsAPIMetric) + .filter((m: M): m is NonNullable => !!m); const { limit, timerange, indexPattern } = options; const metricsApiOptions: MetricsAPIRequest = { diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/layouts/default.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/layouts/default.tsx index d71b90d16725d..5d39995922b93 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/layouts/default.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/layouts/default.tsx @@ -18,7 +18,9 @@ interface Props { } const Container = styled.div` - min-height: calc(100vh - ${(props) => props.theme.eui.euiHeaderChildSize}); + min-height: calc( + 100vh - ${(props) => parseFloat(props.theme.eui.euiHeaderHeightCompensation) * 2}px + ); background: ${(props) => props.theme.eui.euiColorEmptyShade}; display: flex; flex-direction: column; diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/details_page/components/package_policies/package_policies_table.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/details_page/components/package_policies/package_policies_table.tsx index fa9ce24935429..af4c2f78f14a2 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/details_page/components/package_policies/package_policies_table.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/details_page/components/package_policies/package_policies_table.tsx @@ -22,7 +22,6 @@ import { useCapabilities, useLink } from '../../../../../hooks'; import { useAgentPolicyRefresh } from '../../hooks'; interface InMemoryPackagePolicy extends PackagePolicy { - inputTypes: string[]; packageName?: string; packageTitle?: string; packageVersion?: string; @@ -56,11 +55,7 @@ export const PackagePoliciesTable: React.FunctionComponent = ({ // With the package policies provided on input, generate the list of package policies // used in the InMemoryTable (flattens some values for search) as well as // the list of options that will be used in the filters dropdowns - const [packagePolicies, namespaces, inputTypes] = useMemo((): [ - InMemoryPackagePolicy[], - FilterOption[], - FilterOption[] - ] => { + const [packagePolicies, namespaces] = useMemo((): [InMemoryPackagePolicy[], FilterOption[]] => { const namespacesValues: string[] = []; const inputTypesValues: string[] = []; const mappedPackagePolicies = originalPackagePolicies.map( @@ -69,13 +64,8 @@ export const PackagePoliciesTable: React.FunctionComponent = ({ namespacesValues.push(packagePolicy.namespace); } - const dsInputTypes: string[] = []; - - dsInputTypes.sort(stringSortAscending); - return { ...packagePolicy, - inputTypes: dsInputTypes, packageName: packagePolicy.package?.name ?? '', packageTitle: packagePolicy.package?.title ?? '', packageVersion: packagePolicy.package?.version ?? '', @@ -86,11 +76,7 @@ export const PackagePoliciesTable: React.FunctionComponent = ({ namespacesValues.sort(stringSortAscending); inputTypesValues.sort(stringSortAscending); - return [ - mappedPackagePolicies, - namespacesValues.map(toFilterOption), - inputTypesValues.map(toFilterOption), - ]; + return [mappedPackagePolicies, namespacesValues.map(toFilterOption)]; }, [originalPackagePolicies]); const columns = useMemo( @@ -273,13 +259,7 @@ export const PackagePoliciesTable: React.FunctionComponent = ({ name: 'Namespace', options: namespaces, multiSelect: 'or', - }, - { - type: 'field_value_selection', - field: 'inputTypes', - name: 'Input types', - options: inputTypes, - multiSelect: 'or', + operator: 'exact', }, ], }} diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/list_page/index.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/list_page/index.tsx index bb109d766c50a..4e32fa0bbc1b9 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/list_page/index.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/list_page/index.tsx @@ -182,7 +182,12 @@ export const DataStreamListPage: React.FunctionComponent<{}> = () => { [] ); - const filterOptions: { [key: string]: string[] } = { + const filterOptions: { + [key: string]: Array<{ + value: string; + name: string; + }>; + } = { dataset: [], type: [], namespace: [], @@ -190,21 +195,37 @@ export const DataStreamListPage: React.FunctionComponent<{}> = () => { }; if (dataStreamsData && dataStreamsData.data_streams.length) { + const dataValues: { + [key: string]: string[]; + } = { + dataset: [], + type: [], + namespace: [], + package: [], + }; dataStreamsData.data_streams.forEach((stream) => { const { dataset, type, namespace, package: pkg } = stream; - if (!filterOptions.dataset.includes(dataset)) { - filterOptions.dataset.push(dataset); + if (!dataValues.dataset.includes(dataset)) { + dataValues.dataset.push(dataset); } - if (!filterOptions.type.includes(type)) { - filterOptions.type.push(type); + if (!dataValues.type.includes(type)) { + dataValues.type.push(type); } - if (!filterOptions.namespace.includes(namespace)) { - filterOptions.namespace.push(namespace); + if (!dataValues.namespace.includes(namespace)) { + dataValues.namespace.push(namespace); } - if (!filterOptions.package.includes(pkg)) { - filterOptions.package.push(pkg); + if (!dataValues.package.includes(pkg)) { + dataValues.package.push(pkg); } }); + for (const field in dataValues) { + if (filterOptions[field]) { + filterOptions[field] = dataValues[field].sort().map((option) => ({ + value: option, + name: option, + })); + } + } } return ( @@ -266,10 +287,8 @@ export const DataStreamListPage: React.FunctionComponent<{}> = () => { defaultMessage: 'Dataset', }), multiSelect: 'or', - options: filterOptions.dataset.map((option) => ({ - value: option, - name: option, - })), + operator: 'exact', + options: filterOptions.dataset, }, { type: 'field_value_selection', @@ -278,10 +297,8 @@ export const DataStreamListPage: React.FunctionComponent<{}> = () => { defaultMessage: 'Type', }), multiSelect: 'or', - options: filterOptions.type.map((option) => ({ - value: option, - name: option, - })), + operator: 'exact', + options: filterOptions.type, }, { type: 'field_value_selection', @@ -290,10 +307,8 @@ export const DataStreamListPage: React.FunctionComponent<{}> = () => { defaultMessage: 'Namespace', }), multiSelect: 'or', - options: filterOptions.namespace.map((option) => ({ - value: option, - name: option, - })), + operator: 'exact', + options: filterOptions.namespace, }, { type: 'field_value_selection', @@ -302,10 +317,8 @@ export const DataStreamListPage: React.FunctionComponent<{}> = () => { defaultMessage: 'Integration', }), multiSelect: 'or', - options: filterOptions.package.map((option) => ({ - value: option, - name: option, - })), + operator: 'exact', + options: filterOptions.package, }, ], }} diff --git a/x-pack/plugins/ingest_manager/server/plugin.ts b/x-pack/plugins/ingest_manager/server/plugin.ts index e5a06b7e38131..bf5b2aac50643 100644 --- a/x-pack/plugins/ingest_manager/server/plugin.ts +++ b/x-pack/plugins/ingest_manager/server/plugin.ts @@ -14,6 +14,8 @@ import { SavedObjectsServiceStart, HttpServiceSetup, SavedObjectsClientContract, + RequestHandlerContext, + KibanaRequest, } from 'kibana/server'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { DEFAULT_APP_CATEGORIES } from '../../../../src/core/server'; @@ -112,7 +114,11 @@ const allSavedObjectTypes = [ */ export type ExternalCallback = [ 'packagePolicyCreate', - (newPackagePolicy: NewPackagePolicy) => Promise + ( + newPackagePolicy: NewPackagePolicy, + context: RequestHandlerContext, + request: KibanaRequest + ) => Promise ]; export type ExternalCallbacksStorage = Map>; diff --git a/x-pack/plugins/ingest_manager/server/routes/package_policy/handlers.test.ts b/x-pack/plugins/ingest_manager/server/routes/package_policy/handlers.test.ts index db23d6a139f20..44c2ccda3bd2a 100644 --- a/x-pack/plugins/ingest_manager/server/routes/package_policy/handlers.test.ts +++ b/x-pack/plugins/ingest_manager/server/routes/package_policy/handlers.test.ts @@ -168,45 +168,53 @@ describe('When calling package policy', () => { const request = getCreateKibanaRequest(); await routeHandler(context, request, response); expect(response.ok).toHaveBeenCalled(); - expect(callbackOne).toHaveBeenCalledWith({ - policy_id: 'a5ca00c0-b30c-11ea-9732-1bb05811278c', - description: '', - enabled: true, - inputs: [], - name: 'endpoint-1', - namespace: 'default', - output_id: '', - package: { - name: 'endpoint', - title: 'Elastic Endpoint', - version: '0.5.0', + expect(callbackOne).toHaveBeenCalledWith( + { + policy_id: 'a5ca00c0-b30c-11ea-9732-1bb05811278c', + description: '', + enabled: true, + inputs: [], + name: 'endpoint-1', + namespace: 'default', + output_id: '', + package: { + name: 'endpoint', + title: 'Elastic Endpoint', + version: '0.5.0', + }, }, - }); - expect(callbackTwo).toHaveBeenCalledWith({ - policy_id: 'a5ca00c0-b30c-11ea-9732-1bb05811278c', - description: '', - enabled: true, - inputs: [ - { - type: 'endpoint', - enabled: true, - streams: [], - config: { - one: { - value: 'inserted by callbackOne', + context, + request + ); + expect(callbackTwo).toHaveBeenCalledWith( + { + policy_id: 'a5ca00c0-b30c-11ea-9732-1bb05811278c', + description: '', + enabled: true, + inputs: [ + { + type: 'endpoint', + enabled: true, + streams: [], + config: { + one: { + value: 'inserted by callbackOne', + }, }, }, + ], + name: 'endpoint-1', + namespace: 'default', + output_id: '', + package: { + name: 'endpoint', + title: 'Elastic Endpoint', + version: '0.5.0', }, - ], - name: 'endpoint-1', - namespace: 'default', - output_id: '', - package: { - name: 'endpoint', - title: 'Elastic Endpoint', - version: '0.5.0', }, - }); + context, + request + ); }); it('should create with data from callback', async () => { diff --git a/x-pack/plugins/ingest_manager/server/routes/package_policy/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/package_policy/handlers.ts index 183488265e5af..9e582e6960ade 100644 --- a/x-pack/plugins/ingest_manager/server/routes/package_policy/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/routes/package_policy/handlers.ts @@ -90,7 +90,7 @@ export const createPackagePolicyHandler: RequestHandler< try { // ensure that the returned value by the callback passes schema validation updatedNewData = CreatePackagePolicyRequestSchema.body.validate( - await callback(updatedNewData) + await callback(updatedNewData, context, request) ); } catch (error) { // Log the error, but keep going and process the other callbacks diff --git a/x-pack/plugins/ingest_manager/server/services/agent_policy.ts b/x-pack/plugins/ingest_manager/server/services/agent_policy.ts index 19d69a33788c6..b003d16d379ca 100644 --- a/x-pack/plugins/ingest_manager/server/services/agent_policy.ts +++ b/x-pack/plugins/ingest_manager/server/services/agent_policy.ts @@ -83,7 +83,12 @@ class AgentPolicyService { return (await this.get(soClient, id)) as AgentPolicy; } - public async ensureDefaultAgentPolicy(soClient: SavedObjectsClientContract) { + public async ensureDefaultAgentPolicy( + soClient: SavedObjectsClientContract + ): Promise<{ + created: boolean; + defaultAgentPolicy: AgentPolicy; + }> { const agentPolicies = await soClient.find({ type: AGENT_POLICY_SAVED_OBJECT_TYPE, searchFields: ['is_default'], @@ -95,12 +100,18 @@ class AgentPolicyService { ...DEFAULT_AGENT_POLICY, }; - return this.create(soClient, newDefaultAgentPolicy); + return { + created: true, + defaultAgentPolicy: await this.create(soClient, newDefaultAgentPolicy), + }; } return { - id: agentPolicies.saved_objects[0].id, - ...agentPolicies.saved_objects[0].attributes, + created: false, + defaultAgentPolicy: { + id: agentPolicies.saved_objects[0].id, + ...agentPolicies.saved_objects[0].attributes, + }, }; } @@ -404,7 +415,9 @@ class AgentPolicyService { throw new Error('Agent policy not found'); } - const { id: defaultAgentPolicyId } = await this.ensureDefaultAgentPolicy(soClient); + const { + defaultAgentPolicy: { id: defaultAgentPolicyId }, + } = await this.ensureDefaultAgentPolicy(soClient); if (id === defaultAgentPolicyId) { throw new Error('The default agent policy cannot be deleted'); } diff --git a/x-pack/plugins/ingest_manager/server/services/setup.ts b/x-pack/plugins/ingest_manager/server/services/setup.ts index 7f379d3ea4f13..741a23824f010 100644 --- a/x-pack/plugins/ingest_manager/server/services/setup.ts +++ b/x-pack/plugins/ingest_manager/server/services/setup.ts @@ -49,7 +49,11 @@ async function createSetupSideEffects( soClient: SavedObjectsClientContract, callCluster: CallESAsCurrentUser ): Promise { - const [installedPackages, defaultOutput, defaultAgentPolicy] = await Promise.all([ + const [ + installedPackages, + defaultOutput, + { created: defaultAgentPolicyCreated, defaultAgentPolicy }, + ] = await Promise.all([ // packages installed by default ensureInstalledDefaultPackages(soClient, callCluster), outputService.ensureDefaultOutput(soClient), @@ -66,44 +70,46 @@ async function createSetupSideEffects( }), ]); - // ensure default packages are added to the default conifg - const agentPolicyWithPackagePolicies = await agentPolicyService.get( - soClient, - defaultAgentPolicy.id, - true - ); - if (!agentPolicyWithPackagePolicies) { - throw new Error('Policy not found'); - } - if ( - agentPolicyWithPackagePolicies.package_policies.length && - typeof agentPolicyWithPackagePolicies.package_policies[0] === 'string' - ) { - throw new Error('Policy not found'); - } - - for (const installedPackage of installedPackages) { - const packageShouldBeInstalled = DEFAULT_AGENT_POLICIES_PACKAGES.some( - (packageName) => installedPackage.name === packageName + // If we just created the default policy, ensure default packages are added to it + if (defaultAgentPolicyCreated) { + const agentPolicyWithPackagePolicies = await agentPolicyService.get( + soClient, + defaultAgentPolicy.id, + true ); - if (!packageShouldBeInstalled) { - continue; + if (!agentPolicyWithPackagePolicies) { + throw new Error('Policy not found'); + } + if ( + agentPolicyWithPackagePolicies.package_policies.length && + typeof agentPolicyWithPackagePolicies.package_policies[0] === 'string' + ) { + throw new Error('Policy not found'); } - const isInstalled = agentPolicyWithPackagePolicies.package_policies.some( - (d: PackagePolicy | string) => { - return typeof d !== 'string' && d.package?.name === installedPackage.name; + for (const installedPackage of installedPackages) { + const packageShouldBeInstalled = DEFAULT_AGENT_POLICIES_PACKAGES.some( + (packageName) => installedPackage.name === packageName + ); + if (!packageShouldBeInstalled) { + continue; } - ); - if (!isInstalled) { - await addPackageToAgentPolicy( - soClient, - callCluster, - installedPackage, - agentPolicyWithPackagePolicies, - defaultOutput + const isInstalled = agentPolicyWithPackagePolicies.package_policies.some( + (d: PackagePolicy | string) => { + return typeof d !== 'string' && d.package?.name === installedPackage.name; + } ); + + if (!isInstalled) { + await addPackageToAgentPolicy( + soClient, + callCluster, + installedPackage, + agentPolicyWithPackagePolicies, + defaultOutput + ); + } } } diff --git a/x-pack/plugins/lens/public/app_plugin/mounter.tsx b/x-pack/plugins/lens/public/app_plugin/mounter.tsx index 3943cbc54f0b3..ac5d145eedd5b 100644 --- a/x-pack/plugins/lens/public/app_plugin/mounter.tsx +++ b/x-pack/plugins/lens/public/app_plugin/mounter.tsx @@ -97,9 +97,12 @@ export async function mountApp( const redirectTo = (routeProps: RouteComponentProps<{ id?: string }>, savedObjectId?: string) => { if (!savedObjectId) { - routeProps.history.push('/'); + routeProps.history.push({ pathname: '/', search: routeProps.history.location.search }); } else { - routeProps.history.push(`/edit/${savedObjectId}`); + routeProps.history.push({ + pathname: `/edit/${savedObjectId}`, + search: routeProps.history.location.search, + }); } }; diff --git a/x-pack/plugins/lens/public/editor_frame_service/embeddable/embeddable.test.tsx b/x-pack/plugins/lens/public/editor_frame_service/embeddable/embeddable.test.tsx index d91865c21a2a6..3e05d4ddfbc20 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/embeddable/embeddable.test.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/embeddable/embeddable.test.tsx @@ -28,6 +28,7 @@ import { IBasePath } from '../../../../../../src/core/public'; import { AttributeService } from '../../../../../../src/plugins/embeddable/public'; import { LensAttributeService } from '../../lens_attribute_service'; import { OnSaveProps } from '../../../../../../src/plugins/saved_objects/public/save_modal'; +import { act } from 'react-dom/test-utils'; jest.mock('../../../../../../src/plugins/inspector/public/', () => ({ isAvailable: false, @@ -337,10 +338,12 @@ describe('embeddable', () => { } as LensEmbeddableInput); embeddable.render(mountpoint); - embeddable.updateInput({ - timeRange, - query, - filters: [{ meta: { alias: 'test', negate: true, disabled: true } }], + act(() => { + embeddable.updateInput({ + timeRange, + query, + filters: [{ meta: { alias: 'test', negate: true, disabled: true } }], + }); }); expect(expressionRenderer).toHaveBeenCalledTimes(1); @@ -384,7 +387,9 @@ describe('embeddable', () => { } as LensEmbeddableInput); embeddable.render(mountpoint); - autoRefreshFetchSubject.next(); + act(() => { + autoRefreshFetchSubject.next(); + }); expect(expressionRenderer).toHaveBeenCalledTimes(2); }); diff --git a/x-pack/plugins/lens/public/editor_frame_service/format_column.ts b/x-pack/plugins/lens/public/editor_frame_service/format_column.ts deleted file mode 100644 index 2da6e7195a5e1..0000000000000 --- a/x-pack/plugins/lens/public/editor_frame_service/format_column.ts +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { ExpressionFunctionDefinition, Datatable } from 'src/plugins/expressions/public'; - -interface FormatColumn { - format: string; - columnId: string; - decimals?: number; -} - -const supportedFormats: Record string }> = { - number: { - decimalsToPattern: (decimals = 2) => { - if (decimals === 0) { - return `0,0`; - } - return `0,0.${'0'.repeat(decimals)}`; - }, - }, - percent: { - decimalsToPattern: (decimals = 2) => { - if (decimals === 0) { - return `0,0%`; - } - return `0,0.${'0'.repeat(decimals)}%`; - }, - }, - bytes: { - decimalsToPattern: (decimals = 2) => { - if (decimals === 0) { - return `0,0b`; - } - return `0,0.${'0'.repeat(decimals)}b`; - }, - }, -}; - -export const formatColumn: ExpressionFunctionDefinition< - 'lens_format_column', - Datatable, - FormatColumn, - Datatable -> = { - name: 'lens_format_column', - type: 'datatable', - help: '', - args: { - format: { - types: ['string'], - help: '', - required: true, - }, - columnId: { - types: ['string'], - help: '', - required: true, - }, - decimals: { - types: ['number'], - help: '', - }, - }, - inputTypes: ['datatable'], - fn(input, { format, columnId, decimals }: FormatColumn) { - return { - ...input, - columns: input.columns.map((col) => { - if (col.id === columnId) { - if (supportedFormats[format]) { - return { - ...col, - meta: { - ...col.meta, - params: { - id: format, - params: { pattern: supportedFormats[format].decimalsToPattern(decimals) }, - }, - }, - }; - } else { - return { - ...col, - meta: { - ...col.meta, - params: { - id: format, - }, - }, - }; - } - } - return col; - }), - }; - }, -}; diff --git a/x-pack/plugins/lens/public/editor_frame_service/service.tsx b/x-pack/plugins/lens/public/editor_frame_service/service.tsx index e2a382133cb3c..d1df63780594e 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/service.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/service.tsx @@ -23,7 +23,6 @@ import { } from '../types'; import { Document } from '../persistence/saved_object_store'; import { mergeTables } from './merge_tables'; -import { formatColumn } from './format_column'; import { EmbeddableFactory, LensEmbeddableStartServices } from './embeddable/embeddable_factory'; import { UiActionsStart } from '../../../../../src/plugins/ui_actions/public'; import { DashboardStart } from '../../../../../src/plugins/dashboard/public'; @@ -86,7 +85,6 @@ export class EditorFrameService { getAttributeService: () => Promise ): EditorFrameSetup { plugins.expressions.registerFunction(() => mergeTables); - plugins.expressions.registerFunction(() => formatColumn); const getStartServices = async (): Promise => { const [coreStart, deps] = await core.getStartServices(); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index 310548e5ab817..9500d4b44b79e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -435,7 +435,8 @@ export function DimensionEditor(props: DimensionEditorProps) { /> )} - {selectedColumn && selectedColumn.dataType === 'number' ? ( + {selectedColumn && + (selectedColumn.dataType === 'number' || selectedColumn.operationType === 'range') ? ( { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/format_column.ts b/x-pack/plugins/lens/public/indexpattern_datasource/format_column.ts new file mode 100644 index 0000000000000..3666528f43166 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/format_column.ts @@ -0,0 +1,137 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + ExpressionFunctionDefinition, + Datatable, + DatatableColumn, +} from 'src/plugins/expressions/public'; + +interface FormatColumn { + format: string; + columnId: string; + decimals?: number; + parentFormat?: string; +} + +export const supportedFormats: Record< + string, + { decimalsToPattern: (decimals?: number) => string } +> = { + number: { + decimalsToPattern: (decimals = 2) => { + if (decimals === 0) { + return `0,0`; + } + return `0,0.${'0'.repeat(decimals)}`; + }, + }, + percent: { + decimalsToPattern: (decimals = 2) => { + if (decimals === 0) { + return `0,0%`; + } + return `0,0.${'0'.repeat(decimals)}%`; + }, + }, + bytes: { + decimalsToPattern: (decimals = 2) => { + if (decimals === 0) { + return `0,0b`; + } + return `0,0.${'0'.repeat(decimals)}b`; + }, + }, +}; + +export const formatColumn: ExpressionFunctionDefinition< + 'lens_format_column', + Datatable, + FormatColumn, + Datatable +> = { + name: 'lens_format_column', + type: 'datatable', + help: '', + args: { + format: { + types: ['string'], + help: '', + required: true, + }, + columnId: { + types: ['string'], + help: '', + required: true, + }, + decimals: { + types: ['number'], + help: '', + }, + parentFormat: { + types: ['string'], + help: '', + }, + }, + inputTypes: ['datatable'], + fn(input, { format, columnId, decimals, parentFormat }: FormatColumn) { + return { + ...input, + columns: input.columns.map((col) => { + if (col.id === columnId) { + if (!parentFormat) { + if (supportedFormats[format]) { + return withParams(col, { + id: format, + params: { pattern: supportedFormats[format].decimalsToPattern(decimals) }, + }); + } else if (format) { + return withParams(col, { id: format }); + } else { + return col; + } + } + + const parsedParentFormat = JSON.parse(parentFormat); + const parentFormatId = parsedParentFormat.id; + const parentFormatParams = parsedParentFormat.params ?? {}; + + if (!parentFormatId) { + return col; + } + + if (format && supportedFormats[format]) { + return withParams(col, { + id: parentFormatId, + params: { + id: format, + params: { + pattern: supportedFormats[format].decimalsToPattern(decimals), + }, + ...parentFormatParams, + }, + }); + } + if (parentFormatParams) { + const innerParams = (col.meta.params?.params as Record) ?? {}; + return withParams(col, { + ...col.meta.params, + params: { + ...innerParams, + ...parentFormatParams, + }, + }); + } + } + return col; + }), + }; + }, +}; + +function withParams(col: DatatableColumn, params: Record) { + return { ...col, meta: { ...col.meta, params } }; +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/index.ts index 4fbed04112632..35987656f6670 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/index.ts @@ -33,8 +33,11 @@ export class IndexPatternDatasource { { expressions, editorFrame, charts }: IndexPatternDatasourceSetupPlugins ) { editorFrame.registerDatasource(async () => { - const { getIndexPatternDatasource, renameColumns } = await import('../async_services'); + const { getIndexPatternDatasource, renameColumns, formatColumn } = await import( + '../async_services' + ); expressions.registerFunction(renameColumns); + expressions.registerFunction(formatColumn); return core.getStartServices().then(([coreStart, { data }]) => getIndexPatternDatasource({ core: coreStart, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx index 28aeac223e4a6..a6edfd71c93ca 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx @@ -107,6 +107,7 @@ export function uniqueLabels(layers: Record) { } export * from './rename_columns'; +export * from './format_column'; export function getIndexPatternDatasource({ core, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/loader.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/loader.test.ts index 06cfdf7e03481..4222c02388433 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/loader.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/loader.test.ts @@ -117,6 +117,7 @@ const indexPattern2 = ({ title: 'my-fake-restricted-pattern', timeFieldName: 'timestamp', hasRestrictions: true, + fieldFormatMap: { bytes: { id: 'bytes', params: { pattern: '0.0' } } }, fields: [ { name: 'timestamp', diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/loader.ts b/x-pack/plugins/lens/public/indexpattern_datasource/loader.ts index fd8e071d524ee..70079cce6cc46 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/loader.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/loader.ts @@ -103,7 +103,14 @@ export async function loadIndexPatterns({ id: indexPattern.id!, // id exists for sure because we got index patterns by id title, timeFieldName, - fieldFormatMap, + fieldFormatMap: + fieldFormatMap && + Object.fromEntries( + Object.entries(fieldFormatMap).map(([id, format]) => [ + id, + 'toJSON' in format ? format.toJSON() : format, + ]) + ), fields: newFields, hasRestrictions: !!typeMeta?.aggs, }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/mocks.ts b/x-pack/plugins/lens/public/indexpattern_datasource/mocks.ts index 21ed23321cf57..744a9f6743d09 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/mocks.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/mocks.ts @@ -82,6 +82,7 @@ export const createMockedRestrictedIndexPattern = () => ({ title: 'my-fake-restricted-pattern', timeFieldName: 'timestamp', hasRestrictions: true, + fieldFormatMap: { bytes: { id: 'bytes', params: { pattern: '0.0' } } }, fields: [ { name: 'timestamp', diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/advanced_editor.scss b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/advanced_editor.scss index b1658043f3204..4af490e7479da 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/advanced_editor.scss +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/advanced_editor.scss @@ -3,4 +3,8 @@ @include euiFontSizeS; min-height: $euiSizeXL; width: 100%; -} \ No newline at end of file +} + +.lnsRangesOperation__popoverNumberField { + width: 14ch; // Roughly 10 characters plus extra for the padding +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/advanced_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/advanced_editor.tsx index 96f4120e3df78..c6773e806aef8 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/advanced_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/advanced_editor.tsx @@ -20,8 +20,8 @@ import { EuiPopover, EuiToolTip, htmlIdGenerator, + keys, } from '@elastic/eui'; -import { keys } from '@elastic/eui'; import { IFieldFormat } from '../../../../../../../../src/plugins/data/common'; import { RangeTypeLens, isValidRange, isValidNumber } from './ranges'; import { FROM_PLACEHOLDER, TO_PLACEHOLDER, TYPING_DEBOUNCE_TIME } from './constants'; @@ -39,8 +39,8 @@ type LocalRangeType = RangeTypeLens & { id: string }; const getBetterLabel = (range: RangeTypeLens, formatter: IFieldFormat) => range.label || formatter.convert({ - gte: isValidNumber(range.from) ? range.from : FROM_PLACEHOLDER, - lt: isValidNumber(range.to) ? range.to : TO_PLACEHOLDER, + gte: isValidNumber(range.from) ? range.from : -Infinity, + lt: isValidNumber(range.to) ? range.to : Infinity, }); export const RangePopover = ({ @@ -55,7 +55,6 @@ export const RangePopover = ({ Button: React.FunctionComponent<{ onClick: MouseEventHandler }>; isOpenByCreation: boolean; setIsOpenByCreation: (open: boolean) => void; - formatter: IFieldFormat; }) => { const [isPopoverOpen, setIsPopoverOpen] = useState(false); const [tempRange, setTempRange] = useState(range); @@ -112,6 +111,7 @@ export const RangePopover = ({ { const newRange = { @@ -126,7 +126,6 @@ export const RangePopover = ({ {lteAppendLabel} } - fullWidth compressed placeholder={FROM_PLACEHOLDER} isInvalid={!isValidRange(tempRange)} @@ -137,6 +136,7 @@ export const RangePopover = ({ { const newRange = { @@ -151,7 +151,6 @@ export const RangePopover = ({ {ltPrependLabel} } - fullWidth compressed placeholder={TO_PLACEHOLDER} isInvalid={!isValidRange(tempRange)} @@ -180,6 +179,7 @@ export const RangePopover = ({ { defaultMessage: 'Custom label' } )} onSubmit={onSubmit} + compressed dataTestSubj="indexPattern-ranges-label" /> @@ -284,7 +284,6 @@ export const AdvancedRangeEditor = ({ } setLocalRanges(newRanges); }} - formatter={formatter} Button={({ onClick }: { onClick: MouseEventHandler }) => ( { - return { convert: ({ gte, lt }: { gte: string; lt: string }) => `${gte} - ${lt}` }; +dataPluginMockValue.fieldFormats.deserialize = jest.fn().mockImplementation(({ params }) => { + return { + convert: ({ gte, lt }: { gte: string; lt: string }) => { + if (params?.id === 'custom') { + return `Custom format: ${gte} - ${lt}`; + } + if (params?.id === 'bytes') { + return `Bytes format: ${gte} - ${lt}`; + } + return `${gte} - ${lt}`; + }, + }; }); type ReactMouseEvent = React.MouseEvent & @@ -74,7 +90,14 @@ describe('ranges', () => { function getDefaultState(): IndexPatternPrivateState { return { indexPatternRefs: [], - indexPatterns: {}, + indexPatterns: { + '1': { + id: '1', + title: 'my_index_pattern', + hasRestrictions: false, + fields: [{ name: sourceField, type: 'number', displayName: sourceField }], + }, + }, existingFields: {}, currentIndexPatternId: '1', isFirstExistenceFetch: false, @@ -396,7 +419,7 @@ describe('ranges', () => { /> ); - // This series of act clojures are made to make it work properly the update flush + // This series of act closures are made to make it work properly the update flush act(() => { instance.find(EuiButtonEmpty).prop('onClick')!({} as ReactMouseEvent); }); @@ -453,7 +476,7 @@ describe('ranges', () => { /> ); - // This series of act clojures are made to make it work properly the update flush + // This series of act closures are made to make it work properly the update flush act(() => { instance.find(EuiButtonEmpty).prop('onClick')!({} as ReactMouseEvent); }); @@ -510,7 +533,7 @@ describe('ranges', () => { /> ); - // This series of act clojures are made to make it work properly the update flush + // This series of act closures are made to make it work properly the update flush act(() => { instance.find(RangePopover).find(EuiLink).prop('onClick')!({} as ReactMouseEvent); }); @@ -667,6 +690,60 @@ describe('ranges', () => { ); }); }); + + it('should correctly handle the default formatter for the field', () => { + const setStateSpy = jest.fn(); + + // set a default formatter for the sourceField used + state.indexPatterns['1'].fieldFormatMap = { + MyField: { id: 'custom', params: {} }, + }; + + const instance = mount( + + ); + + expect(instance.find(RangePopover).find(EuiText).prop('children')).toMatch( + /^Custom format:/ + ); + }); + + it('should correctly pick the dimension formatter for the field', () => { + const setStateSpy = jest.fn(); + + // set a default formatter for the sourceField used + state.indexPatterns['1'].fieldFormatMap = { + MyField: { id: 'custom', params: {} }, + }; + + // now set a format on the range operation + (state.layers.first.columns.col1 as RangeIndexPatternColumn).params.format = { + id: 'bytes', + params: { decimals: 0 }, + }; + + const instance = mount( + + ); + + expect(instance.find(RangePopover).find(EuiText).prop('children')).toMatch( + /^Bytes format:/ + ); + }); }); }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.tsx index a256f5e4ecfa1..1050eef45a71c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.tsx @@ -13,7 +13,9 @@ import { RangeEditor } from './range_editor'; import { OperationDefinition } from '../index'; import { FieldBasedIndexPatternColumn } from '../column_types'; import { updateColumnParam, changeColumn } from '../../../state_helpers'; +import { supportedFormats } from '../../../format_column'; import { MODES, AUTO_BARS, DEFAULT_INTERVAL, MIN_HISTOGRAM_BARS, SLICES } from './constants'; +import { IndexPattern, IndexPatternField } from '../../../types'; type RangeType = Omit; // Try to cover all possible serialized states for ranges @@ -32,6 +34,11 @@ export interface RangeIndexPatternColumn extends FieldBasedIndexPatternColumn { type: MODES_TYPES; maxBars: typeof AUTO_BARS | number; ranges: RangeTypeLens[]; + format?: { id: string; params?: { decimals: number } }; + parentFormat?: { + id: string; + params?: { id?: string; template?: string; replaceInfinity?: boolean }; + }; }; } @@ -55,6 +62,15 @@ export const isValidRange = (range: RangeTypeLens): boolean => { return true; }; +function getFieldDefaultFormat(indexPattern: IndexPattern, field: IndexPatternField | undefined) { + if (field) { + if (indexPattern.fieldFormatMap && indexPattern.fieldFormatMap[field.name]) { + return indexPattern.fieldFormatMap[field.name]; + } + } + return undefined; +} + function getEsAggsParams({ sourceField, params }: RangeIndexPatternColumn) { if (params.type === MODES.Range) { return { @@ -105,7 +121,7 @@ export const rangeOperation: OperationDefinition { - const rangeFormatter = data.fieldFormats.deserialize({ id: 'range' }); + const indexPattern = state.indexPatterns[state.layers[layerId].indexPatternId]; + const currentField = indexPattern.fields.find( + (field) => field.name === currentColumn.sourceField + ); + const numberFormat = currentColumn.params.format; + const numberFormatterPattern = + numberFormat && + supportedFormats[numberFormat.id] && + supportedFormats[numberFormat.id].decimalsToPattern(numberFormat.params?.decimals || 0); + + const rangeFormatter = data.fieldFormats.deserialize({ + ...currentColumn.params.parentFormat, + params: { + ...currentColumn.params.parentFormat?.params, + ...(numberFormat + ? { id: numberFormat.id, params: { pattern: numberFormatterPattern } } + : getFieldDefaultFormat(indexPattern, currentField)), + }, + }); + const MAX_HISTOGRAM_BARS = uiSettings.get(UI_SETTINGS.HISTOGRAM_MAX_BARS); const granularityStep = (MAX_HISTOGRAM_BARS - MIN_HISTOGRAM_BARS) / SLICES; const maxBarsDefaultValue = (MAX_HISTOGRAM_BARS - MIN_HISTOGRAM_BARS) / 2; @@ -171,6 +208,10 @@ export const rangeOperation: OperationDefinition { const scale = newMode === MODES.Range ? 'ordinal' : 'interval'; const dataType = newMode === MODES.Range ? 'string' : 'number'; + const parentFormat = + newMode === MODES.Range + ? { id: 'range', params: { template: 'arrow_right', replaceInfinity: true } } + : undefined; setState( changeColumn({ state, @@ -184,6 +225,8 @@ export const rangeOperation: OperationDefinition void; @@ -23,6 +24,7 @@ export const LabelInput = ({ inputRef?: React.MutableRefObject; onSubmit?: () => void; dataTestSubj?: string; + compressed?: boolean; }) => { const [inputValue, setInputValue] = useState(value); @@ -57,6 +59,7 @@ export const LabelInput = ({ prepend={i18n.translate('xpack.lens.labelInput.label', { defaultMessage: 'Label', })} + compressed={compressed} /> ); }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts index 1b87c48dc7193..e2c4323b56c2a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts @@ -63,32 +63,50 @@ function getExpressionForLayer( }; }, {} as Record); - type FormattedColumn = Required>; + type FormattedColumn = Required< + Extract< + IndexPatternColumn, + | { + params?: { + format: unknown; + }; + } + // when formatters are nested there's a slightly different format + | { + params: { + format?: unknown; + parentFormat?: unknown; + }; + } + > + >; const columnsWithFormatters = columnEntries.filter( - ([, col]) => col.params && 'format' in col.params && col.params.format + ([, col]) => + col.params && + (('format' in col.params && col.params.format) || + ('parentFormat' in col.params && col.params.parentFormat)) ) as Array<[string, FormattedColumn]>; - const formatterOverrides: ExpressionFunctionAST[] = columnsWithFormatters.map(([id, col]) => { - const format = (col as FormattedColumn).params!.format; - const base: ExpressionFunctionAST = { - type: 'function', - function: 'lens_format_column', - arguments: { - format: [format.id], - columnId: [id], - }, - }; - if (typeof format.params?.decimals === 'number') { - return { - ...base, + const formatterOverrides: ExpressionFunctionAST[] = columnsWithFormatters.map( + ([id, col]: [string, FormattedColumn]) => { + // TODO: improve the type handling here + const parentFormat = 'parentFormat' in col.params ? col.params!.parentFormat! : undefined; + const format = (col as FormattedColumn).params!.format; + + const base: ExpressionFunctionAST = { + type: 'function', + function: 'lens_format_column', arguments: { - ...base.arguments, - decimals: [format.params.decimals], + format: format ? [format.id] : [''], + columnId: [id], + decimals: typeof format?.params?.decimals === 'number' ? [format.params.decimals] : [], + parentFormat: parentFormat ? [JSON.stringify(parentFormat)] : [], }, }; + + return base; } - return base; - }); + ); const allDateHistogramFields = Object.values(columns) .map((column) => diff --git a/x-pack/plugins/lens/public/xy_visualization/visualization.test.ts b/x-pack/plugins/lens/public/xy_visualization/visualization.test.ts index d51b8c195c92c..3706611575c6b 100644 --- a/x-pack/plugins/lens/public/xy_visualization/visualization.test.ts +++ b/x-pack/plugins/lens/public/xy_visualization/visualization.test.ts @@ -312,6 +312,37 @@ describe('xy_visualization', () => { expect(options.map((o) => o.groupId)).toEqual(['x', 'y', 'breakdown']); }); + it('should return the correct labels for the 3 dimensios', () => { + const options = xyVisualization.getConfiguration({ + state: exampleState(), + frame, + layerId: 'first', + }).groups; + expect(options.map((o) => o.groupLabel)).toEqual([ + 'Horizontal axis', + 'Vertical axis', + 'Break down by', + ]); + }); + + it('should return the correct labels for the 3 dimensios for a horizontal chart', () => { + const initialState = exampleState(); + const state = { + ...initialState, + layers: [{ ...initialState.layers[0], seriesType: 'bar_horizontal' as SeriesType }], + }; + const options = xyVisualization.getConfiguration({ + state, + frame, + layerId: 'first', + }).groups; + expect(options.map((o) => o.groupLabel)).toEqual([ + 'Vertical axis', + 'Horizontal axis', + 'Break down by', + ]); + }); + it('should only accept bucketed operations for x', () => { const options = xyVisualization.getConfiguration({ state: exampleState(), diff --git a/x-pack/plugins/lens/public/xy_visualization/visualization.tsx b/x-pack/plugins/lens/public/xy_visualization/visualization.tsx index 76c5a51cb7168..50fbf3f01e34e 100644 --- a/x-pack/plugins/lens/public/xy_visualization/visualization.tsx +++ b/x-pack/plugins/lens/public/xy_visualization/visualization.tsx @@ -156,13 +156,18 @@ export const xyVisualization: Visualization = { getConfiguration(props) { const layer = props.state.layers.find((l) => l.layerId === props.layerId)!; + const isHorizontal = isHorizontalChart(props.state.layers); return { groups: [ { groupId: 'x', - groupLabel: i18n.translate('xpack.lens.xyChart.xAxisLabel', { - defaultMessage: 'X-axis', - }), + groupLabel: isHorizontal + ? i18n.translate('xpack.lens.xyChart.verticalAxisLabel', { + defaultMessage: 'Vertical axis', + }) + : i18n.translate('xpack.lens.xyChart.horizontalAxisLabel', { + defaultMessage: 'Horizontal axis', + }), accessors: layer.xAccessor ? [layer.xAccessor] : [], filterOperations: isBucketed, suggestedPriority: 1, @@ -172,9 +177,13 @@ export const xyVisualization: Visualization = { }, { groupId: 'y', - groupLabel: i18n.translate('xpack.lens.xyChart.yAxisLabel', { - defaultMessage: 'Y-axis', - }), + groupLabel: isHorizontal + ? i18n.translate('xpack.lens.xyChart.horizontalAxisLabel', { + defaultMessage: 'Horizontal axis', + }) + : i18n.translate('xpack.lens.xyChart.verticalAxisLabel', { + defaultMessage: 'Vertical axis', + }), accessors: layer.accessors, filterOperations: isNumericMetric, supportsMoreColumns: true, diff --git a/x-pack/plugins/lens/server/plugin.tsx b/x-pack/plugins/lens/server/plugin.tsx index b801d30f5ba9b..a8f9bef92349c 100644 --- a/x-pack/plugins/lens/server/plugin.tsx +++ b/x-pack/plugins/lens/server/plugin.tsx @@ -29,13 +29,13 @@ export class LensServerPlugin implements Plugin<{}, {}, {}, {}> { private readonly kibanaIndexConfig: Observable<{ kibana: { index: string } }>; private readonly telemetryLogger: Logger; - constructor(initializerContext: PluginInitializerContext) { + constructor(private initializerContext: PluginInitializerContext) { this.kibanaIndexConfig = initializerContext.config.legacy.globalConfig$; this.telemetryLogger = initializerContext.logger.get('usage'); } setup(core: CoreSetup, plugins: PluginSetupContract) { setupSavedObjects(core); - setupRoutes(core); + setupRoutes(core, this.initializerContext.logger.get()); if (plugins.usageCollection && plugins.taskManager) { registerLensUsageCollector( plugins.usageCollection, diff --git a/x-pack/plugins/lens/server/routes/existing_fields.ts b/x-pack/plugins/lens/server/routes/existing_fields.ts index c925517b572da..d3b2314a199cb 100644 --- a/x-pack/plugins/lens/server/routes/existing_fields.ts +++ b/x-pack/plugins/lens/server/routes/existing_fields.ts @@ -7,10 +7,14 @@ import Boom from 'boom'; import { schema } from '@kbn/config-schema'; import { ILegacyScopedClusterClient, SavedObject, RequestHandlerContext } from 'src/core/server'; -import { CoreSetup } from 'src/core/server'; +import { CoreSetup, Logger } from 'src/core/server'; import { BASE_API_URL } from '../../common'; import { IndexPatternAttributes, UI_SETTINGS } from '../../../../../src/plugins/data/server'; +export function isBoomError(error: { isBoom?: boolean }): error is Boom { + return error.isBoom === true; +} + /** * The number of docs to sample to determine field empty status. */ @@ -24,7 +28,7 @@ export interface Field { script?: string; } -export async function existingFieldsRoute(setup: CoreSetup) { +export async function existingFieldsRoute(setup: CoreSetup, logger: Logger) { const router = setup.http.createRouter(); router.post( @@ -52,14 +56,17 @@ export async function existingFieldsRoute(setup: CoreSetup) { }), }); } catch (e) { + logger.info( + `Field existence check failed: ${isBoomError(e) ? e.output.payload.message : e.message}` + ); if (e.status === 404) { - return res.notFound(); + return res.notFound({ body: e.message }); } - if (e.isBoom) { + if (isBoomError(e)) { if (e.output.statusCode === 404) { - return res.notFound(); + return res.notFound({ body: e.output.payload.message }); } - return res.internalError(e.output.message); + return res.internalError({ body: e.output.payload.message }); } else { return res.internalError({ body: Boom.internal(e.message || e.name), diff --git a/x-pack/plugins/lens/server/routes/index.ts b/x-pack/plugins/lens/server/routes/index.ts index 8bc04a56e16d5..01018d8cd7fe5 100644 --- a/x-pack/plugins/lens/server/routes/index.ts +++ b/x-pack/plugins/lens/server/routes/index.ts @@ -4,13 +4,13 @@ * you may not use this file except in compliance with the Elastic License. */ -import { CoreSetup } from 'src/core/server'; +import { CoreSetup, Logger } from 'src/core/server'; import { existingFieldsRoute } from './existing_fields'; import { initFieldsRoute } from './field_stats'; import { initLensUsageRoute } from './telemetry'; -export function setupRoutes(setup: CoreSetup) { - existingFieldsRoute(setup); +export function setupRoutes(setup: CoreSetup, logger: Logger) { + existingFieldsRoute(setup, logger); initFieldsRoute(setup); initLensUsageRoute(setup); } diff --git a/x-pack/plugins/maps/public/actions/layer_actions.ts b/x-pack/plugins/maps/public/actions/layer_actions.ts index 57f83b9533bda..c8c9f6ba40041 100644 --- a/x-pack/plugins/maps/public/actions/layer_actions.ts +++ b/x-pack/plugins/maps/public/actions/layer_actions.ts @@ -441,7 +441,7 @@ export function clearMissingStyleProperties(layerId: string) { const { hasChanges, nextStyleDescriptor, - } = (style as IVectorStyle).getDescriptorWithMissingStylePropsRemoved( + } = await (style as IVectorStyle).getDescriptorWithMissingStylePropsRemoved( nextFields, getMapColors(getState()) ); diff --git a/x-pack/plugins/maps/public/actions/map_actions.ts b/x-pack/plugins/maps/public/actions/map_actions.ts index 4e76bb24c9e34..f26b00cec9fab 100644 --- a/x-pack/plugins/maps/public/actions/map_actions.ts +++ b/x-pack/plugins/maps/public/actions/map_actions.ts @@ -282,12 +282,19 @@ export function setRefreshConfig({ isPaused, interval }: MapRefreshConfig) { } export function triggerRefreshTimer() { - return async (dispatch: ThunkDispatch) => { + return async ( + dispatch: ThunkDispatch, + getState: () => MapStoreState + ) => { dispatch({ type: TRIGGER_REFRESH_TIMER, }); - await dispatch(syncDataForAllLayers()); + if (getMapSettings(getState()).autoFitToDataBounds) { + dispatch(autoFitToBounds()); + } else { + await dispatch(syncDataForAllLayers()); + } }; } diff --git a/x-pack/plugins/maps/public/classes/styles/vector/components/vector_style_editor.js b/x-pack/plugins/maps/public/classes/styles/vector/components/vector_style_editor.js index 53a3fc95adbeb..d577912efb830 100644 --- a/x-pack/plugins/maps/public/classes/styles/vector/components/vector_style_editor.js +++ b/x-pack/plugins/maps/public/classes/styles/vector/components/vector_style_editor.js @@ -20,20 +20,16 @@ import { i18n } from '@kbn/i18n'; import { EuiSpacer, EuiButtonGroup, EuiFormRow, EuiSwitch } from '@elastic/eui'; import { - CATEGORICAL_DATA_TYPES, - ORDINAL_DATA_TYPES, LABEL_BORDER_SIZES, VECTOR_STYLES, STYLE_TYPE, VECTOR_SHAPE_TYPE, } from '../../../../../common/constants'; +import { createStyleFieldsHelper } from '../style_fields_helper'; export class VectorStyleEditor extends Component { state = { - dateFields: [], - numberFields: [], - fields: [], - ordinalAndCategoricalFields: [], + styleFields: [], defaultDynamicProperties: getDefaultDynamicProperties(), defaultStaticProperties: getDefaultStaticProperties(), supportedFeatures: undefined, @@ -56,33 +52,17 @@ export class VectorStyleEditor extends Component { } async _loadFields() { - const getFieldMeta = async (field) => { - return { - label: await field.getLabel(), - name: field.getName(), - origin: field.getOrigin(), - type: await field.getDataType(), - supportsAutoDomain: field.supportsAutoDomain(), - }; - }; - - //These are all fields (only used for text labeling) - const fields = await this.props.layer.getStyleEditorFields(); - const fieldPromises = fields.map(getFieldMeta); - const fieldsArrayAll = await Promise.all(fieldPromises); - if (!this._isMounted || _.isEqual(fieldsArrayAll, this.state.fields)) { + const styleFieldsHelper = await createStyleFieldsHelper( + await this.props.layer.getStyleEditorFields() + ); + const styleFields = styleFieldsHelper.getStyleFields(); + if (!this._isMounted || _.isEqual(styleFields, this.state.styleFields)) { return; } this.setState({ - fields: fieldsArrayAll, - ordinalAndCategoricalFields: fieldsArrayAll.filter((field) => { - return ( - CATEGORICAL_DATA_TYPES.includes(field.type) || ORDINAL_DATA_TYPES.includes(field.type) - ); - }), - dateFields: fieldsArrayAll.filter((field) => field.type === 'date'), - numberFields: fieldsArrayAll.filter((field) => field.type === 'number'), + styleFields, + styleFieldsHelper, }); } @@ -109,12 +89,6 @@ export class VectorStyleEditor extends Component { } } - _getOrdinalFields() { - return [...this.state.dateFields, ...this.state.numberFields].filter((field) => { - return field.supportsAutoDomain; - }); - } - _handleSelectedFeatureChange = (selectedFeature) => { this.setState({ selectedFeature }); }; @@ -165,7 +139,7 @@ export class VectorStyleEditor extends Component { onStaticStyleChange={this._onStaticStyleChange} onDynamicStyleChange={this._onDynamicStyleChange} styleProperty={this.props.styleProperties[VECTOR_STYLES.FILL_COLOR]} - fields={this.state.ordinalAndCategoricalFields} + fields={this.state.styleFieldsHelper.getFieldsForStyle(VECTOR_STYLES.FILL_COLOR)} defaultStaticStyleOptions={ this.state.defaultStaticProperties[VECTOR_STYLES.FILL_COLOR].options } @@ -186,7 +160,7 @@ export class VectorStyleEditor extends Component { onStaticStyleChange={this._onStaticStyleChange} onDynamicStyleChange={this._onDynamicStyleChange} styleProperty={this.props.styleProperties[VECTOR_STYLES.LINE_COLOR]} - fields={this.state.ordinalAndCategoricalFields} + fields={this.state.styleFieldsHelper.getFieldsForStyle(VECTOR_STYLES.LINE_COLOR)} defaultStaticStyleOptions={ this.state.defaultStaticProperties[VECTOR_STYLES.LINE_COLOR].options } @@ -205,7 +179,7 @@ export class VectorStyleEditor extends Component { onStaticStyleChange={this._onStaticStyleChange} onDynamicStyleChange={this._onDynamicStyleChange} styleProperty={this.props.styleProperties[VECTOR_STYLES.LINE_WIDTH]} - fields={this._getOrdinalFields()} + fields={this.state.styleFieldsHelper.getFieldsForStyle(VECTOR_STYLES.LINE_WIDTH)} defaultStaticStyleOptions={ this.state.defaultStaticProperties[VECTOR_STYLES.LINE_WIDTH].options } @@ -225,7 +199,7 @@ export class VectorStyleEditor extends Component { onStaticStyleChange={this._onStaticStyleChange} onDynamicStyleChange={this._onDynamicStyleChange} styleProperty={this.props.styleProperties[VECTOR_STYLES.LABEL_TEXT]} - fields={this.state.fields} + fields={this.state.styleFieldsHelper.getFieldsForStyle(VECTOR_STYLES.LABEL_TEXT)} defaultStaticStyleOptions={ this.state.defaultStaticProperties[VECTOR_STYLES.LABEL_TEXT].options } @@ -242,7 +216,7 @@ export class VectorStyleEditor extends Component { onStaticStyleChange={this._onStaticStyleChange} onDynamicStyleChange={this._onDynamicStyleChange} styleProperty={this.props.styleProperties[VECTOR_STYLES.LABEL_COLOR]} - fields={this.state.ordinalAndCategoricalFields} + fields={this.state.styleFieldsHelper.getFieldsForStyle(VECTOR_STYLES.LABEL_COLOR)} defaultStaticStyleOptions={ this.state.defaultStaticProperties[VECTOR_STYLES.LABEL_COLOR].options } @@ -258,7 +232,7 @@ export class VectorStyleEditor extends Component { onStaticStyleChange={this._onStaticStyleChange} onDynamicStyleChange={this._onDynamicStyleChange} styleProperty={this.props.styleProperties[VECTOR_STYLES.LABEL_SIZE]} - fields={this._getOrdinalFields()} + fields={this.state.styleFieldsHelper.getFieldsForStyle(VECTOR_STYLES.LABEL_SIZE)} defaultStaticStyleOptions={ this.state.defaultStaticProperties[VECTOR_STYLES.LABEL_SIZE].options } @@ -275,7 +249,7 @@ export class VectorStyleEditor extends Component { onStaticStyleChange={this._onStaticStyleChange} onDynamicStyleChange={this._onDynamicStyleChange} styleProperty={this.props.styleProperties[VECTOR_STYLES.LABEL_BORDER_COLOR]} - fields={this.state.ordinalAndCategoricalFields} + fields={this.state.styleFieldsHelper.getFieldsForStyle(VECTOR_STYLES.LABEL_BORDER_COLOR)} defaultStaticStyleOptions={ this.state.defaultStaticProperties[VECTOR_STYLES.LABEL_BORDER_COLOR].options } @@ -309,7 +283,7 @@ export class VectorStyleEditor extends Component { onStaticStyleChange={this._onStaticStyleChange} onDynamicStyleChange={this._onDynamicStyleChange} styleProperty={this.props.styleProperties[VECTOR_STYLES.ICON_ORIENTATION]} - fields={this.state.numberFields} + fields={this.state.styleFieldsHelper.getFieldsForStyle(VECTOR_STYLES.ICON_ORIENTATION)} defaultStaticStyleOptions={ this.state.defaultStaticProperties[VECTOR_STYLES.ICON_ORIENTATION].options } @@ -328,7 +302,7 @@ export class VectorStyleEditor extends Component { onStaticStyleChange={this._onStaticStyleChange} onDynamicStyleChange={this._onDynamicStyleChange} styleProperty={this.props.styleProperties[VECTOR_STYLES.ICON]} - fields={this.state.ordinalAndCategoricalFields} + fields={this.state.styleFieldsHelper.getFieldsForStyle(VECTOR_STYLES.ICON)} defaultStaticStyleOptions={ this.state.defaultStaticProperties[VECTOR_STYLES.ICON].options } @@ -368,7 +342,7 @@ export class VectorStyleEditor extends Component { onStaticStyleChange={this._onStaticStyleChange} onDynamicStyleChange={this._onDynamicStyleChange} styleProperty={this.props.styleProperties[VECTOR_STYLES.ICON_SIZE]} - fields={this._getOrdinalFields()} + fields={this.state.styleFieldsHelper.getFieldsForStyle(VECTOR_STYLES.ICON_SIZE)} defaultStaticStyleOptions={ this.state.defaultStaticProperties[VECTOR_STYLES.ICON_SIZE].options } @@ -409,9 +383,9 @@ export class VectorStyleEditor extends Component { } _renderProperties() { - const { supportedFeatures, selectedFeature } = this.state; + const { supportedFeatures, selectedFeature, styleFieldsHelper } = this.state; - if (!supportedFeatures) { + if (!supportedFeatures || !styleFieldsHelper) { return null; } diff --git a/x-pack/plugins/maps/public/classes/styles/vector/properties/__tests__/test_util.ts b/x-pack/plugins/maps/public/classes/styles/vector/properties/__tests__/test_util.ts index a2dfdc94d8058..a3f63ba7717ea 100644 --- a/x-pack/plugins/maps/public/classes/styles/vector/properties/__tests__/test_util.ts +++ b/x-pack/plugins/maps/public/classes/styles/vector/properties/__tests__/test_util.ts @@ -16,10 +16,38 @@ import { import { AbstractField, IField } from '../../../../fields/field'; import { IStyle } from '../../../style'; -class MockField extends AbstractField { +export class MockField extends AbstractField { + private readonly _dataType: string; + private readonly _supportsAutoDomain: boolean; + + constructor({ + fieldName, + origin = FIELD_ORIGIN.SOURCE, + dataType = 'string', + supportsAutoDomain = true, + }: { + fieldName: string; + origin?: FIELD_ORIGIN; + dataType?: string; + supportsAutoDomain?: boolean; + }) { + super({ fieldName, origin }); + this._dataType = dataType; + this._supportsAutoDomain = supportsAutoDomain; + } + async getLabel(): Promise { return this.getName() + '_label'; } + + async getDataType(): Promise { + return this._dataType; + } + + supportsAutoDomain(): boolean { + return this._supportsAutoDomain; + } + supportsFieldMeta(): boolean { return true; } diff --git a/x-pack/plugins/maps/public/classes/styles/vector/style_fields_helper.ts b/x-pack/plugins/maps/public/classes/styles/vector/style_fields_helper.ts new file mode 100644 index 0000000000000..8613f9e1e946f --- /dev/null +++ b/x-pack/plugins/maps/public/classes/styles/vector/style_fields_helper.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + CATEGORICAL_DATA_TYPES, + FIELD_ORIGIN, + ORDINAL_DATA_TYPES, + VECTOR_STYLES, +} from '../../../../common/constants'; +import { IField } from '../../fields/field'; + +export interface StyleField { + label: string; + name: string; + origin: FIELD_ORIGIN; + type: string; + supportsAutoDomain: boolean; +} + +export async function createStyleFieldsHelper(fields: IField[]): Promise { + const promises: Array> = fields.map(async (field: IField) => { + return { + label: await field.getLabel(), + name: field.getName(), + origin: field.getOrigin(), + type: await field.getDataType(), + supportsAutoDomain: field.supportsAutoDomain(), + }; + }); + const styleFields = await Promise.all(promises); + return new StyleFieldsHelper(styleFields); +} + +class StyleFieldsHelper { + private readonly _styleFields: StyleField[]; + private readonly _ordinalAndCategoricalFields: StyleField[]; + private readonly _numberFields: StyleField[]; + private readonly _ordinalFields: StyleField[]; + + constructor(styleFields: StyleField[]) { + const ordinalAndCategoricalFields: StyleField[] = []; + const numberFields: StyleField[] = []; + const ordinalFields: StyleField[] = []; + + styleFields.forEach((styleField: StyleField) => { + if ( + CATEGORICAL_DATA_TYPES.includes(styleField.type) || + ORDINAL_DATA_TYPES.includes(styleField.type) + ) { + ordinalAndCategoricalFields.push(styleField); + } + + if (styleField.type === 'date' || styleField.type === 'number') { + if (styleField.type === 'number') { + numberFields.push(styleField); + } + if (styleField.supportsAutoDomain) { + ordinalFields.push(styleField); + } + } + }); + + this._styleFields = styleFields; + this._ordinalAndCategoricalFields = ordinalAndCategoricalFields; + this._numberFields = numberFields; + this._ordinalFields = ordinalFields; + } + + getFieldsForStyle(styleName: VECTOR_STYLES): StyleField[] { + switch (styleName) { + case VECTOR_STYLES.ICON_ORIENTATION: + return this._numberFields; + case VECTOR_STYLES.FILL_COLOR: + case VECTOR_STYLES.LINE_COLOR: + case VECTOR_STYLES.LABEL_COLOR: + case VECTOR_STYLES.LABEL_BORDER_COLOR: + case VECTOR_STYLES.ICON: + return this._ordinalAndCategoricalFields; + case VECTOR_STYLES.LINE_WIDTH: + case VECTOR_STYLES.LABEL_SIZE: + case VECTOR_STYLES.ICON_SIZE: + return this._ordinalFields; + case VECTOR_STYLES.LABEL_TEXT: + return this._styleFields; + default: + return []; + } + } + + getStyleFields(): StyleField[] { + return this._styleFields; + } +} diff --git a/x-pack/plugins/maps/public/classes/styles/vector/vector_style.test.js b/x-pack/plugins/maps/public/classes/styles/vector/vector_style.test.js index 28801a402ca14..1dbadc054c8a0 100644 --- a/x-pack/plugins/maps/public/classes/styles/vector/vector_style.test.js +++ b/x-pack/plugins/maps/public/classes/styles/vector/vector_style.test.js @@ -12,23 +12,10 @@ import { VECTOR_SHAPE_TYPE, VECTOR_STYLES, } from '../../../../common/constants'; +import { MockField } from './properties/__tests__/test_util'; jest.mock('../../../kibana_services'); -class MockField { - constructor({ fieldName }) { - this._fieldName = fieldName; - } - - getName() { - return this._fieldName; - } - - isValid() { - return !!this._fieldName; - } -} - class MockSource { constructor({ supportedShapeTypes } = {}) { this._supportedShapeTypes = supportedShapeTypes || Object.values(VECTOR_SHAPE_TYPE); @@ -77,25 +64,25 @@ describe('getDescriptorWithMissingStylePropsRemoved', () => { }); }); - it('Should return no changes when next ordinal fields contain existing style property fields', () => { + it('Should return no changes when next ordinal fields contain existing style property fields', async () => { const vectorStyle = new VectorStyle({ properties }, new MockSource()); - const nextFields = [new MockField({ fieldName })]; - const { hasChanges } = vectorStyle.getDescriptorWithMissingStylePropsRemoved( + const nextFields = [new MockField({ fieldName, dataType: 'number' })]; + const { hasChanges } = await vectorStyle.getDescriptorWithMissingStylePropsRemoved( nextFields, mapColors ); expect(hasChanges).toBe(false); }); - it('Should clear missing fields when next ordinal fields do not contain existing style property fields', () => { + it('Should clear missing fields when next ordinal fields do not contain existing style property fields', async () => { const vectorStyle = new VectorStyle({ properties }, new MockSource()); - const nextFields = [new MockField({ fieldName: 'someOtherField' })]; + const nextFields = [new MockField({ fieldName: 'someOtherField', dataType: 'number' })]; const { hasChanges, nextStyleDescriptor, - } = vectorStyle.getDescriptorWithMissingStylePropsRemoved(nextFields, mapColors); + } = await vectorStyle.getDescriptorWithMissingStylePropsRemoved(nextFields, mapColors); expect(hasChanges).toBe(true); expect(nextStyleDescriptor.properties[VECTOR_STYLES.LINE_COLOR]).toEqual({ options: {}, @@ -110,14 +97,14 @@ describe('getDescriptorWithMissingStylePropsRemoved', () => { }); }); - it('Should convert dynamic styles to static styles when there are no next fields', () => { + it('Should convert dynamic styles to static styles when there are no next fields', async () => { const vectorStyle = new VectorStyle({ properties }, new MockSource()); const nextFields = []; const { hasChanges, nextStyleDescriptor, - } = vectorStyle.getDescriptorWithMissingStylePropsRemoved(nextFields, mapColors); + } = await vectorStyle.getDescriptorWithMissingStylePropsRemoved(nextFields, mapColors); expect(hasChanges).toBe(true); expect(nextStyleDescriptor.properties[VECTOR_STYLES.LINE_COLOR]).toEqual({ options: { @@ -132,6 +119,25 @@ describe('getDescriptorWithMissingStylePropsRemoved', () => { type: 'STATIC', }); }); + + it('Should convert dynamic ICON_SIZE static style when there are no next ordinal fields', async () => { + const vectorStyle = new VectorStyle({ properties }, new MockSource()); + + const nextFields = [ + new MockField({ fieldName, dataType: 'number', supportsAutoDomain: false }), + ]; + const { + hasChanges, + nextStyleDescriptor, + } = await vectorStyle.getDescriptorWithMissingStylePropsRemoved(nextFields, mapColors); + expect(hasChanges).toBe(true); + expect(nextStyleDescriptor.properties[VECTOR_STYLES.ICON_SIZE]).toEqual({ + options: { + size: 6, + }, + type: 'STATIC', + }); + }); }); describe('pluckStyleMetaFromSourceDataRequest', () => { diff --git a/x-pack/plugins/maps/public/classes/styles/vector/vector_style.tsx b/x-pack/plugins/maps/public/classes/styles/vector/vector_style.tsx index acb158636e0b3..5d360f5452d36 100644 --- a/x-pack/plugins/maps/public/classes/styles/vector/vector_style.tsx +++ b/x-pack/plugins/maps/public/classes/styles/vector/vector_style.tsx @@ -77,6 +77,7 @@ import { IField } from '../../fields/field'; import { IVectorLayer } from '../../layers/vector_layer/vector_layer'; import { IVectorSource } from '../../sources/vector_source'; import { ILayer } from '../../layers/layer'; +import { createStyleFieldsHelper } from './style_fields_helper'; const POINTS = [GEO_JSON_TYPE.POINT, GEO_JSON_TYPE.MULTI_POINT]; const LINES = [GEO_JSON_TYPE.LINE_STRING, GEO_JSON_TYPE.MULTI_LINE_STRING]; @@ -90,7 +91,7 @@ export interface IVectorStyle extends IStyle { getDescriptorWithMissingStylePropsRemoved( nextFields: IField[], mapColors: string[] - ): { hasChanges: boolean; nextStyleDescriptor?: VectorStyleDescriptor }; + ): Promise<{ hasChanges: boolean; nextStyleDescriptor?: VectorStyleDescriptor }>; pluckStyleMetaFromSourceDataRequest(sourceDataRequest: DataRequest): Promise; isTimeAware: () => boolean; getIcon: () => ReactElement; @@ -324,7 +325,8 @@ export class VectorStyle implements IVectorStyle { * This method does not update its descriptor. It just returns a new descriptor that the caller * can then use to update store state via dispatch. */ - getDescriptorWithMissingStylePropsRemoved(nextFields: IField[], mapColors: string[]) { + async getDescriptorWithMissingStylePropsRemoved(nextFields: IField[], mapColors: string[]) { + const styleFieldsHelper = await createStyleFieldsHelper(nextFields); const originalProperties = this.getRawProperties(); const updatedProperties = {} as VectorStylePropertiesDescriptor; @@ -346,8 +348,9 @@ export class VectorStyle implements IVectorStyle { }); dynamicProperties.forEach((key: VECTOR_STYLES) => { - // Convert dynamic styling to static stying when there are no nextFields - if (nextFields.length === 0) { + // Convert dynamic styling to static stying when there are no style fields + const styleFields = styleFieldsHelper.getFieldsForStyle(key); + if (styleFields.length === 0) { const staticProperties = getDefaultStaticProperties(mapColors); updatedProperties[key] = staticProperties[key] as any; return; diff --git a/x-pack/plugins/maps/public/connected_components/map_settings_panel/__snapshots__/navigation_panel.test.tsx.snap b/x-pack/plugins/maps/public/connected_components/map_settings_panel/__snapshots__/navigation_panel.test.tsx.snap index a617fbc552854..1859c7d8177f8 100644 --- a/x-pack/plugins/maps/public/connected_components/map_settings_panel/__snapshots__/navigation_panel.test.tsx.snap +++ b/x-pack/plugins/maps/public/connected_components/map_settings_panel/__snapshots__/navigation_panel.test.tsx.snap @@ -25,7 +25,7 @@ exports[`should render 1`] = ` labelType="label" > { + onClick: async () => { if (getHasUnsavedChanges()) { - const navigateAway = window.confirm(unsavedChangesWarning); - if (navigateAway) { + const confirmed = await getCoreOverlays().openConfirm(unsavedChangesWarning, { + title: unsavedChangesTitle, + 'data-test-subj': 'appLeaveConfirmModal', + }); + if (confirmed) { goToSpecifiedPath('/'); } } else { diff --git a/x-pack/plugins/maps/public/routing/routes/maps_app/maps_app_view.tsx b/x-pack/plugins/maps/public/routing/routes/maps_app/maps_app_view.tsx index bd08b2f11fadc..df46d5d6a13ff 100644 --- a/x-pack/plugins/maps/public/routing/routes/maps_app/maps_app_view.tsx +++ b/x-pack/plugins/maps/public/routing/routes/maps_app/maps_app_view.tsx @@ -43,7 +43,7 @@ import { import { MapContainer } from '../../../connected_components/map_container'; import { getIndexPatternsFromIds } from '../../../index_pattern_util'; import { getTopNavConfig } from './top_nav_config'; -import { getBreadcrumbs, unsavedChangesWarning } from './get_breadcrumbs'; +import { getBreadcrumbs, unsavedChangesTitle, unsavedChangesWarning } from './get_breadcrumbs'; import { LayerDescriptor, MapRefreshConfig, @@ -138,9 +138,7 @@ export class MapsAppView extends React.Component { this.props.onAppLeave((actions) => { if (this._hasUnsavedChanges()) { - if (!window.confirm(unsavedChangesWarning)) { - return {} as AppLeaveAction; - } + return actions.confirm(unsavedChangesWarning, unsavedChangesTitle); } return actions.default() as AppLeaveAction; }); diff --git a/x-pack/plugins/maps/server/routes.js b/x-pack/plugins/maps/server/routes.js index 6b19103b59722..5feacfee4d4d2 100644 --- a/x-pack/plugins/maps/server/routes.js +++ b/x-pack/plugins/maps/server/routes.js @@ -189,18 +189,28 @@ export function initRoutes(router, licenseUid, mapConfig, kbnVersion, logger) { return badRequest('map.proxyElasticMapsServiceInMaps disabled'); } - const file = await emsClient.getDefaultFileManifest(); - const layers = file.layers.map((layer) => { - const newLayer = { ...layer }; - const id = encodeURIComponent(layer.layer_id); + const file = await emsClient.getDefaultFileManifest(); //need raw manifest + const fileLayers = await emsClient.getFileLayers(); + + const layers = file.layers.map((layerJson) => { + const newLayerJson = { ...layerJson }; + const id = encodeURIComponent(layerJson.layer_id); + + const fileLayer = fileLayers.find((fileLayer) => fileLayer.getId() === layerJson.layer_id); + const defaultFormat = layerJson.formats.find( + (format) => format.type === fileLayer.getDefaultFormatType() + ); + const newUrl = `${EMS_FILES_DEFAULT_JSON_PATH}?id=${id}`; - newLayer.formats = [ + + //Only proxy default-format. Others are unused in Maps-app + newLayerJson.formats = [ { - ...layer.formats[0], + ...defaultFormat, url: newUrl, }, ]; - return newLayer; + return newLayerJson; }); //rewrite return ok({ diff --git a/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_chart_single_metric.test.js b/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_chart_single_metric.test.js index 99f5c3eff6984..a1e9a9b4760dd 100644 --- a/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_chart_single_metric.test.js +++ b/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_chart_single_metric.test.js @@ -54,6 +54,7 @@ describe('ExplorerChart', () => { ); @@ -79,6 +80,7 @@ describe('ExplorerChart', () => { seriesConfig={config} mlSelectSeverityService={mlSelectSeverityServiceMock} tooltipService={mockTooltipService} + severity={0} /> ); @@ -111,6 +113,7 @@ describe('ExplorerChart', () => { seriesConfig={config} mlSelectSeverityService={mlSelectSeverityServiceMock} tooltipService={mockTooltipService} + severity={0} /> ); diff --git a/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_charts_container.js b/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_charts_container.js index 9c04e8187cd30..ee9869a202f58 100644 --- a/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_charts_container.js +++ b/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_charts_container.js @@ -34,8 +34,7 @@ import { addItemToRecentlyAccessed } from '../../util/recently_accessed'; const textTooManyBuckets = i18n.translate('xpack.ml.explorer.charts.tooManyBucketsDescription', { defaultMessage: - 'This selection contains too many buckets to be displayed.' + - 'The dashboard is best viewed over a shorter time range.', + 'This selection contains too many buckets to be displayed. You should shorten the time range of the view or narrow the selection in the timeline.', }); const textViewButton = i18n.translate( 'xpack.ml.explorer.charts.openInSingleMetricViewerButtonLabel', diff --git a/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_charts_container_service.js b/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_charts_container_service.js index 12e95e859af53..b9634f0eac359 100644 --- a/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_charts_container_service.js +++ b/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_charts_container_service.js @@ -571,11 +571,14 @@ function calculateChartRange( let tooManyBuckets = false; // Calculate the time range for the charts. // Fit in as many points in the available container width plotted at the job bucket span. + // Look for the chart with the shortest bucket span as this determines + // the length of the time range that can be plotted. const midpointMs = Math.ceil((selectedEarliestMs + selectedLatestMs) / 2); + const minBucketSpanMs = Math.min.apply(null, map(seriesConfigs, 'bucketSpanSeconds')) * 1000; const maxBucketSpanMs = Math.max.apply(null, map(seriesConfigs, 'bucketSpanSeconds')) * 1000; const pointsToPlotFullSelection = Math.ceil( - (selectedLatestMs - selectedEarliestMs) / maxBucketSpanMs + (selectedLatestMs - selectedEarliestMs) / minBucketSpanMs ); // Optimally space points 5px apart. @@ -588,16 +591,16 @@ function calculateChartRange( const halfPoints = Math.ceil(plotPoints / 2); const timefilter = getTimefilter(); const bounds = timefilter.getActiveBounds(); + const boundsMin = bounds.min.valueOf(); let chartRange = { - min: Math.max(midpointMs - halfPoints * maxBucketSpanMs, bounds.min.valueOf()), - max: Math.min(midpointMs + halfPoints * maxBucketSpanMs, bounds.max.valueOf()), + min: Math.max(midpointMs - halfPoints * minBucketSpanMs, boundsMin), + max: Math.min(midpointMs + halfPoints * minBucketSpanMs, bounds.max.valueOf()), }; if (plotPoints > CHART_MAX_POINTS) { - tooManyBuckets = true; // For each series being plotted, display the record with the highest score if possible. - const maxTimeSpan = maxBucketSpanMs * CHART_MAX_POINTS; + const maxTimeSpan = minBucketSpanMs * CHART_MAX_POINTS; let minMs = recordsToPlot[0][timeFieldName]; let maxMs = recordsToPlot[0][timeFieldName]; @@ -620,14 +623,33 @@ function calculateChartRange( }); if (maxMs - minMs < maxTimeSpan) { - // Expand out to cover as much as the requested time span as possible. - minMs = Math.max(selectedEarliestMs, minMs - maxTimeSpan); - maxMs = Math.min(selectedLatestMs, maxMs + maxTimeSpan); + // Expand out before and after the span with the highest scoring anomalies, + // covering as much as the requested time span as possible. + // Work out if the high scoring region is nearer the start or end of the selected time span. + const diff = maxTimeSpan - (maxMs - minMs); + if (minMs - 0.5 * diff <= selectedEarliestMs) { + minMs = Math.max(selectedEarliestMs, minMs - 0.5 * diff); + maxMs = minMs + maxTimeSpan; + } else { + maxMs = Math.min(selectedLatestMs, maxMs + 0.5 * diff); + minMs = maxMs - maxTimeSpan; + } } chartRange = { min: minMs, max: maxMs }; } + // Elasticsearch aggregation returns points at start of bucket, + // so align the min to the length of the longest bucket. + chartRange.min = Math.floor(chartRange.min / maxBucketSpanMs) * maxBucketSpanMs; + if (chartRange.min < boundsMin) { + chartRange.min = chartRange.min + maxBucketSpanMs; + } + + if (chartRange.min > selectedEarliestMs || chartRange.max < selectedLatestMs) { + tooManyBuckets = true; + } + return { chartRange, tooManyBuckets, diff --git a/x-pack/plugins/ml/public/application/timeseriesexplorer/_timeseriesexplorer.scss b/x-pack/plugins/ml/public/application/timeseriesexplorer/_timeseriesexplorer.scss index 8d4f6ad0a6844..45678f6e71c2e 100644 --- a/x-pack/plugins/ml/public/application/timeseriesexplorer/_timeseriesexplorer.scss +++ b/x-pack/plugins/ml/public/application/timeseriesexplorer/_timeseriesexplorer.scss @@ -24,10 +24,6 @@ } } - .single-metric-request-callout { - margin: 0 $euiSize; - } - .results-container { .panel-title { font-size: $euiFontSizeM; diff --git a/x-pack/plugins/ml/public/application/timeseriesexplorer/components/entity_control/entity_control.tsx b/x-pack/plugins/ml/public/application/timeseriesexplorer/components/entity_control/entity_control.tsx index 9d2c49a95fec4..e1323019d61db 100644 --- a/x-pack/plugins/ml/public/application/timeseriesexplorer/components/entity_control/entity_control.tsx +++ b/x-pack/plugins/ml/public/application/timeseriesexplorer/components/entity_control/entity_control.tsx @@ -9,13 +9,7 @@ import React, { Component } from 'react'; import { FormattedMessage } from '@kbn/i18n/react'; import { i18n } from '@kbn/i18n'; -import { - EuiComboBox, - EuiComboBoxOptionOption, - EuiFlexItem, - EuiFormRow, - EuiToolTip, -} from '@elastic/eui'; +import { EuiComboBox, EuiComboBoxOptionOption, EuiFlexItem, EuiFormRow } from '@elastic/eui'; export interface Entity { fieldName: string; @@ -156,9 +150,7 @@ export class EntityControl extends Component - - {control} - + {control} ); diff --git a/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer.js b/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer.js index 7d173e161a1cb..9b8764d3f9279 100644 --- a/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer.js +++ b/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer.js @@ -1133,23 +1133,25 @@ export class TimeSeriesExplorer extends React.Component { return ( {fieldNamesWithEmptyValues.length > 0 && ( - - } - color="warning" - iconType="help" - size="s" - /> + <> + + } + color="warning" + iconType="help" + size="s" + /> + + )}
diff --git a/x-pack/plugins/monitoring/public/components/elasticsearch/node/advanced.js b/x-pack/plugins/monitoring/public/components/elasticsearch/node/advanced.js index b2a17515bbb96..67df745e619d4 100644 --- a/x-pack/plugins/monitoring/public/components/elasticsearch/node/advanced.js +++ b/x-pack/plugins/monitoring/public/components/elasticsearch/node/advanced.js @@ -18,8 +18,9 @@ import { import { NodeDetailStatus } from '../node_detail_status'; import { MonitoringTimeseriesContainer } from '../../chart'; import { FormattedMessage } from '@kbn/i18n/react'; +import { AlertsCallout } from '../../../alerts/callout'; -export const AdvancedNode = ({ nodeSummary, metrics, alerts, ...props }) => { +export const AdvancedNode = ({ nodeSummary, metrics, alerts, nodeId, ...props }) => { const metricsToShow = [ metrics.node_gc, metrics.node_gc_time, @@ -50,9 +51,25 @@ export const AdvancedNode = ({ nodeSummary, metrics, alerts, ...props }) => { - + + state.nodeId === nodeId || state.stackProductUuid === nodeId + } + /> + state.nodeId === nodeId || state.stackProductUuid === nodeId} + nextStepsFilter={(nextStep) => { + if (nextStep.text.includes('Elasticsearch nodes')) { + return false; + } + return true; + }} + /> {metricsToShow.map((metric, index) => ( diff --git a/x-pack/plugins/monitoring/public/views/elasticsearch/node/advanced/index.js b/x-pack/plugins/monitoring/public/views/elasticsearch/node/advanced/index.js index 03c0714864f92..8021ae7e5f63c 100644 --- a/x-pack/plugins/monitoring/public/views/elasticsearch/node/advanced/index.js +++ b/x-pack/plugins/monitoring/public/views/elasticsearch/node/advanced/index.js @@ -117,6 +117,7 @@ uiRoutes.when('/elasticsearch/nodes/:node/advanced', { { - let cpuUsage = 0; if (this.config.ui.container.elasticsearch.enabled) { - cpuUsage = + stat.cpuUsage = (stat.containerUsage / (stat.containerPeriods * stat.containerQuota * 1000)) * 100; - } else { - cpuUsage = stat.cpuUsage; } return { instanceKey: `${stat.clusterUuid}:${stat.nodeId}`, clusterUuid: stat.clusterUuid, - shouldFire: cpuUsage > params.threshold, + shouldFire: stat.cpuUsage > params.threshold, severity: AlertSeverity.Danger, meta: stat, ccs: stat.ccs, diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.ts b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.ts index a3ff4b952ce97..0dd5ce291f972 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.ts +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { Collector } from '../../../../../../src/plugins/usage_collection/server'; +import { Collector, UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { KIBANA_SETTINGS_TYPE } from '../../../common/constants'; import { MonitoringConfig } from '../../config'; @@ -48,10 +48,18 @@ export interface KibanaSettingsCollector extends Collector true, + schema: { + xpack: { + default_admin_email: { type: 'text' }, + }, + }, async fetch(this: KibanaSettingsCollector) { let kibanaSettingsData; const defaultAdminEmail = await checkForEmailValue(config); diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.test.ts index 12926a30efa1b..88035c1121848 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.test.ts @@ -97,11 +97,18 @@ describe('fetchCpuUsageNodeStats', () => { }, ], }, - average_usage: { - value: 10, - }, - average_periods: { - value: 5, + histo: { + buckets: [ + null, + { + usage_deriv: { + normalized_value: 10, + }, + periods_deriv: { + normalized_value: 5, + }, + }, + ], }, average_quota: { value: 50, @@ -185,14 +192,14 @@ describe('fetchCpuUsageNodeStats', () => { }); await fetchCpuUsageNodeStats(callCluster, clusters, index, startMs, endMs, size); expect(params).toStrictEqual({ - index, + index: '.monitoring-es-*', filterPath: ['aggregations'], body: { size: 0, query: { bool: { filter: [ - { terms: { cluster_uuid: clusters.map((cluster) => cluster.clusterUuid) } }, + { terms: { cluster_uuid: ['abc123'] } }, { term: { type: 'node_stats' } }, { range: { timestamp: { format: 'epoch_millis', gte: 0, lte: 0 } } }, ], @@ -200,23 +207,38 @@ describe('fetchCpuUsageNodeStats', () => { }, aggs: { clusters: { - terms: { - field: 'cluster_uuid', - size, - include: clusters.map((cluster) => cluster.clusterUuid), - }, + terms: { field: 'cluster_uuid', size: 10, include: ['abc123'] }, aggs: { nodes: { - terms: { field: 'node_stats.node_id', size }, + terms: { field: 'node_stats.node_id', size: 10 }, aggs: { index: { terms: { field: '_index', size: 1 } }, average_cpu: { avg: { field: 'node_stats.process.cpu.percent' } }, - average_usage: { avg: { field: 'node_stats.os.cgroup.cpuacct.usage_nanos' } }, - average_periods: { - avg: { field: 'node_stats.os.cgroup.cpu.stat.number_of_elapsed_periods' }, - }, average_quota: { avg: { field: 'node_stats.os.cgroup.cpu.cfs_quota_micros' } }, name: { terms: { field: 'source_node.name', size: 1 } }, + histo: { + date_histogram: { field: 'timestamp', fixed_interval: '0m' }, + aggs: { + average_periods: { + max: { field: 'node_stats.os.cgroup.cpu.stat.number_of_elapsed_periods' }, + }, + average_usage: { max: { field: 'node_stats.os.cgroup.cpuacct.usage_nanos' } }, + usage_deriv: { + derivative: { + buckets_path: 'average_usage', + gap_policy: 'skip', + unit: '1s', + }, + }, + periods_deriv: { + derivative: { + buckets_path: 'average_periods', + gap_policy: 'skip', + unit: '1s', + }, + }, + }, + }, }, }, }, diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.ts index 4fdb03b61950e..ecd324c083a8c 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.ts @@ -4,6 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ import { get } from 'lodash'; +import moment from 'moment'; +import { NORMALIZED_DERIVATIVE_UNIT } from '../../../common/constants'; import { AlertCluster, AlertCpuUsageNodeStats } from '../../alerts/types'; interface NodeBucketESResponse { @@ -26,6 +28,9 @@ export async function fetchCpuUsageNodeStats( endMs: number, size: number ): Promise { + // Using pure MS didn't seem to work well with the date_histogram interval + // but minutes does + const intervalInMinutes = moment.duration(endMs - startMs).asMinutes(); const filterPath = ['aggregations']; const params = { index, @@ -82,16 +87,6 @@ export async function fetchCpuUsageNodeStats( field: 'node_stats.process.cpu.percent', }, }, - average_usage: { - avg: { - field: 'node_stats.os.cgroup.cpuacct.usage_nanos', - }, - }, - average_periods: { - avg: { - field: 'node_stats.os.cgroup.cpu.stat.number_of_elapsed_periods', - }, - }, average_quota: { avg: { field: 'node_stats.os.cgroup.cpu.cfs_quota_micros', @@ -103,6 +98,38 @@ export async function fetchCpuUsageNodeStats( size: 1, }, }, + histo: { + date_histogram: { + field: 'timestamp', + fixed_interval: `${intervalInMinutes}m`, + }, + aggs: { + average_periods: { + max: { + field: 'node_stats.os.cgroup.cpu.stat.number_of_elapsed_periods', + }, + }, + average_usage: { + max: { + field: 'node_stats.os.cgroup.cpuacct.usage_nanos', + }, + }, + usage_deriv: { + derivative: { + buckets_path: 'average_usage', + gap_policy: 'skip', + unit: NORMALIZED_DERIVATIVE_UNIT, + }, + }, + periods_deriv: { + derivative: { + buckets_path: 'average_periods', + gap_policy: 'skip', + unit: NORMALIZED_DERIVATIVE_UNIT, + }, + }, + }, + }, }, }, }, @@ -120,17 +147,19 @@ export async function fetchCpuUsageNodeStats( ) as ClusterBucketESResponse[]; for (const clusterBucket of clusterBuckets) { for (const node of clusterBucket.nodes.buckets) { + const lastBucket = get(node, 'histo.buckets[1]', {}); const indexName = get(node, 'index.buckets[0].key', ''); - stats.push({ + const stat = { clusterUuid: clusterBucket.key, nodeId: node.key, nodeName: get(node, 'name.buckets[0].key'), cpuUsage: get(node, 'average_cpu.value'), - containerUsage: get(node, 'average_usage.value'), - containerPeriods: get(node, 'average_periods.value'), + containerUsage: get(lastBucket, 'usage_deriv.normalized_value'), + containerPeriods: get(lastBucket, 'periods_deriv.normalized_value'), containerQuota: get(node, 'average_quota.value'), ccs: indexName.includes(':') ? indexName.split(':')[0] : null, - }); + }; + stats.push(stat); } } return stats; diff --git a/x-pack/plugins/painless_lab/public/application/components/editor.tsx b/x-pack/plugins/painless_lab/public/application/components/editor.tsx index b8891ce6524f5..5971c0de5c4ef 100644 --- a/x-pack/plugins/painless_lab/public/application/components/editor.tsx +++ b/x-pack/plugins/painless_lab/public/application/components/editor.tsx @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ import React from 'react'; +import { PainlessLang } from '@kbn/monaco'; import { CodeEditor } from '../../../../../../src/plugins/kibana_react/public'; interface Props { @@ -14,7 +15,7 @@ interface Props { export function Editor({ code, onChange }: Props) { return ( , so we define our own - brackets: [ - ['{', '}', 'delimiter.curly'], - ['[', ']', 'delimiter.square'], - ['(', ')', 'delimiter.parenthesis'], - ], - keywords: [ - 'if', - 'in', - 'else', - 'while', - 'do', - 'for', - 'continue', - 'break', - 'return', - 'new', - 'try', - 'catch', - 'throw', - 'this', - 'instanceof', - ], - primitives: ['void', 'boolean', 'byte', 'short', 'char', 'int', 'long', 'float', 'double', 'def'], - constants: ['true', 'false'], - operators: [ - '=', - '>', - '<', - '!', - '~', - '?', - '?:', - '?.', - ':', - '==', - '===', - '<=', - '>=', - '!=', - '!==', - '&&', - '||', - '++', - '--', - '+', - '-', - '*', - '/', - '&', - '|', - '^', - '%', - '<<', - '>>', - '>>>', - '+=', - '-=', - '*=', - '/=', - '&=', - '|=', - '^=', - '%=', - '<<=', - '>>=', - '>>>=', - '->', - '::', - '=~', - '==~', - ], - symbols: /[=> { @@ -25,8 +24,6 @@ const checkLicenseStatus = (license: ILicense) => { }; export class PainlessLabUIPlugin implements Plugin { - languageService = new LanguageService(); - public setup( { http, getStartServices, uiSettings }: CoreSetup, { devTools, home, licensing }: PluginDependencies @@ -80,8 +77,6 @@ export class PainlessLabUIPlugin implements Plugin { - const blob = new Blob([workerSrc], { type: 'application/javascript' }); - return new Worker(window.URL.createObjectURL(blob)); - }, - }; - } - } - - public stop() { - if (CAN_CREATE_WORKER) { - (window as any).MonacoEnvironment = this.originalMonacoEnvironment; - } - } -} diff --git a/x-pack/plugins/reporting/public/panel_actions/get_csv_panel_action.tsx b/x-pack/plugins/reporting/public/panel_actions/get_csv_panel_action.tsx index 7782576bb5a00..1e3f7e34bebdb 100644 --- a/x-pack/plugins/reporting/public/panel_actions/get_csv_panel_action.tsx +++ b/x-pack/plugins/reporting/public/panel_actions/get_csv_panel_action.tsx @@ -108,7 +108,7 @@ export class GetCsvReportPanelAction implements ActionDefinition const filename = embeddable.getSavedSearch().title; const timezone = kibanaTimezone === 'Browser' ? moment.tz.guess() : kibanaTimezone; const fromTime = dateMath.parse(from); - const toTime = dateMath.parse(to); + const toTime = dateMath.parse(to, { roundUp: true }); if (!fromTime || !toTime) { return this.onGenerationFail( diff --git a/x-pack/plugins/rollup/server/routes/api/index_patterns/register_fields_for_wildcard_route.ts b/x-pack/plugins/rollup/server/routes/api/index_patterns/register_fields_for_wildcard_route.ts index 250947d72c5fa..df9907fbf731a 100644 --- a/x-pack/plugins/rollup/server/routes/api/index_patterns/register_fields_for_wildcard_route.ts +++ b/x-pack/plugins/rollup/server/routes/api/index_patterns/register_fields_for_wildcard_route.ts @@ -8,6 +8,7 @@ import { keyBy } from 'lodash'; import { schema } from '@kbn/config-schema'; import { Field } from '../../../lib/merge_capabilities_with_fields'; import { RouteDependencies } from '../../../types'; +import type { IndexPatternsFetcher as IndexPatternsFetcherType } from '../../../../../../../src/plugins/data/server'; const parseMetaFields = (metaFields: string | string[]) => { let parsedFields: string[] = []; @@ -23,10 +24,10 @@ const getFieldsForWildcardRequest = async ( context: any, request: any, response: any, - IndexPatternsFetcher: any + IndexPatternsFetcher: typeof IndexPatternsFetcherType ) => { - const { callAsCurrentUser } = context.core.elasticsearch.legacy.client; - const indexPatterns = new IndexPatternsFetcher(callAsCurrentUser); + const { asCurrentUser } = context.core.elasticsearch.client; + const indexPatterns = new IndexPatternsFetcher(asCurrentUser); const { pattern, meta_fields: metaFields } = request.query; let parsedFields: string[] = []; diff --git a/x-pack/plugins/security/public/management/role_mappings/role_mappings_grid/role_mappings_grid_page.tsx b/x-pack/plugins/security/public/management/role_mappings/role_mappings_grid/role_mappings_grid_page.tsx index 757e59a4e0583..cf0cb21598c62 100644 --- a/x-pack/plugins/security/public/management/role_mappings/role_mappings_grid/role_mappings_grid_page.tsx +++ b/x-pack/plugins/security/public/management/role_mappings/role_mappings_grid/role_mappings_grid_page.tsx @@ -330,14 +330,16 @@ export class RoleMappingsGridPage extends Component { this.state.roles?.find((r) => r.name === rolename) ?? rolename; return ( - + + + ); }); - return
{roleLinks}
; + return ( + + {roleLinks} + + ); }, }, { diff --git a/x-pack/plugins/security/public/management/roles/roles_grid/roles_grid_page.test.tsx b/x-pack/plugins/security/public/management/roles/roles_grid/roles_grid_page.test.tsx index 005eebbfbf3bb..eb6d25c9ef5ab 100644 --- a/x-pack/plugins/security/public/management/roles/roles_grid/roles_grid_page.test.tsx +++ b/x-pack/plugins/security/public/management/roles/roles_grid/roles_grid_page.test.tsx @@ -66,6 +66,11 @@ describe('', () => { kibana: [{ base: [], spaces: [], feature: {} }], transient_metadata: { enabled: false }, }, + { + name: 'special%chars%role', + elasticsearch: { cluster: [], indices: [], run_as: [] }, + kibana: [{ base: [], spaces: [], feature: {} }], + }, ]); }); @@ -121,7 +126,7 @@ describe('', () => { expect(wrapper.find(PermissionDenied)).toMatchSnapshot(); }); - it('renders role actions as appropriate', async () => { + it('renders role actions as appropriate, escaping when necessary', async () => { const wrapper = mountWithIntl( ', () => { expect(wrapper.find(PermissionDenied)).toHaveLength(0); - const editButton = wrapper.find('EuiButtonIcon[data-test-subj="edit-role-action-test-role-1"]'); + let editButton = wrapper.find('EuiButtonIcon[data-test-subj="edit-role-action-test-role-1"]'); expect(editButton).toHaveLength(1); expect(editButton.prop('href')).toBe('/edit/test-role-1'); - const cloneButton = wrapper.find( - 'EuiButtonIcon[data-test-subj="clone-role-action-test-role-1"]' + editButton = wrapper.find( + 'EuiButtonIcon[data-test-subj="edit-role-action-special%chars%role"]' ); + expect(editButton).toHaveLength(1); + expect(editButton.prop('href')).toBe('/edit/special%25chars%25role'); + + let cloneButton = wrapper.find('EuiButtonIcon[data-test-subj="clone-role-action-test-role-1"]'); expect(cloneButton).toHaveLength(1); expect(cloneButton.prop('href')).toBe('/clone/test-role-1'); + cloneButton = wrapper.find( + 'EuiButtonIcon[data-test-subj="clone-role-action-special%chars%role"]' + ); + expect(cloneButton).toHaveLength(1); + expect(cloneButton.prop('href')).toBe('/clone/special%25chars%25role'); + expect( wrapper.find('EuiButtonIcon[data-test-subj="edit-role-action-disabled-role"]') ).toHaveLength(1); @@ -182,6 +197,11 @@ describe('', () => { kibana: [{ base: [], spaces: [], feature: {} }], metadata: { _reserved: true }, }, + { + name: 'special%chars%role', + elasticsearch: { cluster: [], indices: [], run_as: [] }, + kibana: [{ base: [], spaces: [], feature: {} }], + }, { name: 'test-role-1', elasticsearch: { cluster: [], indices: [], run_as: [] }, @@ -198,6 +218,11 @@ describe('', () => { kibana: [{ base: [], spaces: [], feature: {} }], transient_metadata: { enabled: false }, }, + { + name: 'special%chars%role', + elasticsearch: { cluster: [], indices: [], run_as: [] }, + kibana: [{ base: [], spaces: [], feature: {} }], + }, { name: 'test-role-1', elasticsearch: { cluster: [], indices: [], run_as: [] }, diff --git a/x-pack/plugins/security/public/management/roles/roles_grid/roles_grid_page.tsx b/x-pack/plugins/security/public/management/roles/roles_grid/roles_grid_page.tsx index 4c8f45d7eea43..1e733dfa0767e 100644 --- a/x-pack/plugins/security/public/management/roles/roles_grid/roles_grid_page.tsx +++ b/x-pack/plugins/security/public/management/roles/roles_grid/roles_grid_page.tsx @@ -58,7 +58,7 @@ interface State { } const getRoleManagementHref = (action: 'edit' | 'clone', roleName?: string) => { - return `/${action}${roleName ? `/${roleName}` : ''}`; + return `/${action}${roleName ? `/${encodeURIComponent(roleName)}` : ''}`; }; export class RolesGridPage extends Component { diff --git a/x-pack/plugins/security/public/management/users/users_grid/users_grid_page.test.tsx b/x-pack/plugins/security/public/management/users/users_grid/users_grid_page.test.tsx index df8fe8cee7699..0cc3146172f70 100644 --- a/x-pack/plugins/security/public/management/users/users_grid/users_grid_page.test.tsx +++ b/x-pack/plugins/security/public/management/users/users_grid/users_grid_page.test.tsx @@ -71,6 +71,38 @@ describe('UsersGridPage', () => { expect(findTestSubject(wrapper, 'userDisabled')).toHaveLength(0); }); + it('generates valid links when usernames contain special characters', async () => { + const apiClientMock = userAPIClientMock.create(); + apiClientMock.getUsers.mockImplementation(() => { + return Promise.resolve([ + { + username: 'username with some fun characters!@#$%^&*()', + email: 'foo@bar.net', + full_name: 'foo bar', + roles: ['kibana_user'], + enabled: true, + }, + ]); + }); + + const wrapper = mountWithIntl( + + ); + + await waitForRender(wrapper); + + const link = findTestSubject(wrapper, 'userRowUserName'); + expect(link.props().href).toMatchInlineSnapshot( + `"/edit/username%20with%20some%20fun%20characters!%40%23%24%25%5E%26*()"` + ); + }); + it('renders a forbidden message if user is not authorized', async () => { const apiClient = userAPIClientMock.create(); apiClient.getUsers.mockRejectedValue({ body: { statusCode: 403 } }); diff --git a/x-pack/plugins/security/public/management/users/users_grid/users_grid_page.tsx b/x-pack/plugins/security/public/management/users/users_grid/users_grid_page.tsx index 998739a9a83af..debe1c5e82d3b 100644 --- a/x-pack/plugins/security/public/management/users/users_grid/users_grid_page.tsx +++ b/x-pack/plugins/security/public/management/users/users_grid/users_grid_page.tsx @@ -112,7 +112,7 @@ export class UsersGridPage extends Component { render: (username: string) => ( {username} @@ -151,14 +151,16 @@ export class UsersGridPage extends Component { const roleLinks = rolenames.map((rolename, index) => { const roleDefinition = roles?.find((role) => role.name === rolename) ?? rolename; return ( - + + + ); }); - return
{roleLinks}
; + return ( + + {roleLinks} + + ); }, }, { diff --git a/x-pack/plugins/security_solution/public/common/components/url_state/helpers.ts b/x-pack/plugins/security_solution/public/common/components/url_state/helpers.ts index 05000f91f094c..2be9d27b3fecb 100644 --- a/x-pack/plugins/security_solution/public/common/components/url_state/helpers.ts +++ b/x-pack/plugins/security_solution/public/common/components/url_state/helpers.ts @@ -278,6 +278,7 @@ export const replaceStateInLocation = ({ replaceStateKeyInQueryString(urlStateKey, urlStateToReplace)(getQueryStringFromLocation(search)) ); if (history) { + newLocation.state = history.location.state; history.replace(newLocation); } return newLocation.search; diff --git a/x-pack/plugins/security_solution/public/common/containers/source/index.tsx b/x-pack/plugins/security_solution/public/common/containers/source/index.tsx index c36e2de61fcbf..2cc1c75015e07 100644 --- a/x-pack/plugins/security_solution/public/common/containers/source/index.tsx +++ b/x-pack/plugins/security_solution/public/common/containers/source/index.tsx @@ -194,15 +194,14 @@ export const useIndexFields = (sourcererScopeName: SourcererScopeName) => { const { data, notifications } = useKibana().services; const abortCtrl = useRef(new AbortController()); const dispatch = useDispatch(); - const previousIndexesName = useRef([]); - const indexNamesSelectedSelector = useMemo( () => sourcererSelectors.getIndexNamesSelectedSelector(), [] ); - const indexNames = useShallowEqualSelector((state) => - indexNamesSelectedSelector(state, sourcererScopeName) - ); + const { indexNames, previousIndexNames } = useShallowEqualSelector<{ + indexNames: string[]; + previousIndexNames: string; + }>((state) => indexNamesSelectedSelector(state, sourcererScopeName)); const setLoading = useCallback( (loading: boolean) => { @@ -230,7 +229,6 @@ export const useIndexFields = (sourcererScopeName: SourcererScopeName) => { if (!response.isPartial && !response.isRunning) { if (!didCancel) { const stringifyIndices = response.indicesExist.sort().join(); - previousIndexesName.current = response.indicesExist; dispatch( sourcererActions.setSource({ id: sourcererScopeName, @@ -279,8 +277,8 @@ export const useIndexFields = (sourcererScopeName: SourcererScopeName) => { ); useEffect(() => { - if (!isEmpty(indexNames) && !isEqual(previousIndexesName.current, indexNames)) { + if (!isEmpty(indexNames) && previousIndexNames !== indexNames.sort().join()) { indexFieldsSearch(indexNames); } - }, [indexNames, indexFieldsSearch, previousIndexesName]); + }, [indexNames, indexFieldsSearch, previousIndexNames]); }; diff --git a/x-pack/plugins/security_solution/public/common/containers/sourcerer/index.test.tsx b/x-pack/plugins/security_solution/public/common/containers/sourcerer/index.test.tsx index accfb38bc3dc1..22cb4b91fd839 100644 --- a/x-pack/plugins/security_solution/public/common/containers/sourcerer/index.test.tsx +++ b/x-pack/plugins/security_solution/public/common/containers/sourcerer/index.test.tsx @@ -86,7 +86,29 @@ jest.mock('../../utils/apollo_context', () => ({ })); describe('Sourcerer Hooks', () => { - const state: State = mockGlobalState; + const state: State = { + ...mockGlobalState, + sourcerer: { + ...mockGlobalState.sourcerer, + sourcererScopes: { + ...mockGlobalState.sourcerer.sourcererScopes, + [SourcererScopeName.default]: { + ...mockGlobalState.sourcerer.sourcererScopes[SourcererScopeName.default], + indexPattern: { + fields: [], + title: '', + }, + }, + [SourcererScopeName.timeline]: { + ...mockGlobalState.sourcerer.sourcererScopes[SourcererScopeName.timeline], + indexPattern: { + fields: [], + title: '', + }, + }, + }, + }, + }; const { storage } = createSecuritySolutionStorageMock(); let store = createStore( state, diff --git a/x-pack/plugins/security_solution/public/common/containers/sourcerer/index.tsx b/x-pack/plugins/security_solution/public/common/containers/sourcerer/index.tsx index b02a09625ccf3..d9f2abeb3832e 100644 --- a/x-pack/plugins/security_solution/public/common/containers/sourcerer/index.tsx +++ b/x-pack/plugins/security_solution/public/common/containers/sourcerer/index.tsx @@ -16,6 +16,9 @@ import { ManageScope, SourcererScopeName } from '../../store/sourcerer/model'; import { useIndexFields } from '../source'; import { State } from '../../store'; import { useUserInfo } from '../../../detections/components/user_info'; +import { timelineSelectors } from '../../../timelines/store/timeline'; +import { TimelineId } from '../../../../common/types/timeline'; +import { TimelineModel } from '../../../timelines/store/timeline/model'; export const useInitSourcerer = ( scopeId: SourcererScopeName.default | SourcererScopeName.detections = SourcererScopeName.default @@ -29,6 +32,12 @@ export const useInitSourcerer = ( ); const ConfigIndexPatterns = useSelector(getConfigIndexPatternsSelector, isEqual); + const getTimelineSelector = useMemo(() => timelineSelectors.getTimelineByIdSelector(), []); + const activeTimeline = useSelector( + (state) => getTimelineSelector(state, TimelineId.active), + isEqual + ); + useIndexFields(scopeId); useIndexFields(SourcererScopeName.timeline); @@ -40,7 +49,11 @@ export const useInitSourcerer = ( // Related to timeline useEffect(() => { - if (!loadingSignalIndex && signalIndexName != null) { + if ( + !loadingSignalIndex && + signalIndexName != null && + (activeTimeline == null || (activeTimeline != null && activeTimeline.savedObjectId == null)) + ) { dispatch( sourcererActions.setSelectedIndexPatterns({ id: SourcererScopeName.timeline, @@ -48,7 +61,7 @@ export const useInitSourcerer = ( }) ); } - }, [ConfigIndexPatterns, dispatch, loadingSignalIndex, signalIndexName]); + }, [activeTimeline, ConfigIndexPatterns, dispatch, loadingSignalIndex, signalIndexName]); // Related to the detection page useEffect(() => { diff --git a/x-pack/plugins/security_solution/public/common/store/sourcerer/actions.ts b/x-pack/plugins/security_solution/public/common/store/sourcerer/actions.ts index 0b40586798f09..8e92d7559f1d6 100644 --- a/x-pack/plugins/security_solution/public/common/store/sourcerer/actions.ts +++ b/x-pack/plugins/security_solution/public/common/store/sourcerer/actions.ts @@ -34,3 +34,9 @@ export const setSelectedIndexPatterns = actionCreator<{ selectedPatterns: string[]; eventType?: TimelineEventsType; }>('SET_SELECTED_INDEX_PATTERNS'); + +export const initTimelineIndexPatterns = actionCreator<{ + id: SourcererScopeName; + selectedPatterns: string[]; + eventType?: TimelineEventsType; +}>('INIT_TIMELINE_INDEX_PATTERNS'); diff --git a/x-pack/plugins/security_solution/public/common/store/sourcerer/helpers.ts b/x-pack/plugins/security_solution/public/common/store/sourcerer/helpers.ts index 3ae9740cfd51d..42a4fe73c43ba 100644 --- a/x-pack/plugins/security_solution/public/common/store/sourcerer/helpers.ts +++ b/x-pack/plugins/security_solution/public/common/store/sourcerer/helpers.ts @@ -25,16 +25,7 @@ export const createDefaultIndexPatterns = ({ eventType, id, selectedPatterns, st if (isEmpty(newSelectedPatterns)) { let defaultIndexPatterns = state.configIndexPatterns; if (id === SourcererScopeName.timeline && isEmpty(newSelectedPatterns)) { - if (eventType === 'all' && !isEmpty(state.signalIndexName)) { - defaultIndexPatterns = [...state.configIndexPatterns, state.signalIndexName ?? '']; - } else if (eventType === 'raw') { - defaultIndexPatterns = state.configIndexPatterns; - } else if ( - !isEmpty(state.signalIndexName) && - (eventType === 'signal' || eventType === 'alert') - ) { - defaultIndexPatterns = [state.signalIndexName ?? '']; - } + defaultIndexPatterns = defaultIndexPatternByEventType({ state, eventType }); } else if (id === SourcererScopeName.detections && isEmpty(newSelectedPatterns)) { defaultIndexPatterns = [state.signalIndexName ?? '']; } @@ -42,3 +33,21 @@ export const createDefaultIndexPatterns = ({ eventType, id, selectedPatterns, st } return newSelectedPatterns; }; + +export const defaultIndexPatternByEventType = ({ + state, + eventType, +}: { + state: SourcererModel; + eventType?: TimelineEventsType; +}) => { + let defaultIndexPatterns = state.configIndexPatterns; + if (eventType === 'all' && !isEmpty(state.signalIndexName)) { + defaultIndexPatterns = [...state.configIndexPatterns, state.signalIndexName ?? '']; + } else if (eventType === 'raw') { + defaultIndexPatterns = state.configIndexPatterns; + } else if (!isEmpty(state.signalIndexName) && (eventType === 'signal' || eventType === 'alert')) { + defaultIndexPatterns = [state.signalIndexName ?? '']; + } + return defaultIndexPatterns; +}; diff --git a/x-pack/plugins/security_solution/public/common/store/sourcerer/reducer.ts b/x-pack/plugins/security_solution/public/common/store/sourcerer/reducer.ts index a1112607de24f..0c7c52e5e5733 100644 --- a/x-pack/plugins/security_solution/public/common/store/sourcerer/reducer.ts +++ b/x-pack/plugins/security_solution/public/common/store/sourcerer/reducer.ts @@ -4,8 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -// Prefer importing entire lodash library, e.g. import { get } from "lodash" - +import { isEmpty } from 'lodash/fp'; import { reducerWithInitialState } from 'typescript-fsa-reducers'; import { @@ -14,9 +13,10 @@ import { setSelectedIndexPatterns, setSignalIndexName, setSource, + initTimelineIndexPatterns, } from './actions'; import { initialSourcererState, SourcererModel } from './model'; -import { createDefaultIndexPatterns } from './helpers'; +import { createDefaultIndexPatterns, defaultIndexPatternByEventType } from './helpers'; export type SourcererState = SourcererModel; @@ -52,6 +52,21 @@ export const sourcererReducer = reducerWithInitialState(initialSourcererState) }, }; }) + .case(initTimelineIndexPatterns, (state, { id, selectedPatterns, eventType }) => { + return { + ...state, + sourcererScopes: { + ...state.sourcererScopes, + [id]: { + ...state.sourcererScopes[id], + selectedPatterns: isEmpty(selectedPatterns) + ? defaultIndexPatternByEventType({ state, eventType }) + : selectedPatterns, + }, + }, + }; + }) + .case(setSource, (state, { id, payload }) => { const { ...sourcererScopes } = payload; return { diff --git a/x-pack/plugins/security_solution/public/common/store/sourcerer/selectors.test.ts b/x-pack/plugins/security_solution/public/common/store/sourcerer/selectors.test.ts new file mode 100644 index 0000000000000..e6577f2461a9e --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/store/sourcerer/selectors.test.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { cloneDeep } from 'lodash/fp'; +import { mockGlobalState } from '../../mock'; +import { SourcererScopeName } from './model'; +import { getSourcererScopeSelector } from './selectors'; + +describe('Sourcerer selectors', () => { + describe('getSourcererScopeSelector', () => { + it('Should exclude elastic cloud alias when selected patterns include "logs-*" as an alias', () => { + const mapStateToProps = getSourcererScopeSelector(); + expect( + mapStateToProps(mockGlobalState, SourcererScopeName.default).selectedPatterns + ).toEqual([ + 'apm-*-transaction*', + 'auditbeat-*', + 'endgame-*', + 'filebeat-*', + 'logs-*', + 'packetbeat-*', + 'winlogbeat-*', + '-*elastic-cloud-logs-*', + ]); + }); + + it('Should NOT exclude elastic cloud alias when selected patterns does NOT include "logs-*" as an alias', () => { + const mapStateToProps = getSourcererScopeSelector(); + const myMockGlobalState = cloneDeep(mockGlobalState); + myMockGlobalState.sourcerer.sourcererScopes.default.selectedPatterns = myMockGlobalState.sourcerer.sourcererScopes.default.selectedPatterns.filter( + (index) => !index.includes('logs-*') + ); + expect( + mapStateToProps(myMockGlobalState, SourcererScopeName.default).selectedPatterns + ).toEqual([ + 'apm-*-transaction*', + 'auditbeat-*', + 'endgame-*', + 'filebeat-*', + 'packetbeat-*', + 'winlogbeat-*', + ]); + }); + + it('Should NOT exclude elastic cloud alias when selected patterns include "logs-endpoint.event-*" as an alias', () => { + const mapStateToProps = getSourcererScopeSelector(); + const myMockGlobalState = cloneDeep(mockGlobalState); + myMockGlobalState.sourcerer.sourcererScopes.default.selectedPatterns = [ + ...myMockGlobalState.sourcerer.sourcererScopes.default.selectedPatterns.filter( + (index) => !index.includes('logs-*') + ), + 'logs-endpoint.event-*', + ]; + expect( + mapStateToProps(myMockGlobalState, SourcererScopeName.default).selectedPatterns + ).toEqual([ + 'apm-*-transaction*', + 'auditbeat-*', + 'endgame-*', + 'filebeat-*', + 'packetbeat-*', + 'winlogbeat-*', + 'logs-endpoint.event-*', + ]); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/public/common/store/sourcerer/selectors.ts b/x-pack/plugins/security_solution/public/common/store/sourcerer/selectors.ts index ca9ea26ba5bac..e7bd6234cb207 100644 --- a/x-pack/plugins/security_solution/public/common/store/sourcerer/selectors.ts +++ b/x-pack/plugins/security_solution/public/common/store/sourcerer/selectors.ts @@ -41,13 +41,18 @@ export const getIndexNamesSelectedSelector = () => { const getScopesSelector = scopesSelector(); const getConfigIndexPatternsSelector = configIndexPatternsSelector(); - const mapStateToProps = (state: State, scopeId: SourcererScopeName): string[] => { + const mapStateToProps = ( + state: State, + scopeId: SourcererScopeName + ): { indexNames: string[]; previousIndexNames: string } => { const scope = getScopesSelector(state)[scopeId]; const configIndexPatterns = getConfigIndexPatternsSelector(state); - - return scope.selectedPatterns.length === 0 ? configIndexPatterns : scope.selectedPatterns; + return { + indexNames: + scope.selectedPatterns.length === 0 ? configIndexPatterns : scope.selectedPatterns, + previousIndexNames: scope.indexPattern.title, + }; }; - return mapStateToProps; }; @@ -81,11 +86,18 @@ export const defaultIndexNamesSelector = () => { return mapStateToProps; }; +const EXLCUDE_ELASTIC_CLOUD_INDEX = '-*elastic-cloud-logs-*'; export const getSourcererScopeSelector = () => { const getScopesSelector = scopesSelector(); - const mapStateToProps = (state: State, scopeId: SourcererScopeName): ManageScope => - getScopesSelector(state)[scopeId]; + const mapStateToProps = (state: State, scopeId: SourcererScopeName): ManageScope => ({ + ...getScopesSelector(state)[scopeId], + selectedPatterns: getScopesSelector(state)[scopeId].selectedPatterns.some( + (index) => index === 'logs-*' + ) + ? [...getScopesSelector(state)[scopeId].selectedPatterns, EXLCUDE_ELASTIC_CLOUD_INDEX] + : getScopesSelector(state)[scopeId].selectedPatterns, + }); return mapStateToProps; }; diff --git a/x-pack/plugins/security_solution/public/common/utils/route/spy_routes.tsx b/x-pack/plugins/security_solution/public/common/utils/route/spy_routes.tsx index 589436b945a65..febcf0aee679d 100644 --- a/x-pack/plugins/security_solution/public/common/utils/route/spy_routes.tsx +++ b/x-pack/plugins/security_solution/public/common/utils/route/spy_routes.tsx @@ -11,6 +11,7 @@ import deepEqual from 'fast-deep-equal'; import { SpyRouteProps } from './types'; import { useRouteSpy } from './use_route_spy'; +import { SecurityPageName } from '../../../../common/constants'; export const SpyRouteComponent = memo< SpyRouteProps & { location: H.Location; pageName: string | undefined } @@ -50,6 +51,7 @@ export const SpyRouteComponent = memo< pathName: pathname, state, tabName, + ...(pageName === SecurityPageName.administration ? { search: search ?? '' } : {}), }, }); setIsInitializing(false); diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.test.tsx b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.test.tsx index bfc104b105236..ecc0fc54d0d47 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.test.tsx @@ -47,7 +47,9 @@ describe('alert actions', () => { searchStrategyClient = { aggs: {} as ISearchStart['aggs'], showError: jest.fn(), - search: jest.fn().mockResolvedValue({ data: mockTimelineDetails }), + search: jest + .fn() + .mockImplementation(() => ({ toPromise: () => ({ data: mockTimelineDetails }) })), searchSource: {} as ISearchStart['searchSource'], session: dataPluginMock.createStartContract().search.session, }; @@ -400,6 +402,78 @@ describe('alert actions', () => { expect(createTimeline).toHaveBeenCalledWith(defaultTimelineProps); }); }); + + describe('Eql', () => { + test(' with signal.group.id', async () => { + const ecsDataMock: Ecs = { + ...mockEcsDataWithAlert, + signal: { + rule: { + ...mockEcsDataWithAlert.signal?.rule!, + type: ['eql'], + timeline_id: [''], + }, + group: { + id: ['my-group-id'], + }, + }, + }; + + await sendAlertToTimelineAction({ + createTimeline, + ecsData: ecsDataMock, + nonEcsData: [], + updateTimelineIsLoading, + searchStrategyClient, + }); + + expect(updateTimelineIsLoading).not.toHaveBeenCalled(); + expect(createTimeline).toHaveBeenCalledTimes(1); + expect(createTimeline).toHaveBeenCalledWith({ + ...defaultTimelineProps, + timeline: { + ...defaultTimelineProps.timeline, + dataProviders: [ + { + and: [], + enabled: true, + excluded: false, + id: + 'send-alert-to-timeline-action-default-draggable-event-details-value-formatted-field-value-timeline-1-alert-id-my-group-id', + kqlQuery: '', + name: '1', + queryMatch: { field: 'signal.group.id', operator: ':', value: 'my-group-id' }, + }, + ], + }, + }); + }); + + test(' with NO signal.group.id', async () => { + const ecsDataMock: Ecs = { + ...mockEcsDataWithAlert, + signal: { + rule: { + ...mockEcsDataWithAlert.signal?.rule!, + type: ['eql'], + timeline_id: [''], + }, + }, + }; + + await sendAlertToTimelineAction({ + createTimeline, + ecsData: ecsDataMock, + nonEcsData: [], + updateTimelineIsLoading, + searchStrategyClient, + }); + + expect(updateTimelineIsLoading).not.toHaveBeenCalled(); + expect(createTimeline).toHaveBeenCalledTimes(1); + expect(createTimeline).toHaveBeenCalledWith(defaultTimelineProps); + }); + }); }); describe('determineToAndFrom', () => { diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx index 043a5afc4480d..e3defaea2ec67 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx @@ -150,8 +150,10 @@ export const getThresholdAggregationDataProvider = ( ]; }; -export const isEqlRule = (ecsData: Ecs) => - ecsData.signal?.rule?.type?.length && ecsData.signal?.rule?.type[0] === 'eql'; +export const isEqlRuleWithGroupId = (ecsData: Ecs) => + ecsData.signal?.rule?.type?.length && + ecsData.signal?.rule?.type[0] === 'eql' && + ecsData.signal?.group?.id?.length; export const isThresholdRule = (ecsData: Ecs) => ecsData.signal?.rule?.type?.length && ecsData.signal?.rule?.type[0] === 'threshold'; @@ -181,24 +183,23 @@ export const sendAlertToTimelineAction = async ({ timelineType: TimelineType.template, }, }), - searchStrategyClient.search< - TimelineEventsDetailsRequestOptions, - TimelineEventsDetailsStrategyResponse - >( - { - defaultIndex: [], - docValueFields: [], - indexName: ecsData._index ?? '', - eventId: ecsData._id, - factoryQueryType: TimelineEventsQueries.details, - }, - { - strategy: 'securitySolutionTimelineSearchStrategy', - } - ), + searchStrategyClient + .search( + { + defaultIndex: [], + docValueFields: [], + indexName: ecsData._index ?? '', + eventId: ecsData._id, + factoryQueryType: TimelineEventsQueries.details, + }, + { + strategy: 'securitySolutionTimelineSearchStrategy', + } + ) + .toPromise(), ]); const resultingTimeline: TimelineResult = getOr({}, 'data.getOneTimeline', responseTimeline); - const eventData: TimelineEventsDetailsItem[] = getOr([], 'data', eventDataResp); + const eventData: TimelineEventsDetailsItem[] = eventDataResp.data ?? []; if (!isEmpty(resultingTimeline)) { const timelineTemplate: TimelineResult = omitTypenameInTimeline(resultingTimeline); const { timeline, notes } = formatTimelineResultToModel( @@ -327,7 +328,7 @@ export const sendAlertToTimelineAction = async ({ }, }, ]; - if (isEqlRule(ecsData)) { + if (isEqlRuleWithGroupId(ecsData)) { const signalGroupId = ecsData.signal?.group?.id?.length ? ecsData.signal?.group?.id[0] : 'unknown-signal-group-id'; diff --git a/x-pack/plugins/security_solution/public/detections/components/user_info/index.test.tsx b/x-pack/plugins/security_solution/public/detections/components/user_info/index.test.tsx index 9b15007136b2e..e87303efbe526 100644 --- a/x-pack/plugins/security_solution/public/detections/components/user_info/index.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/user_info/index.test.tsx @@ -42,7 +42,7 @@ describe('useUserInfo', () => { isSignalIndexExists: null, loading: true, signalIndexName: null, - signalIndexTemplateOutdated: null, + signalIndexMappingOutdated: null, }, error: undefined, }); @@ -53,7 +53,7 @@ describe('useUserInfo', () => { const spyOnCreateSignalIndex = jest.spyOn(api, 'createSignalIndex'); const spyOnGetSignalIndex = jest.spyOn(api, 'getSignalIndex').mockResolvedValueOnce({ name: 'mock-signal-index', - template_outdated: true, + index_mapping_outdated: true, }); await act(async () => { const { waitForNextUpdate } = renderHook(() => useUserInfo(), { wrapper: ManageUserInfo }); diff --git a/x-pack/plugins/security_solution/public/detections/components/user_info/index.tsx b/x-pack/plugins/security_solution/public/detections/components/user_info/index.tsx index ac2bf438d7fa6..3b0976f459324 100644 --- a/x-pack/plugins/security_solution/public/detections/components/user_info/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/user_info/index.tsx @@ -20,7 +20,7 @@ export interface State { hasEncryptionKey: boolean | null; loading: boolean; signalIndexName: string | null; - signalIndexTemplateOutdated: boolean | null; + signalIndexMappingOutdated: boolean | null; } export const initialState: State = { @@ -32,7 +32,7 @@ export const initialState: State = { hasEncryptionKey: null, loading: true, signalIndexName: null, - signalIndexTemplateOutdated: null, + signalIndexMappingOutdated: null, }; export type Action = @@ -66,8 +66,8 @@ export type Action = signalIndexName: string | null; } | { - type: 'updateSignalIndexTemplateOutdated'; - signalIndexTemplateOutdated: boolean | null; + type: 'updateSignalIndexMappingOutdated'; + signalIndexMappingOutdated: boolean | null; }; export const userInfoReducer = (state: State, action: Action): State => { @@ -120,10 +120,10 @@ export const userInfoReducer = (state: State, action: Action): State => { signalIndexName: action.signalIndexName, }; } - case 'updateSignalIndexTemplateOutdated': { + case 'updateSignalIndexMappingOutdated': { return { ...state, - signalIndexTemplateOutdated: action.signalIndexTemplateOutdated, + signalIndexMappingOutdated: action.signalIndexMappingOutdated, }; } default: @@ -156,7 +156,7 @@ export const useUserInfo = (): State => { hasEncryptionKey, loading, signalIndexName, - signalIndexTemplateOutdated, + signalIndexMappingOutdated, }, dispatch, ] = useUserData(); @@ -171,7 +171,7 @@ export const useUserInfo = (): State => { loading: indexNameLoading, signalIndexExists: isApiSignalIndexExists, signalIndexName: apiSignalIndexName, - signalIndexTemplateOutdated: apiSignalIndexTemplateOutdated, + signalIndexMappingOutdated: apiSignalIndexMappingOutdated, createDeSignalIndex: createSignalIndex, } = useSignalIndex(); @@ -234,15 +234,15 @@ export const useUserInfo = (): State => { useEffect(() => { if ( !loading && - signalIndexTemplateOutdated !== apiSignalIndexTemplateOutdated && - apiSignalIndexTemplateOutdated != null + signalIndexMappingOutdated !== apiSignalIndexMappingOutdated && + apiSignalIndexMappingOutdated != null ) { dispatch({ - type: 'updateSignalIndexTemplateOutdated', - signalIndexTemplateOutdated: apiSignalIndexTemplateOutdated, + type: 'updateSignalIndexMappingOutdated', + signalIndexMappingOutdated: apiSignalIndexMappingOutdated, }); } - }, [dispatch, loading, signalIndexTemplateOutdated, apiSignalIndexTemplateOutdated]); + }, [dispatch, loading, signalIndexMappingOutdated, apiSignalIndexMappingOutdated]); useEffect(() => { if ( @@ -250,7 +250,7 @@ export const useUserInfo = (): State => { hasEncryptionKey && hasIndexManage && ((isSignalIndexExists != null && !isSignalIndexExists) || - (signalIndexTemplateOutdated != null && signalIndexTemplateOutdated)) && + (signalIndexMappingOutdated != null && signalIndexMappingOutdated)) && createSignalIndex != null ) { createSignalIndex(); @@ -261,7 +261,7 @@ export const useUserInfo = (): State => { hasEncryptionKey, isSignalIndexExists, hasIndexManage, - signalIndexTemplateOutdated, + signalIndexMappingOutdated, ]); return { @@ -273,6 +273,6 @@ export const useUserInfo = (): State => { hasIndexManage, hasIndexWrite, signalIndexName, - signalIndexTemplateOutdated, + signalIndexMappingOutdated, }; }; diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/mock.ts b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/mock.ts index 4fd240348f0f3..21b561ec9cddb 100644 --- a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/mock.ts +++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/mock.ts @@ -980,7 +980,7 @@ export const mockStatusAlertQuery: object = { export const mockSignalIndex: AlertsIndex = { name: 'mock-signal-index', - template_outdated: false, + index_mapping_outdated: false, }; export const mockUserPrivilege: Privilege = { diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/types.ts b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/types.ts index 59ab416ecc824..dadeb1e7958b5 100644 --- a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/types.ts +++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/types.ts @@ -44,7 +44,7 @@ export interface UpdateAlertStatusProps { export interface AlertsIndex { name: string; - template_outdated: boolean; + index_mapping_outdated: boolean; } export interface Privilege { diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_signal_index.test.tsx b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_signal_index.test.tsx index 1db952526414a..07375a31f3bbc 100644 --- a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_signal_index.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_signal_index.test.tsx @@ -26,7 +26,7 @@ describe('useSignalIndex', () => { loading: true, signalIndexExists: null, signalIndexName: null, - signalIndexTemplateOutdated: null, + signalIndexMappingOutdated: null, }); }); }); @@ -43,7 +43,7 @@ describe('useSignalIndex', () => { loading: false, signalIndexExists: true, signalIndexName: 'mock-signal-index', - signalIndexTemplateOutdated: false, + signalIndexMappingOutdated: false, }); }); }); @@ -64,7 +64,7 @@ describe('useSignalIndex', () => { loading: false, signalIndexExists: true, signalIndexName: 'mock-signal-index', - signalIndexTemplateOutdated: false, + signalIndexMappingOutdated: false, }); }); }); @@ -104,7 +104,7 @@ describe('useSignalIndex', () => { loading: false, signalIndexExists: false, signalIndexName: null, - signalIndexTemplateOutdated: null, + signalIndexMappingOutdated: null, }); }); }); @@ -125,7 +125,7 @@ describe('useSignalIndex', () => { loading: false, signalIndexExists: false, signalIndexName: null, - signalIndexTemplateOutdated: null, + signalIndexMappingOutdated: null, }); }); }); diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_signal_index.tsx b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_signal_index.tsx index f7d2202736169..1233456359b7f 100644 --- a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_signal_index.tsx +++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_signal_index.tsx @@ -17,7 +17,7 @@ export interface ReturnSignalIndex { loading: boolean; signalIndexExists: boolean | null; signalIndexName: string | null; - signalIndexTemplateOutdated: boolean | null; + signalIndexMappingOutdated: boolean | null; createDeSignalIndex: Func | null; } @@ -31,7 +31,7 @@ export const useSignalIndex = (): ReturnSignalIndex => { const [signalIndex, setSignalIndex] = useState>({ signalIndexExists: null, signalIndexName: null, - signalIndexTemplateOutdated: null, + signalIndexMappingOutdated: null, createDeSignalIndex: null, }); const [, dispatchToaster] = useStateToaster(); @@ -49,7 +49,7 @@ export const useSignalIndex = (): ReturnSignalIndex => { setSignalIndex({ signalIndexExists: true, signalIndexName: signal.name, - signalIndexTemplateOutdated: signal.template_outdated, + signalIndexMappingOutdated: signal.index_mapping_outdated, createDeSignalIndex: createIndex, }); } @@ -58,7 +58,7 @@ export const useSignalIndex = (): ReturnSignalIndex => { setSignalIndex({ signalIndexExists: false, signalIndexName: null, - signalIndexTemplateOutdated: null, + signalIndexMappingOutdated: null, createDeSignalIndex: createIndex, }); if (isSecurityAppError(error) && error.body.status_code !== 404) { @@ -89,7 +89,7 @@ export const useSignalIndex = (): ReturnSignalIndex => { setSignalIndex({ signalIndexExists: false, signalIndexName: null, - signalIndexTemplateOutdated: null, + signalIndexMappingOutdated: null, createDeSignalIndex: createIndex, }); errorToToaster({ title: i18n.SIGNAL_POST_FAILURE, error, dispatchToaster }); diff --git a/x-pack/plugins/security_solution/public/management/components/management_empty_state.tsx b/x-pack/plugins/security_solution/public/management/components/management_empty_state.tsx index d51a23639f5cb..18745897c594f 100644 --- a/x-pack/plugins/security_solution/public/management/components/management_empty_state.tsx +++ b/x-pack/plugins/security_solution/public/management/components/management_empty_state.tsx @@ -66,7 +66,7 @@ const PolicyEmptyState = React.memo<{ diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/malware.tsx b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/malware.tsx index 1da832fb081ef..6773ed6541927 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/malware.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/malware.tsx @@ -11,6 +11,7 @@ import { EuiRadio, EuiSwitch, EuiTitle, + EuiText, EuiSpacer, htmlIdGenerator, EuiCallOut, @@ -28,6 +29,7 @@ import { policyConfig } from '../../../store/policy_details/selectors'; import { usePolicyDetailsSelector } from '../../policy_hooks'; import { clone } from '../../../models/policy_details_config'; import { LinkToApp } from '../../../../../../common/components/endpoint/link_to_app'; +import { popupVersionsMap } from './popup_options_to_versions'; const ProtectionRadioGroup = styled.div` display: flex; @@ -83,6 +85,25 @@ const ProtectionRadio = React.memo(({ id, label }: { id: ProtectionModes; label: ProtectionRadio.displayName = 'ProtectionRadio'; +const SupportedVersionNotice = ({ optionName }: { optionName: string }) => { + const version = popupVersionsMap.get(optionName); + if (!version) { + return null; + } + + return ( + + + + + + ); +}; + /** The Malware Protections form for policy details * which will configure for all relevant OSes. */ @@ -189,14 +210,15 @@ export const MalwareProtections = React.memo(() => { /> + + ); diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/popup_options_to_versions.ts b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/popup_options_to_versions.ts new file mode 100644 index 0000000000000..d4c7d0102ebd4 --- /dev/null +++ b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/popup_options_to_versions.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +const popupVersions: Array<[string, string]> = [['malware', '7.11+']]; + +export const popupVersionsMap: ReadonlyMap = new Map(popupVersions); diff --git a/x-pack/plugins/security_solution/public/management/pages/trusted_apps/view/trusted_apps_page.tsx b/x-pack/plugins/security_solution/public/management/pages/trusted_apps/view/trusted_apps_page.tsx index 822c57b92b4e5..2d0b9f759f158 100644 --- a/x-pack/plugins/security_solution/public/management/pages/trusted_apps/view/trusted_apps_page.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/trusted_apps/view/trusted_apps_page.tsx @@ -45,9 +45,7 @@ export const TrustedAppsPage = memo(() => { return ; } return null; - // FIXME: Route state is being deleted by some parent component - // eslint-disable-next-line react-hooks/exhaustive-deps - }, []); + }, [routeState]); const addButton = ( = ({ ( - { indices: defaultIndicesName, onlyCheckIfIndicesExist: false }, + { indices: defaultIndicesName, onlyCheckIfIndicesExist: true }, { strategy: 'securitySolutionIndexFields', } diff --git a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_on_origin.ts b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_on_origin.ts index 6fb84eaf7fda6..837d824db8748 100644 --- a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_on_origin.ts +++ b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_on_origin.ts @@ -78,8 +78,7 @@ export function noAncestorsTwoChildrenWithRelatedEventsOnOrigin(): { */ async eventsWithEntityIDAndCategory( entityID: string, - category: string, - after?: string + category: string ): Promise<{ events: SafeResolverEvent[]; nextEvent: string | null }> { const events = entityID === metadata.entityIDs.origin diff --git a/x-pack/plugins/security_solution/public/resolver/mocks/get_ui_settings.ts b/x-pack/plugins/security_solution/public/resolver/mocks/ui_setting.ts similarity index 79% rename from x-pack/plugins/security_solution/public/resolver/mocks/get_ui_settings.ts rename to x-pack/plugins/security_solution/public/resolver/mocks/ui_setting.ts index ab1a5c86859ac..4d173cd270cb8 100644 --- a/x-pack/plugins/security_solution/public/resolver/mocks/get_ui_settings.ts +++ b/x-pack/plugins/security_solution/public/resolver/mocks/ui_setting.ts @@ -4,7 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ -export function getUiSettings(key: string): string | undefined { +/** + * A mock for Kibana UI settings. + */ +export function uiSetting(key: string): string | undefined { if (key === 'dateFormat') { return 'MMM D, YYYY @ HH:mm:ss.SSS'; } diff --git a/x-pack/plugins/security_solution/public/resolver/store/actions.ts b/x-pack/plugins/security_solution/public/resolver/store/actions.ts index 66a32ba29cd74..26a5f8555a81b 100644 --- a/x-pack/plugins/security_solution/public/resolver/store/actions.ts +++ b/x-pack/plugins/security_solution/public/resolver/store/actions.ts @@ -8,16 +8,25 @@ import { DataAction } from './data/action'; /** * When the user wants to bring a node front-and-center on the map. + * @deprecated Nodes are brought into view upon selection instead. See `appReceivedNewExternalProperties` */ interface UserBroughtNodeIntoView { + /** + * @deprecated Nodes are brought into view upon selection instead. See `appReceivedNewExternalProperties` + */ readonly type: 'userBroughtNodeIntoView'; + /** + * @deprecated Nodes are brought into view upon selection instead. See `appReceivedNewExternalProperties` + */ readonly payload: { /** * Used to identify the node that should be brought into view. + * @deprecated Nodes are brought into view upon selection instead. See `appReceivedNewExternalProperties` */ readonly nodeID: string; /** * The time (since epoch in milliseconds) when the action was dispatched. + * @deprecated Nodes are brought into view upon selection instead. See `appReceivedNewExternalProperties` */ readonly time: number; }; diff --git a/x-pack/plugins/security_solution/public/resolver/test_utilities/extend_jest.ts b/x-pack/plugins/security_solution/public/resolver/test_utilities/extend_jest.ts index aa04221361de0..23b651c262cba 100644 --- a/x-pack/plugins/security_solution/public/resolver/test_utilities/extend_jest.ts +++ b/x-pack/plugins/security_solution/public/resolver/test_utilities/extend_jest.ts @@ -34,7 +34,7 @@ expect.extend({ expected: T ): Promise<{ pass: boolean; message: () => string }> { // Used in printing out the pass or fail message - const matcherName = 'toSometimesYieldEqualTo'; + const matcherName = 'toYieldEqualTo'; const options: jest.MatcherHintOptions = { comment: 'deep equality with any yielded value', isNot: this.isNot, @@ -100,9 +100,9 @@ expect.extend({ expected: T ): Promise<{ pass: boolean; message: () => string }> { // Used in printing out the pass or fail message - const matcherName = 'toSometimesYieldEqualTo'; + const matcherName = 'toYieldObjectEqualTo'; const options: jest.MatcherHintOptions = { - comment: 'deep equality with any yielded value', + comment: 'subset equality with any yielded value', isNot: this.isNot, promise: this.promise, }; diff --git a/x-pack/plugins/security_solution/public/resolver/test_utilities/simulator/index.tsx b/x-pack/plugins/security_solution/public/resolver/test_utilities/simulator/index.tsx index 2a399b6844bd7..2a538620dce0b 100644 --- a/x-pack/plugins/security_solution/public/resolver/test_utilities/simulator/index.tsx +++ b/x-pack/plugins/security_solution/public/resolver/test_utilities/simulator/index.tsx @@ -16,7 +16,7 @@ import { MockResolver } from './mock_resolver'; import { ResolverState, DataAccessLayer, SpyMiddleware, SideEffectSimulator } from '../../types'; import { ResolverAction } from '../../store/actions'; import { sideEffectSimulatorFactory } from '../../view/side_effect_simulator_factory'; -import { getUiSettings } from '../../mocks/get_ui_settings'; +import { uiSetting } from '../../mocks/ui_setting'; /** * Test a Resolver instance using jest, enzyme, and a mock data layer. @@ -62,6 +62,13 @@ export class Simulator { return selector; } + /** + * The simulator returns enzyme `ReactWrapper`s from various methods. Use this predicate to determine if they are DOM nodes. + */ + public static isDOM(wrapper: ReactWrapper): boolean { + return typeof wrapper.type() === 'string'; + } + constructor({ dataAccessLayer, resolverComponentInstanceID, @@ -110,7 +117,7 @@ export class Simulator { // Used for `KibanaContextProvider` const coreStart = coreMock.createStart(); - coreStart.uiSettings.get.mockImplementation(getUiSettings); + coreStart.uiSettings.get.mockImplementation(uiSetting); this.sideEffectSimulator = sideEffectSimulatorFactory(); @@ -190,7 +197,7 @@ export class Simulator { * After 10 times, quit. * Use this to continually check a value. See `toYieldEqualTo`. */ - public async *map(mapper: () => R): AsyncIterable { + public async *map(mapper: (() => Promise) | (() => R)): AsyncIterable { let timeoutCount = 0; while (timeoutCount < 10) { timeoutCount++; @@ -267,6 +274,20 @@ export class Simulator { this.sideEffectSimulator.controls.provideAnimationFrame(); } + /** + * The last value written to the clipboard via the `SideEffectors`. + */ + public get clipboardText(): string { + return this.sideEffectSimulator.controls.clipboardText; + } + + /** + * Call this to resolve the promise returned by the `SideEffectors` `writeText` method (which in production points to `navigator.clipboard.writeText`. + */ + confirmTextWrittenToClipboard(): void { + this.sideEffectSimulator.controls.confirmTextWrittenToClipboard(); + } + /** * The 'search' part of the URL. */ @@ -296,13 +317,36 @@ export class Simulator { return this.domNodes(`[data-test-subj="${selector}"]`); } + /** + * Given a `ReactWrapper`, returns a wrapper containing immediately following `dd` siblings. + * `subject` must contain just 1 element. + */ + public descriptionDetails(subject: ReactWrapper): ReactWrapper { + // find the associated DOM nodes, then return an enzyme wrapper that only contains those. + const subjectNode = subject.getDOMNode(); + let current = subjectNode.nextElementSibling; + const associated: Set = new Set(); + // Multiple `dt`s can be associated with a set of `dd`s. Skip immediately following `dt`s. + while (current !== null && current.nodeName === 'DT') { + current = current.nextElementSibling; + } + while (current !== null && current.nodeName === 'DD') { + associated.add(current); + current = current.nextElementSibling; + } + return subject + .closest('dl') + .find('dd') + .filterWhere((candidate) => { + return associated.has(candidate.getDOMNode()); + }); + } + /** * Return DOM nodes that match `enzymeSelector`. */ private domNodes(enzymeSelector: string): ReactWrapper { - return this.wrapper - .find(enzymeSelector) - .filterWhere((wrapper) => typeof wrapper.type() === 'string'); + return this.wrapper.find(enzymeSelector).filterWhere(Simulator.isDOM); } /** @@ -331,7 +375,7 @@ export class Simulator { * Resolve the wrapper returned by `wrapperFactory` only once it has at least 1 element in it. */ public async resolveWrapper( - wrapperFactory: () => ReactWrapper, + wrapperFactory: (() => Promise) | (() => ReactWrapper), predicate: (wrapper: ReactWrapper) => boolean = (wrapper) => wrapper.length > 0 ): Promise { for await (const wrapper of this.map(wrapperFactory)) { diff --git a/x-pack/plugins/security_solution/public/resolver/types.ts b/x-pack/plugins/security_solution/public/resolver/types.ts index fb57f85639e33..7129e3a47120a 100644 --- a/x-pack/plugins/security_solution/public/resolver/types.ts +++ b/x-pack/plugins/security_solution/public/resolver/types.ts @@ -490,9 +490,26 @@ export interface SideEffectors { * A function which returns the time since epoch in milliseconds. Injected because mocking Date is tedious. */ timestamp: () => number; + /** + * Use instead of `window.requestAnimationFrame` + **/ requestAnimationFrame: typeof window.requestAnimationFrame; + /** + * Use instead of `window.cancelAnimationFrame` + **/ cancelAnimationFrame: typeof window.cancelAnimationFrame; + /** + * Use instead of the `ResizeObserver` global. + */ ResizeObserver: ResizeObserverConstructor; + /** + * Use this instead of the Clipboard API's `writeText` method. + */ + writeTextToClipboard(text: string): Promise; + /** + * Use this instead of `Element.prototype.getBoundingClientRect` . + */ + getBoundingClientRect(element: Element): DOMRect; } export interface SideEffectSimulator { @@ -512,6 +529,16 @@ export interface SideEffectSimulator { * Trigger `ResizeObserver` callbacks for `element` and update the mocked value for `getBoundingClientRect`. */ simulateElementResize: (element: Element, contentRect: DOMRect) => void; + + /** + * Get the most recently written clipboard text. This is only updated when `confirmTextWrittenToClipboard` is called. + */ + clipboardText: string; + + /** + * Call this to resolve the promise returned by `writeText`. + */ + confirmTextWrittenToClipboard: () => void; }; /** * Mocked `SideEffectors`. diff --git a/x-pack/plugins/security_solution/public/resolver/view/clickthrough.test.tsx b/x-pack/plugins/security_solution/public/resolver/view/clickthrough.test.tsx index c781832dc8a3b..7739d81269180 100644 --- a/x-pack/plugins/security_solution/public/resolver/view/clickthrough.test.tsx +++ b/x-pack/plugins/security_solution/public/resolver/view/clickthrough.test.tsx @@ -165,10 +165,7 @@ describe('Resolver, when analyzing a tree that has no ancestors and 2 children', ).toYieldEqualTo({ treeCount: 1, nodesOwnedByTrees: 3 }); }); - it(`should show links to the 3 nodes (with icons) in the node list.`, async () => { - await expect( - simulator.map(() => simulator.testSubject('resolver:node-list:node-link:title').length) - ).toYieldEqualTo(3); + it(`should show links to the 3 nodes in the node list.`, async () => { await expect( simulator.map(() => simulator.testSubject('resolver:node-list:node-link:title').length) ).toYieldEqualTo(3); diff --git a/x-pack/plugins/security_solution/public/resolver/view/panels/panel_content_utilities.tsx b/x-pack/plugins/security_solution/public/resolver/view/generated_text.tsx similarity index 59% rename from x-pack/plugins/security_solution/public/resolver/view/panels/panel_content_utilities.tsx rename to x-pack/plugins/security_solution/public/resolver/view/generated_text.tsx index a20498cbfb67b..61a12fa33cc9d 100644 --- a/x-pack/plugins/security_solution/public/resolver/view/panels/panel_content_utilities.tsx +++ b/x-pack/plugins/security_solution/public/resolver/view/generated_text.tsx @@ -4,33 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ +import React from 'react'; /* eslint-disable react/display-name */ - -import { EuiCode } from '@elastic/eui'; -import { i18n } from '@kbn/i18n'; -import styled from 'styled-components'; -import React, { memo } from 'react'; - -/** - * Text to use in place of an undefined timestamp value - */ - -export const noTimestampRetrievedText = i18n.translate( - 'xpack.securitySolution.enpdoint.resolver.panelutils.noTimestampRetrieved', - { - defaultMessage: 'No timestamp retrieved', - } -); - -/** - * A bold version of EuiCode to display certain titles with - */ -export const BoldCode = styled(EuiCode)` - &.euiCodeBlock code.euiCodeBlock__code { - font-weight: 900; - } -`; - /** * A component that renders an element with breaking opportunities (``s) * spliced into text children at word boundaries. @@ -61,12 +36,3 @@ export const GeneratedText = React.memo(function ({ children }) { }); } }); - -/** - * A component to keep time representations in blocks so they don't wrap - * and look bad. - */ -export const StyledTime = memo(styled('time')` - display: inline-block; - text-align: start; -`); diff --git a/x-pack/plugins/security_solution/public/resolver/view/panel.test.tsx b/x-pack/plugins/security_solution/public/resolver/view/panel.test.tsx index 9d72af3109564..3b3651ec2558a 100644 --- a/x-pack/plugins/security_solution/public/resolver/view/panel.test.tsx +++ b/x-pack/plugins/security_solution/public/resolver/view/panel.test.tsx @@ -29,6 +29,20 @@ describe(`Resolver: when analyzing a tree with no ancestors and two children and secondChild: string; }; + /** + * These are the details we expect to see in the node detail view when the origin is selected. + */ + const originEventDetailEntries: ReadonlyMap = new Map([ + ['@timestamp', 'Sep 23, 2020 @ 08:25:32.316'], + ['process.executable', 'executable'], + ['process.pid', '0'], + ['user.name', 'user.name'], + ['user.domain', 'user.domain'], + ['process.parent.pid', '0'], + ['process.hash.md5', 'hash.md5'], + ['process.args', 'args'], + ]); + beforeEach(() => { // create a mock data access layer const { @@ -86,16 +100,7 @@ describe(`Resolver: when analyzing a tree with no ancestors and two children and ).toYieldEqualTo({ title: 'c.ext', titleIcon: 'Running Process', - detailEntries: [ - ['@timestamp', 'Sep 23, 2020 @ 08:25:32.316'], - ['process.executable', 'executable'], - ['process.pid', '0'], - ['user.name', 'user.name'], - ['user.domain', 'user.domain'], - ['process.parent.pid', '0'], - ['process.hash.md5', 'hash.md5'], - ['process.args', 'args'], - ], + detailEntries: [...originEventDetailEntries], }); }); it('should have breaking opportunities (s) in node titles to allow wrapping', async () => { @@ -111,16 +116,46 @@ describe(`Resolver: when analyzing a tree with no ancestors and two children and wordBreaks: 2, }); }); - it('should allow all node details to be copied', async () => { - const copyableFields = await simulator().resolve('resolver:panel:copyable-field'); - copyableFields?.map((copyableField) => { - copyableField.simulate('mouseenter'); - simulator().testSubject('resolver:panel:clipboard').last().simulate('click'); - expect(navigator.clipboard.writeText).toHaveBeenCalledWith(copyableField.text()); - copyableField.simulate('mouseleave'); - }); - }); + /** + * These tests use a statically defined map of fields and expected values. The test finds the `dt` for each field and then finds the related `dd`s. From there it finds a special 'hover area' (via `data-test-subj`) and simulates a `mouseenter` on it. This is because the feature work by adding event listeners to `div`s. There is no way for the user to know that the `div`s are interactable. + + * Finally the test clicks a button and checks that the clipboard was written to. + */ + describe.each([...originEventDetailEntries])( + 'when the user hovers over the description for the field (%p) with their mouse', + (fieldTitleText, value) => { + beforeEach(async () => { + const dt = await simulator().resolveWrapper(() => { + return simulator() + .testSubject('resolver:node-detail:entry-title') + .filterWhere((title) => title.text() === fieldTitleText); + }); + + expect(dt).toHaveLength(1); + + const copyableFieldHoverArea = simulator() + .descriptionDetails(dt!) + // The copyable field popup does not use a button as a trigger. It is instead triggered by mouse interaction with this `div`. + .find(`[data-test-subj="resolver:panel:copyable-field-hover-area"]`) + .filterWhere(Simulator.isDOM); + + expect(copyableFieldHoverArea).toHaveLength(1); + copyableFieldHoverArea!.simulate('mouseenter'); + }); + describe('and when they click the copy-to-clipboard button', () => { + beforeEach(async () => { + const copyButton = await simulator().resolve('resolver:panel:clipboard'); + expect(copyButton).toHaveLength(1); + copyButton!.simulate('click'); + simulator().confirmTextWrittenToClipboard(); + }); + it(`should write ${value} to the clipboard`, async () => { + await expect(simulator().map(() => simulator().clipboardText)).toYieldEqualTo(value); + }); + }); + } + ); }); const queryStringWithFirstChildSelected = urlSearch(resolverComponentInstanceID, { @@ -160,23 +195,43 @@ describe(`Resolver: when analyzing a tree with no ancestors and two children and it('should have 3 nodes (with icons) in the node list', async () => { await expect( - simulator().map(() => simulator().testSubject('resolver:node-list:node-link:title').length) - ).toYieldEqualTo(3); - await expect( - simulator().map(() => simulator().testSubject('resolver:node-list:node-link:icon').length) - ).toYieldEqualTo(3); + simulator().map(() => { + return { + titleCount: simulator().testSubject('resolver:node-list:node-link:title').length, + iconCount: simulator().testSubject('resolver:node-list:node-link:icon').length, + }; + }) + ).toYieldEqualTo({ titleCount: 3, iconCount: 3 }); }); - it('should be able to copy the timestamps for all 3 nodes', async () => { - const copyableFields = await simulator().resolve('resolver:panel:copyable-field'); + describe('when the user hovers over the timestamp for "c.ext" with their mouse', () => { + beforeEach(async () => { + const cExtHoverArea = await simulator().resolveWrapper(async () => { + const nodeLinkTitles = await simulator().resolve('resolver:node-list:node-link:title'); - expect(copyableFields?.length).toBe(3); + expect(nodeLinkTitles).toHaveLength(3); - copyableFields?.map((copyableField) => { - copyableField.simulate('mouseenter'); - simulator().testSubject('resolver:panel:clipboard').last().simulate('click'); - expect(navigator.clipboard.writeText).toHaveBeenCalledWith(copyableField.text()); - copyableField.simulate('mouseleave'); + return ( + nodeLinkTitles! + .filterWhere((linkTitle) => linkTitle.text() === 'c.ext') + // Find the parent `tr` and the find all hover areas in that TR. The test assumes that all cells in a row are associated. + .closest('tr') + // The copyable field popup does not use a button as a trigger. It is instead triggered by mouse interaction with this `div`. + .find('[data-test-subj="resolver:panel:copyable-field-hover-area"]') + .filterWhere(Simulator.isDOM) + ); + }); + cExtHoverArea!.simulate('mouseenter'); + }); + describe('and when the user clicks the copy-to-clipboard button', () => { + beforeEach(async () => { + (await simulator().resolve('resolver:panel:clipboard'))!.simulate('click'); + simulator().confirmTextWrittenToClipboard(); + }); + const expected = 'Sep 23, 2020 @ 08:25:32.316'; + it(`should write "${expected}" to the clipboard`, async () => { + await expect(simulator().map(() => simulator().clipboardText)).toYieldEqualTo(expected); + }); }); }); @@ -191,16 +246,7 @@ describe(`Resolver: when analyzing a tree with no ancestors and two children and it('should show the details for the first node', async () => { await expect( simulator().map(() => simulator().nodeDetailDescriptionListEntries()) - ).toYieldEqualTo([ - ['@timestamp', 'Sep 23, 2020 @ 08:25:32.316'], - ['process.executable', 'executable'], - ['process.pid', '0'], - ['user.name', 'user.name'], - ['user.domain', 'user.domain'], - ['process.parent.pid', '0'], - ['process.hash.md5', 'hash.md5'], - ['process.args', 'args'], - ]); + ).toYieldEqualTo([...originEventDetailEntries]); }); it("should have the first node's ID in the query string", async () => { await expect(simulator().map(() => simulator().historyLocationSearch)).toYieldEqualTo( @@ -278,16 +324,40 @@ describe(`Resolver: when analyzing a tree with no ancestors and two children and simulator().map(() => simulator().testSubject('resolver:panel:event-detail').length) ).toYieldEqualTo(1); }); - it('should allow all fields to be copied', async () => { - const copyableFields = await simulator().resolve('resolver:panel:copyable-field'); - - copyableFields?.map((copyableField) => { - copyableField.simulate('mouseenter'); - simulator().testSubject('resolver:panel:clipboard').last().simulate('click'); - expect(navigator.clipboard.writeText).toHaveBeenCalledWith(copyableField.text()); - copyableField.simulate('mouseleave'); - }); - }); + describe.each([['user.domain', 'user.domain']])( + 'when the user hovers over the description for the field "%p"', + (fieldName, expectedValue) => { + beforeEach(async () => { + const fieldHoverArea = await simulator().resolveWrapper(async () => { + const dt = ( + await simulator().resolve('resolver:panel:event-detail:event-field-title') + )?.filterWhere((title) => title.text() === fieldName); + return ( + simulator() + .descriptionDetails(dt!) + // The copyable field popup does not use a button as a trigger. It is instead triggered by mouse interaction with this `div`. + .find(`[data-test-subj="resolver:panel:copyable-field-hover-area"]`) + .filterWhere(Simulator.isDOM) + ); + }); + expect(fieldHoverArea).toBeTruthy(); + fieldHoverArea?.simulate('mouseenter'); + }); + describe('when the user clicks on the clipboard button', () => { + beforeEach(async () => { + const button = await simulator().resolve('resolver:panel:clipboard'); + expect(button).toBeTruthy(); + button!.simulate('click'); + simulator().confirmTextWrittenToClipboard(); + }); + it(`should write ${expectedValue} to the clipboard`, async () => { + await expect(simulator().map(() => simulator().clipboardText)).toYieldEqualTo( + expectedValue + ); + }); + }); + } + ); }); }); }); diff --git a/x-pack/plugins/security_solution/public/resolver/view/panels/copyable_panel_field.tsx b/x-pack/plugins/security_solution/public/resolver/view/panels/copyable_panel_field.tsx index f6a585ea566bb..6a1667a839548 100644 --- a/x-pack/plugins/security_solution/public/resolver/view/panels/copyable_panel_field.tsx +++ b/x-pack/plugins/security_solution/public/resolver/view/panels/copyable_panel_field.tsx @@ -9,10 +9,11 @@ import { EuiToolTip, EuiButtonIcon, EuiPopover } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import styled from 'styled-components'; -import React, { memo, useState, useCallback } from 'react'; +import React, { memo, useState, useCallback, useContext, useMemo } from 'react'; import { useKibana } from '../../../../../../../src/plugins/kibana_react/public'; import { useColors } from '../use_colors'; import { StyledPanel } from '../styles'; +import { SideEffectContext } from '../side_effect_context'; interface StyledCopyableField { readonly backgroundColor: string; @@ -48,39 +49,41 @@ export const CopyablePanelField = memo( const onMouseEnter = () => setIsOpen(true); const onMouseLeave = () => setIsOpen(false); - const ButtonContent = memo(() => ( - - {content} - - )); + const hoverArea = useMemo( + () => ( + + {content} + + ), + [content, copyableFieldBackground, linkColor] + ); - const onClick = useCallback( - async (event: React.MouseEvent) => { - try { - await navigator.clipboard.writeText(textToCopy); - } catch (error) { - if (toasts) { - toasts.addError(error, { - title: i18n.translate('xpack.securitySolution.resolver.panel.copyFailureTitle', { - defaultMessage: 'Copy Failure', - }), - }); - } + const { writeTextToClipboard } = useContext(SideEffectContext); + + const onClick = useCallback(async () => { + try { + await writeTextToClipboard(textToCopy); + } catch (error) { + if (toasts) { + toasts.addError(error, { + title: i18n.translate('xpack.securitySolution.resolver.panel.copyFailureTitle', { + defaultMessage: 'Copy Failure', + }), + }); } - }, - [textToCopy, toasts] - ); + } + }, [textToCopy, toasts, writeTextToClipboard]); return (
} + button={hoverArea} closePopover={onMouseLeave} hasArrow={false} isOpen={isOpen} diff --git a/x-pack/plugins/security_solution/public/resolver/view/panels/event_detail.tsx b/x-pack/plugins/security_solution/public/resolver/view/panels/event_detail.tsx index e5569b30abb9d..4936cf0cbb80e 100644 --- a/x-pack/plugins/security_solution/public/resolver/view/panels/event_detail.tsx +++ b/x-pack/plugins/security_solution/public/resolver/view/panels/event_detail.tsx @@ -15,12 +15,8 @@ import { EuiSpacer, EuiText, EuiDescriptionList, EuiTextColor, EuiTitle } from ' import styled from 'styled-components'; import { useSelector } from 'react-redux'; import { StyledPanel } from '../styles'; -import { - BoldCode, - StyledTime, - GeneratedText, - noTimestampRetrievedText, -} from './panel_content_utilities'; +import { BoldCode, StyledTime } from './styles'; +import { GeneratedText } from '../generated_text'; import { CopyablePanelField } from './copyable_panel_field'; import { Breadcrumbs } from './breadcrumbs'; import * as eventModel from '../../../../common/endpoint/models/event'; @@ -97,7 +93,12 @@ const EventDetailContents = memo(function ({ processEvent: SafeResolverEvent | null; }) { const timestamp = eventModel.timestampSafeVersion(event); - const formattedDate = useFormattedDate(timestamp) || noTimestampRetrievedText; + const formattedDate = + useFormattedDate(timestamp) || + i18n.translate('xpack.securitySolution.enpdoint.resolver.panelutils.noTimestampRetrieved', { + defaultMessage: 'No timestamp retrieved', + }); + const nodeName = processEvent ? eventModel.processNameSafeVersion(processEvent) : null; return ( @@ -155,15 +156,20 @@ function EventDetailFields({ event }: { event: SafeResolverEvent }) { const section = { // Group the fields by their top-level namespace namespace: {key}, - descriptions: deepObjectEntries(value).map(([path, fieldValue]) => ({ - title: {path.join('.')}, - description: ( - {String(fieldValue)}} - /> - ), - })), + descriptions: deepObjectEntries(value).map(([path, fieldValue]) => { + // The field name is the 'namespace' key as well as the rest of the path, joined with '.' + const fieldName = [key, ...path].join('.'); + + return { + title: {fieldName}, + description: ( + {String(fieldValue)}} + /> + ), + }; + }), }; returnValue.push(section); } @@ -187,7 +193,10 @@ function EventDetailFields({ event }: { event: SafeResolverEvent }) { diff --git a/x-pack/plugins/security_solution/public/resolver/view/panels/node_detail.tsx b/x-pack/plugins/security_solution/public/resolver/view/panels/node_detail.tsx index c7d4f8632659b..5675e29fc2bc1 100644 --- a/x-pack/plugins/security_solution/public/resolver/view/panels/node_detail.tsx +++ b/x-pack/plugins/security_solution/public/resolver/view/panels/node_detail.tsx @@ -16,7 +16,7 @@ import { EuiDescriptionListProps } from '@elastic/eui/src/components/description import { StyledDescriptionList, StyledTitle } from './styles'; import * as selectors from '../../store/selectors'; import * as eventModel from '../../../../common/endpoint/models/event'; -import { GeneratedText } from './panel_content_utilities'; +import { GeneratedText } from '../generated_text'; import { CopyablePanelField } from './copyable_panel_field'; import { Breadcrumbs } from './breadcrumbs'; import { processPath, processPID } from '../../models/process_event'; diff --git a/x-pack/plugins/security_solution/public/resolver/view/panels/node_events_of_type.tsx b/x-pack/plugins/security_solution/public/resolver/view/panels/node_events_of_type.tsx index 17e91902d0c96..c9648c6f562e5 100644 --- a/x-pack/plugins/security_solution/public/resolver/view/panels/node_events_of_type.tsx +++ b/x-pack/plugins/security_solution/public/resolver/view/panels/node_events_of_type.tsx @@ -4,6 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ +/* eslint-disable react/display-name */ + import React, { memo, useCallback, Fragment } from 'react'; import { i18n } from '@kbn/i18n'; import { @@ -18,7 +20,7 @@ import { import { useSelector } from 'react-redux'; import { FormattedMessage } from '@kbn/i18n/react'; import { StyledPanel } from '../styles'; -import { BoldCode, noTimestampRetrievedText, StyledTime } from './panel_content_utilities'; +import { BoldCode, StyledTime } from './styles'; import { Breadcrumbs } from './breadcrumbs'; import * as eventModel from '../../../../common/endpoint/models/event'; import { SafeResolverEvent } from '../../../../common/endpoint/types'; @@ -99,8 +101,6 @@ export const NodeEventsInCategory = memo(function ({ ); }); -NodeEventsInCategory.displayName = 'NodeEventsInCategory'; - /** * Rendered for each event in the list. */ @@ -114,7 +114,11 @@ const NodeEventsListItem = memo(function ({ eventCategory: string; }) { const timestamp = eventModel.eventTimestamp(event); - const date = useFormattedDate(timestamp) || noTimestampRetrievedText; + const date = + useFormattedDate(timestamp) || + i18n.translate('xpack.securitySolution.enpdoint.resolver.panelutils.noTimestampRetrieved', { + defaultMessage: 'No timestamp retrieved', + }); const linkProps = useLinkProps({ panelView: 'eventDetail', panelParameters: { diff --git a/x-pack/plugins/security_solution/public/resolver/view/panels/node_list.tsx b/x-pack/plugins/security_solution/public/resolver/view/panels/node_list.tsx index 9ef72c414bb63..e53cd2cc0860d 100644 --- a/x-pack/plugins/security_solution/public/resolver/view/panels/node_list.tsx +++ b/x-pack/plugins/security_solution/public/resolver/view/panels/node_list.tsx @@ -38,17 +38,14 @@ import { LimitWarning } from '../limit_warnings'; import { ResolverState } from '../../types'; import { useLinkProps } from '../use_link_props'; import { useColors } from '../use_colors'; -import { SafeResolverEvent } from '../../../../common/endpoint/types'; import { ResolverAction } from '../../store/actions'; import { useFormattedDate } from './use_formatted_date'; -import { getEmptyTagValue } from '../../../common/components/empty_value'; import { CopyablePanelField } from './copyable_panel_field'; interface ProcessTableView { name?: string; timestamp?: Date; nodeID: string; - event: SafeResolverEvent; } /** @@ -68,7 +65,7 @@ export const NodeList = memo(() => { sortable: true, truncateText: true, render(name: string | undefined, item: ProcessTableView) { - return ; + return ; }, }, { @@ -101,7 +98,6 @@ export const NodeList = memo(() => { name, timestamp: eventModel.timestampAsDateSafeVersion(processEvent), nodeID, - event: processEvent, }); } } @@ -111,7 +107,7 @@ export const NodeList = memo(() => { const numberOfProcesses = processTableView.length; - const crumbs = useMemo(() => { + const breadcrumbs = useMemo(() => { return [ { text: i18n.translate('xpack.securitySolution.resolver.panel.nodeList.title', { @@ -127,7 +123,7 @@ export const NodeList = memo(() => { const rowProps = useMemo(() => ({ 'data-test-subj': 'resolver:node-list:item' }), []); return ( - + {showWarning && } @@ -141,15 +137,7 @@ export const NodeList = memo(() => { ); }); -function NodeDetailLink({ - name, - nodeID, - event, -}: { - name?: string; - nodeID: string; - event: SafeResolverEvent; -}) { +function NodeDetailLink({ name, nodeID }: { name?: string; nodeID: string }) { const isOrigin = useSelector((state: ResolverState) => { return selectors.originID(state) === nodeID; }); @@ -175,7 +163,7 @@ function NodeDetailLink({ ); return ( - {name === '' ? ( + {name === undefined ? ( {i18n.translate( 'xpack.securitySolution.endpoint.resolver.panel.table.row.valueMissingDescription', @@ -218,6 +206,6 @@ const NodeDetailTimestamp = memo(({ eventDate }: { eventDate: Date | undefined } return formattedDate ? ( ) : ( - getEmptyTagValue() + {'—'} ); }); diff --git a/x-pack/plugins/security_solution/public/resolver/view/panels/styles.tsx b/x-pack/plugins/security_solution/public/resolver/view/panels/styles.tsx index 03826dd38397b..6f9d4bb600fde 100644 --- a/x-pack/plugins/security_solution/public/resolver/view/panels/styles.tsx +++ b/x-pack/plugins/security_solution/public/resolver/view/panels/styles.tsx @@ -4,6 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ +import { EuiCode } from '@elastic/eui'; + /* eslint-disable no-duplicate-imports */ import { EuiBreadcrumbs } from '@elastic/eui'; @@ -89,3 +91,21 @@ export const StyledLabelContainer = styled.div` white-space: nowrap; } `; + +/** + * A bold version of EuiCode to display certain titles with + */ +export const BoldCode = styled(EuiCode)` + &.euiCodeBlock code.euiCodeBlock__code { + font-weight: 900; + } +`; + +/** + * A component to keep time representations in blocks so they don't wrap + * and look bad. + */ +export const StyledTime = styled('time')` + display: inline-block; + text-align: start; +`; diff --git a/x-pack/plugins/security_solution/public/resolver/view/panels/use_formatted_date.test.tsx b/x-pack/plugins/security_solution/public/resolver/view/panels/use_formatted_date.test.tsx index c08c3b370558b..647f7c75d0298 100644 --- a/x-pack/plugins/security_solution/public/resolver/view/panels/use_formatted_date.test.tsx +++ b/x-pack/plugins/security_solution/public/resolver/view/panels/use_formatted_date.test.tsx @@ -4,103 +4,52 @@ * you may not use this file except in compliance with the Elastic License. */ +import { mount } from 'enzyme'; + import React from 'react'; -import { render, RenderResult } from '@testing-library/react'; import { useFormattedDate } from './use_formatted_date'; import { coreMock } from '../../../../../../../src/core/public/mocks'; import { KibanaContextProvider } from '../../../../../../../src/plugins/kibana_react/public'; -import { getUiSettings } from '../../mocks/get_ui_settings'; +import { uiSetting } from '../../mocks/ui_setting'; -describe('useFormattedDate', () => { - let element: HTMLElement; - const testID = 'formattedDate'; - let reactRenderResult: ( - date: ConstructorParameters[0] | Date | undefined - ) => RenderResult; +describe(`useFormattedDate, when the "dateFormat" UI setting is "${uiSetting( + 'dateFormat' +)}" and the "dateFormat:tz" setting is "${uiSetting('dateFormat:tz')}"`, () => { + let formattedDate: (date: ConstructorParameters[0] | Date | undefined) => string; beforeEach(async () => { const mockCoreStart = coreMock.createStart(); - mockCoreStart.uiSettings.get.mockImplementation(getUiSettings); + mockCoreStart.uiSettings.get.mockImplementation(uiSetting); function Test({ date }: { date: ConstructorParameters[0] | Date | undefined }) { - const formattedDate = useFormattedDate(date); - return
{formattedDate}
; + return <>{useFormattedDate(date)}; } - reactRenderResult = ( - date: ConstructorParameters[0] | Date | undefined - ): RenderResult => - render( + formattedDate = (date: ConstructorParameters[0] | Date | undefined): string => + mount( - ); - }); - afterEach(() => { - jest.clearAllMocks(); - }); - - describe('when the provided date is undefined', () => { - it('should return undefined', async () => { - const { findByTestId } = reactRenderResult(undefined); - element = await findByTestId(testID); - - expect(element).toBeEmptyDOMElement(); - }); - }); - - describe('when the provided date is empty', () => { - it('should return undefined', async () => { - const { findByTestId } = reactRenderResult(''); - element = await findByTestId(testID); - - expect(element).toBeEmptyDOMElement(); - }); - }); - - describe('when the provided date is an invalid date', () => { - it('should return the string invalid date', async () => { - const { findByTestId } = reactRenderResult('randomString'); - element = await findByTestId(testID); - - expect(element).toHaveTextContent('Invalid Date'); - }); - }); - - describe('when the provided date is a stringified unix timestamp', () => { - it('should return the string invalid date', async () => { - const { findByTestId } = reactRenderResult('1600863932316'); - element = await findByTestId(testID); - - expect(element).toHaveTextContent('Invalid Date'); - }); - }); - - describe('when the provided date is a valid numerical timestamp', () => { - it('should return the string invalid date', async () => { - const { findByTestId } = reactRenderResult(1600863932316); - element = await findByTestId(testID); - - expect(element).toHaveTextContent('Sep 23, 2020 @ 08:25:32.316'); - }); - }); - - describe('when the provided date is a date string', () => { - it('should return the string invalid date', async () => { - const { findByTestId } = reactRenderResult('2020-09-23T12:25:32Z'); - element = await findByTestId(testID); - - expect(element).toHaveTextContent('Sep 23, 2020 @ 08:25:32.000'); - }); - }); - - describe('when the provided date is a valid date', () => { - it('should return the string invalid date', async () => { - const validDate = new Date(1600863932316); - const { findByTestId } = reactRenderResult(validDate); - element = await findByTestId(testID); - - expect(element).toHaveTextContent('Sep 23, 2020 @ 08:25:32.316'); - }); + ).text(); + }); + + it.each([ + ['randomString', 'an invalid string', 'Invalid Date'], + [ + '1600863932316', + "a string that does't match the configured time format settings", + 'Invalid Date', + ], + [1600863932316, 'a valid unix timestamp', 'Sep 23, 2020 @ 08:25:32.316'], + [undefined, 'undefined', ''], + ['', 'an empty string', ''], + [ + '2020-09-23T12:25:32Z', + 'a string that conforms to the specified format', + 'Sep 23, 2020 @ 08:25:32.000', + ], + [new Date(1600863932316), 'a defined Date object', 'Sep 23, 2020 @ 08:25:32.316'], + ])('when the provided date is %p (%s) it should return %p', (value, _explanation, expected) => { + expect(formattedDate(value)).toBe(expected); }); }); diff --git a/x-pack/plugins/security_solution/public/resolver/view/side_effect_context.ts b/x-pack/plugins/security_solution/public/resolver/view/side_effect_context.ts index ab7f41d815026..71b054948160e 100644 --- a/x-pack/plugins/security_solution/public/resolver/view/side_effect_context.ts +++ b/x-pack/plugins/security_solution/public/resolver/view/side_effect_context.ts @@ -19,6 +19,12 @@ const sideEffectors: SideEffectors = { return window.cancelAnimationFrame(...args); }, ResizeObserver, + writeTextToClipboard(text: string): Promise { + return navigator.clipboard.writeText(text); + }, + getBoundingClientRect(element: Element): DOMRect { + return element.getBoundingClientRect(); + }, }; /** diff --git a/x-pack/plugins/security_solution/public/resolver/view/side_effect_simulator_factory.ts b/x-pack/plugins/security_solution/public/resolver/view/side_effect_simulator_factory.ts index 8517459b8aba3..84da2824962d6 100644 --- a/x-pack/plugins/security_solution/public/resolver/view/side_effect_simulator_factory.ts +++ b/x-pack/plugins/security_solution/public/resolver/view/side_effect_simulator_factory.ts @@ -37,7 +37,7 @@ export const sideEffectSimulatorFactory: () => SideEffectSimulator = () => { /** * Get the simulate `DOMRect` for `element`. */ - const contentRectForElement: (target: Element) => DOMRect = (target) => { + const getBoundingClientRect: (target: Element) => DOMRect = (target) => { if (contentRects.has(target)) { return contentRects.get(target)!; } @@ -58,26 +58,33 @@ export const sideEffectSimulatorFactory: () => SideEffectSimulator = () => { }; /** - * Change `Element.prototype.getBoundingClientRect` to return our faked values. + * Last value written to the clipboard, of '' if no text has been written. Returned by the `controls`. */ - jest - .spyOn(Element.prototype, 'getBoundingClientRect') - .mockImplementation(function (this: Element) { - return contentRectForElement(this); - }); + let clipboardText: string = ''; // the `readText` method of the Clipboard API returns an empty string if the clipboard is empty. + + function confirmTextWrittenToClipboard() { + const next = clipboardWriteTextQueue.shift(); + if (next) { + const [text, resolve] = next; + clipboardText = text; + resolve(); + } + } /** - * Mock the global writeText method as it is not available in jsDOM and alows us to track what was copied + * Queue of `text` waiting to be written to the clipboard. Calling `resolve` will resolve the promise returned by the mock `writeTextToClipboard` method. */ - const MockClipboard: Clipboard = { - writeText: jest.fn(), - readText: jest.fn(), - addEventListener: jest.fn(), - dispatchEvent: jest.fn(), - removeEventListener: jest.fn(), - }; - // @ts-ignore navigator doesn't natively exist on global - global.navigator.clipboard = MockClipboard; + const clipboardWriteTextQueue: Array<[text: string, resolve: () => void]> = []; + + /** + * Mock `writeText` method of the `Clipboard` API. + */ + function writeTextToClipboard(text: string): Promise { + return new Promise((resolve) => { + clipboardWriteTextQueue.push([text, resolve]); + }); + } + /** * A mock implementation of `ResizeObserver` that works with our fake `getBoundingClientRect` and `simulateElementResize` */ @@ -171,12 +178,20 @@ export const sideEffectSimulatorFactory: () => SideEffectSimulator = () => { }, simulateElementResize, + + get clipboardText() { + return clipboardText; + }, + + confirmTextWrittenToClipboard, }, mock: { requestAnimationFrame, cancelAnimationFrame, timestamp, ResizeObserver: MockResizeObserver, + writeTextToClipboard, + getBoundingClientRect, }, }; return retval; diff --git a/x-pack/plugins/security_solution/public/resolver/view/use_camera.test.tsx b/x-pack/plugins/security_solution/public/resolver/view/use_camera.test.tsx index bf72a52559cbd..35cf2c36d6627 100644 --- a/x-pack/plugins/security_solution/public/resolver/view/use_camera.test.tsx +++ b/x-pack/plugins/security_solution/public/resolver/view/use_camera.test.tsx @@ -4,13 +4,15 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { FunctionComponent } from 'react'; -import { render, waitFor, RenderResult, fireEvent } from '@testing-library/react'; -import { renderHook, act } from '@testing-library/react-hooks'; -import { useCamera, useAutoUpdatingClientRect } from './use_camera'; +// Extend jest with a custom matcher +import '../test_utilities/extend_jest'; + +import { mount, ReactWrapper } from 'enzyme'; +import React from 'react'; +import { useCamera } from './use_camera'; import { Provider } from 'react-redux'; import * as selectors from '../store/selectors'; -import { Matrix3, ResolverStore, SideEffectSimulator } from '../types'; +import { Matrix3, ResolverStore, SideEffectors, SideEffectSimulator } from '../types'; import { SafeResolverEvent } from '../../../common/endpoint/types'; import { SideEffectContext } from './side_effect_context'; import { applyMatrix3 } from '../models/vector2'; @@ -22,45 +24,120 @@ import { createStore } from 'redux'; import { resolverReducer } from '../store/reducer'; import { mockTreeFetcherParameters } from '../mocks/tree_fetcher_parameters'; import { entityIDSafeVersion } from '../../../common/endpoint/models/event'; +import { act } from 'react-dom/test-utils'; describe('useCamera on an unpainted element', () => { - let element: HTMLElement; + /** Enzyme full DOM wrapper for the element the camera is attached to. */ + let element: ReactWrapper; + /** + * Enzyme full DOM wrapper for the alternate element that the camera can be attached to. Used for testing that the `ResizeObserver` attaches itself to the latest `ref`. + */ + let alternateElement: ReactWrapper; + /** + * projection matrix returned by camera on last render. + */ let projectionMatrix: Matrix3; + /** + * A `data-test-subj` ID used to identify the element the camera normally attaches to. + */ const testID = 'camera'; - let reactRenderResult: RenderResult; + /** + * A `data-test-subj` ID used to identify the element the camera alternatively attaches to. + */ + const alternateTestID = 'alternate'; + /** + * Returned by the legacy framework's render/mount function. + */ + let wrapper: ReactWrapper; let store: ResolverStore; let simulator: SideEffectSimulator; - beforeEach(async () => { - store = createStore(resolverReducer); - - const Test = function () { - const camera = useCamera(); - const { ref, onMouseDown } = camera; - projectionMatrix = camera.projectionMatrix; - return
; - }; + /** Used to find an element by the data-test-subj attribute. + */ + let domElementByTestSubj: (testSubj: string) => ReactWrapper; - simulator = sideEffectSimulatorFactory(); + /** + * Yield the result of `mapper` over and over, once per event-loop cycle. + * After 10 times, quit. + * Use this to continually check a value. See `toYieldEqualTo`. + */ + async function* map(mapper: () => R): AsyncIterable { + let timeoutCount = 0; + while (timeoutCount < 10) { + timeoutCount++; + yield mapper(); + await new Promise((resolve) => { + setTimeout(() => { + wrapper.update(); + resolve(); + }, 0); + }); + } + } - reactRenderResult = render( - - - + function TestWrapper({ + useSecondElement: useAlternateElement = false, + resolverStore, + sideEffectors, + }: { + /** + * Pass `true`, to attach the camera to an alternate element. Used to test that the `ResizeObserver` attaches itself to the latest `ref`. + */ + useSecondElement?: boolean; + resolverStore: ResolverStore; + sideEffectors: SideEffectors; + }) { + return ( + + + ); + } - const { findByTestId } = reactRenderResult; - element = await findByTestId(testID); - }); - afterEach(() => { - jest.clearAllMocks(); - }); - it('should be usable in React', async () => { - expect(element).toBeInTheDocument(); + function Test({ + useAlternateElement = false, + }: { + /** + * Pass `true`, to attach the camera to an alternate element. Used to test that the `ResizeObserver` attaches itself to the latest `ref`. + */ + useAlternateElement?: boolean; + }) { + const camera = useCamera(); + const { ref, onMouseDown } = camera; + projectionMatrix = camera.projectionMatrix; + return useAlternateElement ? ( + <> +
+
+ + ) : ( + <> +
+
+ + ); + } + + beforeEach(async () => { + store = createStore(resolverReducer); + + simulator = sideEffectSimulatorFactory(); + + wrapper = mount(); + + domElementByTestSubj = (testSubj: string) => + wrapper + .find(`[data-test-subj="${testSubj}"]`) + // Omit React components that may be returned. + .filterWhere((item) => typeof item.type() === 'string'); + + element = domElementByTestSubj(testID); + + alternateElement = domElementByTestSubj(alternateTestID); }); - test('returns a projectionMatrix that changes everything to 0', () => { + it('returns a projectionMatrix that changes everything to 0', () => { expect(applyMatrix3([0, 0], projectionMatrix)).toEqual([0, 0]); }); describe('which has been resized to 800x600', () => { @@ -71,8 +148,8 @@ describe('useCamera on an unpainted element', () => { const centerX = width / 2 + leftMargin; const centerY = height / 2 + topMargin; beforeEach(async () => { - await waitFor(() => { - simulator.controls.simulateElementResize(element, { + act(() => { + simulator.controls.simulateElementResize(element.getDOMNode(), { width, height, left: leftMargin, @@ -87,73 +164,82 @@ describe('useCamera on an unpainted element', () => { }); }); }); - test('should observe all resize reference changes', async () => { - const wrapper: FunctionComponent = ({ children }) => ( - - {children} - - ); - - const { result } = renderHook(() => useAutoUpdatingClientRect(), { wrapper }); - const resizeObserverSpy = jest.spyOn(simulator.mock.ResizeObserver.prototype, 'observe'); - - let [rect, ref] = result.current; - act(() => ref(element)); - expect(resizeObserverSpy).toHaveBeenCalledWith(element); - - const div = document.createElement('div'); - act(() => ref(div)); - expect(resizeObserverSpy).toHaveBeenCalledWith(div); - - [rect, ref] = result.current; - expect(rect?.width).toBe(0); - }); - test('provides a projection matrix that inverts the y axis and translates 400,300 (center of the element)', () => { - expect(applyMatrix3([0, 0], projectionMatrix)).toEqual([400, 300]); + it('provides a projection matrix that inverts the y axis and translates 400,300 (center of the element)', () => { + expect(map(() => applyMatrix3([0, 0], projectionMatrix))).toYieldEqualTo([400, 300]); }); describe('when the user presses the mousedown button in the middle of the element', () => { beforeEach(() => { - fireEvent.mouseDown(element, { + element.simulate('mousedown', { clientX: centerX, clientY: centerY, }); }); describe('when the user moves the mouse 50 pixels to the right', () => { beforeEach(() => { - fireEvent.mouseMove(element, { + element.simulate('mousemove', { clientX: centerX + 50, clientY: centerY, }); }); it('should project [0, 0] in world corrdinates 50 pixels to the right of the center of the element', () => { - expect(applyMatrix3([0, 0], projectionMatrix)).toEqual([450, 300]); + expect(map(() => applyMatrix3([0, 0], projectionMatrix))).toYieldEqualTo([450, 300]); }); }); }); describe('when the user uses the mousewheel w/ ctrl held down', () => { beforeEach(() => { - fireEvent.wheel(element, { + element.simulate('wheel', { ctrlKey: true, deltaY: -10, deltaMode: 0, }); }); it('should zoom in', () => { - expect(projectionMatrix).toMatchInlineSnapshot(` - Array [ - 1.0292841801261479, - 0, - 400, - 0, - -1.0292841801261479, - 300, - 0, - 0, - 0, - ] - `); + expect(map(() => projectionMatrix)).toYieldEqualTo([ + 1.0292841801261479, + 0, + 400, + 0, + -1.0292841801261479, + 300, + 0, + 0, + 0, + ]); + }); + }); + + describe('when the element the camera is attached to is switched', () => { + beforeEach(() => { + wrapper.setProps({ + useAlternateElement: true, + }); + }); + describe('and when that element changes size to 1200x800', () => { + beforeEach(() => { + act(() => { + const alternateElementWidth = 1200; + const alternateElementHeight = 800; + simulator.controls.simulateElementResize(alternateElement.getDOMNode(), { + width: alternateElementWidth, + height: alternateElementHeight, + left: leftMargin, + top: topMargin, + right: leftMargin + alternateElementWidth, + bottom: topMargin + alternateElementHeight, + x: leftMargin, + y: topMargin, + toJSON() { + return this; + }, + }); + }); + }); + it('provides a projection matrix that inverts the y axis and translates 600,400', () => { + expect(map(() => applyMatrix3([0, 0], projectionMatrix))).toYieldEqualTo([600, 400]); + }); }); }); @@ -185,9 +271,7 @@ describe('useCamera on an unpainted element', () => { type: 'serverReturnedResolverData', payload: { result: tree, parameters: mockTreeFetcherParameters() }, }; - await waitFor(() => { - store.dispatch(serverResponseAction); - }); + store.dispatch(serverResponseAction); } else { throw new Error('failed to create tree'); } @@ -210,9 +294,7 @@ describe('useCamera on an unpainted element', () => { nodeID, }, }; - await waitFor(() => { - store.dispatch(cameraAction); - }); + store.dispatch(cameraAction); }); it('should request animation frames in a loop', () => { diff --git a/x-pack/plugins/security_solution/public/resolver/view/use_camera.ts b/x-pack/plugins/security_solution/public/resolver/view/use_camera.ts index 661e038d04e32..c58b9f77d097d 100644 --- a/x-pack/plugins/security_solution/public/resolver/view/use_camera.ts +++ b/x-pack/plugins/security_solution/public/resolver/view/use_camera.ts @@ -280,7 +280,10 @@ export function useCamera(): { * tracked. So if the element's position moves for some reason, be sure to * handle that. */ -export function useAutoUpdatingClientRect(): [DOMRect | null, (node: Element | null) => void] { +function useAutoUpdatingClientRect(): [DOMRect | null, (node: Element | null) => void] { + // Access `getBoundingClientRect` via the `SideEffectContext` (for testing.) + const { getBoundingClientRect } = useContext(SideEffectContext); + // This hooks returns `rect`. const [rect, setRect] = useState(null); @@ -302,9 +305,9 @@ export function useAutoUpdatingClientRect(): [DOMRect | null, (node: Element | n useEffect(() => { if (currentNode !== null) { // When the DOM node is received, immedaiately calculate its DOM Rect and return that - setRect(currentNode.getBoundingClientRect()); + setRect(getBoundingClientRect(currentNode)); } - }, [currentNode]); + }, [currentNode, getBoundingClientRect]); /** * When scroll events occur, recalculate the DOMRect. DOMRect represents the position of an element relative to the viewport, so that may change during scroll (depending on the layout.) @@ -322,7 +325,7 @@ export function useAutoUpdatingClientRect(): [DOMRect | null, (node: Element | n const currentY = window.scrollY; if (currentNode !== null && (previousX !== currentX || previousY !== currentY)) { - setRect(currentNode.getBoundingClientRect()); + setRect(getBoundingClientRect(currentNode)); } previousX = currentX; @@ -334,13 +337,13 @@ export function useAutoUpdatingClientRect(): [DOMRect | null, (node: Element | n return () => { window.removeEventListener('scroll', handleScroll); }; - }, [currentNode, requestAnimationFrame]); + }, [currentNode, requestAnimationFrame, getBoundingClientRect]); useEffect(() => { if (currentNode !== null) { const resizeObserver = new ResizeObserver((entries) => { if (currentNode !== null && currentNode === entries[0].target) { - setRect(currentNode.getBoundingClientRect()); + setRect(getBoundingClientRect(currentNode)); } }); resizeObserver.observe(currentNode); @@ -348,6 +351,6 @@ export function useAutoUpdatingClientRect(): [DOMRect | null, (node: Element | n resizeObserver.disconnect(); }; } - }, [ResizeObserver, currentNode]); + }, [ResizeObserver, currentNode, getBoundingClientRect]); return [rect, ref]; } diff --git a/x-pack/plugins/security_solution/public/timelines/components/edit_data_provider/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/edit_data_provider/index.tsx index 2bc202c65f6ab..8127f2be13d4e 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/edit_data_provider/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/edit_data_provider/index.tsx @@ -189,7 +189,7 @@ export const StatefulEditDataProvider = React.memo( - + ( - + {type !== DataProviderType.template && @@ -245,7 +245,7 @@ export const StatefulEditDataProvider = React.memo( ) : null} - + @@ -265,7 +265,7 @@ export const StatefulEditDataProvider = React.memo( }) || isValueFieldInvalid } onClick={handleSave} - size="s" + size="m" > {i18n.SAVE} diff --git a/x-pack/plugins/security_solution/public/timelines/components/open_timeline/helpers.ts b/x-pack/plugins/security_solution/public/timelines/components/open_timeline/helpers.ts index c89740f667b29..4c3be81a4992a 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/open_timeline/helpers.ts +++ b/x-pack/plugins/security_solution/public/timelines/components/open_timeline/helpers.ts @@ -378,7 +378,7 @@ export const dispatchUpdateTimeline = (dispatch: Dispatch): DispatchUpdateTimeli ruleNote, }: UpdateTimeline): (() => void) => () => { dispatch( - sourcererActions.setSelectedIndexPatterns({ + sourcererActions.initTimelineIndexPatterns({ id: SourcererScopeName.timeline, selectedPatterns: timeline.indexNames, eventType: timeline.eventType, diff --git a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts index 3fc41550a1fc4..2cc8245e521bf 100644 --- a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts +++ b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts @@ -9,11 +9,13 @@ import { SavedObjectsServiceStart, SavedObjectsClientContract, } from 'src/core/server'; +import { SecurityPluginSetup } from '../../../security/server'; import { AgentService, IngestManagerStartContract, PackageService, } from '../../../ingest_manager/server'; +import { PluginStartContract as AlertsPluginStartContract } from '../../../alerts/server'; import { getPackagePolicyCreateCallback } from './ingest_integration'; import { ManifestManager } from './services/artifacts'; import { MetadataQueryStrategy } from './types'; @@ -24,6 +26,8 @@ import { } from './routes/metadata/support/query_strategies'; import { ElasticsearchAssetType } from '../../../ingest_manager/common/types/models'; import { metadataTransformPrefix } from '../../common/endpoint/constants'; +import { AppClientFactory } from '../client'; +import { ConfigType } from '../config'; export interface MetadataService { queryStrategy( @@ -70,6 +74,10 @@ export type EndpointAppContextServiceStartContract = Partial< > & { logger: Logger; manifestManager?: ManifestManager; + appClientFactory: AppClientFactory; + security: SecurityPluginSetup; + alerts: AlertsPluginStartContract; + config: ConfigType; registerIngestCallback?: IngestManagerStartContract['registerExternalCallback']; savedObjectsStart: SavedObjectsServiceStart; }; @@ -93,7 +101,14 @@ export class EndpointAppContextService { if (this.manifestManager && dependencies.registerIngestCallback) { dependencies.registerIngestCallback( 'packagePolicyCreate', - getPackagePolicyCreateCallback(dependencies.logger, this.manifestManager) + getPackagePolicyCreateCallback( + dependencies.logger, + this.manifestManager, + dependencies.appClientFactory, + dependencies.config.maxTimelineImportExportSize, + dependencies.security, + dependencies.alerts + ) ); } } diff --git a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.test.ts b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.test.ts index c28ffcf5b7a3f..1db3e9984284d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.test.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { loggingSystemMock } from 'src/core/server/mocks'; +import { httpServerMock, loggingSystemMock } from 'src/core/server/mocks'; import { createNewPackagePolicyMock } from '../../../ingest_manager/common/mocks'; import { factory as policyConfigFactory } from '../../common/endpoint/models/policy_config'; import { @@ -12,8 +12,23 @@ import { ManifestManagerMockType, } from './services/artifacts/manifest_manager/manifest_manager.mock'; import { getPackagePolicyCreateCallback } from './ingest_integration'; +import { KibanaRequest, RequestHandlerContext } from 'kibana/server'; +import { createMockConfig, requestContextMock } from '../lib/detection_engine/routes/__mocks__'; +import { EndpointAppContextServiceStartContract } from './endpoint_app_context_services'; +import { createMockEndpointAppContextServiceStartContract } from './mocks'; describe('ingest_integration tests ', () => { + let endpointAppContextMock: EndpointAppContextServiceStartContract; + let req: KibanaRequest; + let ctx: RequestHandlerContext; + const maxTimelineImportExportSize = createMockConfig().maxTimelineImportExportSize; + + beforeEach(() => { + endpointAppContextMock = createMockEndpointAppContextServiceStartContract(); + ctx = requestContextMock.createTools().context; + req = httpServerMock.createKibanaRequest(); + }); + describe('ingest_integration sanity checks', () => { test('policy is updated with initial manifest', async () => { const logger = loggingSystemMock.create().get('ingest_integration.test'); @@ -21,9 +36,16 @@ describe('ingest_integration tests ', () => { mockType: ManifestManagerMockType.InitialSystemState, }); - const callback = getPackagePolicyCreateCallback(logger, manifestManager); + const callback = getPackagePolicyCreateCallback( + logger, + manifestManager, + endpointAppContextMock.appClientFactory, + maxTimelineImportExportSize, + endpointAppContextMock.security, + endpointAppContextMock.alerts + ); const policyConfig = createNewPackagePolicyMock(); // policy config without manifest - const newPolicyConfig = await callback(policyConfig); // policy config WITH manifest + const newPolicyConfig = await callback(policyConfig, ctx, req); // policy config WITH manifest expect(newPolicyConfig.inputs[0]!.type).toEqual('endpoint'); expect(newPolicyConfig.inputs[0]!.config!.policy.value).toEqual(policyConfigFactory()); @@ -91,9 +113,16 @@ describe('ingest_integration tests ', () => { manifestManager.pushArtifacts = jest.fn().mockResolvedValue([new Error('error updating')]); const lastComputed = await manifestManager.getLastComputedManifest(); - const callback = getPackagePolicyCreateCallback(logger, manifestManager); + const callback = getPackagePolicyCreateCallback( + logger, + manifestManager, + endpointAppContextMock.appClientFactory, + maxTimelineImportExportSize, + endpointAppContextMock.security, + endpointAppContextMock.alerts + ); const policyConfig = createNewPackagePolicyMock(); - const newPolicyConfig = await callback(policyConfig); + const newPolicyConfig = await callback(policyConfig, ctx, req); expect(newPolicyConfig.inputs[0]!.type).toEqual('endpoint'); expect(newPolicyConfig.inputs[0]!.config!.policy.value).toEqual(policyConfigFactory()); @@ -111,9 +140,16 @@ describe('ingest_integration tests ', () => { expect(lastComputed).toEqual(null); manifestManager.buildNewManifest = jest.fn().mockRejectedValue(new Error('abcd')); - const callback = getPackagePolicyCreateCallback(logger, manifestManager); + const callback = getPackagePolicyCreateCallback( + logger, + manifestManager, + endpointAppContextMock.appClientFactory, + maxTimelineImportExportSize, + endpointAppContextMock.security, + endpointAppContextMock.alerts + ); const policyConfig = createNewPackagePolicyMock(); - const newPolicyConfig = await callback(policyConfig); + const newPolicyConfig = await callback(policyConfig, ctx, req); expect(newPolicyConfig.inputs[0]!.type).toEqual('endpoint'); expect(newPolicyConfig.inputs[0]!.config!.policy.value).toEqual(policyConfigFactory()); @@ -125,9 +161,16 @@ describe('ingest_integration tests ', () => { const lastComputed = await manifestManager.getLastComputedManifest(); manifestManager.buildNewManifest = jest.fn().mockResolvedValue(lastComputed); // no diffs - const callback = getPackagePolicyCreateCallback(logger, manifestManager); + const callback = getPackagePolicyCreateCallback( + logger, + manifestManager, + endpointAppContextMock.appClientFactory, + maxTimelineImportExportSize, + endpointAppContextMock.security, + endpointAppContextMock.alerts + ); const policyConfig = createNewPackagePolicyMock(); - const newPolicyConfig = await callback(policyConfig); + const newPolicyConfig = await callback(policyConfig, ctx, req); expect(newPolicyConfig.inputs[0]!.type).toEqual('endpoint'); expect(newPolicyConfig.inputs[0]!.config!.policy.value).toEqual(policyConfigFactory()); diff --git a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts index 489b146daeb43..279603cd621c8 100644 --- a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts +++ b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts @@ -4,7 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ -import { Logger } from '../../../../../src/core/server'; +import { PluginStartContract as AlertsStartContract } from '../../../alerts/server'; +import { SecurityPluginSetup } from '../../../security/server'; +import { ExternalCallback } from '../../../ingest_manager/server'; +import { KibanaRequest, Logger, RequestHandlerContext } from '../../../../../src/core/server'; import { NewPackagePolicy } from '../../../ingest_manager/common/types/models'; import { factory as policyConfigFactory } from '../../common/endpoint/models/policy_config'; import { NewPolicyData } from '../../common/endpoint/types'; @@ -13,6 +16,10 @@ import { Manifest } from './lib/artifacts'; import { reportErrors } from './lib/artifacts/common'; import { InternalArtifactCompleteSchema } from './schemas/artifacts'; import { manifestDispatchSchema } from '../../common/endpoint/schema/manifest'; +import { AppClientFactory } from '../client'; +import { createDetectionIndex } from '../lib/detection_engine/routes/index/create_index_route'; +import { createPrepackagedRules } from '../lib/detection_engine/routes/rules/add_prepackaged_rules_route'; +import { buildFrameworkRequest } from '../lib/timeline/routes/utils/common'; const getManifest = async (logger: Logger, manifestManager: ManifestManager): Promise => { let manifest: Manifest | null = null; @@ -71,19 +78,52 @@ const getManifest = async (logger: Logger, manifestManager: ManifestManager): Pr */ export const getPackagePolicyCreateCallback = ( logger: Logger, - manifestManager: ManifestManager -): ((newPackagePolicy: NewPackagePolicy) => Promise) => { + manifestManager: ManifestManager, + appClientFactory: AppClientFactory, + maxTimelineImportExportSize: number, + securitySetup: SecurityPluginSetup, + alerts: AlertsStartContract +): ExternalCallback[1] => { const handlePackagePolicyCreate = async ( - newPackagePolicy: NewPackagePolicy + newPackagePolicy: NewPackagePolicy, + context: RequestHandlerContext, + request: KibanaRequest ): Promise => { // We only care about Endpoint package policies if (newPackagePolicy.package?.name !== 'endpoint') { return newPackagePolicy; } - // We cast the type here so that any changes to the Endpoint specific data - // follow the types/schema expected - let updatedPackagePolicy = newPackagePolicy as NewPolicyData; + // prep for detection rules creation + const appClient = appClientFactory.create(request); + const frameworkRequest = await buildFrameworkRequest(context, securitySetup, request); + + // Create detection index & rules (if necessary). move past any failure, this is just a convenience + try { + await createDetectionIndex(context, appClient); + } catch (err) { + if (err.statusCode !== 409) { + // 409 -> detection index already exists, which is fine + logger.warn( + `Possible problem creating detection signals index (${err.statusCode}): ${err.message}` + ); + } + } + try { + // this checks to make sure index exists first, safe to try in case of failure above + // may be able to recover from minor errors + await createPrepackagedRules( + context, + appClient, + alerts.getAlertsClientWithRequest(request), + frameworkRequest, + maxTimelineImportExportSize + ); + } catch (err) { + logger.error( + `Unable to create detection rules automatically (${err.statusCode}): ${err.message}` + ); + } // Get most recent manifest const manifest = await getManifest(logger, manifestManager); @@ -94,6 +134,10 @@ export const getPackagePolicyCreateCallback = ( logger.error('Invalid manifest'); } + // We cast the type here so that any changes to the Endpoint specific data + // follow the types/schema expected + let updatedPackagePolicy = newPackagePolicy as NewPolicyData; + // Until we get the Default Policy Configuration in the Endpoint package, // we will add it here manually at creation time. updatedPackagePolicy = { diff --git a/x-pack/plugins/security_solution/server/endpoint/mocks.ts b/x-pack/plugins/security_solution/server/endpoint/mocks.ts index 9fd1fb26b1c58..98b971a00710d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/mocks.ts +++ b/x-pack/plugins/security_solution/server/endpoint/mocks.ts @@ -6,6 +6,8 @@ import { ILegacyScopedClusterClient, SavedObjectsClientContract } from 'kibana/server'; import { loggingSystemMock, savedObjectsServiceMock } from 'src/core/server/mocks'; +import { securityMock } from '../../../security/server/mocks'; +import { alertsMock } from '../../../alerts/server/mocks'; import { xpackMocks } from '../../../../mocks'; import { AgentService, @@ -14,6 +16,7 @@ import { PackageService, } from '../../../ingest_manager/server'; import { createPackagePolicyServiceMock } from '../../../ingest_manager/server/mocks'; +import { AppClientFactory } from '../client'; import { createMockConfig } from '../lib/detection_engine/routes/__mocks__'; import { EndpointAppContextService, @@ -57,12 +60,19 @@ export const createMockEndpointAppContextService = ( export const createMockEndpointAppContextServiceStartContract = (): jest.Mocked< EndpointAppContextServiceStartContract > => { + const factory = new AppClientFactory(); + const config = createMockConfig(); + factory.setup({ getSpaceId: () => 'mockSpace', config }); return { agentService: createMockAgentService(), packageService: createMockPackageService(), logger: loggingSystemMock.create().get('mock_endpoint_app_context'), savedObjectsStart: savedObjectsServiceMock.createStartContract(), manifestManager: getManifestManagerMock(), + appClientFactory: factory, + security: securityMock.createSetup(), + alerts: alertsMock.createStart(), + config, registerIngestCallback: jest.fn< ReturnType, Parameters diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/check_template_version.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/check_template_version.ts index 473a2dad37f19..e7618f155967b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/check_template_version.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/check_template_version.ts @@ -6,20 +6,19 @@ import { get } from 'lodash'; import { LegacyAPICaller } from '../../../../../../../../src/core/server'; -import { getSignalsTemplate } from './get_signals_template'; import { getTemplateExists } from '../../index/get_template_exists'; +import { SIGNALS_TEMPLATE_VERSION } from './get_signals_template'; export const templateNeedsUpdate = async (callCluster: LegacyAPICaller, index: string) => { const templateExists = await getTemplateExists(callCluster, index); - let existingTemplateVersion: number | undefined; - if (templateExists) { - const existingTemplate: unknown = await callCluster('indices.getTemplate', { - name: index, - }); - existingTemplateVersion = get(existingTemplate, [index, 'version']); + if (!templateExists) { + return true; } - const newTemplate = getSignalsTemplate(index); - if (existingTemplateVersion === undefined || existingTemplateVersion < newTemplate.version) { + const existingTemplate: unknown = await callCluster('indices.getTemplate', { + name: index, + }); + const existingTemplateVersion: number | undefined = get(existingTemplate, [index, 'version']); + if (existingTemplateVersion === undefined || existingTemplateVersion < SIGNALS_TEMPLATE_VERSION) { return true; } return false; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts index a801bc18db439..287459cf5ec9a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts @@ -4,17 +4,19 @@ * you may not use this file except in compliance with the Elastic License. */ -import { IRouter } from '../../../../../../../../src/core/server'; +import { AppClient } from '../../../../types'; +import { IRouter, RequestHandlerContext } from '../../../../../../../../src/core/server'; import { DETECTION_ENGINE_INDEX_URL } from '../../../../../common/constants'; import { transformError, buildSiemResponse } from '../utils'; import { getIndexExists } from '../../index/get_index_exists'; import { getPolicyExists } from '../../index/get_policy_exists'; import { setPolicy } from '../../index/set_policy'; import { setTemplate } from '../../index/set_template'; -import { getSignalsTemplate } from './get_signals_template'; +import { getSignalsTemplate, SIGNALS_TEMPLATE_VERSION } from './get_signals_template'; import { createBootstrapIndex } from '../../index/create_bootstrap_index'; import signalsPolicy from './signals_policy.json'; import { templateNeedsUpdate } from './check_template_version'; +import { getIndexVersion } from './get_index_version'; export const createIndexRoute = (router: IRouter) => { router.post( @@ -29,29 +31,11 @@ export const createIndexRoute = (router: IRouter) => { const siemResponse = buildSiemResponse(response); try { - const clusterClient = context.core.elasticsearch.legacy.client; const siemClient = context.securitySolution?.getAppClient(); - const callCluster = clusterClient.callAsCurrentUser; - if (!siemClient) { return siemResponse.error({ statusCode: 404 }); } - - const index = siemClient.getSignalsIndex(); - const indexExists = await getIndexExists(callCluster, index); - if (await templateNeedsUpdate(callCluster, index)) { - const policyExists = await getPolicyExists(callCluster, index); - if (!policyExists) { - await setPolicy(callCluster, index, signalsPolicy); - } - await setTemplate(callCluster, index, getSignalsTemplate(index)); - if (indexExists) { - await callCluster('indices.rollover', { alias: index }); - } - } - if (!indexExists) { - await createBootstrapIndex(callCluster, index); - } + await createDetectionIndex(context, siemClient!); return response.ok({ body: { acknowledged: true } }); } catch (err) { const error = transformError(err); @@ -63,3 +47,41 @@ export const createIndexRoute = (router: IRouter) => { } ); }; + +class CreateIndexError extends Error { + public readonly statusCode: number; + constructor(message: string, statusCode: number) { + super(message); + this.statusCode = statusCode; + } +} + +export const createDetectionIndex = async ( + context: RequestHandlerContext, + siemClient: AppClient +): Promise => { + const clusterClient = context.core.elasticsearch.legacy.client; + const callCluster = clusterClient.callAsCurrentUser; + + if (!siemClient) { + throw new CreateIndexError('', 404); + } + + const index = siemClient.getSignalsIndex(); + const policyExists = await getPolicyExists(callCluster, index); + if (!policyExists) { + await setPolicy(callCluster, index, signalsPolicy); + } + if (await templateNeedsUpdate(callCluster, index)) { + await setTemplate(callCluster, index, getSignalsTemplate(index)); + } + const indexExists = await getIndexExists(callCluster, index); + if (indexExists) { + const indexVersion = await getIndexVersion(callCluster, index); + if (indexVersion !== SIGNALS_TEMPLATE_VERSION) { + await callCluster('indices.rollover', { alias: index }); + } + } else { + await createBootstrapIndex(callCluster, index); + } +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_index_version.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_index_version.ts new file mode 100644 index 0000000000000..062cffd393555 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_index_version.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { get } from 'lodash'; +import { LegacyAPICaller } from '../../../../../../../../src/core/server'; +import { readIndex } from '../../index/read_index'; + +interface IndicesAliasResponse { + [index: string]: IndexAliasResponse; +} + +interface IndexAliasResponse { + aliases: { + [aliasName: string]: Record; + }; +} + +export const getIndexVersion = async ( + callCluster: LegacyAPICaller, + index: string +): Promise => { + const indexAlias: IndicesAliasResponse = await callCluster('indices.getAlias', { + index, + }); + const writeIndex = Object.keys(indexAlias).find( + (key) => indexAlias[key].aliases[index].is_write_index + ); + if (writeIndex === undefined) { + return undefined; + } + const writeIndexMapping = await readIndex(callCluster, writeIndex); + return get(writeIndexMapping, [writeIndex, 'mappings', '_meta', 'version']); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts index b676ab5705bfc..d1a9b701b2c9d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts @@ -7,8 +7,10 @@ import signalsMapping from './signals_mapping.json'; import ecsMapping from './ecs_mapping.json'; +export const SIGNALS_TEMPLATE_VERSION = 2; +export const MIN_EQL_RULE_INDEX_VERSION = 2; + export const getSignalsTemplate = (index: string) => { - const version = 2; const template = { settings: { index: { @@ -31,10 +33,10 @@ export const getSignalsTemplate = (index: string) => { signal: signalsMapping.mappings.properties.signal, }, _meta: { - version, + version: SIGNALS_TEMPLATE_VERSION, }, }, - version, + version: SIGNALS_TEMPLATE_VERSION, }; return template; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/read_index_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/read_index_route.ts index b9ae8b546b8bd..d1b1a2b4dd0eb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/read_index_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/read_index_route.ts @@ -8,7 +8,8 @@ import { IRouter } from '../../../../../../../../src/core/server'; import { DETECTION_ENGINE_INDEX_URL } from '../../../../../common/constants'; import { transformError, buildSiemResponse } from '../utils'; import { getIndexExists } from '../../index/get_index_exists'; -import { templateNeedsUpdate } from './check_template_version'; +import { SIGNALS_TEMPLATE_VERSION } from './get_signals_template'; +import { getIndexVersion } from './get_index_version'; export const readIndexRoute = (router: IRouter) => { router.get( @@ -32,10 +33,24 @@ export const readIndexRoute = (router: IRouter) => { const index = siemClient.getSignalsIndex(); const indexExists = await getIndexExists(clusterClient.callAsCurrentUser, index); - const templateOutdated = await templateNeedsUpdate(clusterClient.callAsCurrentUser, index); if (indexExists) { - return response.ok({ body: { name: index, template_outdated: templateOutdated } }); + let mappingOutdated: boolean | null = null; + try { + const indexVersion = await getIndexVersion(clusterClient.callAsCurrentUser, index); + mappingOutdated = indexVersion !== SIGNALS_TEMPLATE_VERSION; + } catch (err) { + const error = transformError(err); + // Some users may not have the view_index_metadata permission necessary to check the index mapping version + // so just continue and return null for index_mapping_outdated if the error is a 403 + if (error.statusCode !== 403) { + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + return response.ok({ body: { name: index, index_mapping_outdated: mappingOutdated } }); } else { return siemResponse.error({ statusCode: 404, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.ts index b1f6f73b09627..f885445c29b04 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.ts @@ -4,7 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ -import { IRouter } from '../../../../../../../../src/core/server'; +import { AppClient } from '../../../../types'; +import { IRouter, RequestHandlerContext } from '../../../../../../../../src/core/server'; import { validate } from '../../../../../common/validate'; import { @@ -28,6 +29,8 @@ import { getRulesToUpdate } from '../../rules/get_rules_to_update'; import { getExistingPrepackagedRules } from '../../rules/get_existing_prepackaged_rules'; import { transformError, buildSiemResponse } from '../utils'; +import { AlertsClient } from '../../../../../../alerts/server'; +import { FrameworkRequest } from '../../../framework'; export const addPrepackedRulesRoute = ( router: IRouter, @@ -48,62 +51,20 @@ export const addPrepackedRulesRoute = ( try { const alertsClient = context.alerting?.getAlertsClient(); - const clusterClient = context.core.elasticsearch.legacy.client; - const savedObjectsClient = context.core.savedObjects.client; const siemClient = context.securitySolution?.getAppClient(); if (!siemClient || !alertsClient) { return siemResponse.error({ statusCode: 404 }); } - // This will create the endpoint list if it does not exist yet - await context.lists?.getExceptionListClient().createEndpointList(); - - const rulesFromFileSystem = getPrepackagedRules(); - const prepackagedRules = await getExistingPrepackagedRules({ alertsClient }); - const rulesToInstall = getRulesToInstall(rulesFromFileSystem, prepackagedRules); - const rulesToUpdate = getRulesToUpdate(rulesFromFileSystem, prepackagedRules); - const signalsIndex = siemClient.getSignalsIndex(); - if (rulesToInstall.length !== 0 || rulesToUpdate.length !== 0) { - const signalsIndexExists = await getIndexExists( - clusterClient.callAsCurrentUser, - signalsIndex - ); - if (!signalsIndexExists) { - return siemResponse.error({ - statusCode: 400, - body: `Pre-packaged rules cannot be installed until the signals index is created: ${signalsIndex}`, - }); - } - } - const result = await Promise.all([ - installPrepackagedRules(alertsClient, rulesToInstall, signalsIndex), - installPrepackagedTimelines(config.maxTimelineImportExportSize, frameworkRequest, true), - ]); - const [prepackagedTimelinesResult, timelinesErrors] = validate( - result[1], - importTimelineResultSchema - ); - await updatePrepackagedRules(alertsClient, savedObjectsClient, rulesToUpdate, signalsIndex); - - const prepackagedRulesOutput: PrePackagedRulesAndTimelinesSchema = { - rules_installed: rulesToInstall.length, - rules_updated: rulesToUpdate.length, - timelines_installed: prepackagedTimelinesResult?.timelines_installed ?? 0, - timelines_updated: prepackagedTimelinesResult?.timelines_updated ?? 0, - }; - const [validated, genericErrors] = validate( - prepackagedRulesOutput, - prePackagedRulesAndTimelinesSchema + const validated = await createPrepackagedRules( + context, + siemClient, + alertsClient, + frameworkRequest, + config.maxTimelineImportExportSize ); - if (genericErrors != null && timelinesErrors != null) { - return siemResponse.error({ - statusCode: 500, - body: [genericErrors, timelinesErrors].filter((msg) => msg != null).join(', '), - }); - } else { - return response.ok({ body: validated ?? {} }); - } + return response.ok({ body: validated ?? {} }); } catch (err) { const error = transformError(err); return siemResponse.error({ @@ -114,3 +75,71 @@ export const addPrepackedRulesRoute = ( } ); }; + +class PrepackagedRulesError extends Error { + public readonly statusCode: number; + constructor(message: string, statusCode: number) { + super(message); + this.statusCode = statusCode; + } +} + +export const createPrepackagedRules = async ( + context: RequestHandlerContext, + siemClient: AppClient, + alertsClient: AlertsClient, + frameworkRequest: FrameworkRequest, + maxTimelineImportExportSize: number +): Promise => { + const clusterClient = context.core.elasticsearch.legacy.client; + const savedObjectsClient = context.core.savedObjects.client; + + if (!siemClient || !alertsClient) { + throw new PrepackagedRulesError('', 404); + } + + // This will create the endpoint list if it does not exist yet + await context.lists?.getExceptionListClient().createEndpointList(); + + const rulesFromFileSystem = getPrepackagedRules(); + const prepackagedRules = await getExistingPrepackagedRules({ alertsClient }); + const rulesToInstall = getRulesToInstall(rulesFromFileSystem, prepackagedRules); + const rulesToUpdate = getRulesToUpdate(rulesFromFileSystem, prepackagedRules); + const signalsIndex = siemClient.getSignalsIndex(); + if (rulesToInstall.length !== 0 || rulesToUpdate.length !== 0) { + const signalsIndexExists = await getIndexExists(clusterClient.callAsCurrentUser, signalsIndex); + if (!signalsIndexExists) { + throw new PrepackagedRulesError( + `Pre-packaged rules cannot be installed until the signals index is created: ${signalsIndex}`, + 400 + ); + } + } + const result = await Promise.all([ + installPrepackagedRules(alertsClient, rulesToInstall, signalsIndex), + installPrepackagedTimelines(maxTimelineImportExportSize, frameworkRequest, true), + ]); + const [prepackagedTimelinesResult, timelinesErrors] = validate( + result[1], + importTimelineResultSchema + ); + await updatePrepackagedRules(alertsClient, savedObjectsClient, rulesToUpdate, signalsIndex); + + const prepackagedRulesOutput: PrePackagedRulesAndTimelinesSchema = { + rules_installed: rulesToInstall.length, + rules_updated: rulesToUpdate.length, + timelines_installed: prepackagedTimelinesResult?.timelines_installed ?? 0, + timelines_updated: prepackagedTimelinesResult?.timelines_updated ?? 0, + }; + const [validated, genericErrors] = validate( + prepackagedRulesOutput, + prePackagedRulesAndTimelinesSchema + ); + if (genericErrors != null && timelinesErrors != null) { + throw new PrepackagedRulesError( + [genericErrors, timelinesErrors].filter((msg) => msg != null).join(', '), + 500 + ); + } + return validated; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.test.ts index 6768e9534a87e..977dad680f8a4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.test.ts @@ -7,7 +7,9 @@ import Boom from 'boom'; import { SavedObjectsFindResponse } from 'kibana/server'; -import { IRuleSavedAttributesSavedObjectAttributes, IRuleStatusAttributes } from '../rules/types'; + +import { alertsClientMock } from '../../../../../alerts/server/mocks'; +import { IRuleSavedAttributesSavedObjectAttributes, IRuleStatusSOAttributes } from '../rules/types'; import { BadRequestError } from '../errors/bad_request_error'; import { transformError, @@ -19,8 +21,14 @@ import { transformImportError, convertToSnakeCase, SiemResponseFactory, + mergeStatuses, + getFailingRules, } from './utils'; import { responseMock } from './__mocks__'; +import { exampleRuleStatus, exampleFindRuleStatusResponse } from '../signals/__mocks__/es_results'; +import { getResult } from './__mocks__/request_responses'; + +let alertsClient: ReturnType; describe('utils', () => { describe('transformError', () => { @@ -319,7 +327,7 @@ describe('utils', () => { saved_objects: [], }; expect( - convertToSnakeCase(values.saved_objects[0]?.attributes) // this is undefined, but it says it's not + convertToSnakeCase(values.saved_objects[0]?.attributes) // this is undefined, but it says it's not ).toEqual(null); }); }); @@ -350,4 +358,133 @@ describe('utils', () => { ); }); }); + + describe('mergeStatuses', () => { + it('merges statuses and converts from camelCase saved object to snake_case HTTP response', () => { + const statusOne = exampleRuleStatus(); + statusOne.attributes.status = 'failed'; + const statusTwo = exampleRuleStatus(); + statusTwo.attributes.status = 'failed'; + const currentStatus = exampleRuleStatus(); + const foundRules = exampleFindRuleStatusResponse([currentStatus, statusOne, statusTwo]); + const res = mergeStatuses(currentStatus.attributes.alertId, foundRules.saved_objects, { + 'myfakealertid-8cfac': { + current_status: { + alert_id: 'myfakealertid-8cfac', + status_date: '2020-03-27T22:55:59.517Z', + status: 'succeeded', + last_failure_at: null, + last_success_at: '2020-03-27T22:55:59.517Z', + last_failure_message: null, + last_success_message: 'succeeded', + gap: null, + bulk_create_time_durations: [], + search_after_time_durations: [], + last_look_back_date: null, + }, + failures: [], + }, + }); + expect(res).toEqual({ + 'myfakealertid-8cfac': { + current_status: { + alert_id: 'myfakealertid-8cfac', + status_date: '2020-03-27T22:55:59.517Z', + status: 'succeeded', + last_failure_at: null, + last_success_at: '2020-03-27T22:55:59.517Z', + last_failure_message: null, + last_success_message: 'succeeded', + gap: null, + bulk_create_time_durations: [], + search_after_time_durations: [], + last_look_back_date: null, + }, + failures: [], + }, + 'f4b8e31d-cf93-4bde-a265-298bde885cd7': { + current_status: { + alert_id: 'f4b8e31d-cf93-4bde-a265-298bde885cd7', + status_date: '2020-03-27T22:55:59.517Z', + status: 'succeeded', + last_failure_at: null, + last_success_at: '2020-03-27T22:55:59.517Z', + last_failure_message: null, + last_success_message: 'succeeded', + gap: null, + bulk_create_time_durations: [], + search_after_time_durations: [], + last_look_back_date: null, + }, + failures: [ + { + alert_id: 'f4b8e31d-cf93-4bde-a265-298bde885cd7', + status_date: '2020-03-27T22:55:59.517Z', + status: 'failed', + last_failure_at: null, + last_success_at: '2020-03-27T22:55:59.517Z', + last_failure_message: null, + last_success_message: 'succeeded', + gap: null, + bulk_create_time_durations: [], + search_after_time_durations: [], + last_look_back_date: null, + }, + { + alert_id: 'f4b8e31d-cf93-4bde-a265-298bde885cd7', + status_date: '2020-03-27T22:55:59.517Z', + status: 'failed', + last_failure_at: null, + last_success_at: '2020-03-27T22:55:59.517Z', + last_failure_message: null, + last_success_message: 'succeeded', + gap: null, + bulk_create_time_durations: [], + search_after_time_durations: [], + last_look_back_date: null, + }, + ], + }, + }); + }); + }); + + describe('getFailingRules', () => { + beforeEach(() => { + alertsClient = alertsClientMock.create(); + }); + it('getFailingRules finds no failing rules', async () => { + alertsClient.get.mockResolvedValue(getResult()); + const res = await getFailingRules(['my-fake-id'], alertsClient); + expect(res).toEqual({}); + }); + it('getFailingRules finds a failing rule', async () => { + const foundRule = getResult(); + foundRule.executionStatus = { + status: 'error', + lastExecutionDate: foundRule.executionStatus.lastExecutionDate, + error: { + reason: 'read', + message: 'oops', + }, + }; + alertsClient.get.mockResolvedValue(foundRule); + const res = await getFailingRules([foundRule.id], alertsClient); + expect(res).toEqual({ [foundRule.id]: foundRule }); + }); + it('getFailingRules throws an error', async () => { + alertsClient.get.mockImplementation(() => { + throw new Error('my test error'); + }); + let error; + try { + await getFailingRules(['my-fake-id'], alertsClient); + } catch (exc) { + error = exc; + } + expect(error.message).toEqual( + 'Failed to get executionStatus with AlertsClient: my test error' + ); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.ts index 96f96d7ebcc9e..72be7a3c0fa08 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.ts @@ -17,7 +17,7 @@ import { } from '../../../../../../../src/core/server'; import { AlertsClient } from '../../../../../alerts/server'; import { BadRequestError } from '../errors/bad_request_error'; -import { RuleStatusResponse, IRuleStatusAttributes } from '../rules/types'; +import { RuleStatusResponse, IRuleStatusSOAttributes } from '../rules/types'; export interface OutputError { message: string; @@ -294,39 +294,53 @@ export const convertToSnakeCase = >( }, {}); }; +/** + * + * @param id rule id + * @param currentStatusAndFailures array of rule statuses where the 0th status is the current status and 1-5 positions are the historical failures + * @param acc accumulated rule id : statuses + */ export const mergeStatuses = ( id: string, - failures: Array>, + currentStatusAndFailures: Array>, acc: RuleStatusResponse -) => { - if (failures.length === 0) { +): RuleStatusResponse => { + if (currentStatusAndFailures.length === 0) { return { ...acc, }; } - const convertedCurrentStatus = convertToSnakeCase(failures[0].attributes); + const convertedCurrentStatus = convertToSnakeCase( + currentStatusAndFailures[0].attributes + ); return { ...acc, [id]: { current_status: convertedCurrentStatus, - failures: failures.map((errorItem) => - convertToSnakeCase(errorItem.attributes) - ), + failures: currentStatusAndFailures + .slice(1) + .map((errorItem) => convertToSnakeCase(errorItem.attributes)), }, } as RuleStatusResponse; }; -export const getFailingRules = (ids: string[], alertsClient: AlertsClient) => - Promise.all( - ids.map(async (id) => - alertsClient.get({ - id, - }) - ) - ) - .then((rules) => rules.filter((rule) => rule.executionStatus.status === 'error')) - .then((rules) => - rules.reduce((acc, failingRule) => { +export type GetFailingRulesResult = Record; + +export const getFailingRules = async ( + ids: string[], + alertsClient: AlertsClient +): Promise => { + try { + const errorRules = await Promise.all( + ids.map(async (id) => + alertsClient.get({ + id, + }) + ) + ); + return errorRules + .filter((rule) => rule.executionStatus.status === 'error') + .reduce((acc, failingRule) => { const accum = acc; const theRule = failingRule; return { @@ -335,5 +349,8 @@ export const getFailingRules = (ids: string[], alertsClient: AlertsClient) => }, ...accum, }; - }, {} as Record) - ); + }, {}); + } catch (exc) { + throw new Error(`Failed to get executionStatus with AlertsClient: ${exc.message}`); + } +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_403_response_to_a_post.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_403_response_to_a_post.json index 9139ca82cc7d8..44deb00140654 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_403_response_to_a_post.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_403_response_to_a_post.json @@ -20,9 +20,9 @@ "rule_id": "a87a4e42-1d82-44bd-b0bf-d9b7f91fb89e", "severity": "medium", "tags": [ - "APM", - "Elastic" + "Elastic", + "APM" ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_405_response_method_not_allowed.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_405_response_method_not_allowed.json index 2eb7d711e5fb8..5ff5cb39a6b21 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_405_response_method_not_allowed.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_405_response_method_not_allowed.json @@ -20,9 +20,9 @@ "rule_id": "75ee75d8-c180-481c-ba88-ee50129a6aef", "severity": "medium", "tags": [ - "APM", - "Elastic" + "Elastic", + "APM" ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_null_user_agent.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_null_user_agent.json index e78395be8fb1b..75bbfe477732b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_null_user_agent.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_null_user_agent.json @@ -38,9 +38,9 @@ "rule_id": "43303fd4-4839-4e48-b2b2-803ab060758d", "severity": "medium", "tags": [ - "APM", - "Elastic" + "Elastic", + "APM" ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_sqlmap_user_agent.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_sqlmap_user_agent.json index aaaab6b5c6031..10185c28a8e52 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_sqlmap_user_agent.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_sqlmap_user_agent.json @@ -20,9 +20,9 @@ "rule_id": "d49cc73f-7a16-4def-89ce-9fc7127d7820", "severity": "medium", "tags": [ - "APM", - "Elastic" + "Elastic", + "APM" ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_network_connection_from_windows_binary.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_network_connection_from_windows_binary.json deleted file mode 100644 index 5652f025952d7..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_network_connection_from_windows_binary.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "author": [ - "Elastic" - ], - "description": "Identifies network activity from unexpected system applications. This may indicate adversarial activity as these applications are often leveraged by adversaries to execute code and evade detection.", - "from": "now-9m", - "index": [ - "logs-endpoint.events.*", - "winlogbeat-*" - ], - "language": "eql", - "license": "Elastic License", - "name": "Unusual Network Activity from a Windows System Binary", - "query": "sequence by process.entity_id with maxspan=5m\n [process where event.type in (\"start\", \"process_started\") and\n\n /* known applocker bypasses */\n process.name in (\"bginfo.exe\",\n \"cdb.exe\",\n \"control.exe\",\n \"cmstp.exe\",\n \"csi.exe\",\n \"dnx.exe\",\n \"fsi.exe\",\n \"ieexec.exe\",\n \"iexpress.exe\",\n \"installutil.exe\",\n \"Microsoft.Workflow.Compiler.exe\",\n \"MSBuild.exe\",\n \"msdt.exe\",\n \"mshta.exe\",\n \"msiexec.exe\",\n \"msxsl.exe\",\n \"odbcconf.exe\",\n \"rcsi.exe\",\n \"regsvr32.exe\",\n \"xwizard.exe\")]\n [network where event.type == \"connection\" and\n process.name in (\"bginfo.exe\",\n \"cdb.exe\",\n \"control.exe\",\n \"cmstp.exe\",\n \"csi.exe\",\n \"dnx.exe\",\n \"fsi.exe\",\n \"ieexec.exe\",\n \"iexpress.exe\",\n \"installutil.exe\",\n \"Microsoft.Workflow.Compiler.exe\",\n \"MSBuild.exe\",\n \"msdt.exe\",\n \"mshta.exe\",\n \"msiexec.exe\",\n \"msxsl.exe\",\n \"odbcconf.exe\",\n \"rcsi.exe\",\n \"regsvr32.exe\",\n \"xwizard.exe\")]\n", - "risk_score": 21, - "rule_id": "1fe3b299-fbb5-4657-a937-1d746f2c711a", - "severity": "medium", - "tags": [ - "Elastic", - "Windows" - ], - "threat": [ - { - "framework": "MITRE ATT&CK", - "tactic": { - "id": "TA0005", - "name": "Defense Evasion", - "reference": "https://attack.mitre.org/tactics/TA0005/" - }, - "technique": [ - { - "id": "T1127", - "name": "Trusted Developer Utilities Proxy Execution", - "reference": "https://attack.mitre.org/techniques/T1127/" - } - ] - } - ], - "type": "eql", - "version": 1 -} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_cloudtrail_logging_created.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_cloudtrail_logging_created.json index 3dc084a3af54b..7c98f47fd05e2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_cloudtrail_logging_created.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_cloudtrail_logging_created.json @@ -25,11 +25,12 @@ "rule_id": "594e0cbf-86cc-45aa-9ff7-ff27db27d3ed", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Logging", - "Continuous Monitoring" + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_gcp_pub_sub_subscription_creation.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_gcp_pub_sub_subscription_creation.json index 720c6f71dafdd..6b90ec776926c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_gcp_pub_sub_subscription_creation.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_gcp_pub_sub_subscription_creation.json @@ -22,10 +22,11 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", - "Logging" + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_gcp_pub_sub_topic_creation.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_gcp_pub_sub_topic_creation.json index 93695334faae2..e53c36b236639 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_gcp_pub_sub_topic_creation.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_gcp_pub_sub_topic_creation.json @@ -22,10 +22,11 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", - "Logging" + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_update_event_hub_auth_rule.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_update_event_hub_auth_rule.json index cddc98ba2e6d7..d65a0bcdbc6d0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_update_event_hub_auth_rule.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_update_event_hub_auth_rule.json @@ -22,11 +22,12 @@ "rule_id": "b6dce542-2b75-4ffb-b7d6-38787298ba9d", "severity": "medium", "tags": [ - "Azure", "Elastic", - "SecOps", + "Cloud", + "Azure", "Continuous Monitoring", - "Logging" + "SecOps", + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_certutil_network_connection.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_certutil_network_connection.json index f32877da78d99..e88297aa2c813 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_certutil_network_connection.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_certutil_network_connection.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Command and Control" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_cobalt_strike_beacon.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_cobalt_strike_beacon.json index 7ebc13ac8079b..27ad410df1fa2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_cobalt_strike_beacon.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_cobalt_strike_beacon.json @@ -23,7 +23,9 @@ "severity": "high", "tags": [ "Elastic", - "Network" + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_dns_directly_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_dns_directly_to_the_internet.json index 79ec202c41ffb..3df567b09055a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_dns_directly_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_dns_directly_to_the_internet.json @@ -23,7 +23,9 @@ "severity": "medium", "tags": [ "Elastic", - "Network" + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { @@ -43,5 +45,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_download_rar_powershell_from_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_download_rar_powershell_from_internet.json index 7b739f005a0cb..92411011ba66e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_download_rar_powershell_from_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_download_rar_powershell_from_internet.json @@ -23,7 +23,9 @@ "severity": "medium", "tags": [ "Elastic", - "Network" + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_fin7_c2_behavior.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_fin7_c2_behavior.json index 04d68aff0da1c..1ea40aad7861a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_fin7_c2_behavior.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_fin7_c2_behavior.json @@ -22,7 +22,9 @@ "severity": "high", "tags": [ "Elastic", - "Network" + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_ftp_file_transfer_protocol_activity_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_ftp_file_transfer_protocol_activity_to_the_internet.json index 9a009ffd3fd21..c73fdf1bded9d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_ftp_file_transfer_protocol_activity_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_ftp_file_transfer_protocol_activity_to_the_internet.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { @@ -54,5 +58,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_halfbaked_beacon.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_halfbaked_beacon.json index 7dacb9afcbd60..19c2832b4b82e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_halfbaked_beacon.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_halfbaked_beacon.json @@ -23,7 +23,9 @@ "severity": "high", "tags": [ "Elastic", - "Network" + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_irc_internet_relay_chat_protocol_activity_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_irc_internet_relay_chat_protocol_activity_to_the_internet.json index 0e35d4b1c5ca0..f1901fa70def2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_irc_internet_relay_chat_protocol_activity_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_irc_internet_relay_chat_protocol_activity_to_the_internet.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_nat_traversal_port_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_nat_traversal_port_activity.json index e42bf4029eb01..5afdd1f629ae4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_nat_traversal_port_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_nat_traversal_port_activity.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { @@ -39,5 +43,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_port_26_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_port_26_activity.json index 1cdfd44eb2adf..edd913da4d2b3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_port_26_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_port_26_activity.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -23,7 +24,10 @@ "severity": "low", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_port_8000_activity_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_port_8000_activity_to_the_internet.json index 319f95ed88e08..0c35bd5e23ed5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_port_8000_activity_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_port_8000_activity_to_the_internet.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { @@ -39,5 +43,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_pptp_point_to_point_tunneling_protocol_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_pptp_point_to_point_tunneling_protocol_activity.json index bd478f2b23fc0..c706a5b7248c8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_pptp_point_to_point_tunneling_protocol_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_pptp_point_to_point_tunneling_protocol_activity.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { @@ -39,5 +43,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_proxy_port_activity_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_proxy_port_activity_to_the_internet.json index ee02505300611..8535a9591b88f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_proxy_port_activity_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_proxy_port_activity_to_the_internet.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { @@ -39,5 +43,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_rdp_remote_desktop_protocol_from_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_rdp_remote_desktop_protocol_from_the_internet.json index 87544647b17e1..4a3fd026f54a7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_rdp_remote_desktop_protocol_from_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_rdp_remote_desktop_protocol_from_the_internet.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { @@ -69,5 +73,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_remote_file_copy_desktopimgdownldr.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_remote_file_copy_desktopimgdownldr.json index d5b21dfe2db18..596c4bbac57ba 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_remote_file_copy_desktopimgdownldr.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_remote_file_copy_desktopimgdownldr.json @@ -11,7 +11,7 @@ "language": "kuery", "license": "Elastic License", "name": "Remote File Download via Desktopimgdownldr Utility", - "query": "event.category:process and event.type:(start or process_started) and (process.name:desktopimgdownldr.exe or process.pe.original_file_name:desktopimgdownldr.exe) and process.args:/lockscreenurl\\:http*", + "query": "event.category:process and event.type:(start or process_started) and (process.name:desktopimgdownldr.exe or process.pe.original_file_name:desktopimgdownldr.exe or winlog.event_data.OriginalFileName:desktopimgdownldr.exe) and process.args:/lockscreenurl\\:http*", "references": [ "https://labs.sentinelone.com/living-off-windows-land-a-new-native-file-downldr/" ], @@ -20,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Command and Control" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_remote_file_copy_mpcmdrun.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_remote_file_copy_mpcmdrun.json index aeadc849eac17..9eef2fbbc62a6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_remote_file_copy_mpcmdrun.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_remote_file_copy_mpcmdrun.json @@ -12,7 +12,7 @@ "license": "Elastic License", "name": "Remote File Download via MpCmdRun", "note": "### Investigating Remote File Download via MpCmdRun\nVerify details such as the parent process, URL reputation, and downloaded file details. Additionally, `MpCmdRun` logs this information in the Appdata Temp folder in `MpCmdRun.log`.", - "query": "event.category:process and event.type:(start or process_started) and (process.name:MpCmdRun.exe or process.pe.original_file_name:MpCmdRun.exe) and process.args:((\"-DownloadFile\" or \"-downloadfile\") and \"-url\" and \"-path\")", + "query": "event.category:process and event.type:(start or process_started) and (process.name:MpCmdRun.exe or process.pe.original_file_name:MpCmdRun.exe or winlog.event_data.OriginalFileName:MpCmdRun.exe) and process.args:((\"-DownloadFile\" or \"-downloadfile\") and \"-url\" and \"-path\")", "references": [ "https://twitter.com/mohammadaskar2/status/1301263551638761477", "https://www.bleepingcomputer.com/news/microsoft/microsoft-defender-can-ironically-be-used-to-download-malware/" @@ -22,7 +22,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Command and Control" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_smtp_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_smtp_to_the_internet.json index 3a082c29a4cf1..f041255374f12 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_smtp_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_smtp_to_the_internet.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { @@ -54,5 +58,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_sql_server_port_activity_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_sql_server_port_activity_to_the_internet.json index 95ac4d8836800..7e4f3907fc31e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_sql_server_port_activity_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_sql_server_port_activity_to_the_internet.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { @@ -39,5 +43,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_ssh_secure_shell_from_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_ssh_secure_shell_from_the_internet.json index fe5608459ffce..08ab14aeb5c7c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_ssh_secure_shell_from_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_ssh_secure_shell_from_the_internet.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { @@ -69,5 +73,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_ssh_secure_shell_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_ssh_secure_shell_to_the_internet.json index 9ecfe39a79303..4bc48ebe0c316 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_ssh_secure_shell_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_ssh_secure_shell_to_the_internet.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { @@ -39,5 +43,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_teamviewer_remote_file_copy.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_teamviewer_remote_file_copy.json index 793ff4ebda72f..c597f6b2718b9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_teamviewer_remote_file_copy.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_teamviewer_remote_file_copy.json @@ -20,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Command and Control" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_telnet_port_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_telnet_port_activity.json index 4455d8adfdf83..9007db322ae58 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_telnet_port_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_telnet_port_activity.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_tor_activity_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_tor_activity_to_the_internet.json index b278c36d01c1b..e82106a87bc2e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_tor_activity_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_tor_activity_to_the_internet.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { @@ -54,5 +58,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_vnc_virtual_network_computing_from_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_vnc_virtual_network_computing_from_the_internet.json index 97d2b940a6949..9321d2a2103de 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_vnc_virtual_network_computing_from_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_vnc_virtual_network_computing_from_the_internet.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "high", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_vnc_virtual_network_computing_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_vnc_virtual_network_computing_to_the_internet.json index 97757af22be0c..38f38e9762645 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_vnc_virtual_network_computing_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_vnc_virtual_network_computing_to_the_internet.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_attempted_bypass_of_okta_mfa.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_attempted_bypass_of_okta_mfa.json index 118f8f6b2ad4a..fb8256bf2509c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_attempted_bypass_of_okta_mfa.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_attempted_bypass_of_okta_mfa.json @@ -21,10 +21,11 @@ "severity": "high", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Identity and Access", - "Continuous Monitoring" + "Identity and Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_attempts_to_brute_force_okta_user_account.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_attempts_to_brute_force_okta_user_account.json index 5aae95476e9da..d8d5b5305aaaa 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_attempts_to_brute_force_okta_user_account.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_attempts_to_brute_force_okta_user_account.json @@ -22,6 +22,7 @@ "severity": "medium", "tags": [ "Elastic", + "Identity", "Okta", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_aws_iam_assume_role_brute_force.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_aws_iam_assume_role_brute_force.json index e350c3697f685..b7e89a9120bbb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_aws_iam_assume_role_brute_force.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_aws_iam_assume_role_brute_force.json @@ -21,11 +21,12 @@ "rule_id": "ea248a02-bc47-4043-8e94-2885b19b2636", "severity": "medium", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Identity and Access", - "Continuous Monitoring" + "Identity and Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/exfiltration_compress_credentials_keychains.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_compress_credentials_keychains.json similarity index 95% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/exfiltration_compress_credentials_keychains.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_compress_credentials_keychains.json index bf2a52066ae1c..c13ac69e50987 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/exfiltration_compress_credentials_keychains.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_compress_credentials_keychains.json @@ -20,7 +20,10 @@ "severity": "high", "tags": [ "Elastic", - "MacOS" + "Host", + "macOS", + "Threat Detection", + "Credential Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_credential_dumping_msbuild.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_credential_dumping_msbuild.json index a67fa01ab371a..0761ba515d9b1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_credential_dumping_msbuild.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_credential_dumping_msbuild.json @@ -20,7 +20,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Credential Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_domain_backup_dpapi_private_keys.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_domain_backup_dpapi_private_keys.json index dc4f5e11754d3..eefd6ee9e601b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_domain_backup_dpapi_private_keys.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_domain_backup_dpapi_private_keys.json @@ -22,7 +22,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Credential Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_gcp_iam_service_account_key_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_gcp_iam_service_account_key_deletion.json index 63d5081869f1b..5db891caa2857 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_gcp_iam_service_account_key_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_gcp_iam_service_account_key_deletion.json @@ -23,6 +23,7 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_gcp_key_created_for_service_account.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_gcp_key_created_for_service_account.json index c1ae7f5fc1953..a6d45b7465771 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_gcp_key_created_for_service_account.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_gcp_key_created_for_service_account.json @@ -23,6 +23,7 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_iam_user_addition_to_group.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_iam_user_addition_to_group.json index 7c5aa9bc7f3a8..8244cb755787f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_iam_user_addition_to_group.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_iam_user_addition_to_group.json @@ -24,11 +24,12 @@ "rule_id": "333de828-8190-4cf5-8d7c-7575846f6fe0", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Identity and Access", - "Continuous Monitoring" + "Identity and Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_iis_apppoolsa_pwd_appcmd.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_iis_apppoolsa_pwd_appcmd.json index dd7bc43c58382..6a182617945f1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_iis_apppoolsa_pwd_appcmd.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_iis_apppoolsa_pwd_appcmd.json @@ -12,7 +12,7 @@ "license": "Elastic License", "max_signals": 33, "name": "Microsoft IIS Service Account Password Dumped", - "query": "event.category:process AND event.type:(start OR process_started) AND (process.name:appcmd.exe OR process.pe.original_file_name:appcmd.exe) AND process.args:(/[lL][iI][sS][tT]/ AND /\\/[tT][eE][xX][tT]\\:[pP][aA][sS][sS][wW][oO][rR][dD]/)", + "query": "event.category:process AND event.type:(start OR process_started) AND (process.name:appcmd.exe OR process.pe.original_file_name:appcmd.exe or winlog.event_data.OriginalFileName:appcmd.exe) AND process.args:(/[lL][iI][sS][tT]/ AND /\\/[tT][eE][xX][tT]\\:[pP][aA][sS][sS][wW][oO][rR][dD]/)", "references": [ "https://blog.netspi.com/decrypting-iis-passwords-to-break-out-of-the-dmz-part-1/" ], @@ -21,7 +21,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Credential Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_iis_connectionstrings_dumping.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_iis_connectionstrings_dumping.json index 2735fcbbd6130..f750a0f5594b4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_iis_connectionstrings_dumping.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_iis_connectionstrings_dumping.json @@ -12,7 +12,7 @@ "license": "Elastic License", "max_signals": 33, "name": "Microsoft IIS Connection Strings Decryption", - "query": "event.category:process and event.type:(start or process_started) and (process.name:aspnet_regiis.exe or process.pe.original_file_name:aspnet_regiis.exe) and process.args:(connectionStrings and \"-pdf\")", + "query": "event.category:process and event.type:(start or process_started) and (process.name:aspnet_regiis.exe or process.pe.original_file_name:aspnet_regiis.exe or winlog.event_data.OriginalFileName:aspnet_regiis.exe) and process.args:(connectionStrings and \"-pdf\")", "references": [ "https://blog.netspi.com/decrypting-iis-passwords-to-break-out-of-the-dmz-part-1/", "https://symantec-enterprise-blogs.security.com/blogs/threat-intelligence/greenbug-espionage-telco-south-asia" @@ -22,7 +22,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Credential Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_kerberosdump_kcc.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_kerberosdump_kcc.json index 4713d09f8adec..dda2f67c11663 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_kerberosdump_kcc.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_kerberosdump_kcc.json @@ -21,7 +21,10 @@ "severity": "high", "tags": [ "Elastic", - "MacOS" + "Host", + "macOS", + "Threat Detection", + "Credential Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_key_vault_modified.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_key_vault_modified.json index a45591c73dcb3..33df4e5930066 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_key_vault_modified.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_key_vault_modified.json @@ -24,9 +24,10 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "Azure", - "SecOps", "Continuous Monitoring", + "SecOps", "Data Protection" ], "threat": [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_mimikatz_memssp_default_logs.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_mimikatz_memssp_default_logs.json index fa1f99eef7f00..ae9b63371a452 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_mimikatz_memssp_default_logs.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_mimikatz_memssp_default_logs.json @@ -17,7 +17,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Credential Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_okta_brute_force_or_password_spraying.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_okta_brute_force_or_password_spraying.json index c36f878792ccf..9e10dd6dae522 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_okta_brute_force_or_password_spraying.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_okta_brute_force_or_password_spraying.json @@ -24,10 +24,11 @@ "severity": "medium", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Identity and Access", - "Continuous Monitoring" + "Identity and Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_root_console_failure_brute_force.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_root_console_failure_brute_force.json similarity index 96% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_root_console_failure_brute_force.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_root_console_failure_brute_force.json index 5f7781be82efd..f65a7b11d11ad 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_root_console_failure_brute_force.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_root_console_failure_brute_force.json @@ -23,11 +23,12 @@ "rule_id": "4d50a94f-2844-43fa-8395-6afbd5e1c5ef", "severity": "high", "tags": [ + "Elastic", + "Cloud", "AWS", "Continuous Monitoring", - "Elastic", - "Identity and Access", - "SecOps" + "SecOps", + "Identity and Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_secretsmanager_getsecretvalue.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_secretsmanager_getsecretvalue.json index 879e93750df9c..a33593f701ba7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_secretsmanager_getsecretvalue.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_secretsmanager_getsecretvalue.json @@ -26,11 +26,12 @@ "rule_id": "a00681e3-9ed6-447c-ab2c-be648821c622", "severity": "high", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Data Protection", - "Continuous Monitoring" + "Data Protection" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_storage_account_key_regenerated.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_storage_account_key_regenerated.json index 2a3dc85294a9d..62e1aab700680 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_storage_account_key_regenerated.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_storage_account_key_regenerated.json @@ -22,10 +22,11 @@ "rule_id": "1e0b832e-957e-43ae-b319-db82d228c908", "severity": "low", "tags": [ - "Azure", "Elastic", - "SecOps", + "Cloud", + "Azure", "Continuous Monitoring", + "SecOps", "Identity and Access" ], "threat": [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_tcpdump_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_tcpdump_activity.json index 306a38f5d2a28..67ab62e8d0ee5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_tcpdump_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_tcpdump_activity.json @@ -20,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Credential Access" ], "threat": [ { @@ -55,5 +58,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_adding_the_hidden_file_attribute_with_via_attribexe.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_adding_the_hidden_file_attribute_with_via_attribexe.json index c80f24a21d958..a1ff4bfc890a1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_adding_the_hidden_file_attribute_with_via_attribexe.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_adding_the_hidden_file_attribute_with_via_attribexe.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -52,5 +55,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_attempt_to_disable_iptables_or_firewall.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_attempt_to_disable_iptables_or_firewall.json index 4d4f10bbaa599..b17e4979a885c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_attempt_to_disable_iptables_or_firewall.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_attempt_to_disable_iptables_or_firewall.json @@ -17,7 +17,10 @@ "severity": "medium", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_attempt_to_disable_syslog_service.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_attempt_to_disable_syslog_service.json index 3c34b04a77a50..960000c91e4fa 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_attempt_to_disable_syslog_service.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_attempt_to_disable_syslog_service.json @@ -17,7 +17,10 @@ "severity": "medium", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_azure_diagnostic_settings_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_azure_diagnostic_settings_deletion.json index 49d98813dc040..7721790b5cf97 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_azure_diagnostic_settings_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_azure_diagnostic_settings_deletion.json @@ -23,9 +23,10 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "Azure", - "SecOps", "Continuous Monitoring", + "SecOps", "Monitoring" ], "threat": [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_base16_or_base32_encoding_or_decoding_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_base16_or_base32_encoding_or_decoding_activity.json index 3cdfac92572b1..140e1ccd8e890 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_base16_or_base32_encoding_or_decoding_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_base16_or_base32_encoding_or_decoding_activity.json @@ -20,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -55,5 +58,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_base64_encoding_or_decoding_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_base64_encoding_or_decoding_activity.json index 2d26d867b8718..fa322fca5db8a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_base64_encoding_or_decoding_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_base64_encoding_or_decoding_activity.json @@ -20,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -55,5 +58,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_clearing_windows_event_logs.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_clearing_windows_event_logs.json index 60ce575148f4c..11d57b855f974 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_clearing_windows_event_logs.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_clearing_windows_event_logs.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cloudtrail_logging_deleted.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cloudtrail_logging_deleted.json index 64261af2a3105..169f429a6dd26 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cloudtrail_logging_deleted.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cloudtrail_logging_deleted.json @@ -25,11 +25,12 @@ "rule_id": "7024e2a0-315d-4334-bb1a-441c593e16ab", "severity": "medium", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Logging", - "Continuous Monitoring" + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cloudtrail_logging_suspended.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cloudtrail_logging_suspended.json index 090073698026d..cbd040a7f7a30 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cloudtrail_logging_suspended.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cloudtrail_logging_suspended.json @@ -25,11 +25,12 @@ "rule_id": "1aa8fa52-44a7-4dae-b058-f3333b91c8d7", "severity": "medium", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Logging", - "Continuous Monitoring" + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cloudwatch_alarm_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cloudwatch_alarm_deletion.json index aeaf0a4168814..e18deb65c497b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cloudwatch_alarm_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cloudwatch_alarm_deletion.json @@ -25,11 +25,12 @@ "rule_id": "f772ec8a-e182-483c-91d2-72058f76a44c", "severity": "medium", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Monitoring", - "Continuous Monitoring" + "Monitoring" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_code_injection_conhost.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_code_injection_conhost.json index 63c7ea12b3b6b..ed1b8978c6329 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_code_injection_conhost.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_code_injection_conhost.json @@ -21,7 +21,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_config_service_rule_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_config_service_rule_deletion.json index 268f52a8efd5e..b7d9321814fd7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_config_service_rule_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_config_service_rule_deletion.json @@ -25,11 +25,12 @@ "rule_id": "7024e2a0-315d-4334-bb1a-552d604f27bc", "severity": "medium", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Monitoring", - "Continuous Monitoring" + "Monitoring" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_configuration_recorder_stopped.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_configuration_recorder_stopped.json index b926937450f5a..b28572deaf204 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_configuration_recorder_stopped.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_configuration_recorder_stopped.json @@ -25,11 +25,12 @@ "rule_id": "fbd44836-0d69-4004-a0b4-03c20370c435", "severity": "high", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Monitoring", - "Continuous Monitoring" + "Monitoring" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cve_2020_0601.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cve_2020_0601.json index 2abad3c255f15..3beb71763f1ae 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cve_2020_0601.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cve_2020_0601.json @@ -15,7 +15,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -35,5 +38,5 @@ } ], "type": "query", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_delete_volume_usn_journal_with_fsutil.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_delete_volume_usn_journal_with_fsutil.json index 50213b9f1a42c..5fde3c462eded 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_delete_volume_usn_journal_with_fsutil.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_delete_volume_usn_journal_with_fsutil.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_deleting_backup_catalogs_with_wbadmin.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_deleting_backup_catalogs_with_wbadmin.json index 026735f413eab..554ccc6972e5d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_deleting_backup_catalogs_with_wbadmin.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_deleting_backup_catalogs_with_wbadmin.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_deletion_of_bash_command_line_history.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_deletion_of_bash_command_line_history.json index 85d8bdcb2582f..eef37499c8eb5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_deletion_of_bash_command_line_history.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_deletion_of_bash_command_line_history.json @@ -17,7 +17,10 @@ "severity": "medium", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_disable_selinux_attempt.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_disable_selinux_attempt.json index d107c0b262091..35476a76fd4b5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_disable_selinux_attempt.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_disable_selinux_attempt.json @@ -17,7 +17,10 @@ "severity": "medium", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_disable_windows_firewall_rules_with_netsh.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_disable_windows_firewall_rules_with_netsh.json index 6fbf9ca800f79..a69fde9f6a5cc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_disable_windows_firewall_rules_with_netsh.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_disable_windows_firewall_rules_with_netsh.json @@ -17,7 +17,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_dotnet_compiler_parent_process.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_dotnet_compiler_parent_process.json similarity index 93% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_dotnet_compiler_parent_process.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_dotnet_compiler_parent_process.json index 0a675bd7aab74..08cbb33710b26 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_dotnet_compiler_parent_process.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_dotnet_compiler_parent_process.json @@ -17,7 +17,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_ec2_flow_log_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_ec2_flow_log_deletion.json index 9f3d4e6b5e379..d36294684698e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_ec2_flow_log_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_ec2_flow_log_deletion.json @@ -25,11 +25,12 @@ "rule_id": "9395fd2c-9947-4472-86ef-4aceb2f7e872", "severity": "high", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Logging", - "Continuous Monitoring" + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_ec2_network_acl_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_ec2_network_acl_deletion.json index 6ecc9ad3d558d..b6ac9be800807 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_ec2_network_acl_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_ec2_network_acl_deletion.json @@ -27,11 +27,12 @@ "rule_id": "8623535c-1e17-44e1-aa97-7a0699c3037d", "severity": "medium", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Network", - "Continuous Monitoring" + "Network Security" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_encoding_or_decoding_files_via_certutil.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_encoding_or_decoding_files_via_certutil.json index 0d47aab2c64bd..f00ac3f90527b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_encoding_or_decoding_files_via_certutil.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_encoding_or_decoding_files_via_certutil.json @@ -17,7 +17,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_event_hub_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_event_hub_deletion.json index 29df07cced4d7..d09edf473c939 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_event_hub_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_event_hub_deletion.json @@ -24,11 +24,12 @@ "rule_id": "e0f36de1-0342-453d-95a9-a068b257b053", "severity": "medium", "tags": [ - "Azure", "Elastic", - "SecOps", + "Cloud", + "Azure", "Continuous Monitoring", - "Logging" + "SecOps", + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_office_app.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_office_app.json index a987c00b392ec..c41b3aad2f42c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_office_app.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_office_app.json @@ -23,7 +23,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_script.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_script.json index 0537f27bad463..a8e16d8bda238 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_script.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_script.json @@ -20,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_system_process.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_system_process.json index 11fdd128475dc..60c9a317bbb6c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_system_process.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_system_process.json @@ -20,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_renamed.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_renamed.json index a90e5ebc57800..5daab573db5bd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_renamed.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_renamed.json @@ -20,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_unusal_process.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_unusal_process.json index 4f353a6ff9e6f..09247d2f21323 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_unusal_process.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_unusal_process.json @@ -23,7 +23,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -43,5 +46,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_suspicious_explorer_winword.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_suspicious_explorer_winword.json index 69d334cf13fdb..7d9f190ba7be2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_suspicious_explorer_winword.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_suspicious_explorer_winword.json @@ -11,13 +11,16 @@ "language": "kuery", "license": "Elastic License", "name": "Potential DLL SideLoading via Trusted Microsoft Programs", - "query": "event.category:process and event.type:(start or process_started) and process.pe.original_file_name:(WinWord.exe or EXPLORER.EXE or w3wp.exe or DISM.EXE) and not (process.name:(winword.exe or WINWORD.EXE or explorer.exe or w3wp.exe or Dism.exe) or process.executable:(\"C:\\Windows\\explorer.exe\" or C\\:\\\\Program?Files\\\\Microsoft?Office\\\\root\\\\Office*\\\\WINWORD.EXE or C\\:\\\\Program?Files?\\(x86\\)\\\\Microsoft?Office\\\\root\\\\Office*\\\\WINWORD.EXE or \"C:\\Windows\\System32\\Dism.exe\" or \"C:\\Windows\\SysWOW64\\Dism.exe\" or \"C:\\Windows\\System32\\inetsrv\\w3wp.exe\"))", + "query": "event.category:process and event.type:(start or process_started) and (process.pe.original_file_name:(WinWord.exe or EXPLORER.EXE or w3wp.exe or DISM.EXE) or winlog.event_data.OriginalFileName:(WinWord.exe or EXPLORER.EXE or w3wp.exe or DISM.EXE)) and not (process.name:(winword.exe or WINWORD.EXE or explorer.exe or w3wp.exe or Dism.exe) or process.executable:(\"C:\\Windows\\explorer.exe\" or C\\:\\\\Program?Files\\\\Microsoft?Office\\\\root\\\\Office*\\\\WINWORD.EXE or C\\:\\\\Program?Files?\\(x86\\)\\\\Microsoft?Office\\\\root\\\\Office*\\\\WINWORD.EXE or \"C:\\Windows\\System32\\Dism.exe\" or \"C:\\Windows\\SysWOW64\\Dism.exe\" or \"C:\\Windows\\System32\\inetsrv\\w3wp.exe\"))", "risk_score": 73, "rule_id": "1160dcdb-0a0a-4a79-91d8-9b84616edebd", "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_via_trusted_developer_utilities.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_via_trusted_developer_utilities.json index a1d14155cc3b3..7963c03699f78 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_via_trusted_developer_utilities.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_via_trusted_developer_utilities.json @@ -7,7 +7,8 @@ "These programs may be used by Windows developers but use by non-engineers is unusual." ], "index": [ - "winlogbeat-*" + "winlogbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -18,7 +19,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_file_deletion_via_shred.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_file_deletion_via_shred.json index 5b02f63a1c7f7..dc73b7bc1eb76 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_file_deletion_via_shred.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_file_deletion_via_shred.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_file_mod_writable_dir.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_file_mod_writable_dir.json index 8ee2d4fda7bf8..b97e5c1d6a182 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_file_mod_writable_dir.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_file_mod_writable_dir.json @@ -20,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -40,5 +43,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_firewall_policy_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_firewall_policy_deletion.json index 759fc9d5ecb1f..69a123ba678fd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_firewall_policy_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_firewall_policy_deletion.json @@ -22,11 +22,12 @@ "rule_id": "e02bd3ea-72c6-4181-ac2b-0f83d17ad969", "severity": "low", "tags": [ - "Azure", "Elastic", - "SecOps", + "Cloud", + "Azure", "Continuous Monitoring", - "Network" + "SecOps", + "Network Security" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_firewall_rule_created.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_firewall_rule_created.json index b80a5f0e17949..dc08dace20bfc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_firewall_rule_created.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_firewall_rule_created.json @@ -22,6 +22,7 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_firewall_rule_deleted.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_firewall_rule_deleted.json index 64c8d01df47e9..7ee5af109f37b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_firewall_rule_deleted.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_firewall_rule_deleted.json @@ -22,6 +22,7 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_firewall_rule_modified.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_firewall_rule_modified.json index b2c0e259b45e0..b4107fb9f08fd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_firewall_rule_modified.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_firewall_rule_modified.json @@ -22,6 +22,7 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_logging_bucket_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_logging_bucket_deletion.json index 62447b789d632..079a87b5c615b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_logging_bucket_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_logging_bucket_deletion.json @@ -23,10 +23,11 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", - "Logging" + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_logging_sink_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_logging_sink_deletion.json index 0fc83070ffbb7..8466b618fab98 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_logging_sink_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_logging_sink_deletion.json @@ -22,10 +22,11 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", - "Logging" + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_pub_sub_subscription_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_pub_sub_subscription_deletion.json index 2ae47140b66a5..5b87b8722595c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_pub_sub_subscription_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_pub_sub_subscription_deletion.json @@ -22,10 +22,11 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", - "Logging" + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_pub_sub_topic_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_pub_sub_topic_deletion.json index f276af3e21862..5a681a35006a7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_pub_sub_topic_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_pub_sub_topic_deletion.json @@ -22,10 +22,11 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", - "Logging" + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_storage_bucket_configuration_modified.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_storage_bucket_configuration_modified.json index 3b18732137c32..5992beef9873e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_storage_bucket_configuration_modified.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_storage_bucket_configuration_modified.json @@ -22,6 +22,7 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_storage_bucket_permissions_modified.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_storage_bucket_permissions_modified.json index ad6beb9383eea..0687bb1e5178a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_storage_bucket_permissions_modified.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_storage_bucket_permissions_modified.json @@ -22,6 +22,7 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_guardduty_detector_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_guardduty_detector_deletion.json index 3910b8e4039ad..50ee5a902b144 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_guardduty_detector_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_guardduty_detector_deletion.json @@ -25,11 +25,12 @@ "rule_id": "523116c0-d89d-4d7c-82c2-39e6845a78ef", "severity": "high", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Monitoring", - "Continuous Monitoring" + "Monitoring" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_hex_encoding_or_decoding_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_hex_encoding_or_decoding_activity.json index f5345b2276e8a..6d3d6f456da4c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_hex_encoding_or_decoding_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_hex_encoding_or_decoding_activity.json @@ -20,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -55,5 +58,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_hidden_file_dir_tmp.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_hidden_file_dir_tmp.json index e66968a50709e..c21c15909d82a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_hidden_file_dir_tmp.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_hidden_file_dir_tmp.json @@ -21,7 +21,10 @@ "severity": "medium", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -56,5 +59,5 @@ } ], "type": "query", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_iis_httplogging_disabled.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_iis_httplogging_disabled.json index 507260f04d016..7d75f50856125 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_iis_httplogging_disabled.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_iis_httplogging_disabled.json @@ -12,13 +12,16 @@ "license": "Elastic License", "max_signals": 33, "name": "IIS HTTP Logging Disabled", - "query": "event.category:process and event.type:(start or process_started) and (process.name:appcmd.exe or process.pe.original_file_name:appcmd.exe) and process.args:/dontLog\\:\\\"True\\\" and not process.parent.name:iissetup.exe", + "query": "event.category:process and event.type:(start or process_started) and (process.name:appcmd.exe or process.pe.original_file_name:appcmd.exe or winlog.event_data.OriginalFileName:appcmd.exe) and process.args:/dontLog\\:\\\"True\\\" and not process.parent.name:iissetup.exe", "risk_score": 73, "rule_id": "ebf1adea-ccf2-4943-8b96-7ab11ca173a5", "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_injection_msbuild.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_injection_msbuild.json index 9abce01769e92..7978ef9004271 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_injection_msbuild.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_injection_msbuild.json @@ -18,7 +18,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -53,5 +56,5 @@ } ], "type": "query", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_installutil_beacon.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_installutil_beacon.json similarity index 67% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_installutil_beacon.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_installutil_beacon.json index 7437bf27141ec..231ed1b36dc0f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_installutil_beacon.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_installutil_beacon.json @@ -11,13 +11,16 @@ "language": "eql", "license": "Elastic License", "name": "InstallUtil Process Making Network Connections", - "query": "/* this can be done without a sequence however, this does include more info on the process */\n\nsequence by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and process.name == \"installutil.exe\"]\n [network where event.type == \"connection\" and process.name == \"installutil.exe\" and network.direction == \"outgoing\"]\n", + "query": "/* the benefit of doing this as an eql sequence vs kql is this will limit to alerting only on the first network connection */\n\nsequence by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and process.name : \"installutil.exe\"]\n [network where event.type == \"connection\" and process.name : \"installutil.exe\" and network.direction == \"outgoing\"]\n", "risk_score": 21, "rule_id": "a13167f1-eec2-4015-9631-1fee60406dcf", "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_kernel_module_removal.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_kernel_module_removal.json index ad751a1031437..5c38974b46525 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_kernel_module_removal.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_kernel_module_removal.json @@ -23,7 +23,10 @@ "severity": "high", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -58,5 +61,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_as_elastic_endpoint_process.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_as_elastic_endpoint_process.json index 8b7ef47443e2f..163c7e834ba34 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_as_elastic_endpoint_process.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_as_elastic_endpoint_process.json @@ -17,7 +17,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_renamed_autoit.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_renamed_autoit.json index cc964bfdd3e92..be83f8c41a2ea 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_renamed_autoit.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_renamed_autoit.json @@ -11,13 +11,16 @@ "language": "lucene", "license": "Elastic License", "name": "Renamed AutoIt Scripts Interpreter", - "query": "event.category:process AND event.type:(start OR process_started) AND process.pe.original_file_name:/[aA][uU][tT][oO][iI][tT]\\d\\.[eE][xX][eE]/ AND NOT process.name:/[aA][uU][tT][oO][iI][tT]\\d{1,3}\\.[eE][xX][eE]/", + "query": "event.category:process AND event.type:(start OR process_started) AND (process.pe.original_file_name:/[aA][uU][tT][oO][iI][tT]\\d\\.[eE][xX][eE]/ OR winlog.event_data.OriginalFileName:/[aA][uU][tT][oO][iI][tT]\\d\\.[eE][xX][eE]/) AND NOT process.name:/[aA][uU][tT][oO][iI][tT]\\d{1,3}\\.[eE][xX][eE]/", "risk_score": 47, "rule_id": "2e1e835d-01e5-48ca-b9fc-7a61f7f11902", "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_suspicious_werfault_childproc.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_suspicious_werfault_childproc.json index 3000e7ac86daa..df0ecf1d534a1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_suspicious_werfault_childproc.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_suspicious_werfault_childproc.json @@ -24,7 +24,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_werfault.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_werfault.json index db421146085ff..9f5615d466374 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_werfault.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_werfault.json @@ -24,7 +24,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_misc_lolbin_connecting_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_misc_lolbin_connecting_to_the_internet.json index 5b5f69a0aef74..3d87720818ff5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_misc_lolbin_connecting_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_misc_lolbin_connecting_to_the_internet.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -52,5 +55,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_modification_of_boot_config.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_modification_of_boot_config.json index 6025fc5ca6452..84ccc52249622 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_modification_of_boot_config.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_modification_of_boot_config.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_msbuild_beacon_sequence.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_msbuild_beacon_sequence.json similarity index 79% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_msbuild_beacon_sequence.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_msbuild_beacon_sequence.json index 59295c3735a3a..0740e26d3bba6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_msbuild_beacon_sequence.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_msbuild_beacon_sequence.json @@ -11,13 +11,16 @@ "language": "eql", "license": "Elastic License", "name": "MsBuild Network Connection Sequence", - "query": "sequence by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and process.name == \"MSBuild.exe\"]\n [network where process.name == \"MSBuild.exe\" and\n not (destination.address == \"127.0.0.1\" and source.address == \"127.0.0.1\")]\n", + "query": "sequence by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and process.name : \"MSBuild.exe\"]\n [network where process.name : \"MSBuild.exe\" and\n not (destination.ip == \"127.0.0.1\" and source.ip == \"127.0.0.1\")]\n", "risk_score": 21, "rule_id": "9dc6ed5d-62a9-4feb-a903-fafa1d33b8e9", "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_mshta_beacon.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_mshta_beacon.json similarity index 69% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_mshta_beacon.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_mshta_beacon.json index 105f536628777..fd19942a33d48 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_mshta_beacon.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_mshta_beacon.json @@ -11,13 +11,16 @@ "language": "eql", "license": "Elastic License", "name": "Mshta Making Network Connections", - "query": "sequence by process.entity_id with maxspan=2h\n [process where event.type in (\"start\", \"process_started\") and process.name == \"mshta.exe\" and\n process.parent.name != \"Microsoft.ConfigurationManagement.exe\" and\n process.parent.executable not in (\"C:\\\\Amazon\\\\Amazon Assistant\\\\amazonAssistantService.exe\",\n \"C:\\\\TeamViewer\\\\TeamViewer.exe\") and\n process.args != \"ADSelfService_Enroll.hta\"]\n [network where process.name == \"mshta.exe\"]\n", + "query": "sequence by process.entity_id with maxspan=2h\n [process where event.type in (\"start\", \"process_started\") and process.name : \"mshta.exe\" and\n not process.parent.name : \"Microsoft.ConfigurationManagement.exe\" and\n not (process.parent.executable : \"C:\\\\Amazon\\\\Amazon Assistant\\\\amazonAssistantService.exe\" or\n process.parent.executable : \"C:\\\\TeamViewer\\\\TeamViewer.exe\") and\n not process.args : \"ADSelfService_Enroll.hta\"]\n [network where process.name : \"mshta.exe\"]\n", "risk_score": 21, "rule_id": "c2d90150-0133-451c-a783-533e736c12d7", "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_msxsl_beacon.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_msxsl_beacon.json similarity index 79% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_msxsl_beacon.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_msxsl_beacon.json index 27704b3e182ed..c188387304214 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_msxsl_beacon.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_msxsl_beacon.json @@ -11,13 +11,16 @@ "language": "eql", "license": "Elastic License", "name": "MsXsl Making Network Connections", - "query": "sequence by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and process.name == \"msxsl.exe\"]\n [network where event.type == \"connection\" and process.name == \"msxsl.exe\" and network.direction == \"outgoing\"]\n", + "query": "sequence by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and process.name : \"msxsl.exe\"]\n [network where event.type == \"connection\" and process.name : \"msxsl.exe\" and network.direction == \"outgoing\"]\n", "risk_score": 21, "rule_id": "870d1753-1078-403e-92d4-735f142edcca", "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_network_connection_from_windows_binary.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_network_connection_from_windows_binary.json new file mode 100644 index 0000000000000..4e1d0cad0b5da --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_network_connection_from_windows_binary.json @@ -0,0 +1,44 @@ +{ + "author": [ + "Elastic" + ], + "description": "Identifies network activity from unexpected system applications. This may indicate adversarial activity as these applications are often leveraged by adversaries to execute code and evade detection.", + "from": "now-9m", + "index": [ + "logs-endpoint.events.*", + "winlogbeat-*" + ], + "language": "eql", + "license": "Elastic License", + "name": "Unusual Network Activity from a Windows System Binary", + "query": "sequence by process.entity_id with maxspan=5m\n [process where event.type in (\"start\", \"process_started\") and\n\n /* known applocker bypasses */\n (process.name : \"bginfo.exe\" or\n process.name : \"cdb.exe\" or\n process.name : \"control.exe\" or\n process.name : \"cmstp.exe\" or\n process.name : \"csi.exe\" or\n process.name : \"dnx.exe\" or\n process.name : \"fsi.exe\" or\n process.name : \"ieexec.exe\" or\n process.name : \"iexpress.exe\" or\n process.name : \"installutil.exe\" or\n process.name : \"Microsoft.Workflow.Compiler.exe\" or\n process.name : \"MSBuild.exe\" or\n process.name : \"msdt.exe\" or\n process.name : \"mshta.exe\" or\n process.name : \"msiexec.exe\" or\n process.name : \"msxsl.exe\" or\n process.name : \"odbcconf.exe\" or\n process.name : \"rcsi.exe\" or\n process.name : \"regsvr32.exe\" or\n process.name : \"xwizard.exe\")]\n [network where event.type == \"connection\" and\n (process.name : \"bginfo.exe\" or\n process.name : \"cdb.exe\" or\n process.name : \"control.exe\" or\n process.name : \"cmstp.exe\" or\n process.name : \"csi.exe\" or\n process.name : \"dnx.exe\" or\n process.name : \"fsi.exe\" or\n process.name : \"ieexec.exe\" or\n process.name : \"iexpress.exe\" or\n process.name : \"installutil.exe\" or\n process.name : \"Microsoft.Workflow.Compiler.exe\" or\n process.name : \"MSBuild.exe\" or\n process.name : \"msdt.exe\" or\n process.name : \"mshta.exe\" or\n process.name : \"msiexec.exe\" or\n process.name : \"msxsl.exe\" or\n process.name : \"odbcconf.exe\" or\n process.name : \"rcsi.exe\" or\n process.name : \"regsvr32.exe\" or\n process.name : \"xwizard.exe\")]\n", + "risk_score": 21, + "rule_id": "1fe3b299-fbb5-4657-a937-1d746f2c711a", + "severity": "medium", + "tags": [ + "Elastic", + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" + ], + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0005", + "name": "Defense Evasion", + "reference": "https://attack.mitre.org/tactics/TA0005/" + }, + "technique": [ + { + "id": "T1127", + "name": "Trusted Developer Utilities Proxy Execution", + "reference": "https://attack.mitre.org/techniques/T1127/" + } + ] + } + ], + "type": "eql", + "version": 1 +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_network_watcher_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_network_watcher_deletion.json index 09bbba5a049e4..0e6d9172eb2c1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_network_watcher_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_network_watcher_deletion.json @@ -23,10 +23,11 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "Azure", - "SecOps", "Continuous Monitoring", - "Network" + "SecOps", + "Network Security" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_reg_beacon.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_reg_beacon.json similarity index 66% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_reg_beacon.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_reg_beacon.json index 332c719eaa41d..aa4f9985f6e2c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_reg_beacon.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_reg_beacon.json @@ -11,13 +11,16 @@ "language": "eql", "license": "Elastic License", "name": "Registration Tool Making Network Connections", - "query": "sequence by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and\n process.name in (\"regasm.exe\", \"regsvcs.exe\", \"regsvr32.exe\")]\n [network where event.type == \"connection\" and process.name in (\"regasm.exe\", \"regsvcs.exe\", \"regsvr32.exe\")]\nuntil\n [process where event.type == \"end\" and process.name in (\"regasm.exe\", \"regsvcs.exe\", \"regsvr32.exe\")]\n", + "query": "sequence by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and\n (process.name : \"RegAsm.exe\" or process.name : \"regsvcs.exe\" or process.name : \"regsvr32.exe\")]\n [network where event.type == \"connection\" and\n (process.name : \"RegAsm.exe\" or process.name : \"regsvcs.exe\" or process.name : \"regsvr32.exe\")]\nuntil\n [process where event.type == \"end\" and\n (process.name : \"RegAsm.exe\" or process.name : \"regsvcs.exe\" or process.name : \"regsvr32.exe\")]\n", "risk_score": 21, "rule_id": "6d3456a5-4a42-49d1-aaf2-7b1fd475b2c6", "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/evasion_rundll32_no_arguments.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_rundll32_no_arguments.json similarity index 61% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/evasion_rundll32_no_arguments.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_rundll32_no_arguments.json index 06fad7e0f630b..2950b792219b6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/evasion_rundll32_no_arguments.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_rundll32_no_arguments.json @@ -11,13 +11,16 @@ "language": "eql", "license": "Elastic License", "name": "Unusual Child Processes of RunDLL32", - "query": "sequence with maxspan=1h\n [process where event.type in (\"start\", \"process_started\") and\n (process.name == \"rundll32.exe\" or process.pe.original_file_name == \"rundll32.exe\") and\n\n /* zero arguments excluding the binary itself (and accounting for when the binary may not be logged in args) */\n ((process.args == \"rundll32.exe\" and process.args_count == 1) or\n (process.args != \"rundll32.exe\" and process.args_count == 0))\n\n ] by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and\n (process.name == \"rundll32.exe\" or process.pe.original_file_name == \"rundll32.exe\")\n ] by process.parent.entity_id\n", + "query": "sequence with maxspan=1h\n [process where event.type in (\"start\", \"process_started\") and\n /* uncomment once in winlogbeat */\n (process.name : \"rundll32.exe\" /* or process.pe.original_file_name == \"RUNDLL32.EXE\" */ ) and\n process.args_count < 2\n ] by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and\n /* uncomment once in winlogbeat */\n (process.name : \"rundll32.exe\" /* or process.pe.original_file_name == \"RUNDLL32.EXE\" */ )\n ] by process.parent.entity_id\n", "risk_score": 21, "rule_id": "f036953a-4615-4707-a1ca-dc53bf69dcd5", "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_rundll32_sequence.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_rundll32_sequence.json similarity index 67% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_rundll32_sequence.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_rundll32_sequence.json index 6f465325039a6..c022d0a603858 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_rundll32_sequence.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_rundll32_sequence.json @@ -11,13 +11,16 @@ "language": "eql", "license": "Elastic License", "name": "Unusual Network Connection Sequence via RunDLL32", - "query": "sequence by process.entity_id with maxspan=2h\n [process where event.type in (\"start\", \"process_started\") and\n (process.name == \"rundll32.exe\" or process.pe.original_file_name == \"rundll32.exe\") and\n\n /* zero arguments excluding the binary itself (and accounting for when the binary may not be logged in args) */\n ((process.args == \"rundll32.exe\" and process.args_count == 1) or\n (process.args != \"rundll32.exe\" and process.args_count == 0))]\n\n [network where event.type == \"connection\" and\n (process.name == \"rundll32.exe\" or process.pe.original_file_name == \"rundll32.exe\")]\n", + "query": "sequence by process.entity_id with maxspan=2h\n [process where event.type in (\"start\", \"process_started\") and\n /* uncomment once in winlogbeat */\n (process.name : \"rundll32.exe\" /* or process.pe.original_file_name == \"RUNDLL32.EXE\" */ ) and\n process.args_count < 2]\n [network where event.type == \"connection\" and\n /* uncomment once in winlogbeat */\n (process.name : \"rundll32.exe\" /* or process.pe.original_file_name == \"RUNDLL32.EXE\" */ )]\n", "risk_score": 21, "rule_id": "2b347f66-6739-4ae3-bd94-195036dde8b3", "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_s3_bucket_configuration_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_s3_bucket_configuration_deletion.json index adbe310b784e5..6cfdf40781397 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_s3_bucket_configuration_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_s3_bucket_configuration_deletion.json @@ -28,11 +28,12 @@ "rule_id": "227dc608-e558-43d9-b521-150772250bae", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Asset Visibility", - "Continuous Monitoring" + "Asset Visibility" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_sdelete_like_filename_rename.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_sdelete_like_filename_rename.json index ec3030d44ff29..6fea9a3c78945 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_sdelete_like_filename_rename.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_sdelete_like_filename_rename.json @@ -18,7 +18,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_managedcode_host_process.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_managedcode_host_process.json index 1bb3f26c0298f..85eac81808a71 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_managedcode_host_process.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_managedcode_host_process.json @@ -20,7 +20,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_suspicious_okta_user_password_reset_or_unlock_attempts.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_okta_user_password_reset_or_unlock_attempts.json similarity index 99% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_suspicious_okta_user_password_reset_or_unlock_attempts.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_okta_user_password_reset_or_unlock_attempts.json index 5f115416fa032..fedeaca68ab64 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_suspicious_okta_user_password_reset_or_unlock_attempts.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_okta_user_password_reset_or_unlock_attempts.json @@ -25,6 +25,7 @@ "severity": "medium", "tags": [ "Elastic", + "Identity", "Okta", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/evasion_suspicious_scrobj_load.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_scrobj_load.json similarity index 51% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/evasion_suspicious_scrobj_load.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_scrobj_load.json index 7880b86533b53..16364f590cd0e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/evasion_suspicious_scrobj_load.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_scrobj_load.json @@ -11,13 +11,16 @@ "language": "eql", "license": "Elastic License", "name": "Windows Suspicious Script Object Execution", - "query": "sequence by process.entity_id with maxspan=2m\n [process where event.type in (\"start\", \"process_started\") and\n /* process.code_signature.* fields need to be populated for 7.10 */\n process.code_signature.subject_name == \"Microsoft Corporation\" and process.code_signature.trusted == true and\n process.name not in (\"cscript.exe\",\n \"iexplore.exe\",\n \"MicrosoftEdge.exe\",\n \"msiexec.exe\",\n \"smartscreen.exe\",\n \"taskhostw.exe\",\n \"w3wp.exe\",\n \"wscript.exe\")]\n [library where event.type == \"start\" and file.name == \"scrobj.dll\"]\n", + "query": "/* add winlogbeat-* when process.code_signature.* fields are populated */\n\nsequence by process.entity_id with maxspan=2m\n [process where event.type in (\"start\", \"process_started\") and\n /* uncomment once in winlogbeat */\n /* process.code_signature.subject_name == \"Microsoft Corporation\" and process.code_signature.trusted == true and */\n not (process.name : \"cscript.exe\" or\n process.name : \"iexplore.exe\" or\n process.name : \"MicrosoftEdge.exe\" or\n process.name : \"msiexec.exe\" or\n process.name : \"smartscreen.exe\" or\n process.name : \"taskhostw.exe\" or\n process.name : \"w3wp.exe\" or\n process.name : \"wscript.exe\")]\n [library where event.type == \"start\" and file.name : \"scrobj.dll\"]\n", "risk_score": 21, "rule_id": "4ed678a9-3a4f-41fb-9fea-f85a6e0a0dff", "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/evasion_suspicious_wmi_script.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_wmi_script.json similarity index 59% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/evasion_suspicious_wmi_script.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_wmi_script.json index 943471f5801c2..e922416264359 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/evasion_suspicious_wmi_script.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_wmi_script.json @@ -5,19 +5,21 @@ "description": "Identifies WMIC whitelisting bypass techniques by alerting on suspicious execution of scripts. When WMIC loads scripting libraries it may be indicative of a whitelist bypass.", "from": "now-9m", "index": [ - "logs-endpoint.events.*", - "winlogbeat-*" + "logs-endpoint.events.*" ], "language": "eql", "license": "Elastic License", "name": "Suspicious WMIC XSL Script Execution", - "query": "/* lots of wildcards in the args\n need to verify args cleanup is accurate\n*/\n\nsequence by process.entity_id with maxspan=2m\n[process where event.type in (\"start\", \"process_started\") and\n (process.name == \"wmic.exe\" or process.pe.original_file_name == \"wmic.exe\") and\n wildcard(process.args, \"format*:*\", \"/format*:*\", \"*-format*:*\") and\n not process.args in (\"/format:table\", \"/format:table\") or wildcard(process.args, \"format*:*\")]\n[library where event.type == \"start\" and file.name in (\"jscript.dll\", \"vbscript.dll\")]\n", + "query": "sequence by process.entity_id with maxspan=2m\n[process where event.type in (\"start\", \"process_started\") and\n (process.name : \"WMIC.exe\" or process.pe.original_file_name == \"wmic.exe\") and\n wildcard(process.args, \"format*:*\", \"/format*:*\", \"*-format*:*\") and\n not wildcard(process.command_line, \"* /format:table *\")]\n[library where event.type == \"start\" and file.name in (\"jscript.dll\", \"vbscript.dll\")]\n", "risk_score": 21, "rule_id": "7f370d54-c0eb-4270-ac5a-9a6020585dc6", "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_zoom_child_process.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_zoom_child_process.json index 7b08f5a565424..f3c20e5251184 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_zoom_child_process.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_zoom_child_process.json @@ -20,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_system_critical_proc_abnormal_file_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_system_critical_proc_abnormal_file_activity.json index 6fea3a75c8e62..31702332d7cd4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_system_critical_proc_abnormal_file_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_system_critical_proc_abnormal_file_activity.json @@ -17,7 +17,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_unusual_system_vp_child_program.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_unusual_system_vp_child_program.json index 4efec948f49a7..65439d6d653b6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_unusual_system_vp_child_program.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_unusual_system_vp_child_program.json @@ -17,7 +17,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_via_filter_manager.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_via_filter_manager.json index 210e9c778afef..3092ea653aa4b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_via_filter_manager.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_via_filter_manager.json @@ -4,7 +4,8 @@ ], "description": "The Filter Manager Control Program (fltMC.exe) binary may be abused by adversaries to unload a filter driver and evade defenses.", "index": [ - "winlogbeat-*" + "winlogbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -15,7 +16,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_volume_shadow_copy_deletion_via_vssadmin.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_volume_shadow_copy_deletion_via_vssadmin.json index 8a504281b03f7..493ee919000dc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_volume_shadow_copy_deletion_via_vssadmin.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_volume_shadow_copy_deletion_via_vssadmin.json @@ -17,7 +17,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_volume_shadow_copy_deletion_via_wmic.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_volume_shadow_copy_deletion_via_wmic.json index 2ae938bb34104..49a68f4bfcf66 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_volume_shadow_copy_deletion_via_wmic.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_volume_shadow_copy_deletion_via_wmic.json @@ -17,7 +17,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_waf_acl_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_waf_acl_deletion.json index 32101029fb107..86ed9e2f9c042 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_waf_acl_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_waf_acl_deletion.json @@ -25,11 +25,12 @@ "rule_id": "91d04cd4-47a9-4334-ab14-084abe274d49", "severity": "medium", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Network", - "Continuous Monitoring" + "Network Security" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_waf_rule_or_rule_group_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_waf_rule_or_rule_group_deletion.json index a08c05d0d6ca7..302e89d416f4b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_waf_rule_or_rule_group_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_waf_rule_or_rule_group_deletion.json @@ -25,11 +25,12 @@ "rule_id": "5beaebc1-cc13-4bfc-9949-776f9e0dc318", "severity": "medium", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Network", - "Continuous Monitoring" + "Network Security" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_blob_container_access_mod.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_blob_container_access_mod.json index 7e601c9928d08..16db02338de55 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_blob_container_access_mod.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_blob_container_access_mod.json @@ -23,9 +23,10 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "Azure", - "SecOps", "Continuous Monitoring", + "SecOps", "Asset Visibility" ], "threat": [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_kernel_module_enumeration.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_kernel_module_enumeration.json index af9c4b5409964..f2681f5501af0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_kernel_module_enumeration.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_kernel_module_enumeration.json @@ -20,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Discovery" ], "threat": [ { @@ -40,5 +43,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_net_command_system_account.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_net_command_system_account.json index 96c300cfde016..682434f28f745 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_net_command_system_account.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_net_command_system_account.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Discovery" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_post_exploitation_public_ip_reconnaissance.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_post_exploitation_public_ip_reconnaissance.json index 952d70ee3589a..36e95cf4ade41 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_post_exploitation_public_ip_reconnaissance.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_post_exploitation_public_ip_reconnaissance.json @@ -24,8 +24,8 @@ "tags": [ "Elastic", "Network", - "Threat Detection, Preventing and Hunting", - "Post-Execution" + "Threat Detection", + "Discovery" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_process_discovery_via_tasklist_command.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_process_discovery_via_tasklist_command.json index c2d95de4129f9..59ddc00d6525d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_process_discovery_via_tasklist_command.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_process_discovery_via_tasklist_command.json @@ -7,7 +7,8 @@ "Administrators may use the tasklist command to display a list of currently running processes. By itself, it does not indicate malicious activity. After obtaining a foothold, it's possible adversaries may use discovery commands like tasklist to get information about running processes." ], "index": [ - "winlogbeat-*" + "winlogbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -18,7 +19,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Discovery" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_virtual_machine_fingerprinting.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_virtual_machine_fingerprinting.json index d913a92e2ee0e..66b744667f568 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_virtual_machine_fingerprinting.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_virtual_machine_fingerprinting.json @@ -20,7 +20,10 @@ "severity": "high", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Discovery" ], "threat": [ { @@ -40,5 +43,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_whoami_command_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_whoami_command_activity.json index cb330879be9b3..2e577e44c960e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_whoami_command_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_whoami_command_activity.json @@ -7,7 +7,8 @@ "Some normal use of this program, at varying levels of frequency, may originate from scripts, automation tools and frameworks. Usage by non-engineers and ordinary users is unusual." ], "index": [ - "winlogbeat-*" + "winlogbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -18,7 +19,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Discovery" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_whoami_commmand.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_whoami_commmand.json index a8b34362d9579..ddf309b377609 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_whoami_commmand.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_whoami_commmand.json @@ -20,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Discovery" ], "threat": [ { @@ -40,5 +43,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/elastic_endpoint.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/elastic_endpoint.json index f3acc5d3a2b5e..d6e7419391f9c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/elastic_endpoint.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/elastic_endpoint.json @@ -60,7 +60,7 @@ ], "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "timestamp_override": "event.ingested", "type": "query", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_adversary_behavior_detected.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_adversary_behavior_detected.json index b3bac305bc1f1..8084067b3a6d2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_adversary_behavior_detected.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_adversary_behavior_detected.json @@ -17,7 +17,7 @@ "severity": "medium", "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "type": "query", "version": 4 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_dumping_detected.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_dumping_detected.json index 2f91c1fe813f9..9c28d065b322d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_dumping_detected.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_dumping_detected.json @@ -17,7 +17,7 @@ "severity": "high", "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "type": "query", "version": 4 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_dumping_prevented.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_dumping_prevented.json index 75488c2d3a5ed..352712e38f42d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_dumping_prevented.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_dumping_prevented.json @@ -17,7 +17,7 @@ "severity": "medium", "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "type": "query", "version": 4 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_manipulation_detected.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_manipulation_detected.json index adc29d9106774..259bcd51aeb3e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_manipulation_detected.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_manipulation_detected.json @@ -17,7 +17,7 @@ "severity": "high", "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "type": "query", "version": 4 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_manipulation_prevented.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_manipulation_prevented.json index 99def69978a48..19348062b10f1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_manipulation_prevented.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_manipulation_prevented.json @@ -17,7 +17,7 @@ "severity": "medium", "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "type": "query", "version": 4 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_exploit_detected.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_exploit_detected.json index 80eb3ce637f30..2fd3aaa0d8a57 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_exploit_detected.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_exploit_detected.json @@ -17,7 +17,7 @@ "severity": "high", "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "type": "query", "version": 4 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_exploit_prevented.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_exploit_prevented.json index 50444904654de..8f90e1162546b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_exploit_prevented.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_exploit_prevented.json @@ -17,7 +17,7 @@ "severity": "medium", "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "type": "query", "version": 4 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_malware_detected.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_malware_detected.json index bb2ddf92a83e7..3d740f8b7064f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_malware_detected.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_malware_detected.json @@ -17,7 +17,7 @@ "severity": "critical", "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "type": "query", "version": 4 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_malware_prevented.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_malware_prevented.json index fae8a3a0ab5a6..33195c7fcbecc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_malware_prevented.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_malware_prevented.json @@ -17,7 +17,7 @@ "severity": "high", "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "type": "query", "version": 4 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_permission_theft_detected.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_permission_theft_detected.json index 821c3b0d8a63b..fac13a6d358dd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_permission_theft_detected.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_permission_theft_detected.json @@ -17,7 +17,7 @@ "severity": "high", "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "type": "query", "version": 4 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_permission_theft_prevented.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_permission_theft_prevented.json index e38afe19e7d38..a2d8700076c23 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_permission_theft_prevented.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_permission_theft_prevented.json @@ -17,7 +17,7 @@ "severity": "medium", "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "type": "query", "version": 4 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_process_injection_detected.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_process_injection_detected.json index 52eb3c2d96bf7..ef4f29067b0c5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_process_injection_detected.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_process_injection_detected.json @@ -17,7 +17,7 @@ "severity": "high", "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "type": "query", "version": 4 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_process_injection_prevented.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_process_injection_prevented.json index 76aff15e1588c..b22751e35c053 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_process_injection_prevented.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_process_injection_prevented.json @@ -17,7 +17,7 @@ "severity": "medium", "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "type": "query", "version": 4 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_ransomware_detected.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_ransomware_detected.json index 29efdd910904d..3b973f42bbca5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_ransomware_detected.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_ransomware_detected.json @@ -17,7 +17,7 @@ "severity": "critical", "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "type": "query", "version": 4 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_ransomware_prevented.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_ransomware_prevented.json index c603e503c5dad..b6458b73e8015 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_ransomware_prevented.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_ransomware_prevented.json @@ -17,7 +17,7 @@ "severity": "high", "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "type": "query", "version": 4 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/escalation_uac_sdclt.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/escalation_uac_sdclt.json deleted file mode 100644 index 843ba3401b4e4..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/escalation_uac_sdclt.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "author": [ - "Elastic" - ], - "description": "Identifies User Account Control (UAC) bypass via sdclt.exe. Attackers bypass UAC to stealthily execute code with elevated permissions.", - "from": "now-9m", - "index": [ - "logs-endpoint.events.*", - "winlogbeat-*" - ], - "language": "eql", - "license": "Elastic License", - "name": "Bypass UAC via Sdclt", - "query": "sequence with maxspan=1m\n [process where event.type in (\"start\", \"process_started\") and process.name == \"sdclt.exe\" and\n /* process.code_signature.* fields need to be populated for 7.10 */\n process.code_signature.subject_name == \"Microsoft Corporation\" and process.code_signature.trusted == true and\n process.args == \"/kickoffelev\"\n ] by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and process.parent.name == \"sdclt.exe\" and\n process.executable not in (\"C:\\\\Windows\\\\System32\\\\sdclt.exe\",\n \"C:\\\\Windows\\\\System32\\\\control.exe\",\n \"C:\\\\Windows\\\\SysWOW64\\\\sdclt.exe\",\n \"C:\\\\Windows\\\\SysWOW64\\\\control.exe\")\n ] by process.parent.entity_id\n", - "risk_score": 21, - "rule_id": "9b54e002-034a-47ac-9307-ad12c03fa900", - "severity": "high", - "tags": [ - "Elastic", - "Windows" - ], - "threat": [ - { - "framework": "MITRE ATT&CK", - "tactic": { - "id": "TA0004", - "name": "Privilege Escalation", - "reference": "https://attack.mitre.org/tactics/TA0004/" - }, - "technique": [ - { - "id": "T1088", - "name": "Bypass User Account Control", - "reference": "https://attack.mitre.org/techniques/T1088/" - } - ] - } - ], - "type": "eql", - "version": 1 -} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_prompt_connecting_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_prompt_connecting_to_the_internet.json index e9989fe50019e..d73b1a4cab008 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_prompt_connecting_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_prompt_connecting_to_the_internet.json @@ -20,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_shell_started_by_powershell.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_shell_started_by_powershell.json index bb252b1416832..220a7f94dce9a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_shell_started_by_powershell.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_shell_started_by_powershell.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_shell_started_by_svchost.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_shell_started_by_svchost.json index aeae5518fece1..d6b1e67b11982 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_shell_started_by_svchost.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_shell_started_by_svchost.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_shell_started_by_unusual_process.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_shell_started_by_unusual_process.json index 577bee1ffe6de..566a5963c122a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_shell_started_by_unusual_process.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_shell_started_by_unusual_process.json @@ -17,7 +17,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_virtual_machine.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_virtual_machine.json index 5e7852e1c1b13..31c4d488c6960 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_virtual_machine.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_virtual_machine.json @@ -24,11 +24,12 @@ "rule_id": "60884af6-f553-4a6c-af13-300047455491", "severity": "medium", "tags": [ - "Azure", "Elastic", - "SecOps", + "Cloud", + "Azure", "Continuous Monitoring", - "Logging" + "SecOps", + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_html_help_executable_program_connecting_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_html_help_executable_program_connecting_to_the_internet.json index 963c6b2e53ed6..d33f2287c7d8b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_html_help_executable_program_connecting_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_html_help_executable_program_connecting_to_the_internet.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { @@ -52,5 +55,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_local_service_commands.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_local_service_commands.json index 7b20cefdc67f0..693ca83e387b3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_local_service_commands.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_local_service_commands.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_ms_office_written_file.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_ms_office_written_file.json index 35d7a7c969ee7..2db46080a4e75 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_ms_office_written_file.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_ms_office_written_file.json @@ -11,13 +11,16 @@ "language": "eql", "license": "Elastic License", "name": "Execution of File Written or Modified by Microsoft Office", - "query": "sequence with maxspan=2h\n [file where event.type != \"delete\" and file.extension == \"exe\" and\n process.name in (\"winword.exe\",\n \"excel.exe\",\n \"outlook.exe\",\n \"powerpnt.exe\",\n \"eqnedt32.exe\",\n \"fltldr.exe\",\n \"mspub.exe\",\n \"msaccess.exe\")\n ] by host.id, file.path\n [process where event.type in (\"start\", \"process_started\")] by host.id, process.executable\n", + "query": "sequence with maxspan=2h\n [file where event.type != \"deletion\" and file.extension : \"exe\" and\n (process.name : \"WINWORD.EXE\" or\n process.name : \"EXCEL.EXE\" or\n process.name : \"OUTLOOK.EXE\" or\n process.name : \"POWERPNT.EXE\" or\n process.name : \"eqnedt32.exe\" or\n process.name : \"fltldr.exe\" or\n process.name : \"MSPUB.EXE\" or\n process.name : \"MSACCESS.EXE\")\n ] by host.id, file.path\n [process where event.type in (\"start\", \"process_started\")] by host.id, process.executable\n", "risk_score": 21, "rule_id": "0d8ad79f-9025-45d8-80c1-4f0cd3c5e8e5", "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_msbuild_making_network_connections.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_msbuild_making_network_connections.json index fcbbfbdb3d686..7fd2933fe46f1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_msbuild_making_network_connections.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_msbuild_making_network_connections.json @@ -17,7 +17,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_mshta_making_network_connections.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_mshta_making_network_connections.json index 7af823070889f..9b863c2ed5ee4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_mshta_making_network_connections.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_mshta_making_network_connections.json @@ -20,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { @@ -40,5 +43,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_msxsl_network.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_msxsl_network.json index 1dc75575636fb..17987218af0ae 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_msxsl_network.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_msxsl_network.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_pdf_written_file.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_pdf_written_file.json index 3963b3d594902..a807052cf7b0d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_pdf_written_file.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_pdf_written_file.json @@ -11,13 +11,16 @@ "language": "eql", "license": "Elastic License", "name": "Execution of File Written or Modified by PDF Reader", - "query": "sequence with maxspan=2h\n [file where event.type != \"delete\" and file.extension == \"exe\" and\n process.name in (\"acrord32.exe\", \"rdrcef.exe\", \"foxitphantomPDF.exe\", \"foxitreader.exe\") and\n file.name not in (\"foxitphantomPDF.exe\",\n \"FoxitPhantomPDFUpdater.exe\",\n \"foxitreader.exe\",\n \"FoxitReaderUpdater.exe\",\n \"acrord32.exe\",\n \"rdrcef.exe\")\n ] by host.id, file.path\n [process where event.type in (\"start\", \"process_started\")] by host.id, process.executable\n", + "query": "sequence with maxspan=2h\n [file where event.type != \"deletion\" and file.extension : \"exe\" and\n (process.name : \"AcroRd32.exe\" or\n process.name : \"rdrcef.exe\" or\n process.name : \"FoxitPhantomPDF.exe\" or\n process.name : \"FoxitReader.exe\") and\n not (file.name : \"FoxitPhantomPDF.exe\" or\n file.name : \"FoxitPhantomPDFUpdater.exe\" or\n file.name : \"FoxitReader.exe\" or\n file.name : \"FoxitReaderUpdater.exe\" or\n file.name : \"AcroRd32.exe\" or\n file.name : \"rdrcef.exe\")\n ] by host.id, file.path\n [process where event.type in (\"start\", \"process_started\")] by host.id, process.executable\n", "risk_score": 21, "rule_id": "1defdd62-cd8d-426e-a246-81a37751bb2b", "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_perl_tty_shell.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_perl_tty_shell.json index 4502f42bbb4c4..ee6cee941bef2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_perl_tty_shell.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_perl_tty_shell.json @@ -17,7 +17,10 @@ "severity": "high", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_psexec_lateral_movement_command.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_psexec_lateral_movement_command.json index f647d8d00e084..8505a837ad591 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_psexec_lateral_movement_command.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_psexec_lateral_movement_command.json @@ -20,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { @@ -55,5 +58,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_python_tty_shell.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_python_tty_shell.json index 0e8b5f0218d00..151c3fbb8df46 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_python_tty_shell.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_python_tty_shell.json @@ -17,7 +17,10 @@ "severity": "high", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_register_server_program_connecting_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_register_server_program_connecting_to_the_internet.json index 899bb1c20e711..bf1a30f11137e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_register_server_program_connecting_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_register_server_program_connecting_to_the_internet.json @@ -20,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_script_executing_powershell.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_script_executing_powershell.json index 6d7f11f01fae0..843cf322e5849 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_script_executing_powershell.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_script_executing_powershell.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_ms_office_child_process.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_ms_office_child_process.json index 005a0c38c8a8b..90c60ceea37ab 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_ms_office_child_process.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_ms_office_child_process.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_ms_outlook_child_process.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_ms_outlook_child_process.json index 74e21c7d17479..d5ee8fa818367 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_ms_outlook_child_process.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_ms_outlook_child_process.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_pdf_reader.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_pdf_reader.json index adf1a76bfb901..bf37f5a803e58 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_pdf_reader.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_pdf_reader.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_suspicious_psexesvc.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_psexesvc.json similarity index 76% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_suspicious_psexesvc.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_psexesvc.json index 51396fb5995f6..205b5148f2fb4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_suspicious_psexesvc.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_psexesvc.json @@ -11,13 +11,16 @@ "language": "kuery", "license": "Elastic License", "name": "Suspicious Process Execution via Renamed PsExec Executable", - "query": "event.category:process and event.type:(start or process_started) and process.pe.original_file_name:(psexesvc.exe or PSEXESVC.exe) and process.parent.name:services.exe and not process.name:(psexesvc.exe or PSEXESVC.exe)", + "query": "event.category:process and event.type:(start or process_started) and (process.pe.original_file_name:(psexesvc.exe or PSEXESVC.exe) or winlog.event_data.OriginalFileName:(psexesvc.exe or PSEXESVC.exe)) and process.parent.name:services.exe and not process.name:(psexesvc.exe or PSEXESVC.exe)", "risk_score": 47, "rule_id": "e2f9fdf5-8076-45ad-9427-41e0e03dc9c2", "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_dns_service_children.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_dns_service_children.json index 96305b2197bfc..52e67b0c7bcff 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_dns_service_children.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_dns_service_children.json @@ -7,7 +7,8 @@ "Werfault.exe will legitimately spawn when dns.exe crashes, but the DNS service is very stable and so this is a low occurring event. Denial of Service (DoS) attempts by intentionally crashing the service will also cause werfault.exe to spawn." ], "index": [ - "winlogbeat-*" + "winlogbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -24,7 +25,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_dns_service_file_writes.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_dns_service_file_writes.json index c175ecbfa78b6..229fc28beee9c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_dns_service_file_writes.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_dns_service_file_writes.json @@ -4,7 +4,8 @@ ], "description": "Identifies an unexpected file being modified by dns.exe, the process responsible for Windows DNS Server services, which may indicate activity related to remote code execution or other forms of exploitation.", "index": [ - "winlogbeat-*" + "winlogbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -20,7 +21,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_network_connection_via_rundll32.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_network_connection_via_rundll32.json index 1104159350655..2763f69e1f8e8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_network_connection_via_rundll32.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_network_connection_via_rundll32.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 5 + "version": 6 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_process_network_connection.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_process_network_connection.json index 774e8e9189ced..877c489b0d187 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_process_network_connection.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_process_network_connection.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_compiled_html_file.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_compiled_html_file.json index fe3e110830420..276e5c18335f5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_compiled_html_file.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_compiled_html_file.json @@ -7,7 +7,8 @@ "The HTML Help executable program (hh.exe) runs whenever a user clicks a compiled help (.chm) file or menu item that opens the help file inside the Help Viewer. This is not always malicious, but adversaries may abuse this technology to conceal malicious code." ], "index": [ - "winlogbeat-*" + "winlogbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -18,7 +19,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_hidden_shell_conhost.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_hidden_shell_conhost.json index 7fbf962469f71..b6f6502e603aa 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_hidden_shell_conhost.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_hidden_shell_conhost.json @@ -20,7 +20,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_net_com_assemblies.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_net_com_assemblies.json index d9dcbfe25a4c2..0a21599c31a4a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_net_com_assemblies.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_net_com_assemblies.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { @@ -52,5 +55,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_system_manager.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_system_manager.json index 081ebcb518999..13493a90e3e50 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_system_manager.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_system_manager.json @@ -24,11 +24,12 @@ "rule_id": "37b211e8-4e2f-440f-86d8-06cc8f158cfa", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Logging", - "Continuous Monitoring" + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_xp_cmdshell_mssql_stored_procedure.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_xp_cmdshell_mssql_stored_procedure.json index 8769e641fad90..91097b2050d17 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_xp_cmdshell_mssql_stored_procedure.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_xp_cmdshell_mssql_stored_procedure.json @@ -17,7 +17,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_wpad_exploitation.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_wpad_exploitation.json index 03c4482b60340..a09bce5119ecf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_wpad_exploitation.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_wpad_exploitation.json @@ -5,19 +5,21 @@ "description": "Identifies probable exploitation of the Web Proxy Auto-Discovery Protocol (WPAD) service. Attackers who have access to the local network or upstream DNS traffic can inject malicious JavaScript to the WPAD service which can lead to a full system compromise.", "from": "now-9m", "index": [ - "logs-endpoint.events.*", - "winlogbeat-*" + "logs-endpoint.events.*" ], "language": "eql", "license": "Elastic License", "name": "WPAD Service Exploit", - "query": "/* preference would be to use user.sid rather than domain+name, once it is available in ECS + datasources */\n\nsequence with maxspan=5s\n [process where event.type in (\"start\", \"process_started\") and process.name == \"svchost.exe\" and\n user.domain == \"NT AUTHORITY\" and user.name == \"LOCAL SERVICE\"] by process.entity_id\n [network where network.protocol == \"dns\" and process.name == \"svchost.exe\" and\n dns.question.name == \"wpad\" and process.name == \"svchost.exe\"] by process.entity_id\n [network where event.type == \"connection\" and process.name == \"svchost.exe\"\n and network.direction == \"outgoing\" and destination.port == 80] by process.entity_id\n [library where event.type == \"start\" and process.name == \"svchost.exe\" and\n file.name == \"jscript.dll\" and process.name == \"svchost.exe\"] by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and\n process.parent.name == \"svchost.exe\"] by process.parent.entity_id\n", + "query": "/* preference would be to use user.sid rather than domain+name, once it is available in ECS + datasources */\n\nsequence with maxspan=5s\n [process where event.type in (\"start\", \"process_started\") and process.name : \"svchost.exe\" and\n user.domain == \"NT AUTHORITY\" and user.name == \"LOCAL SERVICE\"] by process.entity_id\n [network where network.protocol == \"dns\" and process.name : \"svchost.exe\" and\n dns.question.name : \"wpad\" and process.name : \"svchost.exe\"] by process.entity_id\n [network where event.type == \"connection\" and process.name : \"svchost.exe\"\n and network.direction == \"outgoing\" and destination.port == 80] by process.entity_id\n [library where event.type == \"start\" and process.name : \"svchost.exe\" and\n file.name : \"jscript.dll\" and process.name : \"svchost.exe\"] by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and\n process.parent.name : \"svchost.exe\"] by process.parent.entity_id\n", "risk_score": 21, "rule_id": "ec328da1-d5df-482b-866c-4a435692b1f3", "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/exfiltration_ec2_snapshot_change_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/exfiltration_ec2_snapshot_change_activity.json index fc18a516be0f4..abf64d55ab99a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/exfiltration_ec2_snapshot_change_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/exfiltration_ec2_snapshot_change_activity.json @@ -25,11 +25,12 @@ "rule_id": "98fd7407-0bd5-5817-cda0-3fcc33113a56", "severity": "medium", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Asset Visibility", - "Continuous Monitoring" + "Asset Visibility" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/exfiltration_gcp_logging_sink_modification.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/exfiltration_gcp_logging_sink_modification.json index 4e8954c3441cd..5e3cc2da2f871 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/exfiltration_gcp_logging_sink_modification.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/exfiltration_gcp_logging_sink_modification.json @@ -22,10 +22,11 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", - "Logging" + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/external_alerts.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/external_alerts.json index 6bc14f4e5af8a..a8a86348a1019 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/external_alerts.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/external_alerts.json @@ -54,9 +54,14 @@ } ], "tags": [ - "Elastic" + "Elastic", + "Network", + "Windows", + "APM", + "macOS", + "Linux" ], "timestamp_override": "event.ingested", "type": "query", - "version": 1 + "version": 2 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_attempt_to_revoke_okta_api_token.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_attempt_to_revoke_okta_api_token.json index f9d71a2e1cbff..f2ad30fa26020 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_attempt_to_revoke_okta_api_token.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_attempt_to_revoke_okta_api_token.json @@ -24,10 +24,11 @@ "severity": "low", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Monitoring", - "Continuous Monitoring" + "Monitoring" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_azure_automation_runbook_deleted.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_azure_automation_runbook_deleted.json index 662709774f5ba..f474357cc6e2c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_azure_automation_runbook_deleted.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_azure_automation_runbook_deleted.json @@ -23,6 +23,7 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "Azure", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_cloudtrail_logging_updated.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_cloudtrail_logging_updated.json index c648ae1ea4b5b..d7f4c2b19bc0f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_cloudtrail_logging_updated.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_cloudtrail_logging_updated.json @@ -25,11 +25,12 @@ "rule_id": "3e002465-876f-4f04-b016-84ef48ce7e5d", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Logging", - "Continuous Monitoring" + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_cloudwatch_log_group_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_cloudwatch_log_group_deletion.json index f7e9077c14314..ae978f6564d67 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_cloudwatch_log_group_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_cloudwatch_log_group_deletion.json @@ -25,11 +25,12 @@ "rule_id": "68a7a5a5-a2fc-4a76-ba9f-26849de881b4", "severity": "medium", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Logging", - "Continuous Monitoring" + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_cloudwatch_log_stream_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_cloudwatch_log_stream_deletion.json index b50efb21e42f0..7b985fdb6f693 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_cloudwatch_log_stream_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_cloudwatch_log_stream_deletion.json @@ -25,11 +25,12 @@ "rule_id": "d624f0ae-3dd1-4856-9aad-ccfe4d4bfa17", "severity": "medium", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Logging", - "Continuous Monitoring" + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_ec2_disable_ebs_encryption.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_ec2_disable_ebs_encryption.json index 370a65c31e7c1..c60619e894717 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_ec2_disable_ebs_encryption.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_ec2_disable_ebs_encryption.json @@ -26,11 +26,12 @@ "rule_id": "bb9b13b2-1700-48a8-a750-b43b0a72ab69", "severity": "medium", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Data Protection", - "Continuous Monitoring" + "Data Protection" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_iam_role_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_iam_role_deletion.json index 2c67be7408d1d..09a9996680155 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_iam_role_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_iam_role_deletion.json @@ -22,6 +22,7 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_service_account_deleted.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_service_account_deleted.json index 2aa702e5ca4d1..9d34f31c1700d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_service_account_deleted.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_service_account_deleted.json @@ -22,6 +22,7 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_service_account_disabled.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_service_account_disabled.json index 9b5188f43633d..606ebd1e6128e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_service_account_disabled.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_service_account_disabled.json @@ -22,6 +22,7 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_storage_bucket_deleted.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_storage_bucket_deleted.json index 6adad4b687de7..859c59ff8a325 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_storage_bucket_deleted.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_storage_bucket_deleted.json @@ -22,6 +22,7 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_virtual_private_cloud_network_deleted.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_virtual_private_cloud_network_deleted.json index c5dc9f25f893f..7f702f11a9515 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_virtual_private_cloud_network_deleted.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_virtual_private_cloud_network_deleted.json @@ -22,6 +22,7 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_virtual_private_cloud_route_created.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_virtual_private_cloud_route_created.json index 5e8fea09befc4..1da90189f96b8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_virtual_private_cloud_route_created.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_virtual_private_cloud_route_created.json @@ -23,6 +23,7 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_virtual_private_cloud_route_deleted.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_virtual_private_cloud_route_deleted.json index 8482e0efbb036..c379f07f021a6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_virtual_private_cloud_route_deleted.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_virtual_private_cloud_route_deleted.json @@ -23,6 +23,7 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_hosts_file_modified.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_hosts_file_modified.json index bf04626dee277..5d7e0bec4332c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_hosts_file_modified.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_hosts_file_modified.json @@ -22,9 +22,12 @@ "severity": "medium", "tags": [ "Elastic", + "Host", "Linux", "Windows", - "macOS" + "macOS", + "Threat Detection", + "Impact" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_iam_deactivate_mfa_device.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_iam_deactivate_mfa_device.json index ca5cfd4ae596b..83e184d9bf805 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_iam_deactivate_mfa_device.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_iam_deactivate_mfa_device.json @@ -25,11 +25,12 @@ "rule_id": "d8fc1cca-93ed-43c1-bbb6-c0dd3eff2958", "severity": "medium", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Monitoring", - "Continuous Monitoring" + "Monitoring" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_iam_group_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_iam_group_deletion.json index 00a10772d4d9a..6212aa70747a5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_iam_group_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_iam_group_deletion.json @@ -25,11 +25,12 @@ "rule_id": "867616ec-41e5-4edc-ada2-ab13ab45de8a", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Monitoring", - "Continuous Monitoring" + "Monitoring" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_possible_okta_dos_attack.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_possible_okta_dos_attack.json index 9bc44bf4e6da9..d1852478c666f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_possible_okta_dos_attack.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_possible_okta_dos_attack.json @@ -21,10 +21,11 @@ "severity": "medium", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Monitoring", - "Continuous Monitoring" + "Monitoring" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_rds_cluster_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_rds_cluster_deletion.json index 829c244dd45c0..116483173564e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_rds_cluster_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_rds_cluster_deletion.json @@ -27,11 +27,12 @@ "rule_id": "9055ece6-2689-4224-a0e0-b04881e1f8ad", "severity": "medium", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Asset Visibility", - "Continuous Monitoring" + "Asset Visibility" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_rds_instance_cluster_stoppage.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_rds_instance_cluster_stoppage.json index 68459d0e777b9..3227649149380 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_rds_instance_cluster_stoppage.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_rds_instance_cluster_stoppage.json @@ -27,11 +27,12 @@ "rule_id": "ecf2b32c-e221-4bd4-aa3b-c7d59b3bc01d", "severity": "medium", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Asset Visibility", - "Continuous Monitoring" + "Asset Visibility" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_resource_group_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_resource_group_deletion.json index a0c56c19b964e..8086c09e4b174 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_resource_group_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_resource_group_deletion.json @@ -23,10 +23,11 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "Azure", - "SecOps", "Continuous Monitoring", - "Logging" + "SecOps", + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/index.ts index 6e376930617de..5fec97e83bad4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/index.ts @@ -221,14 +221,14 @@ import rule209 from './credential_access_okta_brute_force_or_password_spraying.j import rule210 from './execution_unusual_dns_service_children.json'; import rule211 from './execution_unusual_dns_service_file_writes.json'; import rule212 from './lateral_movement_dns_server_overflow.json'; -import rule213 from './initial_access_root_console_failure_brute_force.json'; +import rule213 from './credential_access_root_console_failure_brute_force.json'; import rule214 from './initial_access_unsecure_elasticsearch_node.json'; import rule215 from './credential_access_domain_backup_dpapi_private_keys.json'; -import rule216 from './lateral_movement_gpo_schtask_service_creation.json'; -import rule217 from './credential_access_kerberosdump_kcc.json'; -import rule218 from './defense_evasion_execution_suspicious_psexesvc.json'; -import rule219 from './execution_via_xp_cmdshell_mssql_stored_procedure.json'; -import rule220 from './exfiltration_compress_credentials_keychains.json'; +import rule216 from './persistence_gpo_schtask_service_creation.json'; +import rule217 from './credential_access_compress_credentials_keychains.json'; +import rule218 from './credential_access_kerberosdump_kcc.json'; +import rule219 from './execution_suspicious_psexesvc.json'; +import rule220 from './execution_via_xp_cmdshell_mssql_stored_procedure.json'; import rule221 from './privilege_escalation_printspooler_service_suspicious_file.json'; import rule222 from './privilege_escalation_printspooler_suspicious_spl_file.json'; import rule223 from './defense_evasion_azure_diagnostic_settings_deletion.json'; @@ -250,16 +250,16 @@ import rule238 from './persistence_azure_automation_webhook_created.json'; import rule239 from './privilege_escalation_uac_bypass_diskcleanup_hijack.json'; import rule240 from './credential_access_attempts_to_brute_force_okta_user_account.json'; import rule241 from './credential_access_storage_account_key_regenerated.json'; -import rule242 from './credential_access_suspicious_okta_user_password_reset_or_unlock_attempts.json'; +import rule242 from './defense_evasion_suspicious_okta_user_password_reset_or_unlock_attempts.json'; import rule243 from './defense_evasion_system_critical_proc_abnormal_file_activity.json'; import rule244 from './defense_evasion_unusual_system_vp_child_program.json'; -import rule245 from './defense_evasion_mfa_disabled_for_azure_user.json'; -import rule246 from './discovery_blob_container_access_mod.json'; +import rule245 from './discovery_blob_container_access_mod.json'; +import rule246 from './persistence_mfa_disabled_for_azure_user.json'; import rule247 from './persistence_user_added_as_owner_for_azure_application.json'; import rule248 from './persistence_user_added_as_owner_for_azure_service_principal.json'; -import rule249 from './defense_evasion_suspicious_managedcode_host_process.json'; -import rule250 from './execution_command_shell_started_by_unusual_process.json'; -import rule251 from './execution_suspicious_dotnet_compiler_parent_process.json'; +import rule249 from './defense_evasion_dotnet_compiler_parent_process.json'; +import rule250 from './defense_evasion_suspicious_managedcode_host_process.json'; +import rule251 from './execution_command_shell_started_by_unusual_process.json'; import rule252 from './defense_evasion_masquerading_as_elastic_endpoint_process.json'; import rule253 from './defense_evasion_masquerading_suspicious_werfault_childproc.json'; import rule254 from './defense_evasion_masquerading_werfault.json'; @@ -268,28 +268,28 @@ import rule256 from './credential_access_mimikatz_memssp_default_logs.json'; import rule257 from './defense_evasion_code_injection_conhost.json'; import rule258 from './defense_evasion_network_watcher_deletion.json'; import rule259 from './initial_access_external_guest_user_invite.json'; -import rule260 from './defense_evasion_azure_conditional_access_policy_modified.json'; -import rule261 from './defense_evasion_azure_privileged_identity_management_role_modified.json'; -import rule262 from './defense_evasion_masquerading_renamed_autoit.json'; -import rule263 from './impact_azure_automation_runbook_deleted.json'; -import rule264 from './initial_access_consent_grant_attack_via_azure_registered_application.json'; -import rule265 from './c2_installutil_beacon.json'; -import rule266 from './c2_msbuild_beacon_sequence.json'; -import rule267 from './c2_mshta_beacon.json'; -import rule268 from './c2_msxsl_beacon.json'; -import rule269 from './c2_network_connection_from_windows_binary.json'; -import rule270 from './c2_reg_beacon.json'; -import rule271 from './c2_rundll32_sequence.json'; -import rule272 from './command_and_control_teamviewer_remote_file_copy.json'; -import rule273 from './escalation_uac_sdclt.json'; -import rule274 from './evasion_rundll32_no_arguments.json'; -import rule275 from './evasion_suspicious_scrobj_load.json'; -import rule276 from './evasion_suspicious_wmi_script.json'; -import rule277 from './execution_ms_office_written_file.json'; -import rule278 from './execution_pdf_written_file.json'; -import rule279 from './execution_wpad_exploitation.json'; -import rule280 from './lateral_movement_cmd_service.json'; -import rule281 from './persistence_app_compat_shim.json'; +import rule260 from './defense_evasion_masquerading_renamed_autoit.json'; +import rule261 from './impact_azure_automation_runbook_deleted.json'; +import rule262 from './initial_access_consent_grant_attack_via_azure_registered_application.json'; +import rule263 from './persistence_azure_conditional_access_policy_modified.json'; +import rule264 from './persistence_azure_privileged_identity_management_role_modified.json'; +import rule265 from './command_and_control_teamviewer_remote_file_copy.json'; +import rule266 from './defense_evasion_installutil_beacon.json'; +import rule267 from './defense_evasion_msbuild_beacon_sequence.json'; +import rule268 from './defense_evasion_mshta_beacon.json'; +import rule269 from './defense_evasion_msxsl_beacon.json'; +import rule270 from './defense_evasion_network_connection_from_windows_binary.json'; +import rule271 from './defense_evasion_reg_beacon.json'; +import rule272 from './defense_evasion_rundll32_no_arguments.json'; +import rule273 from './defense_evasion_rundll32_sequence.json'; +import rule274 from './defense_evasion_suspicious_scrobj_load.json'; +import rule275 from './defense_evasion_suspicious_wmi_script.json'; +import rule276 from './execution_ms_office_written_file.json'; +import rule277 from './execution_pdf_written_file.json'; +import rule278 from './execution_wpad_exploitation.json'; +import rule279 from './lateral_movement_cmd_service.json'; +import rule280 from './persistence_app_compat_shim.json'; +import rule281 from './privilege_escalation_uac_sdclt.json'; import rule282 from './command_and_control_remote_file_copy_desktopimgdownldr.json'; import rule283 from './command_and_control_remote_file_copy_mpcmdrun.json'; import rule284 from './defense_evasion_execution_suspicious_explorer_winword.json'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_consent_grant_attack_via_azure_registered_application.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_consent_grant_attack_via_azure_registered_application.json index 8147859fa4e6f..1dab4e8df71b4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_consent_grant_attack_via_azure_registered_application.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_consent_grant_attack_via_azure_registered_application.json @@ -20,6 +20,7 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "Azure", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_console_login_root.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_console_login_root.json index 621881e264138..1d6f5b2c90f26 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_console_login_root.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_console_login_root.json @@ -24,11 +24,12 @@ "rule_id": "e2a67480-3b79-403d-96e3-fdd2992c50ef", "severity": "high", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Identity and Access", - "Continuous Monitoring" + "Identity and Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_external_guest_user_invite.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_external_guest_user_invite.json index 392e0ec745fc2..455fc3c762978 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_external_guest_user_invite.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_external_guest_user_invite.json @@ -23,9 +23,10 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "Azure", - "SecOps", "Continuous Monitoring", + "SecOps", "Identity and Access" ], "threat": [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_gcp_iam_custom_role_creation.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_gcp_iam_custom_role_creation.json index 0eab41ad8c4bd..ff7ad0e8d29a2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_gcp_iam_custom_role_creation.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_gcp_iam_custom_role_creation.json @@ -22,6 +22,7 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_password_recovery.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_password_recovery.json index 2f0eed31d05be..13279ea170358 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_password_recovery.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_password_recovery.json @@ -24,11 +24,12 @@ "rule_id": "69c420e8-6c9e-4d28-86c0-8a2be2d1e78c", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Identity and Access", - "Continuous Monitoring" + "Identity and Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_rdp_remote_desktop_protocol_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_rdp_remote_desktop_protocol_to_the_internet.json index 15c3c81a551bd..ce0f44713523f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_rdp_remote_desktop_protocol_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_rdp_remote_desktop_protocol_to_the_internet.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Initial Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_rpc_remote_procedure_call_from_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_rpc_remote_procedure_call_from_the_internet.json index 7c61f95f9e9f3..b8f3e01823312 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_rpc_remote_procedure_call_from_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_rpc_remote_procedure_call_from_the_internet.json @@ -5,7 +5,8 @@ "description": "This rule detects network events that may indicate the use of RPC traffic from the Internet. RPC is commonly used by system administrators to remotely control a system for maintenance or to use shared resources. It should almost never be directly exposed to the Internet, as it is frequently targeted and exploited by threat actors as an initial access or back-door vector.", "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -16,7 +17,10 @@ "severity": "high", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Initial Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_rpc_remote_procedure_call_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_rpc_remote_procedure_call_to_the_internet.json index e8da93ed9d1c7..e8e4ea4eb3746 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_rpc_remote_procedure_call_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_rpc_remote_procedure_call_to_the_internet.json @@ -5,7 +5,8 @@ "description": "This rule detects network events that may indicate the use of RPC traffic to the Internet. RPC is commonly used by system administrators to remotely control a system for maintenance or to use shared resources. It should almost never be directly exposed to the Internet, as it is frequently targeted and exploited by threat actors as an initial access or back-door vector.", "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -16,7 +17,10 @@ "severity": "high", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Initial Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_smb_windows_file_sharing_activity_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_smb_windows_file_sharing_activity_to_the_internet.json index aff8a415b7e35..fec0f308a8d27 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_smb_windows_file_sharing_activity_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_smb_windows_file_sharing_activity_to_the_internet.json @@ -5,7 +5,8 @@ "description": "This rule detects network events that may indicate the use of Windows file sharing (also called SMB or CIFS) traffic to the Internet. SMB is commonly used within networks to share files, printers, and other system resources amongst trusted systems. It should almost never be directly exposed to the Internet, as it is frequently targeted and exploited by threat actors as an initial access or back-door vector or for data exfiltration.", "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -16,7 +17,10 @@ "severity": "high", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Initial Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_suspicious_activity_reported_by_okta_user.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_suspicious_activity_reported_by_okta_user.json index 24837084c8381..5b1946dc7c07d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_suspicious_activity_reported_by_okta_user.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_suspicious_activity_reported_by_okta_user.json @@ -24,10 +24,11 @@ "severity": "medium", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Monitoring", - "Continuous Monitoring" + "Monitoring" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_unsecure_elasticsearch_node.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_unsecure_elasticsearch_node.json index e6d718a23eb96..a577a3413e5df 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_unsecure_elasticsearch_node.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_unsecure_elasticsearch_node.json @@ -23,7 +23,9 @@ "severity": "medium", "tags": [ "Elastic", - "Network" + "Network", + "Threat Detection", + "Initial Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_cmd_service.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_cmd_service.json index bd14db77b9fe9..f4e137663762a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_cmd_service.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_cmd_service.json @@ -11,13 +11,16 @@ "language": "eql", "license": "Elastic License", "name": "Service Command Lateral Movement", - "query": "/* dependent on a wildcard for remote path */\n\nsequence by process.entity_id with maxspan=1m\n [process where event.type in (\"start\", \"process_started\") and\n (process.name == \"sc.exe\" or process.pe.original_file_name == \"sc.exe\") and\n wildcard(process.args, \"\\\\\\\\*\") and wildcard(process.args, \"binPath*\", \"binpath*\") and\n process.args in (\"create\", \"config\", \"failure\", \"start\")]\n [network where event.type == \"connection\" and process.name == \"sc.exe\" and destination.address != \"127.0.0.1\"]\n", + "query": "sequence by process.entity_id with maxspan=1m\n [process where event.type in (\"start\", \"process_started\") and\n /* uncomment once in winlogbeat */\n (process.name == \"sc.exe\" /* or process.pe.original_file_name == \"sc.exe\" */ ) and\n /* case insensitive */\n wildcard(process.args, \"\\\\\\\\*\") and wildcard(process.args, \"binPath=*\", \"binpath=*\") and \n (process.args : \"create\" or\n process.args : \"config\" or\n process.args : \"failure\" or\n process.args : \"start\")]\n [network where event.type == \"connection\" and process.name : \"sc.exe\" and destination.ip != \"127.0.0.1\"]\n", "risk_score": 21, "rule_id": "d61cbcf8-1bc1-4cff-85ba-e7b21c5beedc", "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Lateral Movement" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_direct_outbound_smb_connection.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_direct_outbound_smb_connection.json index e4014b22a6c09..39f45a736383a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_direct_outbound_smb_connection.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_direct_outbound_smb_connection.json @@ -17,7 +17,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Lateral Movement" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_dns_server_overflow.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_dns_server_overflow.json index 2a86dcac12e7b..052d0de3ef79e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_dns_server_overflow.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_dns_server_overflow.json @@ -26,7 +26,8 @@ "tags": [ "Elastic", "Network", - "Windows" + "Threat Detection", + "Lateral Movement" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_remote_ssh_login_enabled.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_remote_ssh_login_enabled.json index f1ce68abf8302..149dab112929c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_remote_ssh_login_enabled.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_remote_ssh_login_enabled.json @@ -21,7 +21,10 @@ "severity": "medium", "tags": [ "Elastic", - "MacOS" + "Host", + "macOS", + "Threat Detection", + "Lateral Movement" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_telnet_network_activity_external.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_telnet_network_activity_external.json index e4804329c0f30..ff62251a465f7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_telnet_network_activity_external.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_telnet_network_activity_external.json @@ -20,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Lateral Movement" ], "threat": [ { @@ -40,5 +43,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_telnet_network_activity_internal.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_telnet_network_activity_internal.json index 30312987d166c..bb6c03e9410e5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_telnet_network_activity_internal.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_telnet_network_activity_internal.json @@ -20,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Lateral Movement" ], "threat": [ { @@ -40,5 +43,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_hping_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_hping_activity.json index 3a5c4d9e69d49..775d201cd1808 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_hping_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_hping_activity.json @@ -23,8 +23,10 @@ "severity": "high", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection" ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_iodine_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_iodine_activity.json index 63c82c5662df6..510eecb9d2fee 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_iodine_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_iodine_activity.json @@ -23,8 +23,10 @@ "severity": "high", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection" ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_mknod_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_mknod_activity.json index 99d087fe675a6..a81567740590d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_mknod_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_mknod_activity.json @@ -23,7 +23,9 @@ "severity": "low", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection" ], "type": "query", "version": 5 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_netcat_network_connection.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_netcat_network_connection.json index bce10f640691b..ebe884debc9a0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_netcat_network_connection.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_netcat_network_connection.json @@ -25,8 +25,10 @@ "severity": "medium", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection" ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_nmap_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_nmap_activity.json index 5d9e338425bda..76866f67d7288 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_nmap_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_nmap_activity.json @@ -23,8 +23,10 @@ "severity": "low", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection" ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_nping_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_nping_activity.json index bd019c9a80c4c..cc744614d68af 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_nping_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_nping_activity.json @@ -23,8 +23,10 @@ "severity": "medium", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection" ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_process_started_in_temp_directory.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_process_started_in_temp_directory.json index f0bbc892d7d9c..1662d63066feb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_process_started_in_temp_directory.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_process_started_in_temp_directory.json @@ -20,8 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection" ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_socat_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_socat_activity.json index fac03d31b57bf..07cd9e4a065b3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_socat_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_socat_activity.json @@ -23,8 +23,10 @@ "severity": "medium", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection" ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_strace_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_strace_activity.json index c1b782d612ccb..06be108db2f14 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_strace_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_strace_activity.json @@ -23,8 +23,10 @@ "severity": "low", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection" ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_error_message_spike.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_error_message_spike.json index 0730c421cf5f2..890b777751a03 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_error_message_spike.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_error_message_spike.json @@ -20,10 +20,11 @@ "rule_id": "78d3d8d9-b476-451d-a9e0-7a5addd70670", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", "ML" ], "type": "machine_learning", - "version": 1 + "version": 2 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_error_code.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_error_code.json index 8003cdd7504c7..62254d970ca7e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_error_code.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_error_code.json @@ -20,10 +20,11 @@ "rule_id": "19de8096-e2b0-4bd8-80c9-34a820813fff", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", "ML" ], "type": "machine_learning", - "version": 1 + "version": 2 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_city.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_city.json index 2c54dbd03daba..5c1189c989bac 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_city.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_city.json @@ -20,10 +20,11 @@ "rule_id": "809b70d3-e2c3-455e-af1b-2626a5a1a276", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", "ML" ], "type": "machine_learning", - "version": 1 + "version": 2 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_country.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_country.json index 68cbf4979a933..149ebd1a84fd4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_country.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_country.json @@ -20,10 +20,11 @@ "rule_id": "dca28dee-c999-400f-b640-50a081cc0fd1", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", "ML" ], "type": "machine_learning", - "version": 1 + "version": 2 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_user.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_user.json index e4ec651e71934..a8c0bce169432 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_user.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_user.json @@ -20,10 +20,11 @@ "rule_id": "ac706eae-d5ec-4b14-b4fd-e8ba8086f0e1", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", "ML" ], "type": "machine_learning", - "version": 1 + "version": 2 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_compiler_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_compiler_activity.json index eb764c5e40817..592bd723e1a0c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_compiler_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_compiler_activity.json @@ -17,7 +17,9 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "type": "machine_learning", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_kernel_module_arguments.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_kernel_module_arguments.json index d289e0ba6f008..52a1d6dd5c60a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_kernel_module_arguments.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_kernel_module_arguments.json @@ -20,7 +20,9 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "threat": [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_metadata_process.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_metadata_process.json index c1cc619164b1f..ec033a609cac1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_metadata_process.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_metadata_process.json @@ -11,13 +11,15 @@ "interval": "15m", "license": "Elastic License", "machine_learning_job_id": "linux_rare_metadata_process", - "name": "Unusual Process Calling the Metadata Service", + "name": "Unusual Linux Process Calling the Metadata Service", "risk_score": 21, "rule_id": "9d302377-d226-4e12-b54c-1906b5aec4f6", "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "type": "machine_learning", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_metadata_user.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_metadata_user.json index 59a04dd54dd89..a4a6b38bd32b8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_metadata_user.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_metadata_user.json @@ -17,7 +17,9 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "type": "machine_learning", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_activity.json index bf86f78fe3e72..04df4adab8525 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_activity.json @@ -18,9 +18,11 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_port_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_port_activity.json index a588a6f5bcb0a..c5a0891a7bbc4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_port_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_port_activity.json @@ -20,9 +20,11 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_service.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_service.json index 5c56845024eb2..81dff605dc779 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_service.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_service.json @@ -20,9 +20,11 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_url_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_url_activity.json index 3b3f751dfc60b..5336c31d4a741 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_url_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_url_activity.json @@ -20,9 +20,11 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_process_all_hosts.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_process_all_hosts.json index 8475410735f34..04d112c26e886 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_process_all_hosts.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_process_all_hosts.json @@ -21,9 +21,11 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_sudo_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_sudo_activity.json index 8f03b24a6bd18..57d5260d111db 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_sudo_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_sudo_activity.json @@ -17,7 +17,9 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "threat": [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_user_name.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_user_name.json index 3e4b1f15fdce4..3f889dc529317 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_user_name.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_user_name.json @@ -21,9 +21,11 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_information_discovery.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_information_discovery.json index 40f117c6a5708..cdd1ee8d46653 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_information_discovery.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_information_discovery.json @@ -17,7 +17,9 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "threat": [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_network_configuration_discovery.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_network_configuration_discovery.json index 326024114f145..b143293ac995b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_network_configuration_discovery.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_network_configuration_discovery.json @@ -17,7 +17,9 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "threat": [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_network_connection_discovery.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_network_connection_discovery.json index 881a2f9fa3410..ffab1c9eb8495 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_network_connection_discovery.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_network_connection_discovery.json @@ -17,7 +17,9 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "threat": [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_process_discovery.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_process_discovery.json index 66859e2f9ccbf..69d2940029011 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_process_discovery.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_process_discovery.json @@ -17,7 +17,9 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "threat": [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_user_discovery.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_user_discovery.json index 4437334b0aa1f..b5c5e66af6e7e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_user_discovery.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_user_discovery.json @@ -17,7 +17,9 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "threat": [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_dns_tunneling.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_dns_tunneling.json index 1352fde91b59b..fbce00ebaf5a3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_dns_tunneling.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_dns_tunneling.json @@ -20,9 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Packetbeat" + "Network", + "Threat Detection", + "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_dns_question.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_dns_question.json index b16e67052a212..744e9fbfc18d8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_dns_question.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_dns_question.json @@ -20,9 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Packetbeat" + "Network", + "Threat Detection", + "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_server_domain.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_server_domain.json index a8971300fe11b..a672985655859 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_server_domain.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_server_domain.json @@ -20,9 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Packetbeat" + "Network", + "Threat Detection", + "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_urls.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_urls.json index 469f5d741ef6e..5600fbe74c7fc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_urls.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_urls.json @@ -20,9 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Packetbeat" + "Network", + "Threat Detection", + "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_user_agent.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_user_agent.json index ebcf4f987e9de..5cce9c63fd88e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_user_agent.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_user_agent.json @@ -20,9 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Packetbeat" + "Network", + "Threat Detection", + "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_rare_process_by_host_linux.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_rare_process_by_host_linux.json index 385158dd6b65d..5c25b47833112 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_rare_process_by_host_linux.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_rare_process_by_host_linux.json @@ -21,9 +21,11 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_rare_process_by_host_windows.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_rare_process_by_host_windows.json index d0a99b32d4713..e86bc96fddae5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_rare_process_by_host_windows.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_rare_process_by_host_windows.json @@ -21,9 +21,11 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Windows" + "Host", + "Windows", + "Threat Detection", + "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_suspicious_login_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_suspicious_login_activity.json index f309debcdffe9..ae0ac2b716e32 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_suspicious_login_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_suspicious_login_activity.json @@ -20,9 +20,11 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_metadata_process.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_metadata_process.json index 56874ec371b43..3e35b2af5e3b8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_metadata_process.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_metadata_process.json @@ -17,8 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Windows" + "Host", + "Windows", + "Threat Detection", + "ML" ], "type": "machine_learning", "version": 1 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_metadata_user.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_metadata_user.json index f124cda7717c3..7c45ab4dfa08b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_metadata_user.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_metadata_user.json @@ -17,8 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Windows" + "Host", + "Windows", + "Threat Detection", + "ML" ], "type": "machine_learning", "version": 1 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_network_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_network_activity.json index 0ab591097f975..937fcbc079eb8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_network_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_network_activity.json @@ -21,9 +21,11 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Windows" + "Host", + "Windows", + "Threat Detection", + "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_path_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_path_activity.json index a7b309e6d7fcd..d45661f737b5d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_path_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_path_activity.json @@ -20,9 +20,11 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Windows" + "Host", + "Windows", + "Threat Detection", + "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_process_all_hosts.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_process_all_hosts.json index bc6346f457b65..88764ed1bf46d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_process_all_hosts.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_process_all_hosts.json @@ -21,9 +21,11 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Windows" + "Host", + "Windows", + "Threat Detection", + "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_process_creation.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_process_creation.json index 97351a1f517b3..2ebc7ee667d4e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_process_creation.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_process_creation.json @@ -20,9 +20,11 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Windows" + "Host", + "Windows", + "Threat Detection", + "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_script.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_script.json index d0dc8d7e40fa2..91714b10d2ab6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_script.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_script.json @@ -20,9 +20,11 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Windows" + "Host", + "Windows", + "Threat Detection", + "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_service.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_service.json index b7e7a0357e118..3aea73eb489e5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_service.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_service.json @@ -20,9 +20,11 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Windows" + "Host", + "Windows", + "Threat Detection", + "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_user_name.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_user_name.json index 26bd6837cbde5..749cbb218bcd8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_user_name.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_user_name.json @@ -21,9 +21,11 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Windows" + "Host", + "Windows", + "Threat Detection", + "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_rare_user_runas_event.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_rare_user_runas_event.json index 1af765f568bb1..8ed8a005d9aa5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_rare_user_runas_event.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_rare_user_runas_event.json @@ -20,9 +20,11 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Windows" + "Host", + "Windows", + "Threat Detection", + "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_rare_user_type10_remote_login.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_rare_user_type10_remote_login.json index b69e759120ce4..dda4e0bbad88d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_rare_user_type10_remote_login.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_rare_user_type10_remote_login.json @@ -21,9 +21,11 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Windows" + "Host", + "Windows", + "Threat Detection", + "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_deactivate_okta_mfa_rule.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_deactivate_okta_mfa_rule.json index c503d2298adad..0ee0bbd6d6226 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_deactivate_okta_mfa_rule.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_deactivate_okta_mfa_rule.json @@ -24,10 +24,11 @@ "severity": "low", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Identity and Access", - "Continuous Monitoring" + "Identity and Access" ], "type": "query", "version": 2 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_delete_okta_policy.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_delete_okta_policy.json index d095d7c1166de..211fdb1ae3474 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_delete_okta_policy.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_delete_okta_policy.json @@ -24,10 +24,11 @@ "severity": "low", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Monitoring", - "Continuous Monitoring" + "Monitoring" ], "type": "query", "version": 2 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_okta_mfa_rule.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_okta_mfa_rule.json index 2fe27575b7b2a..eb726e24c89da 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_okta_mfa_rule.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_okta_mfa_rule.json @@ -24,10 +24,11 @@ "severity": "low", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Identity and Access", - "Continuous Monitoring" + "Identity and Access" ], "type": "query", "version": 2 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_okta_network_zone.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_okta_network_zone.json index 23b3313488847..682dc17f0ed49 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_okta_network_zone.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_okta_network_zone.json @@ -24,10 +24,11 @@ "severity": "medium", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Network", - "Continuous Monitoring" + "Network Security" ], "type": "query", "version": 2 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_okta_policy.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_okta_policy.json index 5b19031046b66..88e556d37a27c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_okta_policy.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_okta_policy.json @@ -24,10 +24,11 @@ "severity": "low", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Monitoring", - "Continuous Monitoring" + "Monitoring" ], "type": "query", "version": 2 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_or_delete_application_sign_on_policy.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_or_delete_application_sign_on_policy.json index 58ba13e147a38..262a91f8e25c9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_or_delete_application_sign_on_policy.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_or_delete_application_sign_on_policy.json @@ -24,10 +24,11 @@ "severity": "medium", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Identity and Access", - "Continuous Monitoring" + "Identity and Access" ], "type": "query", "version": 2 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_threat_detected_by_okta_threatinsight.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_threat_detected_by_okta_threatinsight.json index 1efcf0474c049..0101ae0459454 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_threat_detected_by_okta_threatinsight.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_threat_detected_by_okta_threatinsight.json @@ -21,10 +21,11 @@ "severity": "medium", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Monitoring", - "Continuous Monitoring" + "Monitoring" ], "type": "query", "version": 2 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_administrator_privileges_assigned_to_okta_group.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_administrator_privileges_assigned_to_okta_group.json index 87d5bf3e0f48c..fad3e3c922478 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_administrator_privileges_assigned_to_okta_group.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_administrator_privileges_assigned_to_okta_group.json @@ -24,10 +24,11 @@ "severity": "low", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Monitoring", - "Continuous Monitoring" + "Monitoring" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_adobe_hijack_persistence.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_adobe_hijack_persistence.json index a4c62b98fb060..077147a8ed1a6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_adobe_hijack_persistence.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_adobe_hijack_persistence.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Persistence" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_app_compat_shim.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_app_compat_shim.json index e9e2e044ddc04..5c467c39f5128 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_app_compat_shim.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_app_compat_shim.json @@ -11,13 +11,16 @@ "language": "eql", "license": "Elastic License", "name": "Installation of Custom Shim Databases", - "query": "/* dependent on wildcard for registry.value */\n\nsequence by process.entity_id with maxspan=5m\n [process where event.type in (\"start\", \"process_started\") and\n not (process.name == \"sdbinst.exe\" and process.parent.name == \"msiexec.exe\")]\n [registry where event.type in (\"creation\", \"change\") and\n wildcard(registry.path, \"HKLM\\\\SOFTWARE\\\\Microsoft\\\\Windows NT\\\\CurrentVersion\\\\AppCompatFlags\\\\Custom\\\\*.sdb\")]\n", + "query": "sequence by process.entity_id with maxspan=5m\n [process where event.type in (\"start\", \"process_started\") and\n not (process.name : \"sdbinst.exe\" and process.parent.name : \"msiexec.exe\")]\n [registry where event.type in (\"creation\", \"change\") and\n wildcard(registry.path, \"HKLM\\\\SOFTWARE\\\\Microsoft\\\\Windows NT\\\\CurrentVersion\\\\AppCompatFlags\\\\Custom\\\\*.sdb\")]\n", "risk_score": 21, "rule_id": "c5ce48a6-7f57-4ee8-9313-3d0024caee10", "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Persistence" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_create_okta_api_token.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_create_okta_api_token.json index c1d7d51f1401e..9d1a7c7aef464 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_create_okta_api_token.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_create_okta_api_token.json @@ -24,10 +24,11 @@ "severity": "low", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Monitoring", - "Continuous Monitoring" + "Monitoring" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_deactivate_mfa_for_okta_user_account.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_deactivate_mfa_for_okta_user_account.json index 9cd9572400a6c..764c60b829498 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_deactivate_mfa_for_okta_user_account.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_deactivate_mfa_for_okta_user_account.json @@ -24,10 +24,11 @@ "severity": "low", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Identity and Access", - "Continuous Monitoring" + "Identity and Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_deactivate_okta_policy.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_deactivate_okta_policy.json index d5c9e505659f6..9003f6877341f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_deactivate_okta_policy.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_deactivate_okta_policy.json @@ -24,10 +24,11 @@ "severity": "low", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Monitoring", - "Continuous Monitoring" + "Monitoring" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_reset_mfa_factors_for_okta_user_account.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_reset_mfa_factors_for_okta_user_account.json index 302618773e323..4fef3e833a7b6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_reset_mfa_factors_for_okta_user_account.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_reset_mfa_factors_for_okta_user_account.json @@ -24,10 +24,11 @@ "severity": "low", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Identity and Access", - "Continuous Monitoring" + "Identity and Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_automation_account_created.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_automation_account_created.json index 645c025ec4738..5c000967ce44d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_automation_account_created.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_automation_account_created.json @@ -22,11 +22,12 @@ "rule_id": "df26fd74-1baa-4479-b42e-48da84642330", "severity": "low", "tags": [ + "Elastic", + "Cloud", "Azure", "Continuous Monitoring", - "Elastic", - "Identity and Access", - "SecOps" + "SecOps", + "Identity and Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_automation_runbook_created_or_modified.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_automation_runbook_created_or_modified.json index e96700e409090..28a5864353942 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_automation_runbook_created_or_modified.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_automation_runbook_created_or_modified.json @@ -23,6 +23,7 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "Azure", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_automation_webhook_created.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_automation_webhook_created.json index f31fdcc18978e..5dde815022283 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_automation_webhook_created.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_automation_webhook_created.json @@ -23,6 +23,7 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "Azure", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_azure_conditional_access_policy_modified.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_conditional_access_policy_modified.json similarity index 99% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_azure_conditional_access_policy_modified.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_conditional_access_policy_modified.json index 8d4d4b971316e..14d6d3d479c6a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_azure_conditional_access_policy_modified.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_conditional_access_policy_modified.json @@ -20,6 +20,7 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "Azure", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_pim_user_added_global_admin.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_pim_user_added_global_admin.json index b8ea2c55dd3f9..24411a40ffc46 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_pim_user_added_global_admin.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_pim_user_added_global_admin.json @@ -22,6 +22,7 @@ "severity": "high", "tags": [ "Elastic", + "Cloud", "Azure", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_azure_privileged_identity_management_role_modified.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_privileged_identity_management_role_modified.json similarity index 99% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_azure_privileged_identity_management_role_modified.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_privileged_identity_management_role_modified.json index f675a490c4e05..77e955727b2d8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_azure_privileged_identity_management_role_modified.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_privileged_identity_management_role_modified.json @@ -21,6 +21,7 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "Azure", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_ec2_network_acl_creation.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_ec2_network_acl_creation.json index 1b98b9744cd5b..8deaa9924cc1d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_ec2_network_acl_creation.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_ec2_network_acl_creation.json @@ -27,11 +27,12 @@ "rule_id": "39144f38-5284-4f8e-a2ae-e3fd628d90b0", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Network", - "Continuous Monitoring" + "Network Security" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_gcp_service_account_created.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_gcp_service_account_created.json index aa9d48459262e..62e28e588cd0a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_gcp_service_account_created.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_gcp_service_account_created.json @@ -22,6 +22,7 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_gpo_schtask_service_creation.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_gpo_schtask_service_creation.json similarity index 96% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_gpo_schtask_service_creation.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_gpo_schtask_service_creation.json index fbf6fddcb8c00..536eda4e21476 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_gpo_schtask_service_creation.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_gpo_schtask_service_creation.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Persistence" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_iam_group_creation.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_iam_group_creation.json index 0addb86b8d031..963ac46b7ed94 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_iam_group_creation.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_iam_group_creation.json @@ -25,11 +25,12 @@ "rule_id": "169f3a93-efc7-4df2-94d6-0d9438c310d1", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Identity and Access", - "Continuous Monitoring" + "Identity and Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_kernel_module_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_kernel_module_activity.json index e3dedeef07eb5..b29a8b2384f95 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_kernel_module_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_kernel_module_activity.json @@ -23,7 +23,10 @@ "severity": "low", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Persistence" ], "threat": [ { @@ -43,5 +46,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_local_scheduled_task_commands.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_local_scheduled_task_commands.json index 78f035318c614..9e5a6a0eae041 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_local_scheduled_task_commands.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_local_scheduled_task_commands.json @@ -20,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Persistence" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_mfa_disabled_for_azure_user.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_mfa_disabled_for_azure_user.json similarity index 99% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_mfa_disabled_for_azure_user.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_mfa_disabled_for_azure_user.json index eda6f5b2bdf62..8a9f4d4c661e9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_mfa_disabled_for_azure_user.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_mfa_disabled_for_azure_user.json @@ -17,6 +17,7 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "Azure", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_priv_escalation_via_accessibility_features.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_priv_escalation_via_accessibility_features.json index 5899b58bce4d5..e7f4598a19f33 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_priv_escalation_via_accessibility_features.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_priv_escalation_via_accessibility_features.json @@ -4,7 +4,8 @@ ], "description": "Windows contains accessibility features that may be launched with a key combination before a user has logged in. An adversary can modify the way these programs are launched to get a command prompt or backdoor without logging in to the system.", "index": [ - "winlogbeat-*" + "winlogbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -15,7 +16,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Persistence" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_rds_cluster_creation.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_rds_cluster_creation.json index eb77c183d90ea..06ca022726aad 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_rds_cluster_creation.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_rds_cluster_creation.json @@ -27,11 +27,12 @@ "rule_id": "e14c5fd7-fdd7-49c2-9e5b-ec49d817bc8d", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Asset Visibility", - "Continuous Monitoring" + "Asset Visibility" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_shell_activity_by_web_server.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_shell_activity_by_web_server.json index 2aaf0012acabf..ea10fa9bdf865 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_shell_activity_by_web_server.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_shell_activity_by_web_server.json @@ -23,7 +23,10 @@ "severity": "medium", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Persistence" ], "threat": [ { @@ -43,5 +46,5 @@ } ], "type": "query", - "version": 5 + "version": 6 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_system_shells_via_services.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_system_shells_via_services.json index 32d78480325e6..880101e8d9338 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_system_shells_via_services.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_system_shells_via_services.json @@ -17,7 +17,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Persistence" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_account_creation.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_account_creation.json index 3f2e00f0976de..9bc603b5ad547 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_account_creation.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_account_creation.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Persistence" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_added_as_owner_for_azure_application.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_added_as_owner_for_azure_application.json index 8882b87e91291..3fddde78beb33 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_added_as_owner_for_azure_application.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_added_as_owner_for_azure_application.json @@ -17,6 +17,7 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "Azure", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_added_as_owner_for_azure_service_principal.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_added_as_owner_for_azure_service_principal.json index f7c0af67692e7..de6482f14d2f0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_added_as_owner_for_azure_service_principal.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_added_as_owner_for_azure_service_principal.json @@ -20,6 +20,7 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "Azure", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_application_shimming.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_application_shimming.json index f20cc75dfa38b..c9d56a9c68edb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_application_shimming.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_application_shimming.json @@ -4,7 +4,8 @@ ], "description": "The Application Shim was created to allow for backward compatibility of software as the operating system codebase changes over time. This Windows functionality has been abused by attackers to stealthily gain persistence and arbitrary code execution in legitimate Windows processes.", "index": [ - "winlogbeat-*" + "winlogbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -15,7 +16,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Persistence" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_telemetrycontroller_scheduledtask_hijack.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_telemetrycontroller_scheduledtask_hijack.json index b7f4ec5d8a73c..0622309387f35 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_telemetrycontroller_scheduledtask_hijack.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_telemetrycontroller_scheduledtask_hijack.json @@ -20,7 +20,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Persistence" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_update_orchestrator_service_hijack.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_update_orchestrator_service_hijack.json index e512e92a31560..bdddf2eb7e8c7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_update_orchestrator_service_hijack.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_update_orchestrator_service_hijack.json @@ -20,7 +20,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Persistence" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_printspooler_service_suspicious_file.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_printspooler_service_suspicious_file.json index 1a76e077a7465..398e4ca22b757 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_printspooler_service_suspicious_file.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_printspooler_service_suspicious_file.json @@ -21,7 +21,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Privilege Escalation" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_printspooler_suspicious_spl_file.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_printspooler_suspicious_spl_file.json index c5ffe5a9f6a11..18da4c8075c8f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_printspooler_suspicious_spl_file.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_printspooler_suspicious_spl_file.json @@ -21,7 +21,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Privilege Escalation" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_root_login_without_mfa.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_root_login_without_mfa.json index 16389d43945f1..16af96e98448c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_root_login_without_mfa.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_root_login_without_mfa.json @@ -24,11 +24,12 @@ "rule_id": "bc0c6f0d-dab0-47a3-b135-0925f0a333bc", "severity": "high", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Identity and Access", - "Continuous Monitoring" + "Identity and Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_setgid_bit_set_via_chmod.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_setgid_bit_set_via_chmod.json index e72e58132adee..ff63d1e38d950 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_setgid_bit_set_via_chmod.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_setgid_bit_set_via_chmod.json @@ -18,7 +18,10 @@ "severity": "low", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Privilege Escalation" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_setuid_bit_set_via_chmod.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_setuid_bit_set_via_chmod.json index 5e560097d2545..47490fe08ff12 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_setuid_bit_set_via_chmod.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_setuid_bit_set_via_chmod.json @@ -18,7 +18,10 @@ "severity": "low", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Privilege Escalation" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_sudoers_file_mod.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_sudoers_file_mod.json index 73a804fcbda8f..5519f6ce3a9ec 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_sudoers_file_mod.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_sudoers_file_mod.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Privilege Escalation" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_uac_bypass_diskcleanup_hijack.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_uac_bypass_diskcleanup_hijack.json index b22457db49e49..80b01f90d3cf4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_uac_bypass_diskcleanup_hijack.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_uac_bypass_diskcleanup_hijack.json @@ -17,7 +17,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Privilege Escalation" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_uac_bypass_event_viewer.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_uac_bypass_event_viewer.json index 740ff47e5abe5..415111c725828 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_uac_bypass_event_viewer.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_uac_bypass_event_viewer.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Privilege Escalation" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_uac_sdclt.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_uac_sdclt.json new file mode 100644 index 0000000000000..ce82db3a96e11 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_uac_sdclt.json @@ -0,0 +1,44 @@ +{ + "author": [ + "Elastic" + ], + "description": "Identifies User Account Control (UAC) bypass via sdclt.exe. Attackers bypass UAC to stealthily execute code with elevated permissions.", + "from": "now-9m", + "index": [ + "logs-endpoint.events.*", + "winlogbeat-*" + ], + "language": "eql", + "license": "Elastic License", + "name": "Bypass UAC via Sdclt", + "query": "/* add winlogbeat-* when process.code_signature.* fields are populated */\n\nsequence with maxspan=1m\n [process where event.type in (\"start\", \"process_started\") and process.name : \"sdclt.exe\" and\n /* uncomment once in winlogbeat */\n /* process.code_signature.subject_name == \"Microsoft Corporation\" and process.code_signature.trusted == true and */\n process.args : \"/kickoffelev\"\n ] by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and process.parent.name : \"sdclt.exe\" and\n not (process.executable : \"C:\\\\Windows\\\\System32\\\\sdclt.exe\" or\n process.executable : \"C:\\\\Windows\\\\System32\\\\control.exe\" or\n process.executable : \"C:\\\\Windows\\\\SysWOW64\\\\sdclt.exe\" or\n process.executable : \"C:\\\\Windows\\\\SysWOW64\\\\control.exe\")\n ] by process.parent.entity_id\n", + "risk_score": 21, + "rule_id": "9b54e002-034a-47ac-9307-ad12c03fa900", + "severity": "high", + "tags": [ + "Elastic", + "Host", + "Windows", + "Threat Detection", + "Privilege Escalation" + ], + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0004", + "name": "Privilege Escalation", + "reference": "https://attack.mitre.org/tactics/TA0004/" + }, + "technique": [ + { + "id": "T1088", + "name": "Bypass User Account Control", + "reference": "https://attack.mitre.org/techniques/T1088/" + } + ] + } + ], + "type": "eql", + "version": 1 +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_unusual_parentchild_relationship.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_unusual_parentchild_relationship.json index 0cac8561c7e9c..ad871716a67aa 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_unusual_parentchild_relationship.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_unusual_parentchild_relationship.json @@ -17,7 +17,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Privilege Escalation" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_updateassumerolepolicy.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_updateassumerolepolicy.json index 55947e00170ae..3d1e66497bd2e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_updateassumerolepolicy.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_updateassumerolepolicy.json @@ -24,11 +24,12 @@ "rule_id": "a60326d7-dca7-4fb7-93eb-1ca03a1febbd", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Identity and Access", - "Continuous Monitoring" + "Identity and Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts index 8af622e6a128b..fb4763a982f43 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts @@ -105,7 +105,7 @@ export interface RuleAlertType extends Alert { } // eslint-disable-next-line @typescript-eslint/no-explicit-any -export interface IRuleStatusAttributes extends Record { +export interface IRuleStatusSOAttributes extends Record { alertId: string; // created alert id. statusDate: StatusDate; lastFailureAt: LastFailureAt | null | undefined; @@ -119,21 +119,35 @@ export interface IRuleStatusAttributes extends Record { searchAfterTimeDurations: string[] | null | undefined; } +export interface IRuleStatusResponseAttributes { + alert_id: string; // created alert id. + status_date: StatusDate; + last_failure_at: LastFailureAt | null | undefined; + last_failure_message: LastFailureMessage | null | undefined; + last_success_at: LastSuccessAt | null | undefined; + last_success_message: LastSuccessMessage | null | undefined; + status: JobStatus | null | undefined; + last_look_back_date: string | null | undefined; + gap: string | null | undefined; + bulk_create_time_durations: string[] | null | undefined; + search_after_time_durations: string[] | null | undefined; +} + export interface RuleStatusResponse { [key: string]: { - current_status: IRuleStatusAttributes | null | undefined; - failures: IRuleStatusAttributes[] | null | undefined; + current_status: IRuleStatusResponseAttributes | null | undefined; + failures: IRuleStatusResponseAttributes[] | null | undefined; }; } export interface IRuleSavedAttributesSavedObjectAttributes - extends IRuleStatusAttributes, + extends IRuleStatusSOAttributes, SavedObjectAttributes {} export interface IRuleStatusSavedObject { type: string; id: string; - attributes: Array>; + attributes: Array>; references: unknown[]; updated_at: string; version: string; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index cbf70f3119b31..4559a658c9583 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -19,7 +19,7 @@ import { } from '../../../../../../../../src/core/server'; import { loggingSystemMock } from '../../../../../../../../src/core/server/mocks'; import { RuleTypeParams } from '../../types'; -import { IRuleStatusAttributes } from '../../rules/types'; +import { IRuleStatusSOAttributes } from '../../rules/types'; import { ruleStatusSavedObjectType } from '../../rules/saved_object_mappings'; import { getListArrayMock } from '../../../../../common/detection_engine/schemas/types/lists.mock'; import { RulesSchema } from '../../../../../common/detection_engine/schemas/response'; @@ -555,7 +555,7 @@ export const sampleDocSearchResultsWithSortId = ( export const sampleRuleGuid = '04128c15-0d1b-4716-a4c5-46997ac7f3bd'; export const sampleIdGuid = 'e1e08ddc-5e37-49ff-a258-5393aa44435a'; -export const exampleRuleStatus: () => SavedObject = () => ({ +export const exampleRuleStatus: () => SavedObject = () => ({ type: ruleStatusSavedObjectType, id: '042e6d90-7069-11ea-af8b-0f8ae4fa817e', attributes: { @@ -577,8 +577,10 @@ export const exampleRuleStatus: () => SavedObject = () => }); export const exampleFindRuleStatusResponse: ( - mockStatuses: Array> -) => SavedObjectsFindResponse = (mockStatuses = [exampleRuleStatus()]) => ({ + mockStatuses: Array> +) => SavedObjectsFindResponse = ( + mockStatuses = [exampleRuleStatus()] +) => ({ total: 1, per_page: 6, page: 1, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_or_create_rule_statuses.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_or_create_rule_statuses.ts index 913efbe04aa16..1ddec9cd15148 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_or_create_rule_statuses.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_or_create_rule_statuses.ts @@ -6,7 +6,7 @@ import { SavedObject } from 'src/core/server'; -import { IRuleStatusAttributes } from '../rules/types'; +import { IRuleStatusSOAttributes } from '../rules/types'; import { RuleStatusSavedObjectsClient } from './rule_status_saved_objects_client'; import { getRuleStatusSavedObjects } from './get_rule_status_saved_objects'; @@ -18,7 +18,7 @@ interface RuleStatusParams { export const createNewRuleStatus = async ({ alertId, ruleStatusClient, -}: RuleStatusParams): Promise> => { +}: RuleStatusParams): Promise> => { const now = new Date().toISOString(); return ruleStatusClient.create({ alertId, @@ -38,7 +38,7 @@ export const createNewRuleStatus = async ({ export const getOrCreateRuleStatuses = async ({ alertId, ruleStatusClient, -}: RuleStatusParams): Promise>> => { +}: RuleStatusParams): Promise>> => { const ruleStatuses = await getRuleStatusSavedObjects({ alertId, ruleStatusClient, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_rule_status_saved_objects.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_rule_status_saved_objects.ts index 828b4ea41096e..72a271fb2606f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_rule_status_saved_objects.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_rule_status_saved_objects.ts @@ -5,7 +5,7 @@ */ import { SavedObjectsFindResponse } from 'kibana/server'; -import { IRuleStatusAttributes } from '../rules/types'; +import { IRuleStatusSOAttributes } from '../rules/types'; import { MAX_RULE_STATUSES } from './rule_status_service'; import { RuleStatusSavedObjectsClient } from './rule_status_saved_objects_client'; @@ -17,7 +17,7 @@ interface GetRuleStatusSavedObject { export const getRuleStatusSavedObjects = async ({ alertId, ruleStatusClient, -}: GetRuleStatusSavedObject): Promise> => { +}: GetRuleStatusSavedObject): Promise> => { return ruleStatusClient.find({ perPage: MAX_RULE_STATUSES, sortField: 'statusDate', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/rule_status_saved_objects_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/rule_status_saved_objects_client.ts index 4b5faeb5b9d27..f6a08852ac8d5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/rule_status_saved_objects_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/rule_status_saved_objects_client.ts @@ -12,17 +12,17 @@ import { SavedObjectsFindResponse, } from '../../../../../../../src/core/server'; import { ruleStatusSavedObjectType } from '../rules/saved_object_mappings'; -import { IRuleStatusAttributes } from '../rules/types'; +import { IRuleStatusSOAttributes } from '../rules/types'; export interface RuleStatusSavedObjectsClient { find: ( options?: Omit - ) => Promise>; - create: (attributes: IRuleStatusAttributes) => Promise>; + ) => Promise>; + create: (attributes: IRuleStatusSOAttributes) => Promise>; update: ( id: string, - attributes: Partial - ) => Promise>; + attributes: Partial + ) => Promise>; delete: (id: string) => Promise<{}>; } @@ -30,7 +30,10 @@ export const ruleStatusSavedObjectsClientFactory = ( savedObjectsClient: SavedObjectsClientContract ): RuleStatusSavedObjectsClient => ({ find: (options) => - savedObjectsClient.find({ ...options, type: ruleStatusSavedObjectType }), + savedObjectsClient.find({ + ...options, + type: ruleStatusSavedObjectType, + }), create: (attributes) => savedObjectsClient.create(ruleStatusSavedObjectType, attributes), update: (id, attributes) => savedObjectsClient.update(ruleStatusSavedObjectType, id, attributes), delete: (id) => savedObjectsClient.delete(ruleStatusSavedObjectType, id), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/rule_status_service.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/rule_status_service.ts index 8fdbe282eece5..433ad4e2affea 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/rule_status_service.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/rule_status_service.ts @@ -6,7 +6,7 @@ import { assertUnreachable } from '../../../../common/utility_types'; import { JobStatus } from '../../../../common/detection_engine/schemas/common/schemas'; -import { IRuleStatusAttributes } from '../rules/types'; +import { IRuleStatusSOAttributes } from '../rules/types'; import { getOrCreateRuleStatuses } from './get_or_create_rule_statuses'; import { RuleStatusSavedObjectsClient } from './rule_status_saved_objects_client'; @@ -30,9 +30,9 @@ export const buildRuleStatusAttributes: ( status: JobStatus, message?: string, attributes?: Attributes -) => Partial = (status, message, attributes = {}) => { +) => Partial = (status, message, attributes = {}) => { const now = new Date().toISOString(); - const baseAttributes: Partial = { + const baseAttributes: Partial = { ...attributes, status, statusDate: now, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 838ac2558b038..bb3a0b4fa6f08 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -8,7 +8,6 @@ import { Logger, KibanaRequest } from 'src/core/server'; -import { get } from 'lodash'; import { SIGNALS_ID, DEFAULT_SEARCH_AFTER_PAGE_SIZE, @@ -62,6 +61,8 @@ import { buildEqlSearchRequest } from '../../../../common/detection_engine/get_q import { bulkInsertSignals } from './single_bulk_create'; import { buildSignalFromEvent, buildSignalGroupFromSequence } from './build_bulk_body'; import { createThreatSignals } from './threat_mapping/create_threat_signals'; +import { getIndexVersion } from '../routes/index/get_index_version'; +import { MIN_EQL_RULE_INDEX_VERSION } from '../routes/index/get_signals_template'; export const signalRulesAlertType = ({ logger, @@ -119,17 +120,6 @@ export const signalRulesAlertType = ({ type, exceptionsList, } = params; - const outputIndexTemplateMapping: unknown = await services.callCluster( - 'indices.getTemplate', - { name: outputIndex } - ); - const signalMappingVersion: number | undefined = get(outputIndexTemplateMapping, [ - outputIndex, - 'version', - ]); - if (signalMappingVersion !== undefined && typeof signalMappingVersion !== 'number') { - throw new Error('Found non-numeric value for "version" in output index template'); - } const searchAfterSize = Math.min(maxSignals, DEFAULT_SEARCH_AFTER_PAGE_SIZE); let hasError: boolean = false; @@ -457,14 +447,24 @@ export const signalRulesAlertType = ({ if (query === undefined) { throw new Error('EQL query rule must have a query defined'); } - const MIN_EQL_RULE_TEMPLATE_VERSION = 2; - if ( - signalMappingVersion === undefined || - signalMappingVersion < MIN_EQL_RULE_TEMPLATE_VERSION - ) { - throw new Error( - `EQL based rules require an update to version ${MIN_EQL_RULE_TEMPLATE_VERSION} of the detection alerts index mapping` - ); + try { + const signalIndexVersion = await getIndexVersion(services.callCluster, outputIndex); + if ( + signalIndexVersion === undefined || + signalIndexVersion < MIN_EQL_RULE_INDEX_VERSION + ) { + throw new Error( + `EQL based rules require an update to version ${MIN_EQL_RULE_INDEX_VERSION} of the detection alerts index mapping` + ); + } + } catch (err) { + if (err.statusCode === 403) { + throw new Error( + `EQL based rules require the user that created it to have the view_index_metadata, read, and write permissions for index: ${outputIndex}` + ); + } else { + throw err; + } } const inputIndex = await getInputIndex(services, version, index); const request = buildEqlSearchRequest( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts index 7b7c40f0c4355..c4869f024a977 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts @@ -105,7 +105,9 @@ describe('singleSearchAfter', () => { timestampOverride: undefined, buildRuleMessage, }); - expect(searchErrors).toEqual(['reason: some reason, type: some type, caused by: some reason']); + expect(searchErrors).toEqual([ + 'reason: "some reason" type: "some type" caused by reason: "some reason" caused by type: "some type"', + ]); }); test('if singleSearchAfter works with a given sort id', async () => { const searchAfterSortId = '1234567891111'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts index 157f741439bd2..894e934ff0247 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts @@ -878,7 +878,7 @@ describe('utils', () => { ]; const createdErrors = createErrorsFromShard({ errors }); expect(createdErrors).toEqual([ - 'reason: some reason, type: some type, caused by: some reason', + 'reason: "some reason" type: "some type" caused by reason: "some reason" caused by type: "some type"', ]); }); @@ -917,8 +917,54 @@ describe('utils', () => { ]; const createdErrors = createErrorsFromShard({ errors }); expect(createdErrors).toEqual([ - 'reason: some reason, type: some type, caused by: some reason', - 'reason: some reason 2, type: some type 2, caused by: some reason 2', + 'reason: "some reason" type: "some type" caused by reason: "some reason" caused by type: "some type"', + 'reason: "some reason 2" type: "some type 2" caused by reason: "some reason 2" caused by type: "some type 2"', + ]); + }); + + test('You can have missing values for the shard errors and get the expected output of an empty string', () => { + const errors: ShardError[] = [ + { + shard: 1, + index: 'index-123', + node: 'node-123', + reason: {}, + }, + ]; + const createdErrors = createErrorsFromShard({ errors }); + expect(createdErrors).toEqual(['']); + }); + + test('You can have a single value for the shard errors and get expected output without extra spaces anywhere', () => { + const errors: ShardError[] = [ + { + shard: 1, + index: 'index-123', + node: 'node-123', + reason: { + reason: 'some reason something went wrong', + }, + }, + ]; + const createdErrors = createErrorsFromShard({ errors }); + expect(createdErrors).toEqual(['reason: "some reason something went wrong"']); + }); + + test('You can have two values for the shard errors and get expected output with one space exactly between the two values', () => { + const errors: ShardError[] = [ + { + shard: 1, + index: 'index-123', + node: 'node-123', + reason: { + reason: 'some reason something went wrong', + caused_by: { type: 'some type' }, + }, + }, + ]; + const createdErrors = createErrorsFromShard({ errors }); + expect(createdErrors).toEqual([ + 'reason: "some reason something went wrong" caused by type: "some type"', ]); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index ac10f5ed9a72d..e2b39b8d0a8c8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -511,7 +511,23 @@ export const getSignalTimeTuples = ({ */ export const createErrorsFromShard = ({ errors }: { errors: ShardError[] }): string[] => { return errors.map((error) => { - return `reason: ${error.reason.reason}, type: ${error.reason.caused_by.type}, caused by: ${error.reason.caused_by.reason}`; + const { + reason: { + reason, + type, + caused_by: { reason: causedByReason, type: causedByType } = { + reason: undefined, + type: undefined, + }, + } = {}, + } = error; + + return [ + ...(reason != null ? [`reason: "${reason}"`] : []), + ...(type != null ? [`type: "${type}"`] : []), + ...(causedByReason != null ? [`caused by reason: "${causedByReason}"`] : []), + ...(causedByType != null ? [`caused by type: "${causedByType}"`] : []), + ].join(' '); }); }; diff --git a/x-pack/plugins/security_solution/server/lib/framework/kibana_framework_adapter.ts b/x-pack/plugins/security_solution/server/lib/framework/kibana_framework_adapter.ts index 6d9e9b13bc356..e36fb1144e93f 100644 --- a/x-pack/plugins/security_solution/server/lib/framework/kibana_framework_adapter.ts +++ b/x-pack/plugins/security_solution/server/lib/framework/kibana_framework_adapter.ts @@ -149,14 +149,7 @@ export class KibanaBackendFrameworkAdapter implements FrameworkAdapter { } public getIndexPatternsService(request: FrameworkRequest): FrameworkIndexPatternsService { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const callCluster = async (endpoint: string, params?: Record) => - this.callWithRequest(request, endpoint, { - ...params, - allowNoIndices: true, - }); - - return new IndexPatternsFetcher(callCluster); + return new IndexPatternsFetcher(request.context.core.elasticsearch.client.asCurrentUser, true); } } diff --git a/x-pack/plugins/security_solution/server/lib/types.ts b/x-pack/plugins/security_solution/server/lib/types.ts index 29db38bbbea68..c735412aedbf5 100644 --- a/x-pack/plugins/security_solution/server/lib/types.ts +++ b/x-pack/plugins/security_solution/server/lib/types.ts @@ -48,21 +48,26 @@ export interface ShardsResponse { failures?: ShardError[]; } -export interface ShardError { +/** + * This type is being very conservative with the partials to not expect anything to + * be guaranteed on the type as we don't have regular and proper types of ShardError. + * Once we do, remove this type for the regular ShardError type from the elastic library. + */ +export type ShardError = Partial<{ shard: number; index: string; node: string; - reason: { + reason: Partial<{ type: string; reason: string; index_uuid: string; index: string; - caused_by: { + caused_by: Partial<{ type: string; reason: string; - }; - }; -} + }>; + }>; +}>; export interface SearchResponse { took: number; diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index f5e1c6936cbd6..43f87a0c69ab3 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -23,7 +23,10 @@ import { PluginStart as DataPluginStart, } from '../../../../src/plugins/data/server'; import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/server'; -import { PluginSetupContract as AlertingSetup } from '../../alerts/server'; +import { + PluginSetupContract as AlertingSetup, + PluginStartContract as AlertPluginStartContract, +} from '../../alerts/server'; import { SecurityPluginSetup as SecuritySetup } from '../../security/server'; import { PluginSetupContract as FeaturesSetup } from '../../features/server'; import { MlPluginSetup as MlSetup } from '../../ml/server'; @@ -88,6 +91,7 @@ export interface SetupPlugins { } export interface StartPlugins { + alerts: AlertPluginStartContract; data: DataPluginStart; ingestManager?: IngestManagerStartContract; taskManager?: TaskManagerStartContract; @@ -113,8 +117,10 @@ const securitySubPlugins = [ export class Plugin implements IPlugin { private readonly logger: Logger; private readonly config$: Observable; + private config?: ConfigType; private context: PluginInitializerContext; private appClientFactory: AppClientFactory; + private setupPlugins?: SetupPlugins; private readonly endpointAppContextService = new EndpointAppContextService(); private readonly telemetryEventsSender: TelemetryEventsSender; @@ -137,8 +143,10 @@ export class Plugin implements IPlugin, plugins: SetupPlugins) { this.logger.debug('plugin setup'); + this.setupPlugins = plugins; const config = await this.config$.pipe(first()).toPromise(); + this.config = config; const globalConfig = await this.context.config.legacy.globalConfig$.pipe(first()).toPromise(); initSavedObjects(core.savedObjects); @@ -337,6 +345,10 @@ export class Plugin implements IPlugin(async (resolve) => { const { elasticsearch } = context.core; - const indexPatternsFetcher = new IndexPatternsFetcher( - elasticsearch.legacy.client.callAsCurrentUser - ); + const indexPatternsFetcher = new IndexPatternsFetcher(elasticsearch.client.asCurrentUser); const dedupeIndices = dedupeIndexName(request.indices); const responsesIndexFields = await Promise.all( diff --git a/x-pack/plugins/task_manager/server/MONITORING.md b/x-pack/plugins/task_manager/server/MONITORING.md new file mode 100644 index 0000000000000..4960086411e9a --- /dev/null +++ b/x-pack/plugins/task_manager/server/MONITORING.md @@ -0,0 +1,265 @@ +# Task Manager Monitoring + +Task Manager has an internal monitoring mechanism in which keeps track of a variety of metrics which are exposed via a `health` api endpoint and Kibana Server Log debug messaging. + +## Exposed Metrics +There are three different sections to the stats returned by the `health` api. +- `configuration`: Summarizes Task Manager's current configuration. +- `workload`: Summarizes the workload in the current deployment. +- `runtime`: Tracks Task Manager's performance. + +### Configuring the Stats +There are four new configurations: + +- `xpack.task_manager.monitored_stats_required_freshness` - The _required freshness_ of critical "Hot" stats, which means that if key stats (last polling cycle time, for example) haven't been refreshed within the specified duration, the `_health` endpoint and service will report an `Error` status. By default this is inferred from the configured `poll_interval` and is set to `poll_interval` plus a `1s` buffer. +- `xpack.task_manager.monitored_aggregated_stats_refresh_rate` - Dictates how often we refresh the "Cold" metrics. These metrics require an aggregation against Elasticsearch and add load to the system, hence we want to limit how often we execute these. We also inffer the _required freshness_ of these "Cold" metrics from this configuration, which means that if these stats have not been updated within the required duration then the `_health` endpoint and service will report an `Error` status. This covers the entire `workload` section of the stats. By default this is configured to `60s`, and as a result the _required freshness_ defaults to `61s` (refresh plus a `1s` buffer). +- `xpack.task_manager.monitored_stats_running_average_window`- Dictates the size of the window used to calculate the running average of various "Hot" stats, such as the time it takes to run a task, the _drift_ that tasks experience etc. These stats are collected throughout the lifecycle of tasks and this window will dictate how large the queue we keep in memory would be, and how many values we need to calculate the average against. We do not calculate the average on *every* new value, but rather only when the time comes to summarize the stats before logging them or returning them to the API endpoint. +- `xpack.task_manager.monitored_task_execution_thresholds`- Configures the threshold of failed task executions at which point the `warn` or `error` health status will be set either at a default level or a custom level for specific task types. This will allow you to mark the health as `error` when any task type failes 90% of the time, but set it to `error` at 50% of the time for task types that you consider critical. This value can be set to any number between 0 to 100, and a threshold is hit when the value *exceeds* this number. This means that you can avoid setting the status to `error` by setting the threshold at 100, or hit `error` the moment any task failes by setting the threshold to 0 (as it will exceed 0 once a single failer occurs). + +For example, in your `Kibana.yml`: +``` +xpack.task_manager.monitored_stats_required_freshness: 5000 +xpack.task_manager.monitored_aggregated_stats_refresh_rate: 60000 +xpack.task_manager.monitored_stats_running_average_window: 50 +xpack.task_manager.monitored_task_execution_thresholds: + default: + error_threshold: 70 + warn_threshold: 50 + custom: + "alerting:always-firing": + error_threshold: 50 + warn_threshold: 0 +``` + +## Consuming Health Stats +Task Manager exposes a `/api/task_manager/_health` api which returns the _latest_ stats. +Calling this API is designed to be fast and doesn't actually perform any checks- rather it returns the result of the latest stats in the system, and is design in such a way that you could call it from an external service on a regular basis without worrying that you'll be adding substantial load to the system. + +Additionally, the metrics are logged out into Task Manager's `DEBUG` logger at a regular cadence (dictated by the Polling Interval). +If you wish to enable DEBUG logging in your Kibana instance, you will need to add the following to your `Kibana.yml`: +``` +logging: + loggers: + - context: plugins.taskManager + appenders: [console] + level: debug +``` + +Please bear in mind that these stats are logged as often as your `poll_interval` configuration, which means it could add substantial noise to your logs. +We would recommend only enabling this level of logging temporarily. + +### Understanding the Exposed Stats + +As mentioned above, the `health` api exposes three sections: `configuration`, `workload` and `runtime`. +Each section has a `timestamp` and a `status` which indicates when the last update to this setion took place and whether the health of this section was evaluated as `OK`, `Warning` or `Error`. + +The root has its own `status` which indicate the state of the system overall as infered from the `status` of the section. +An `Error` status in any section will cause the whole system to display as `Error`. +A `Warning` status in any section will cause the whole system to display as `Warning`. +An `OK` status will only be displayed when all sections are marked as `OK`. + +The root `timestamp` is the time in which the summary was exposed (either to the DEBUG logger or the http api) and the `last_update` is the last time any one of the sections was updated. + +#### The Configuration Section +The `configuration` section summarizes Task Manager's current configuration, including dynamic configurations which change over time, such as `poll_interval` and `max_workers` which adjust in reaction to changing load on the system. + +These are "Hot" stats which are updated whenever a change happens in the configuration. + +#### The Workload Section +The `workload` which summarizes the work load in the current deployment, listing the tasks in the system, their types and what their current status is. + +It includes three sub sections: + - The number of tasks scheduled in the system, broken down by type and status. + - The number of idle `overdue` tasks, whose `runAt` has expired. + - Execution density in the next minute or so (configurable), which shows how many tasks are scheduled to execute in the scope of each polling interval. This can give us an idea of how much load there is on the current Kibana deployment. + +These are "Cold" stat which are updated at a regular cadence, configured by the `monitored_aggregated_stats_refresh_rate` config. + +#### The Runtime Section +The `runtime` tracks Task Manager's performance as it runs, making note of task execution time, _drift_ etc. +These include: + - The time it takes a task to run (mean and median, using a configurable running average window, `50` by default) + - The average _drift_ that tasks experience (mean and median, using the same configurable running average window as above). Drift tells us how long after a task's scheduled a task typically executes. + - The polling rate (the timestamp of the last time a polling cycle completed) and the result [`No tasks | Filled task pool | Unexpectedly ran out of workers`] frequency the past 50 polling cycles (using the same window size as the one used for running averages) + - The `Success | Retry | Failure ratio` by task type. This is different than the workload stats which tell you what's in the queue, but ca't keep track of retries and of non recurring tasks as they're wiped off the index when completed. + +These are "Hot" stats which are updated reactively as Tasks are executed and interacted with. + +### Example Stats + +For example, if you _curl_ the `/api/task_manager/_health` endpoint, you might get these stats: +``` +{ + /* the time these stats were returned by the api */ + "timestamp": "2020-10-05T18:26:11.346Z", + /* the overall status of the system */ + "status": "OK", + /* last time any stat was updated in this output */ + "last_update": "2020-10-05T17:57:55.411Z", + "stats": { + "configuration": { /* current configuration of TM */ + "timestamp": "2020-10-05T17:56:06.507Z", + "status": "OK", + "value": { + "max_workers": 10, + "poll_interval": 3000, + "request_capacity": 1000, + "max_poll_inactivity_cycles": 10, + "monitored_aggregated_stats_refresh_rate": 60000, + "monitored_stats_running_average_window": 50 + } + }, + "workload": { /* The workload of this deployment */ + "timestamp": "2020-10-05T17:57:06.534Z", + "status": "OK", + "value": { + "count": 6, /* count of tasks in the system */ + "task_types": { /* what tasks are there and what status are they in */ + "actions_telemetry": { + "count": 1, + "status": { + "idle": 1 + } + }, + "alerting_telemetry": { + "count": 1, + "status": { + "idle": 1 + } + }, + "apm-telemetry-task": { + "count": 1, + "status": { + "idle": 1 + } + }, + "endpoint:user-artifact-packager": { + "count": 1, + "status": { + "idle": 1 + } + }, + "lens_telemetry": { + "count": 1, + "status": { + "idle": 1 + } + }, + "session_cleanup": { + "count": 1, + "status": { + "idle": 1 + } + } + }, + + /* Frequency of recurring tasks schedules */ + "schedule": [ + ["60s", 1], /* 1 task, every 60s */ + ["3600s", 3], /* 3 tasks every hour */ + ["720m", 1] + ], + /* There are no overdue tasks in this system at the moment */ + "overdue": 0, + /* This is the schedule density, it shows a histogram of all the polling intervals in the next minute (or, if + pollInterval is configured unusually high it will show a min of 2 refresh intervals into the future, and a max of 50 buckets). + Here we see that on the 3rd polling interval from *now* (which is ~9 seconds from now, as pollInterval is `3s`) there is one task due to run. + We also see that there are 5 due two intervals later, which is fine as we have a max workers of `10` + */ + "estimated_schedule_density": [0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + } + }, + "runtime": { + "timestamp": "2020-10-05T17:57:55.411Z", + "status": "OK", + "value": { + "polling": { + /* When was the last polling cycle? */ + "last_successful_poll": "2020-10-05T17:57:55.411Z", + /* What is the frequency of polling cycle result? + Here we see 94% of "NoTasksClaimed" and 6% "PoolFilled" */ + "result_frequency_percent_as_number": { + "NoTasksClaimed": 94, + "RanOutOfCapacity": 0, /* This is a legacy result, we might want to rename - it tells us when a polling cycle resulted in claiming more tasks than we had workers for, butt he name doesn't make much sense outside of the context of the code */ + "PoolFilled": 6 + } + }, + /* on average, the tasks in this deployment run 1.7s after their scheduled time */ + "drift": { + "mean": 1720, + "median": 2276 + }, + "execution": { + "duration": { + /* on average, the `endpoint:user-artifact-packager` tasks take 15ms to run */ + "endpoint:user-artifact-packager": { + "mean": 15, + "median": 14.5 + }, + "session_cleanup": { + "mean": 28, + "median": 28 + }, + "lens_telemetry": { + "mean": 100, + "median": 100 + }, + "actions_telemetry": { + "mean": 135, + "median": 135 + }, + "alerting_telemetry": { + "mean": 197, + "median": 197 + }, + "apm-telemetry-task": { + "mean": 1347, + "median": 1347 + } + }, + "result_frequency_percent_as_number": { + /* and 100% of `endpoint:user-artifact-packager` have completed in success (within the running average window, so the past 50 runs (by default, configrable by `monitored_stats_running_average_window`) */ + "endpoint:user-artifact-packager": { + "status": "OK", + "Success": 100, + "RetryScheduled": 0, + "Failed": 0 + }, + "session_cleanup": { + /* `error` status as 90% of results are `Failed` */ + "status": "error", + "Success": 5, + "RetryScheduled": 5, + "Failed": 90 + }, + "lens_telemetry": { + "status": "OK", + "Success": 100, + "RetryScheduled": 0, + "Failed": 0 + }, + "actions_telemetry": { + "status": "OK", + "Success": 100, + "RetryScheduled": 0, + "Failed": 0 + }, + "alerting_telemetry": { + "status": "OK", + "Success": 100, + "RetryScheduled": 0, + "Failed": 0 + }, + "apm-telemetry-task": { + "status": "OK", + "Success": 100, + "RetryScheduled": 0, + "Failed": 0 + } + } + } + } + } + } +} +``` diff --git a/x-pack/plugins/task_manager/server/README.md b/x-pack/plugins/task_manager/server/README.md index fd2409a7db0a5..a0b35ad094537 100644 --- a/x-pack/plugins/task_manager/server/README.md +++ b/x-pack/plugins/task_manager/server/README.md @@ -48,6 +48,10 @@ The task_manager can be configured via `taskManager` config options (e.g. `taskM - `override_num_workers`: An object of `taskType: number` that overrides the `num_workers` for tasks - For example: `task_manager.override_num_workers.reporting: 2` would override the number of workers occupied by tasks of type `reporting` - This allows sysadmins to tweak the operational performance of Kibana, allowing more or fewer tasks of a specific type to run simultaneously +- `monitored_aggregated_stats_refresh_rate` - Dictates how often we refresh the "Cold" metrics. Learn More: [./MONITORING](./MONITORING.MD) +- `monitored_stats_running_average_window`- Dictates the size of the window used to calculate the running average of various "Hot" stats. Learn More: [./MONITORING](./MONITORING.MD) +- `monitored_stats_required_freshness` - Dictates the _required freshness_ of critical "Hot" stats. Learn More: [./MONITORING](./MONITORING.MD) +- `monitored_task_execution_thresholds`- Dictates the threshold of failed task executions. Learn More: [./MONITORING](./MONITORING.MD) ## Task definitions @@ -460,3 +464,9 @@ The task manager's public API is create / delete / list. Updates aren't directly node scripts/functional_tests_server.js --config x-pack/test/plugin_api_integration/config.ts node scripts/functional_test_runner --config x-pack/test/plugin_api_integration/config.ts ``` + +## Monitoring + +Task Manager exposes runtime statistics which enable basic observability into its inner workings and makes it possible to monitor the system from external services. + +Learn More: [./MONITORING](./MONITORING.MD) \ No newline at end of file diff --git a/x-pack/plugins/task_manager/server/config.test.ts b/x-pack/plugins/task_manager/server/config.test.ts index d5bbbe65582f1..d2d5ac8f22a1f 100644 --- a/x-pack/plugins/task_manager/server/config.test.ts +++ b/x-pack/plugins/task_manager/server/config.test.ts @@ -15,6 +15,16 @@ describe('config validation', () => { "max_attempts": 3, "max_poll_inactivity_cycles": 10, "max_workers": 10, + "monitored_aggregated_stats_refresh_rate": 60000, + "monitored_stats_required_freshness": 4000, + "monitored_stats_running_average_window": 50, + "monitored_task_execution_thresholds": Object { + "custom": Object {}, + "default": Object { + "error_threshold": 90, + "warn_threshold": 80, + }, + }, "poll_interval": 3000, "request_capacity": 1000, } @@ -31,4 +41,147 @@ describe('config validation', () => { `"[index]: \\".tasks\\" is an invalid Kibana Task Manager index, as it is already in use by the ElasticSearch Tasks Manager"` ); }); + + test('the required freshness of the monitored stats config must always be less-than-equal to the poll interval', () => { + const config: Record = { + monitored_stats_required_freshness: 100, + }; + expect(() => { + configSchema.validate(config); + }).toThrowErrorMatchingInlineSnapshot( + `"The specified monitored_stats_required_freshness (100) is invalid, as it is below the poll_interval (3000)"` + ); + }); + + test('the default required freshness of the monitored stats is poll interval with a slight buffer', () => { + const config: Record = {}; + expect(configSchema.validate(config)).toMatchInlineSnapshot(` + Object { + "enabled": true, + "index": ".kibana_task_manager", + "max_attempts": 3, + "max_poll_inactivity_cycles": 10, + "max_workers": 10, + "monitored_aggregated_stats_refresh_rate": 60000, + "monitored_stats_required_freshness": 4000, + "monitored_stats_running_average_window": 50, + "monitored_task_execution_thresholds": Object { + "custom": Object {}, + "default": Object { + "error_threshold": 90, + "warn_threshold": 80, + }, + }, + "poll_interval": 3000, + "request_capacity": 1000, + } + `); + }); + + test('the custom monitored_task_execution_thresholds can be configured at task type', () => { + const config: Record = { + monitored_task_execution_thresholds: { + custom: { + 'alerting:always-fires': { + error_threshold: 50, + warn_threshold: 30, + }, + }, + }, + }; + expect(configSchema.validate(config)).toMatchInlineSnapshot(` + Object { + "enabled": true, + "index": ".kibana_task_manager", + "max_attempts": 3, + "max_poll_inactivity_cycles": 10, + "max_workers": 10, + "monitored_aggregated_stats_refresh_rate": 60000, + "monitored_stats_required_freshness": 4000, + "monitored_stats_running_average_window": 50, + "monitored_task_execution_thresholds": Object { + "custom": Object { + "alerting:always-fires": Object { + "error_threshold": 50, + "warn_threshold": 30, + }, + }, + "default": Object { + "error_threshold": 90, + "warn_threshold": 80, + }, + }, + "poll_interval": 3000, + "request_capacity": 1000, + } + `); + }); + + test('the monitored_task_execution_thresholds ensures that the default warn_threshold is lt the default error_threshold', () => { + const config: Record = { + monitored_task_execution_thresholds: { + default: { + warn_threshold: 80, + error_threshold: 70, + }, + }, + }; + expect(() => { + configSchema.validate(config); + }).toThrowErrorMatchingInlineSnapshot( + `"[monitored_task_execution_thresholds.default]: warn_threshold (80) must be less than, or equal to, error_threshold (70)"` + ); + }); + + test('the monitored_task_execution_thresholds allows the default warn_threshold to equal the default error_threshold', () => { + const config: Record = { + monitored_task_execution_thresholds: { + default: { + warn_threshold: 70, + error_threshold: 70, + }, + }, + }; + expect(() => { + configSchema.validate(config); + }).not.toThrowError(); + }); + + test('the monitored_task_execution_thresholds ensures that the warn_threshold is lte error_threshold on custom thresholds', () => { + const config: Record = { + monitored_task_execution_thresholds: { + custom: { + 'alerting:always-fires': { + error_threshold: 80, + warn_threshold: 90, + }, + }, + }, + }; + expect(() => { + configSchema.validate(config); + }).toThrowErrorMatchingInlineSnapshot( + `"[monitored_task_execution_thresholds.custom.alerting:always-fires]: warn_threshold (90) must be less than, or equal to, error_threshold (80)"` + ); + }); + + test('the monitored_task_execution_thresholds allows a custom error_threshold which is lower than the default warn_threshold', () => { + const config: Record = { + monitored_task_execution_thresholds: { + default: { + warn_threshold: 80, + error_threshold: 90, + }, + custom: { + 'alerting:always-fires': { + error_threshold: 60, + warn_threshold: 50, + }, + }, + }, + }; + expect(() => { + configSchema.validate(config); + }).not.toThrowError(); + }); }); diff --git a/x-pack/plugins/task_manager/server/config.ts b/x-pack/plugins/task_manager/server/config.ts index aa78cf3baa96d..157f01281836d 100644 --- a/x-pack/plugins/task_manager/server/config.ts +++ b/x-pack/plugins/task_manager/server/config.ts @@ -10,44 +10,109 @@ export const DEFAULT_MAX_WORKERS = 10; export const DEFAULT_POLL_INTERVAL = 3000; export const DEFAULT_MAX_POLL_INACTIVITY_CYCLES = 10; -export const configSchema = schema.object({ - enabled: schema.boolean({ defaultValue: true }), - /* The maximum number of times a task will be attempted before being abandoned as failed */ - max_attempts: schema.number({ - defaultValue: 3, - min: 1, - }), - /* How often, in milliseconds, the task manager will look for more work. */ - poll_interval: schema.number({ - defaultValue: DEFAULT_POLL_INTERVAL, - min: 100, - }), - /* How many poll interval cycles can work take before it's timed out. */ - max_poll_inactivity_cycles: schema.number({ - defaultValue: DEFAULT_MAX_POLL_INACTIVITY_CYCLES, - min: 1, - }), - /* How many requests can Task Manager buffer before it rejects new requests. */ - request_capacity: schema.number({ - // a nice round contrived number, feel free to change as we learn how it behaves - defaultValue: 1000, - min: 1, - }), - /* The name of the index used to store task information. */ - index: schema.string({ - defaultValue: '.kibana_task_manager', - validate: (val) => { - if (val.toLowerCase() === '.tasks') { - return `"${val}" is an invalid Kibana Task Manager index, as it is already in use by the ElasticSearch Tasks Manager`; +// Monitoring Constants +// =================== +// Refresh aggregated monitored stats at a default rate of once a minute +export const DEFAULT_MONITORING_REFRESH_RATE = 60 * 1000; +export const DEFAULT_MONITORING_STATS_RUNNING_AVERGAE_WINDOW = 50; + +export const taskExecutionFailureThresholdSchema = schema.object( + { + error_threshold: schema.number({ + defaultValue: 90, + min: 0, + }), + warn_threshold: schema.number({ + defaultValue: 80, + min: 0, + }), + }, + { + validate(config) { + if (config.error_threshold < config.warn_threshold) { + return `warn_threshold (${config.warn_threshold}) must be less than, or equal to, error_threshold (${config.error_threshold})`; + } + }, + } +); + +export const configSchema = schema.object( + { + enabled: schema.boolean({ defaultValue: true }), + /* The maximum number of times a task will be attempted before being abandoned as failed */ + max_attempts: schema.number({ + defaultValue: 3, + min: 1, + }), + /* How often, in milliseconds, the task manager will look for more work. */ + poll_interval: schema.number({ + defaultValue: DEFAULT_POLL_INTERVAL, + min: 100, + }), + /* How many poll interval cycles can work take before it's timed out. */ + max_poll_inactivity_cycles: schema.number({ + defaultValue: DEFAULT_MAX_POLL_INACTIVITY_CYCLES, + min: 1, + }), + /* How many requests can Task Manager buffer before it rejects new requests. */ + request_capacity: schema.number({ + // a nice round contrived number, feel free to change as we learn how it behaves + defaultValue: 1000, + min: 1, + }), + /* The name of the index used to store task information. */ + index: schema.string({ + defaultValue: '.kibana_task_manager', + validate: (val) => { + if (val.toLowerCase() === '.tasks') { + return `"${val}" is an invalid Kibana Task Manager index, as it is already in use by the ElasticSearch Tasks Manager`; + } + }, + }), + /* The maximum number of tasks that this Kibana instance will run simultaneously. */ + max_workers: schema.number({ + defaultValue: DEFAULT_MAX_WORKERS, + // disable the task manager rather than trying to specify it with 0 workers + min: 1, + }), + /* The rate at which we emit fresh monitored stats. By default we'll use the poll_interval (+ a slight buffer) */ + monitored_stats_required_freshness: schema.number({ + defaultValue: (config?: unknown) => + ((config as { poll_interval: number })?.poll_interval ?? DEFAULT_POLL_INTERVAL) + 1000, + min: 100, + }), + /* The rate at which we refresh monitored stats that require aggregation queries against ES. */ + monitored_aggregated_stats_refresh_rate: schema.number({ + defaultValue: DEFAULT_MONITORING_REFRESH_RATE, + /* don't run monitored stat aggregations any faster than once every 5 seconds */ + min: 5000, + }), + /* The size of the running average window for monitored stats. */ + monitored_stats_running_average_window: schema.number({ + defaultValue: DEFAULT_MONITORING_STATS_RUNNING_AVERGAE_WINDOW, + max: 100, + min: 10, + }), + /* Task Execution result warn & error thresholds. */ + monitored_task_execution_thresholds: schema.object({ + default: taskExecutionFailureThresholdSchema, + custom: schema.recordOf(schema.string(), taskExecutionFailureThresholdSchema, { + defaultValue: {}, + }), + }), + }, + { + validate: (config) => { + if ( + config.monitored_stats_required_freshness && + config.poll_interval && + config.monitored_stats_required_freshness < config.poll_interval + ) { + return `The specified monitored_stats_required_freshness (${config.monitored_stats_required_freshness}) is invalid, as it is below the poll_interval (${config.poll_interval})`; } }, - }), - /* The maximum number of tasks that this Kibana instance will run simultaneously. */ - max_workers: schema.number({ - defaultValue: DEFAULT_MAX_WORKERS, - // disable the task manager rather than trying to specify it with 0 workers - min: 1, - }), -}); + } +); export type TaskManagerConfig = TypeOf; +export type TaskExecutionFailureThreshold = TypeOf; diff --git a/x-pack/plugins/task_manager/server/integration_tests/managed_configuration.test.ts b/x-pack/plugins/task_manager/server/integration_tests/managed_configuration.test.ts index 11f6ccc881850..01326c73bd680 100644 --- a/x-pack/plugins/task_manager/server/integration_tests/managed_configuration.test.ts +++ b/x-pack/plugins/task_manager/server/integration_tests/managed_configuration.test.ts @@ -30,7 +30,17 @@ describe('managed configuration', () => { max_attempts: 9, poll_interval: 3000, max_poll_inactivity_cycles: 10, + monitored_aggregated_stats_refresh_rate: 60000, + monitored_stats_required_freshness: 4000, + monitored_stats_running_average_window: 50, request_capacity: 1000, + monitored_task_execution_thresholds: { + default: { + error_threshold: 90, + warn_threshold: 80, + }, + custom: {}, + }, }); logger = context.logger.get('taskManager'); diff --git a/x-pack/plugins/task_manager/server/lib/bulk_operation_buffer.ts b/x-pack/plugins/task_manager/server/lib/bulk_operation_buffer.ts index 6df5b064f2792..4de92ffc77030 100644 --- a/x-pack/plugins/task_manager/server/lib/bulk_operation_buffer.ts +++ b/x-pack/plugins/task_manager/server/lib/bulk_operation_buffer.ts @@ -4,11 +4,11 @@ * you may not use this file except in compliance with the Elastic License. */ +import { Logger } from 'src/core/server'; import { map } from 'lodash'; import { Subject, race, from } from 'rxjs'; import { bufferWhen, filter, bufferCount, flatMap, mapTo, first } from 'rxjs/operators'; import { either, Result, asOk, asErr, Ok, Err } from './result_type'; -import { Logger } from '../../../../../src/core/server'; export interface BufferOptions { bufferMaxDuration?: number; diff --git a/x-pack/plugins/task_manager/server/lib/correct_deprecated_fields.ts b/x-pack/plugins/task_manager/server/lib/correct_deprecated_fields.ts index 9e5f4b7c143a2..a15682a9d3f38 100644 --- a/x-pack/plugins/task_manager/server/lib/correct_deprecated_fields.ts +++ b/x-pack/plugins/task_manager/server/lib/correct_deprecated_fields.ts @@ -4,8 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ +import { Logger } from 'src/core/server'; import { TaskInstance, TaskInstanceWithDeprecatedFields } from '../task'; -import { Logger } from '../../../../../src/core/server'; export function ensureDeprecatedFieldsAreCorrected( { id, taskType, interval, schedule, ...taskInstance }: TaskInstanceWithDeprecatedFields, diff --git a/x-pack/plugins/task_manager/server/lib/intervals.test.ts b/x-pack/plugins/task_manager/server/lib/intervals.test.ts index ac28b81eaf490..e79694915f926 100644 --- a/x-pack/plugins/task_manager/server/lib/intervals.test.ts +++ b/x-pack/plugins/task_manager/server/lib/intervals.test.ts @@ -7,13 +7,13 @@ import _ from 'lodash'; import sinon from 'sinon'; import { - assertValidInterval, + parseIntervalAsSecond, + parseIntervalAsMillisecond, intervalFromNow, intervalFromDate, - minutesFromNow, - minutesFromDate, secondsFromNow, secondsFromDate, + asInterval, } from './intervals'; let fakeTimer: sinon.SinonFakeTimers; @@ -25,32 +25,100 @@ beforeAll(() => { afterAll(() => fakeTimer.restore()); describe('taskIntervals', () => { - describe('assertValidInterval', () => { + describe('parseIntervalAsSecond', () => { test('it accepts intervals in the form `Nm`', () => { - expect(() => assertValidInterval(`${_.random(1, 1000)}m`)).not.toThrow(); + expect(() => parseIntervalAsSecond(`${_.random(1, 1000)}m`)).not.toThrow(); }); test('it accepts intervals in the form `Ns`', () => { - expect(() => assertValidInterval(`${_.random(1, 1000)}s`)).not.toThrow(); + expect(() => parseIntervalAsSecond(`${_.random(1, 1000)}s`)).not.toThrow(); }); test('it rejects 0 based intervals', () => { - expect(() => assertValidInterval('0m')).toThrow( + expect(() => parseIntervalAsSecond('0m')).toThrow( /Invalid interval "0m"\. Intervals must be of the form {number}m. Example: 5m/ ); - expect(() => assertValidInterval('0s')).toThrow( + expect(() => parseIntervalAsSecond('0s')).toThrow( /Invalid interval "0s"\. Intervals must be of the form {number}m. Example: 5m/ ); }); test('it rejects intervals are not of the form `Nm` or `Ns`', () => { - expect(() => assertValidInterval(`5m 2s`)).toThrow( + expect(() => parseIntervalAsSecond(`5m 2s`)).toThrow( /Invalid interval "5m 2s"\. Intervals must be of the form {number}m. Example: 5m/ ); - expect(() => assertValidInterval(`hello`)).toThrow( + expect(() => parseIntervalAsSecond(`hello`)).toThrow( /Invalid interval "hello"\. Intervals must be of the form {number}m. Example: 5m/ ); }); + + test('returns an interval as s', () => { + expect(parseIntervalAsSecond('5s')).toEqual(5); + expect(parseIntervalAsSecond('15s')).toEqual(15); + expect(parseIntervalAsSecond('20m')).toEqual(20 * 60); + expect(parseIntervalAsSecond('61m')).toEqual(61 * 60); + expect(parseIntervalAsSecond('90m')).toEqual(90 * 60); + }); + }); + + describe('parseIntervalAsMillisecond', () => { + test('it accepts intervals in the form `Nm`', () => { + expect(() => parseIntervalAsMillisecond(`${_.random(1, 1000)}m`)).not.toThrow(); + }); + + test('it accepts intervals in the form `Ns`', () => { + expect(() => parseIntervalAsMillisecond(`${_.random(1, 1000)}s`)).not.toThrow(); + }); + + test('it rejects 0 based intervals', () => { + expect(() => parseIntervalAsMillisecond('0m')).toThrow( + /Invalid interval "0m"\. Intervals must be of the form {number}m. Example: 5m/ + ); + expect(() => parseIntervalAsMillisecond('0s')).toThrow( + /Invalid interval "0s"\. Intervals must be of the form {number}m. Example: 5m/ + ); + }); + + test('it rejects intervals are not of the form `Nm` or `Ns`', () => { + expect(() => parseIntervalAsMillisecond(`5m 2s`)).toThrow( + /Invalid interval "5m 2s"\. Intervals must be of the form {number}m. Example: 5m/ + ); + expect(() => parseIntervalAsMillisecond(`hello`)).toThrow( + /Invalid interval "hello"\. Intervals must be of the form {number}m. Example: 5m/ + ); + }); + + test('returns an interval as ms', () => { + expect(parseIntervalAsMillisecond('5s')).toEqual(5 * 1000); + expect(parseIntervalAsMillisecond('15s')).toEqual(15 * 1000); + expect(parseIntervalAsMillisecond('20m')).toEqual(20 * 60 * 1000); + expect(parseIntervalAsMillisecond('61m')).toEqual(61 * 60 * 1000); + expect(parseIntervalAsMillisecond('90m')).toEqual(90 * 60 * 1000); + }); + }); + + describe('asInterval', () => { + test('returns a ms interval when ms duration can only divide by ms', () => { + expect(asInterval(500)).toEqual('500ms'); + expect(asInterval(1500)).toEqual('1500ms'); + expect(asInterval(1001)).toEqual('1001ms'); + expect(asInterval(2001)).toEqual('2001ms'); + expect(asInterval(61001)).toEqual('61001ms'); + expect(asInterval(90001)).toEqual('90001ms'); + }); + + test('returns a seconds interval when ms duration divides by seconds', () => { + expect(asInterval(1000)).toEqual('1s'); + expect(asInterval(2000)).toEqual('2s'); + expect(asInterval(61000)).toEqual('61s'); + expect(asInterval(99000)).toEqual('99s'); + expect(asInterval(90000)).toEqual('90s'); + }); + + test('returns a minutes interval when ms duration divides by minutes', () => { + expect(asInterval(60000)).toEqual('1m'); + expect(asInterval(120000)).toEqual('2m'); + }); }); describe('intervalFromNow', () => { @@ -125,25 +193,6 @@ describe('taskIntervals', () => { }); }); - describe('minutesFromNow', () => { - test('it returns the current date plus a number of minutes', () => { - const mins = _.random(1, 100); - const expected = Date.now() + mins * 60 * 1000; - const nextRun = minutesFromNow(mins).getTime(); - expect(nextRun).toEqual(expected); - }); - }); - - describe('minutesFromDate', () => { - test('it returns the given date plus a number of minutes', () => { - const originalDate = new Date(2019, 1, 1); - const mins = _.random(1, 100); - const expected = originalDate.valueOf() + mins * 60 * 1000; - const nextRun = minutesFromDate(originalDate, mins).getTime(); - expect(expected).toEqual(nextRun); - }); - }); - describe('secondsFromNow', () => { test('it returns the current date plus a number of seconds', () => { const secs = _.random(1, 100); diff --git a/x-pack/plugins/task_manager/server/lib/intervals.ts b/x-pack/plugins/task_manager/server/lib/intervals.ts index 9009be5f78220..a28dfa62a501f 100644 --- a/x-pack/plugins/task_manager/server/lib/intervals.ts +++ b/x-pack/plugins/task_manager/server/lib/intervals.ts @@ -4,6 +4,28 @@ * you may not use this file except in compliance with the Elastic License. */ +import { memoize } from 'lodash'; + +export enum IntervalCadence { + Minute = 'm', + Second = 's', +} +const VALID_CADENCE = new Set(Object.values(IntervalCadence)); +const CADENCE_IN_MS: Record = { + [IntervalCadence.Second]: 1000, + [IntervalCadence.Minute]: 60 * 1000, +}; + +function isCadence(cadence: IntervalCadence | string): cadence is IntervalCadence { + return VALID_CADENCE.has(cadence as IntervalCadence); +} + +export function asInterval(ms: number): string { + const secondsRemainder = ms % 1000; + const minutesRemainder = ms % 60000; + return secondsRemainder ? `${ms}ms` : minutesRemainder ? `${ms / 1000}s` : `${ms / 60000}m`; +} + /** * Returns a date that is the specified interval from now. Currently, * only minute-intervals and second-intervals are supported. @@ -14,14 +36,7 @@ export function intervalFromNow(interval?: string): Date | undefined { if (interval === undefined) { return; } - - assertValidInterval(interval); - - if (isSeconds(interval)) { - return secondsFromNow(parseInterval(interval)); - } - - return minutesFromNow(parseInterval(interval)); + return secondsFromNow(parseIntervalAsSecond(interval)); } /** @@ -35,37 +50,7 @@ export function intervalFromDate(date: Date, interval?: string): Date | undefine if (interval === undefined) { return; } - - assertValidInterval(interval); - - if (isSeconds(interval)) { - return secondsFromDate(date, parseInterval(interval)); - } - - return minutesFromDate(date, parseInterval(interval)); -} - -/** - * Returns a date that is mins minutes from now. - * - * @param mins The number of mintues from now - */ -export function minutesFromNow(mins: number): Date { - return minutesFromDate(new Date(), mins); -} - -/** - * Returns a date that is mins minutes from given date. - * - * @param date The date to add minutes to - * @param mins The number of mintues from given date - */ -export function minutesFromDate(date: Date, mins: number): Date { - const result = new Date(date.valueOf()); - - result.setMinutes(result.getMinutes() + mins); - - return result; + return secondsFromDate(date, parseIntervalAsSecond(interval)); } /** @@ -85,9 +70,7 @@ export function secondsFromNow(secs: number): Date { */ export function secondsFromDate(date: Date, secs: number): Date { const result = new Date(date.valueOf()); - result.setSeconds(result.getSeconds() + secs); - return result; } @@ -95,29 +78,22 @@ export function secondsFromDate(date: Date, secs: number): Date { * Verifies that the specified interval matches our expected format. * * @param {string} interval - An interval such as `5m` or `10s` + * @returns {number} The interval as seconds */ -export function assertValidInterval(interval: string) { - if (isMinutes(interval)) { - return interval; +export const parseIntervalAsSecond = memoize((interval: string): number => { + return Math.round(parseIntervalAsMillisecond(interval) / 1000); +}); + +export const parseIntervalAsMillisecond = memoize((interval: string): number => { + const numericAsStr: string = interval.slice(0, -1); + const numeric: number = parseInt(numericAsStr, 10); + const cadence: IntervalCadence | string = interval.slice(-1); + if (!isCadence(cadence) || isNaN(numeric) || numeric <= 0 || !isNumeric(numericAsStr)) { + throw new Error( + `Invalid interval "${interval}". Intervals must be of the form {number}m. Example: 5m.` + ); } + return numeric * CADENCE_IN_MS[cadence]; +}); - if (isSeconds(interval)) { - return interval; - } - - throw new Error( - `Invalid interval "${interval}". Intervals must be of the form {number}m. Example: 5m.` - ); -} - -function parseInterval(interval: string) { - return parseInt(interval, 10); -} - -function isMinutes(interval: string) { - return /^[1-9][0-9]*m$/.test(interval); -} - -function isSeconds(interval: string) { - return /^[1-9][0-9]*s$/.test(interval); -} +const isNumeric = (numAsStr: string) => /^\d+$/.test(numAsStr); diff --git a/x-pack/plugins/task_manager/server/monitoring/configuration_statistics.test.ts b/x-pack/plugins/task_manager/server/monitoring/configuration_statistics.test.ts new file mode 100644 index 0000000000000..f97861901b5b5 --- /dev/null +++ b/x-pack/plugins/task_manager/server/monitoring/configuration_statistics.test.ts @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Subject } from 'rxjs'; +import { take, bufferCount } from 'rxjs/operators'; +import { createConfigurationAggregator } from './configuration_statistics'; +import { TaskManagerConfig } from '../config'; + +describe('Configuration Statistics Aggregator', () => { + test('merges the static config with the merged configs', async () => { + const configuration: TaskManagerConfig = { + enabled: true, + max_workers: 10, + index: 'foo', + max_attempts: 9, + poll_interval: 6000000, + monitored_stats_required_freshness: 6000000, + max_poll_inactivity_cycles: 10, + request_capacity: 1000, + monitored_aggregated_stats_refresh_rate: 5000, + monitored_stats_running_average_window: 50, + monitored_task_execution_thresholds: { + default: { + error_threshold: 90, + warn_threshold: 80, + }, + custom: {}, + }, + }; + + const managedConfig = { + maxWorkersConfiguration$: new Subject(), + pollIntervalConfiguration$: new Subject(), + }; + + return new Promise(async (resolve, reject) => { + createConfigurationAggregator(configuration, managedConfig) + .pipe(take(3), bufferCount(3)) + .subscribe(([initial, updatedWorkers, updatedInterval]) => { + expect(initial.value).toEqual({ + max_workers: 10, + poll_interval: 6000000, + max_poll_inactivity_cycles: 10, + request_capacity: 1000, + monitored_aggregated_stats_refresh_rate: 5000, + monitored_stats_running_average_window: 50, + monitored_task_execution_thresholds: { + default: { + error_threshold: 90, + warn_threshold: 80, + }, + custom: {}, + }, + }); + + expect(updatedWorkers.value).toEqual({ + max_workers: 8, + poll_interval: 6000000, + max_poll_inactivity_cycles: 10, + request_capacity: 1000, + monitored_aggregated_stats_refresh_rate: 5000, + monitored_stats_running_average_window: 50, + monitored_task_execution_thresholds: { + default: { + error_threshold: 90, + warn_threshold: 80, + }, + custom: {}, + }, + }); + + expect(updatedInterval.value).toEqual({ + max_workers: 8, + poll_interval: 3000, + max_poll_inactivity_cycles: 10, + request_capacity: 1000, + monitored_aggregated_stats_refresh_rate: 5000, + monitored_stats_running_average_window: 50, + monitored_task_execution_thresholds: { + default: { + error_threshold: 90, + warn_threshold: 80, + }, + custom: {}, + }, + }); + resolve(); + }, reject); + + managedConfig.maxWorkersConfiguration$.next(8); + + managedConfig.pollIntervalConfiguration$.next(3000); + }); + }); +}); diff --git a/x-pack/plugins/task_manager/server/monitoring/configuration_statistics.ts b/x-pack/plugins/task_manager/server/monitoring/configuration_statistics.ts new file mode 100644 index 0000000000000..22b08bc5c88db --- /dev/null +++ b/x-pack/plugins/task_manager/server/monitoring/configuration_statistics.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { combineLatest, of } from 'rxjs'; +import { pick, merge } from 'lodash'; +import { map, startWith } from 'rxjs/operators'; +import { AggregatedStatProvider } from './runtime_statistics_aggregator'; +import { TaskManagerConfig } from '../config'; +import { ManagedConfiguration } from '../lib/create_managed_configuration'; + +const CONFIG_FIELDS_TO_EXPOSE = [ + 'request_capacity', + 'max_poll_inactivity_cycles', + 'monitored_aggregated_stats_refresh_rate', + 'monitored_stats_running_average_window', + 'monitored_task_execution_thresholds', +] as const; + +export type ConfigStat = Pick< + TaskManagerConfig, + 'max_workers' | 'poll_interval' | typeof CONFIG_FIELDS_TO_EXPOSE[number] +>; + +export function createConfigurationAggregator( + config: TaskManagerConfig, + managedConfig: ManagedConfiguration +): AggregatedStatProvider { + return combineLatest([ + of(pick(config, ...CONFIG_FIELDS_TO_EXPOSE)), + managedConfig.pollIntervalConfiguration$.pipe( + startWith(config.poll_interval), + map>((pollInterval) => ({ + poll_interval: pollInterval, + })) + ), + managedConfig.maxWorkersConfiguration$.pipe( + startWith(config.max_workers), + map>((maxWorkers) => ({ + max_workers: maxWorkers, + })) + ), + ]).pipe( + map((configurations) => ({ + key: 'configuration', + value: merge({}, ...configurations), + })) + ); +} diff --git a/x-pack/plugins/task_manager/server/monitoring/index.ts b/x-pack/plugins/task_manager/server/monitoring/index.ts new file mode 100644 index 0000000000000..8e71ce2519a7c --- /dev/null +++ b/x-pack/plugins/task_manager/server/monitoring/index.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Logger } from 'src/core/server'; +import { Observable } from 'rxjs'; +import { TaskManagerConfig } from '../config'; +import { + MonitoringStats, + createAggregators, + createMonitoringStatsStream, +} from './monitoring_stats_stream'; +import { TaskStore } from '../task_store'; +import { TaskPollingLifecycle } from '../polling_lifecycle'; +import { ManagedConfiguration } from '../lib/create_managed_configuration'; + +export { + MonitoringStats, + HealthStatus, + RawMonitoringStats, + summarizeMonitoringStats, + createAggregators, + createMonitoringStatsStream, +} from './monitoring_stats_stream'; + +export function createMonitoringStats( + taskPollingLifecycle: TaskPollingLifecycle, + taskStore: TaskStore, + config: TaskManagerConfig, + managedConfig: ManagedConfiguration, + logger: Logger +): Observable { + return createMonitoringStatsStream( + createAggregators(taskPollingLifecycle, taskStore, config, managedConfig, logger), + config + ); +} diff --git a/x-pack/plugins/task_manager/server/monitoring/monitoring_stats_stream.test.ts b/x-pack/plugins/task_manager/server/monitoring/monitoring_stats_stream.test.ts new file mode 100644 index 0000000000000..8479def5deeeb --- /dev/null +++ b/x-pack/plugins/task_manager/server/monitoring/monitoring_stats_stream.test.ts @@ -0,0 +1,173 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { TaskManagerConfig } from '../config'; +import { of, Subject } from 'rxjs'; +import { take, bufferCount } from 'rxjs/operators'; +import { createMonitoringStatsStream, AggregatedStat } from './monitoring_stats_stream'; +import { JsonValue } from 'src/plugins/kibana_utils/common'; + +beforeEach(() => { + jest.resetAllMocks(); +}); + +describe('createMonitoringStatsStream', () => { + const configuration: TaskManagerConfig = { + enabled: true, + max_workers: 10, + index: 'foo', + max_attempts: 9, + poll_interval: 6000000, + monitored_stats_required_freshness: 6000000, + max_poll_inactivity_cycles: 10, + request_capacity: 1000, + monitored_aggregated_stats_refresh_rate: 5000, + monitored_stats_running_average_window: 50, + monitored_task_execution_thresholds: { + default: { + error_threshold: 90, + warn_threshold: 80, + }, + custom: {}, + }, + }; + + it('returns the initial config used to configure Task Manager', async () => { + return new Promise((resolve) => { + createMonitoringStatsStream(of(), configuration) + .pipe(take(1)) + .subscribe((firstValue) => { + expect(firstValue.stats).toEqual({}); + resolve(); + }); + }); + }); + + it('incrementally updates the stats returned by the endpoint', async () => { + const aggregatedStats$ = new Subject(); + + return new Promise((resolve) => { + createMonitoringStatsStream(aggregatedStats$, configuration) + .pipe(take(3), bufferCount(3)) + .subscribe(([initialValue, secondValue, thirdValue]) => { + expect(initialValue.stats).toMatchObject({ + lastUpdate: expect.any(String), + stats: { + configuration: { + value: { + max_workers: 10, + poll_interval: 6000000, + max_poll_inactivity_cycles: 10, + request_capacity: 1000, + monitored_aggregated_stats_refresh_rate: 5000, + monitored_stats_running_average_window: 50, + monitored_task_execution_thresholds: { + default: { + error_threshold: 90, + warn_threshold: 80, + }, + custom: {}, + }, + }, + }, + }, + }); + + expect(secondValue).toMatchObject({ + lastUpdate: expect.any(String), + stats: { + newAggregatedStat: { + timestamp: expect.any(String), + value: { + some: { + complex: { + value: 123, + }, + }, + }, + }, + configuration: { + timestamp: expect.any(String), + value: { + max_workers: 10, + poll_interval: 6000000, + max_poll_inactivity_cycles: 10, + request_capacity: 1000, + monitored_aggregated_stats_refresh_rate: 5000, + monitored_stats_running_average_window: 50, + monitored_task_execution_thresholds: { + default: { + error_threshold: 90, + warn_threshold: 80, + }, + custom: {}, + }, + }, + }, + }, + }); + + expect(thirdValue).toMatchObject({ + lastUpdate: expect.any(String), + stats: { + newAggregatedStat: { + timestamp: expect.any(String), + value: { + some: { + updated: { + value: 456, + }, + }, + }, + }, + configuration: { + timestamp: expect.any(String), + value: { + max_workers: 10, + poll_interval: 6000000, + max_poll_inactivity_cycles: 10, + request_capacity: 1000, + monitored_aggregated_stats_refresh_rate: 5000, + monitored_stats_running_average_window: 50, + monitored_task_execution_thresholds: { + default: { + error_threshold: 90, + warn_threshold: 80, + }, + custom: {}, + }, + }, + }, + }, + }); + }); + + aggregatedStats$.next({ + key: 'newAggregatedStat', + value: { + some: { + complex: { + value: 123, + }, + }, + } as JsonValue, + }); + + aggregatedStats$.next({ + key: 'newAggregatedStat', + value: { + some: { + updated: { + value: 456, + }, + }, + } as JsonValue, + }); + + resolve(); + }); + }); +}); diff --git a/x-pack/plugins/task_manager/server/monitoring/monitoring_stats_stream.ts b/x-pack/plugins/task_manager/server/monitoring/monitoring_stats_stream.ts new file mode 100644 index 0000000000000..374660a257c59 --- /dev/null +++ b/x-pack/plugins/task_manager/server/monitoring/monitoring_stats_stream.ts @@ -0,0 +1,148 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { merge, of, Observable } from 'rxjs'; +import { map, scan } from 'rxjs/operators'; +import { set } from '@elastic/safer-lodash-set'; +import { Logger } from 'src/core/server'; +import { JsonObject } from 'src/plugins/kibana_utils/common'; +import { TaskStore } from '../task_store'; +import { TaskPollingLifecycle } from '../polling_lifecycle'; +import { + createWorkloadAggregator, + summarizeWorkloadStat, + WorkloadStat, +} from './workload_statistics'; +import { + createTaskRunAggregator, + summarizeTaskRunStat, + TaskRunStat, + SummarizedTaskRunStat, +} from './task_run_statistics'; +import { ConfigStat, createConfigurationAggregator } from './configuration_statistics'; +import { TaskManagerConfig } from '../config'; +import { AggregatedStatProvider } from './runtime_statistics_aggregator'; +import { ManagedConfiguration } from '../lib/create_managed_configuration'; + +export { AggregatedStatProvider, AggregatedStat } from './runtime_statistics_aggregator'; + +export interface MonitoringStats { + last_update: string; + stats: { + configuration?: MonitoredStat; + workload?: MonitoredStat; + runtime?: MonitoredStat; + }; +} + +export enum HealthStatus { + OK = 'OK', + Warning = 'warn', + Error = 'error', +} + +interface MonitoredStat { + timestamp: string; + value: T; +} +type RawMonitoredStat = MonitoredStat & { + status: HealthStatus; +}; + +export interface RawMonitoringStats { + last_update: string; + stats: { + configuration?: RawMonitoredStat; + workload?: RawMonitoredStat; + runtime?: RawMonitoredStat; + }; +} + +export function createAggregators( + taskPollingLifecycle: TaskPollingLifecycle, + taskStore: TaskStore, + config: TaskManagerConfig, + managedConfig: ManagedConfiguration, + logger: Logger +): AggregatedStatProvider { + return merge( + createConfigurationAggregator(config, managedConfig), + createTaskRunAggregator(taskPollingLifecycle, config.monitored_stats_running_average_window), + createWorkloadAggregator( + taskStore, + config.monitored_aggregated_stats_refresh_rate, + config.poll_interval, + logger + ) + ); +} + +export function createMonitoringStatsStream( + provider$: AggregatedStatProvider, + config: TaskManagerConfig +): Observable { + const initialStats = { + last_update: new Date().toISOString(), + stats: {}, + }; + return merge( + // emit the initial stats + of(initialStats), + // emit updated stats whenever a provider updates a specific key on the stats + provider$.pipe( + map(({ key, value }) => { + return { + value: { timestamp: new Date().toISOString(), value }, + key, + }; + }), + scan((monitoringStats: MonitoringStats, { key, value }) => { + // incrementally merge stats as they come in + set(monitoringStats.stats, key, value); + monitoringStats.last_update = new Date().toISOString(); + return monitoringStats; + }, initialStats) + ) + ); +} + +export function summarizeMonitoringStats( + { + // eslint-disable-next-line @typescript-eslint/naming-convention + last_update, + stats: { runtime, workload, configuration }, + }: MonitoringStats, + config: TaskManagerConfig +): RawMonitoringStats { + return { + last_update, + stats: { + ...(configuration + ? { + configuration: { + ...configuration, + status: HealthStatus.OK, + }, + } + : {}), + ...(runtime + ? { + runtime: { + timestamp: runtime.timestamp, + ...summarizeTaskRunStat(runtime.value, config), + }, + } + : {}), + ...(workload + ? { + workload: { + timestamp: workload.timestamp, + ...summarizeWorkloadStat(workload.value), + }, + } + : {}), + }, + }; +} diff --git a/x-pack/plugins/task_manager/server/monitoring/runtime_statistics_aggregator.ts b/x-pack/plugins/task_manager/server/monitoring/runtime_statistics_aggregator.ts new file mode 100644 index 0000000000000..bd2b3845f2526 --- /dev/null +++ b/x-pack/plugins/task_manager/server/monitoring/runtime_statistics_aggregator.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { Observable } from 'rxjs'; +import { JsonValue } from 'src/plugins/kibana_utils/common'; + +export interface AggregatedStat { + key: string; + value: Stat; +} + +export type AggregatedStatProvider = Observable< + AggregatedStat +>; diff --git a/x-pack/plugins/task_manager/server/monitoring/task_run_calcultors.test.ts b/x-pack/plugins/task_manager/server/monitoring/task_run_calcultors.test.ts new file mode 100644 index 0000000000000..eb8cabd9f3a8f --- /dev/null +++ b/x-pack/plugins/task_manager/server/monitoring/task_run_calcultors.test.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import uuid from 'uuid'; + +import { + calculateRunningAverage, + calculateFrequency, + createRunningAveragedStat, + createMapOfRunningAveragedStats, +} from './task_run_calcultors'; + +describe('calculateRunningAverage', () => { + test('calculates the running average and median of a window of values', async () => { + expect(calculateRunningAverage([2, 2, 4, 6, 6])).toMatchInlineSnapshot(` + Object { + "p50": 4, + "p90": 6, + "p95": 6, + "p99": 6, + } + `); + }); +}); + +describe('calculateFrequency', () => { + test('calculates the frequency of each terms in the list as a percentage', async () => { + const [term1, term2, term3] = [uuid.v4(), uuid.v4(), uuid.v4()]; + expect( + calculateFrequency([term1, term2, term2, term3, term1, term1, term2, term1, term3]) + ).toEqual({ + [term3]: 22, + [term1]: 44, + [term2]: 33, + }); + }); +}); + +describe('createRunningAveragedStat', () => { + test('create a function which tracks a window of values', async () => { + const queue = createRunningAveragedStat(3); + expect(queue(1)).toEqual([1]); + expect(queue(2)).toEqual([1, 2]); + expect(queue(3)).toEqual([1, 2, 3]); + expect(queue(4)).toEqual([2, 3, 4]); + expect(queue(5)).toEqual([3, 4, 5]); + }); +}); + +describe('createMapOfRunningAveragedStats', () => { + test('create a function which tracks multiple window of values by key', async () => { + const [term1, term2, term3] = [uuid.v4(), uuid.v4(), uuid.v4()]; + const mappedQueues = createMapOfRunningAveragedStats(3); + expect(mappedQueues(term1, 1)).toEqual({ [term1]: [1] }); + expect(mappedQueues(term1, 2)).toEqual({ [term1]: [1, 2] }); + expect(mappedQueues(term2, 3)).toEqual({ [term1]: [1, 2], [term2]: [3] }); + expect(mappedQueues(term3, 4)).toEqual({ [term1]: [1, 2], [term2]: [3], [term3]: [4] }); + expect(mappedQueues(term2, 5)).toEqual({ [term1]: [1, 2], [term2]: [3, 5], [term3]: [4] }); + expect(mappedQueues(term2, 6)).toEqual({ [term1]: [1, 2], [term2]: [3, 5, 6], [term3]: [4] }); + expect(mappedQueues(term1, 7)).toEqual({ + [term1]: [1, 2, 7], + [term2]: [3, 5, 6], + [term3]: [4], + }); + expect(mappedQueues(term1, 8)).toEqual({ + [term1]: [2, 7, 8], + [term2]: [3, 5, 6], + [term3]: [4], + }); + expect(mappedQueues(term1, 9)).toEqual({ + [term1]: [7, 8, 9], + [term2]: [3, 5, 6], + [term3]: [4], + }); + }); +}); diff --git a/x-pack/plugins/task_manager/server/monitoring/task_run_calcultors.ts b/x-pack/plugins/task_manager/server/monitoring/task_run_calcultors.ts new file mode 100644 index 0000000000000..67b77a29b1c76 --- /dev/null +++ b/x-pack/plugins/task_manager/server/monitoring/task_run_calcultors.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import stats from 'stats-lite'; +import { JsonObject } from 'src/plugins/kibana_utils/common'; +import { isUndefined, countBy, mapValues } from 'lodash'; + +export interface AveragedStat extends JsonObject { + p50: number; + p90: number; + p95: number; + p99: number; +} + +export function calculateRunningAverage(values: number[]): AveragedStat { + return { + p50: stats.percentile(values, 0.5), + p90: stats.percentile(values, 0.9), + p95: stats.percentile(values, 0.95), + p99: stats.percentile(values, 0.99), + }; +} + +/** + * Calculate the frequency of each term in a list of terms. + * @param values + */ +export function calculateFrequency(values: T[]): JsonObject { + return values.length + ? mapValues(countBy(values), (count) => Math.round((count * 100) / values.length)) + : {}; +} + +/** + * Utility to keep track of a bounded array of values which changes over time + * dropping older values as they slide out of the window we wish to track + */ +export function createRunningAveragedStat(runningAverageWindowSize: number) { + const list = new Array(); + return (value?: T) => { + if (!isUndefined(value)) { + if (list.length === runningAverageWindowSize) { + list.shift(); + } + list.push(value); + } + // clone list to ensure it isn't mutated externally + return [...list]; + }; +} + +export function createMapOfRunningAveragedStats(runningAverageWindowSize: number) { + const mappedQueue: Record T[]> = {}; + const asRecordOfValues = () => mapValues(mappedQueue, (queue) => queue()); + return (key?: string, value?: T) => { + if (!isUndefined(key)) { + mappedQueue[key] = mappedQueue[key] ?? createRunningAveragedStat(runningAverageWindowSize); + mappedQueue[key](value); + } + return asRecordOfValues(); + }; +} diff --git a/x-pack/plugins/task_manager/server/monitoring/task_run_statistics.test.ts b/x-pack/plugins/task_manager/server/monitoring/task_run_statistics.test.ts new file mode 100644 index 0000000000000..a931f0ff7c304 --- /dev/null +++ b/x-pack/plugins/task_manager/server/monitoring/task_run_statistics.test.ts @@ -0,0 +1,501 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import uuid from 'uuid'; +import { Subject, Observable } from 'rxjs'; +import stats from 'stats-lite'; +import sinon from 'sinon'; +import { take, tap, bufferCount, skip, map } from 'rxjs/operators'; + +import { ConcreteTaskInstance, TaskStatus } from '../task'; +import { asTaskRunEvent, asTaskPollingCycleEvent, TaskTiming } from '../task_events'; +import { asOk } from '../lib/result_type'; +import { TaskLifecycleEvent } from '../polling_lifecycle'; +import { TaskRunResult } from '../task_runner'; +import { + createTaskRunAggregator, + summarizeTaskRunStat, + TaskRunStat, + SummarizedTaskRunStat, +} from './task_run_statistics'; +import { AggregatedStat } from './runtime_statistics_aggregator'; +import { FillPoolResult } from '../lib/fill_pool'; +import { taskPollingLifecycleMock } from '../polling_lifecycle.mock'; +import { configSchema } from '../config'; + +describe('Task Run Statistics', () => { + let fakeTimer: sinon.SinonFakeTimers; + + beforeAll(() => { + fakeTimer = sinon.useFakeTimers(); + }); + + afterAll(() => fakeTimer.restore()); + + test('returns a running average of task drift', async () => { + const runAtDrift = [1000, 2000, 500, 300, 400, 15000, 20000, 200]; + const events$ = new Subject(); + const taskPollingLifecycle = taskPollingLifecycleMock.create({ + events$: events$ as Observable, + }); + + const runningAverageWindowSize = 5; + const taskRunAggregator = createTaskRunAggregator( + taskPollingLifecycle, + runningAverageWindowSize + ); + + function expectWindowEqualsUpdate( + taskStat: AggregatedStat, + window: number[] + ) { + expect(taskStat.value.drift).toMatchObject({ + p50: stats.percentile(window, 0.5), + p90: stats.percentile(window, 0.9), + p95: stats.percentile(window, 0.95), + p99: stats.percentile(window, 0.99), + }); + } + + return new Promise((resolve) => { + taskRunAggregator + .pipe( + // skip initial stat which is just initialized data which + // ensures we don't stall on combineLatest + skip(1), + // Use 'summarizeTaskRunStat' to receive summarize stats + map(({ key, value }: AggregatedStat) => ({ + key, + value: summarizeTaskRunStat(value, getTaskManagerConfig()).value, + })), + take(runAtDrift.length), + bufferCount(runAtDrift.length) + ) + .subscribe((taskStats: Array>) => { + expectWindowEqualsUpdate(taskStats[0], runAtDrift.slice(0, 1)); + expectWindowEqualsUpdate(taskStats[1], runAtDrift.slice(0, 2)); + expectWindowEqualsUpdate(taskStats[2], runAtDrift.slice(0, 3)); + expectWindowEqualsUpdate(taskStats[3], runAtDrift.slice(0, 4)); + expectWindowEqualsUpdate(taskStats[4], runAtDrift.slice(0, 5)); + // from the 6th value, begin to drop old values as out window is 5 + expectWindowEqualsUpdate(taskStats[5], runAtDrift.slice(1, 6)); + expectWindowEqualsUpdate(taskStats[6], runAtDrift.slice(2, 7)); + expectWindowEqualsUpdate(taskStats[7], runAtDrift.slice(3, 8)); + resolve(); + }); + + const now = Date.now(); + for (const drift of runAtDrift) { + const start = Math.floor(Math.random() * 1000); + events$.next( + mockTaskRunEvent( + { runAt: runAtMillisecondsAgo(drift + start) }, + { start: runAtMillisecondsAgo(start).getTime(), stop: now } + ) + ); + } + }); + }); + + test('returns a running average of task run duration', async () => { + const runDurations = [1000, 2000, 500, 300, 400, 15000, 20000, 200]; + const runDurationsInReverse = runDurations.reverse(); + const events$ = new Subject(); + const taskPollingLifecycle = taskPollingLifecycleMock.create({ + events$: events$ as Observable, + }); + + const runningAverageWindowSize = 5; + const taskRunAggregator = createTaskRunAggregator( + taskPollingLifecycle, + runningAverageWindowSize + ); + + function expectWindowEqualsUpdate( + taskStat: AggregatedStat, + windows: Record + ) { + for (const [type, window] of Object.entries(windows)) { + expect(taskStat.value.execution.duration[type]).toMatchObject({ + p50: stats.percentile(window, 0.5), + p90: stats.percentile(window, 0.9), + p95: stats.percentile(window, 0.95), + p99: stats.percentile(window, 0.99), + }); + } + } + + return new Promise((resolve, reject) => { + taskRunAggregator + .pipe( + // skip initial stat which is just initialized data which + // ensures we don't stall on combineLatest + skip(1), + // Use 'summarizeTaskRunStat' to receive summarize stats + map(({ key, value }: AggregatedStat) => ({ + key, + value: summarizeTaskRunStat(value, getTaskManagerConfig()).value, + })), + take(runDurations.length * 2), + bufferCount(runDurations.length * 2) + ) + .subscribe((taskStats: Array>) => { + try { + expectWindowEqualsUpdate(taskStats[0], { 'alerting:test': runDurations.slice(0, 1) }); + expectWindowEqualsUpdate(taskStats[1], { 'alerting:test': runDurations.slice(0, 2) }); + expectWindowEqualsUpdate(taskStats[2], { 'alerting:test': runDurations.slice(0, 3) }); + expectWindowEqualsUpdate(taskStats[3], { 'alerting:test': runDurations.slice(0, 4) }); + expectWindowEqualsUpdate(taskStats[4], { 'alerting:test': runDurations.slice(0, 5) }); + // from the 6th value, begin to drop old values as out window is 5 + expectWindowEqualsUpdate(taskStats[5], { 'alerting:test': runDurations.slice(1, 6) }); + expectWindowEqualsUpdate(taskStats[6], { 'alerting:test': runDurations.slice(2, 7) }); + expectWindowEqualsUpdate(taskStats[7], { 'alerting:test': runDurations.slice(3, 8) }); + expectWindowEqualsUpdate(taskStats[8], { + 'actions:test': runDurations.slice(0, 1), + 'alerting:test': runDurations.slice(3, 8), + }); + expectWindowEqualsUpdate(taskStats[9], { + 'actions:test': runDurations.slice(0, 2), + 'alerting:test': runDurations.slice(3, 8), + }); + expectWindowEqualsUpdate(taskStats[10], { + 'actions:test': runDurations.slice(0, 3), + 'alerting:test': runDurations.slice(3, 8), + }); + expectWindowEqualsUpdate(taskStats[11], { + 'actions:test': runDurations.slice(0, 4), + 'alerting:test': runDurations.slice(3, 8), + }); + expectWindowEqualsUpdate(taskStats[12], { + 'actions:test': runDurations.slice(0, 5), + 'alerting:test': runDurations.slice(3, 8), + }); + // from the 6th value, begin to drop old values as out window is 5 + expectWindowEqualsUpdate(taskStats[13], { + 'actions:test': runDurations.slice(1, 6), + 'alerting:test': runDurations.slice(3, 8), + }); + expectWindowEqualsUpdate(taskStats[14], { + 'actions:test': runDurations.slice(2, 7), + 'alerting:test': runDurations.slice(3, 8), + }); + expectWindowEqualsUpdate(taskStats[15], { + 'actions:test': runDurations.slice(3, 8), + 'alerting:test': runDurations.slice(3, 8), + }); + resolve(); + } catch (e) { + reject(e); + } + }); + + const now = Date.now(); + for (const runDuration of runDurations) { + events$.next( + mockTaskRunEvent( + { taskType: 'alerting:test' }, + { start: runAtMillisecondsAgo(runDuration).getTime(), stop: now } + ) + ); + } + for (const runDuration of runDurationsInReverse) { + events$.next( + mockTaskRunEvent( + { taskType: 'actions:test' }, + { start: runAtMillisecondsAgo(runDuration).getTime(), stop: now } + ) + ); + } + }); + }); + + test('returns the frequency of task run results', async () => { + const events$ = new Subject(); + const taskPollingLifecycle = taskPollingLifecycleMock.create({ + events$: events$ as Observable, + }); + + const runningAverageWindowSize = 5; + const taskRunAggregator = createTaskRunAggregator( + taskPollingLifecycle, + runningAverageWindowSize + ); + + return new Promise((resolve, reject) => { + taskRunAggregator + .pipe( + // skip initial stat which is just initialized data which + // ensures we don't stall on combineLatest + skip(1), + // Use 'summarizeTaskRunStat' to receive summarize stats + map(({ key, value }: AggregatedStat) => ({ + key, + value: summarizeTaskRunStat(value, getTaskManagerConfig()).value, + })), + take(10), + bufferCount(10) + ) + .subscribe((taskStats: Array>) => { + try { + /** + * At any given time we only keep track of the last X Polling Results + * In the tests this is ocnfiugured to a window size of 5 + */ + expect( + taskStats.map( + (taskStat) => + taskStat.value.execution.result_frequency_percent_as_number['alerting:test'] + ) + ).toEqual([ + // Success + { Success: 100, RetryScheduled: 0, Failed: 0, status: 'OK' }, + // Success, Success, + { Success: 100, RetryScheduled: 0, Failed: 0, status: 'OK' }, + // Success, Success, Success + { Success: 100, RetryScheduled: 0, Failed: 0, status: 'OK' }, + // Success, Success, Success, Failed + { Success: 75, RetryScheduled: 0, Failed: 25, status: 'OK' }, + // Success, Success, Success, Failed, Failed + { Success: 60, RetryScheduled: 0, Failed: 40, status: 'OK' }, + // Success, Success, Failed, Failed, Failed + { Success: 40, RetryScheduled: 0, Failed: 60, status: 'OK' }, + // Success, Failed, Failed, Failed, RetryScheduled + { Success: 20, RetryScheduled: 20, Failed: 60, status: 'OK' }, + // Failed, Failed, Failed, RetryScheduled, RetryScheduled + { Success: 0, RetryScheduled: 40, Failed: 60, status: 'OK' }, + // Failed, Failed, RetryScheduled, RetryScheduled, Success + { Success: 20, RetryScheduled: 40, Failed: 40, status: 'OK' }, + // Failed, RetryScheduled, RetryScheduled, Success, Success + { Success: 40, RetryScheduled: 40, Failed: 20, status: 'OK' }, + ]); + resolve(); + } catch (e) { + reject(e); + } + }); + + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Success)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Success)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Success)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Failed)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Failed)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Failed)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.RetryScheduled)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.RetryScheduled)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Success)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Success)); + }); + }); + + test('frequency of task run results return an error health status when failure is above a certain threshold', async () => { + const events$ = new Subject(); + + const taskPollingLifecycle = taskPollingLifecycleMock.create({ + events$: events$ as Observable, + }); + + const runningAverageWindowSize = 5; + const taskRunAggregator = createTaskRunAggregator( + taskPollingLifecycle, + runningAverageWindowSize + ); + + return new Promise((resolve, reject) => { + taskRunAggregator + .pipe( + // skip initial stat which is just initialized data which + // ensures we don't stall on combineLatest + skip(1), + // Use 'summarizeTaskRunStat' to receive summarize stats + map(({ key, value }: AggregatedStat) => ({ + key, + value: summarizeTaskRunStat( + value, + getTaskManagerConfig({ + monitored_task_execution_thresholds: { + custom: { + 'alerting:test': { + error_threshold: 59, + warn_threshold: 39, + }, + }, + }, + }) + ).value, + })), + take(10), + bufferCount(10) + ) + .subscribe((taskStats: Array>) => { + try { + /** + * At any given time we only keep track of the last X Polling Results + * In the tests this is ocnfiugured to a window size of 5 + */ + expect( + taskStats.map( + (taskStat) => + taskStat.value.execution.result_frequency_percent_as_number['alerting:test'] + ) + ).toEqual([ + // Success + { Success: 100, RetryScheduled: 0, Failed: 0, status: 'OK' }, + // Success, Success, + { Success: 100, RetryScheduled: 0, Failed: 0, status: 'OK' }, + // Success, Success, Success + { Success: 100, RetryScheduled: 0, Failed: 0, status: 'OK' }, + // Success, Success, Success, Failed + { Success: 75, RetryScheduled: 0, Failed: 25, status: 'OK' }, + // Success, Success, Success, Failed, Failed + { Success: 60, RetryScheduled: 0, Failed: 40, status: 'warn' }, + // Success, Success, Failed, Failed, Failed + { Success: 40, RetryScheduled: 0, Failed: 60, status: 'error' }, + // Success, Failed, Failed, Failed, RetryScheduled + { Success: 20, RetryScheduled: 20, Failed: 60, status: 'error' }, + // Failed, Failed, Failed, RetryScheduled, RetryScheduled + { Success: 0, RetryScheduled: 40, Failed: 60, status: 'error' }, + // Failed, Failed, RetryScheduled, RetryScheduled, Success + { Success: 20, RetryScheduled: 40, Failed: 40, status: 'warn' }, + // Failed, RetryScheduled, RetryScheduled, Success, Success + { Success: 40, RetryScheduled: 40, Failed: 20, status: 'OK' }, + ]); + resolve(); + } catch (e) { + reject(e); + } + }); + + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Success)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Success)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Success)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Failed)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Failed)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Failed)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.RetryScheduled)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.RetryScheduled)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Success)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Success)); + }); + }); + + test('returns polling stats', async () => { + const expectedTimestamp: string[] = []; + const events$ = new Subject(); + const taskPollingLifecycle = taskPollingLifecycleMock.create({ + events$: events$ as Observable, + }); + + const runningAverageWindowSize = 5; + const taskRunAggregator = createTaskRunAggregator( + taskPollingLifecycle, + runningAverageWindowSize + ); + + return new Promise((resolve, reject) => { + taskRunAggregator + .pipe( + // skip initial stat which is just initialized data which + // ensures we don't stall on combineLatest + skip(1), + // Use 'summarizeTaskRunStat' to receive summarize stats + map(({ key, value }: AggregatedStat) => ({ + key, + value: summarizeTaskRunStat(value, getTaskManagerConfig()).value, + })), + tap(() => { + expectedTimestamp.push(new Date().toISOString()); + // each event is a second after the previous one + fakeTimer.tick(1000); + }), + take(10), + bufferCount(10) + ) + .subscribe((taskStats: Array>) => { + try { + expect( + taskStats.map((taskStat) => taskStat.value.polling.last_successful_poll) + ).toEqual(expectedTimestamp); + + /** + * At any given time we only keep track of the last X Polling Results + * In the tests this is ocnfiugured to a window size of 5 + */ + expect( + taskStats.map((taskStat) => taskStat.value.polling.result_frequency_percent_as_number) + ).toEqual([ + // NoTasksClaimed + { NoTasksClaimed: 100, RanOutOfCapacity: 0, PoolFilled: 0 }, + // NoTasksClaimed, NoTasksClaimed, + { NoTasksClaimed: 100, RanOutOfCapacity: 0, PoolFilled: 0 }, + // NoTasksClaimed, NoTasksClaimed, NoTasksClaimed + { NoTasksClaimed: 100, RanOutOfCapacity: 0, PoolFilled: 0 }, + // NoTasksClaimed, NoTasksClaimed, NoTasksClaimed, PoolFilled + { NoTasksClaimed: 75, RanOutOfCapacity: 0, PoolFilled: 25 }, + // NoTasksClaimed, NoTasksClaimed, NoTasksClaimed, PoolFilled, PoolFilled + { NoTasksClaimed: 60, RanOutOfCapacity: 0, PoolFilled: 40 }, + // NoTasksClaimed, NoTasksClaimed, PoolFilled, PoolFilled, PoolFilled + { NoTasksClaimed: 40, RanOutOfCapacity: 0, PoolFilled: 60 }, + // NoTasksClaimed, PoolFilled, PoolFilled, PoolFilled, RanOutOfCapacity + { NoTasksClaimed: 20, RanOutOfCapacity: 20, PoolFilled: 60 }, + // PoolFilled, PoolFilled, PoolFilled, RanOutOfCapacity, RanOutOfCapacity + { NoTasksClaimed: 0, RanOutOfCapacity: 40, PoolFilled: 60 }, + // PoolFilled, PoolFilled, RanOutOfCapacity, RanOutOfCapacity, NoTasksClaimed + { NoTasksClaimed: 20, RanOutOfCapacity: 40, PoolFilled: 40 }, + // PoolFilled, RanOutOfCapacity, RanOutOfCapacity, NoTasksClaimed, NoTasksClaimed + { NoTasksClaimed: 40, RanOutOfCapacity: 40, PoolFilled: 20 }, + ]); + resolve(); + } catch (e) { + reject(e); + } + }); + + events$.next(asTaskPollingCycleEvent(asOk(FillPoolResult.NoTasksClaimed))); + events$.next(asTaskPollingCycleEvent(asOk(FillPoolResult.NoTasksClaimed))); + events$.next(asTaskPollingCycleEvent(asOk(FillPoolResult.NoTasksClaimed))); + events$.next(asTaskPollingCycleEvent(asOk(FillPoolResult.PoolFilled))); + events$.next(asTaskPollingCycleEvent(asOk(FillPoolResult.PoolFilled))); + events$.next(asTaskPollingCycleEvent(asOk(FillPoolResult.PoolFilled))); + events$.next(asTaskPollingCycleEvent(asOk(FillPoolResult.RanOutOfCapacity))); + events$.next(asTaskPollingCycleEvent(asOk(FillPoolResult.RanOutOfCapacity))); + events$.next(asTaskPollingCycleEvent(asOk(FillPoolResult.NoTasksClaimed))); + events$.next(asTaskPollingCycleEvent(asOk(FillPoolResult.NoTasksClaimed))); + }); + }); +}); + +function runAtMillisecondsAgo(ms: number): Date { + return new Date(Date.now() - ms); +} + +const mockTaskRunEvent = ( + overrides: Partial = {}, + timing: TaskTiming, + result: TaskRunResult = TaskRunResult.Success +) => { + const task = mockTaskInstance(overrides); + return asTaskRunEvent(task.id, asOk({ task, result }), timing); +}; + +const mockTaskInstance = (overrides: Partial = {}): ConcreteTaskInstance => ({ + id: uuid.v4(), + attempts: 0, + status: TaskStatus.Running, + version: '123', + runAt: new Date(), + scheduledAt: new Date(), + startedAt: new Date(), + retryAt: new Date(Date.now() + 5 * 60 * 1000), + state: {}, + taskType: 'alerting:test', + params: { + alertId: '1', + }, + ownerId: null, + ...overrides, +}); + +const getTaskManagerConfig = (overrides: unknown = {}) => configSchema.validate(overrides); diff --git a/x-pack/plugins/task_manager/server/monitoring/task_run_statistics.ts b/x-pack/plugins/task_manager/server/monitoring/task_run_statistics.ts new file mode 100644 index 0000000000000..6dd533177a86e --- /dev/null +++ b/x-pack/plugins/task_manager/server/monitoring/task_run_statistics.ts @@ -0,0 +1,224 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { combineLatest, Observable } from 'rxjs'; +import { filter, startWith, map } from 'rxjs/operators'; +import { JsonObject } from 'src/plugins/kibana_utils/common'; +import { mapValues } from 'lodash'; +import { AggregatedStatProvider, AggregatedStat } from './runtime_statistics_aggregator'; +import { TaskLifecycleEvent } from '../polling_lifecycle'; +import { + isTaskRunEvent, + isTaskPollingCycleEvent, + TaskRun, + ErroredTask, + RanTask, + TaskTiming, +} from '../task_events'; +import { isOk, Ok, unwrap } from '../lib/result_type'; +import { ConcreteTaskInstance } from '../task'; +import { TaskRunResult } from '../task_runner'; +import { FillPoolResult } from '../lib/fill_pool'; +import { + AveragedStat, + calculateRunningAverage, + calculateFrequency, + createRunningAveragedStat, + createMapOfRunningAveragedStats, +} from './task_run_calcultors'; +import { HealthStatus } from './monitoring_stats_stream'; +import { TaskPollingLifecycle } from '../polling_lifecycle'; +import { TaskExecutionFailureThreshold, TaskManagerConfig } from '../config'; + +interface FillPoolStat extends JsonObject { + last_successful_poll: string; + result_frequency_percent_as_number: FillPoolResult[]; +} + +interface ExecutionStat extends JsonObject { + duration: Record; + result_frequency_percent_as_number: Record; +} + +export interface TaskRunStat extends JsonObject { + drift: number[]; + execution: ExecutionStat; + polling: FillPoolStat | Omit; +} + +interface FillPoolRawStat extends JsonObject { + last_successful_poll: string; + result_frequency_percent_as_number: { + [FillPoolResult.NoTasksClaimed]: number; + [FillPoolResult.RanOutOfCapacity]: number; + [FillPoolResult.PoolFilled]: number; + }; +} + +interface ResultFrequency extends JsonObject { + [TaskRunResult.Success]: number; + [TaskRunResult.SuccessRescheduled]: number; + [TaskRunResult.RetryScheduled]: number; + [TaskRunResult.Failed]: number; +} + +type ResultFrequencySummary = ResultFrequency & { + status: HealthStatus; +}; + +export interface SummarizedTaskRunStat extends JsonObject { + drift: AveragedStat; + execution: { + duration: Record; + result_frequency_percent_as_number: Record; + }; + polling: FillPoolRawStat | Omit; +} + +export function createTaskRunAggregator( + taskPollingLifecycle: TaskPollingLifecycle, + runningAverageWindowSize: number +): AggregatedStatProvider { + const taskRunEventToStat = createTaskRunEventToStat(runningAverageWindowSize); + const taskRunEvents$: Observable> = taskPollingLifecycle.events.pipe( + filter((taskEvent: TaskLifecycleEvent) => isTaskRunEvent(taskEvent) && hasTiming(taskEvent)), + map((taskEvent: TaskLifecycleEvent) => { + const { task, result }: RanTask | ErroredTask = unwrap((taskEvent as TaskRun).event); + return taskRunEventToStat(task, taskEvent.timing!, result); + }) + ); + + const resultFrequencyQueue = createRunningAveragedStat(runningAverageWindowSize); + const taskPollingEvents$: Observable> = taskPollingLifecycle.events.pipe( + filter( + (taskEvent: TaskLifecycleEvent) => + isTaskPollingCycleEvent(taskEvent) && isOk(taskEvent.event) + ), + map((taskEvent: TaskLifecycleEvent) => { + return { + polling: { + last_successful_poll: new Date().toISOString(), + result_frequency_percent_as_number: resultFrequencyQueue( + ((taskEvent.event as unknown) as Ok).value + ), + }, + }; + }) + ); + + return combineLatest([ + taskRunEvents$.pipe( + startWith({ drift: [], execution: { duration: {}, result_frequency_percent_as_number: {} } }) + ), + taskPollingEvents$.pipe( + startWith({ + polling: { result_frequency_percent_as_number: [] }, + }) + ), + ]).pipe( + map(([taskRun, polling]: [Omit, Pick]) => { + return { + key: 'runtime', + value: { + ...taskRun, + ...polling, + }, + } as AggregatedStat; + }) + ); +} + +function hasTiming(taskEvent: TaskLifecycleEvent) { + return !!taskEvent?.timing; +} + +function createTaskRunEventToStat(runningAverageWindowSize: number) { + const driftQueue = createRunningAveragedStat(runningAverageWindowSize); + const taskRunDurationQueue = createMapOfRunningAveragedStats(runningAverageWindowSize); + const resultFrequencyQueue = createMapOfRunningAveragedStats( + runningAverageWindowSize + ); + return ( + task: ConcreteTaskInstance, + timing: TaskTiming, + result: TaskRunResult + ): Omit => ({ + drift: driftQueue(timing!.start - task.runAt.getTime()), + execution: { + duration: taskRunDurationQueue(task.taskType, timing!.stop - timing!.start), + result_frequency_percent_as_number: resultFrequencyQueue(task.taskType, result), + }, + }); +} + +const DEFAULT_TASK_RUN_FREQUENCIES = { + [TaskRunResult.Success]: 0, + [TaskRunResult.SuccessRescheduled]: 0, + [TaskRunResult.RetryScheduled]: 0, + [TaskRunResult.Failed]: 0, +}; +const DEFAULT_POLLING_FREQUENCIES = { + [FillPoolResult.NoTasksClaimed]: 0, + [FillPoolResult.RanOutOfCapacity]: 0, + [FillPoolResult.PoolFilled]: 0, +}; + +export function summarizeTaskRunStat( + { + // eslint-disable-next-line @typescript-eslint/naming-convention + polling: { last_successful_poll, result_frequency_percent_as_number: pollingResultFrequency }, + drift, + execution: { duration, result_frequency_percent_as_number: executionResultFrequency }, + }: TaskRunStat, + config: TaskManagerConfig +): { value: SummarizedTaskRunStat; status: HealthStatus } { + return { + value: { + polling: { + ...(last_successful_poll ? { last_successful_poll } : {}), + result_frequency_percent_as_number: { + ...DEFAULT_POLLING_FREQUENCIES, + ...calculateFrequency(pollingResultFrequency as FillPoolResult[]), + }, + }, + drift: calculateRunningAverage(drift), + execution: { + duration: mapValues(duration, (typedDurations) => calculateRunningAverage(typedDurations)), + result_frequency_percent_as_number: mapValues( + executionResultFrequency, + (typedResultFrequencies, taskType) => + summarizeTaskExecutionResultFrequencyStat( + { + ...DEFAULT_TASK_RUN_FREQUENCIES, + ...calculateFrequency(typedResultFrequencies), + }, + config.monitored_task_execution_thresholds.custom[taskType] ?? + config.monitored_task_execution_thresholds.default + ) + ), + }, + }, + status: HealthStatus.OK, + }; +} + +function summarizeTaskExecutionResultFrequencyStat( + resultFrequencySummary: ResultFrequency, + executionErrorThreshold: TaskExecutionFailureThreshold +): ResultFrequencySummary { + return { + ...resultFrequencySummary, + status: + resultFrequencySummary.Failed > executionErrorThreshold.warn_threshold + ? resultFrequencySummary.Failed > executionErrorThreshold.error_threshold + ? HealthStatus.Error + : HealthStatus.Warning + : HealthStatus.OK, + }; +} diff --git a/x-pack/plugins/task_manager/server/monitoring/workload_statistics.test.ts b/x-pack/plugins/task_manager/server/monitoring/workload_statistics.test.ts new file mode 100644 index 0000000000000..d9af3307e75cb --- /dev/null +++ b/x-pack/plugins/task_manager/server/monitoring/workload_statistics.test.ts @@ -0,0 +1,773 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { first, take, bufferCount } from 'rxjs/operators'; +import { loggingSystemMock } from '../../../../../src/core/server/mocks'; +import { + WorkloadAggregation, + createWorkloadAggregator, + padBuckets, + estimateRecurringTaskScheduling, +} from './workload_statistics'; +import { ConcreteTaskInstance } from '../task'; +import { ESSearchResponse } from '../../../apm/typings/elasticsearch'; +import { AggregationResultOf } from '../../../apm/typings/elasticsearch/aggregations'; +import { times } from 'lodash'; +import { taskStoreMock } from '../task_store.mock'; + +type MockESResult = ESSearchResponse< + ConcreteTaskInstance, + { + body: WorkloadAggregation; + } +>; + +describe('Workload Statistics Aggregator', () => { + test('queries the Task Store at a fixed interval for the current workload', async () => { + const taskStore = taskStoreMock.create({}); + taskStore.aggregate.mockResolvedValue({ + hits: { + hits: [], + max_score: 0, + total: { value: 0, relation: 'eq' }, + }, + took: 1, + timed_out: false, + _shards: { + total: 1, + successful: 1, + skipped: 1, + failed: 0, + }, + aggregations: { + taskType: { + buckets: [], + }, + schedule: { + buckets: [], + }, + idleTasks: { + doc_count: 0, + overdue: { + doc_count: 0, + }, + scheduleDensity: { + buckets: [ + { + key: '2020-10-02T15:18:37.274Z-2020-10-02T15:19:36.274Z', + from: 1.601651917274e12, + from_as_string: '2020-10-02T15:18:37.274Z', + to: 1.601651976274e12, + to_as_string: '2020-10-02T15:19:36.274Z', + doc_count: 0, + histogram: { + buckets: [], + }, + }, + ], + }, + }, + }, + } as MockESResult); + + const workloadAggregator = createWorkloadAggregator( + taskStore, + 10, + 3000, + loggingSystemMock.create().get() + ); + + return new Promise((resolve) => { + workloadAggregator.pipe(first()).subscribe(() => { + expect(taskStore.aggregate).toHaveBeenCalledWith({ + aggs: { + taskType: { + terms: { field: 'task.taskType' }, + aggs: { + status: { + terms: { field: 'task.status' }, + }, + }, + }, + schedule: { + terms: { + field: 'task.schedule.interval', + }, + }, + idleTasks: { + filter: { + term: { 'task.status': 'idle' }, + }, + aggs: { + scheduleDensity: { + range: { + field: 'task.runAt', + ranges: [{ from: 'now', to: 'now+1m' }], + }, + aggs: { + histogram: { + date_histogram: { + field: 'task.runAt', + fixed_interval: '3s', + }, + aggs: { + interval: { + terms: { + field: 'task.schedule.interval', + }, + }, + }, + }, + }, + }, + overdue: { + filter: { + range: { + 'task.runAt': { lt: 'now' }, + }, + }, + }, + }, + }, + }, + }); + resolve(); + }); + }); + }); + + const mockAggregatedResult: () => MockESResult = () => + ({ + hits: { + hits: [], + max_score: 0, + total: { value: 4, relation: 'eq' }, + }, + took: 1, + timed_out: false, + _shards: { + total: 1, + successful: 1, + skipped: 1, + failed: 0, + }, + aggregations: { + schedule: { + buckets: [ + { + key: '3600s', + doc_count: 1, + }, + { + key: '60s', + doc_count: 1, + }, + { + key: '720m', + doc_count: 1, + }, + ], + }, + taskType: { + buckets: [ + { + key: 'actions_telemetry', + doc_count: 2, + status: { + buckets: [ + { + key: 'idle', + doc_count: 2, + }, + ], + }, + }, + { + key: 'alerting_telemetry', + doc_count: 1, + status: { + buckets: [ + { + key: 'idle', + doc_count: 1, + }, + ], + }, + }, + { + key: 'session_cleanup', + doc_count: 1, + status: { + buckets: [ + { + key: 'idle', + doc_count: 1, + }, + ], + }, + }, + ], + }, + idleTasks: { + doc_count: 13, + overdue: { + doc_count: 6, + }, + scheduleDensity: { + buckets: [ + mockHistogram(0, 7 * 3000 + 500, 60 * 1000, 3000, [2, 2, 5, 0, 0, 0, 0, 0, 0, 1]), + ], + }, + }, + }, + } as MockESResult); + + test('returns a summary of the workload by task type', async () => { + const taskStore = taskStoreMock.create({}); + taskStore.aggregate.mockResolvedValue(mockAggregatedResult()); + + const workloadAggregator = createWorkloadAggregator( + taskStore, + 10, + 3000, + loggingSystemMock.create().get() + ); + + return new Promise((resolve) => { + workloadAggregator.pipe(first()).subscribe((result) => { + expect(result.key).toEqual('workload'); + expect(result.value).toMatchObject({ + count: 4, + task_types: { + actions_telemetry: { count: 2, status: { idle: 2 } }, + alerting_telemetry: { count: 1, status: { idle: 1 } }, + session_cleanup: { count: 1, status: { idle: 1 } }, + }, + }); + resolve(); + }); + }); + }); + + test('returns a count of the overdue workload', async () => { + const taskStore = taskStoreMock.create({}); + taskStore.aggregate.mockResolvedValue(mockAggregatedResult()); + + const workloadAggregator = createWorkloadAggregator( + taskStore, + 10, + 3000, + loggingSystemMock.create().get() + ); + + return new Promise((resolve) => { + workloadAggregator.pipe(first()).subscribe((result) => { + expect(result.key).toEqual('workload'); + expect(result.value).toMatchObject({ + overdue: 6, + }); + resolve(); + }); + }); + }); + + test('returns a histogram of the upcoming workload for the upcoming minute when refresh rate is high', async () => { + const taskStore = taskStoreMock.create({}); + taskStore.aggregate.mockResolvedValue(mockAggregatedResult()); + + const workloadAggregator = createWorkloadAggregator( + taskStore, + 10, + 3000, + loggingSystemMock.create().get() + ); + + return new Promise((resolve) => { + workloadAggregator.pipe(first()).subscribe((result) => { + expect(result.key).toEqual('workload'); + expect(result.value).toMatchObject({ + // we have intervals every 3s, so we aggregate buckets 3s apart + // in this mock, Elasticsearch found tasks scheduled in 21 (8th bucket), 24, 27 and 48s seconds from now + // 0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 30, 33, 36, 39, 42, 45, 48, 51, 54, 57 + // [0, 0, 0, 0, 0, 0, 0, 2, 2, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0 ] + // Above you see each bucket and the number of scheduled tasks we expect to have in them + estimated_schedule_density: [0, 0, 0, 0, 0, 0, 0, 2, 2, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0], + }); + resolve(); + }); + }); + }); + + test('returns a histogram of the upcoming workload for twice refresh rate when rate is low', async () => { + const taskStore = taskStoreMock.create({}); + taskStore.aggregate.mockResolvedValue(mockAggregatedResult()); + + const workloadAggregator = createWorkloadAggregator( + taskStore, + 60 * 1000, + 3000, + loggingSystemMock.create().get() + ); + + return new Promise((resolve) => { + workloadAggregator.pipe(first()).subscribe(() => { + expect(taskStore.aggregate.mock.calls[0][0]).toMatchObject({ + aggs: { + idleTasks: { + aggs: { + scheduleDensity: { + range: { + field: 'task.runAt', + ranges: [ + { + from: 'now', + to: 'now+2m', + }, + ], + }, + }, + }, + }, + }, + }); + resolve(); + }); + }); + }); + + test('returns a histogram of the upcoming workload maxed out at 50 buckets when rate is too low', async () => { + const taskStore = taskStoreMock.create({}); + taskStore.aggregate.mockResolvedValue(mockAggregatedResult()); + + const workloadAggregator = createWorkloadAggregator( + taskStore, + 15 * 60 * 1000, + 3000, + loggingSystemMock.create().get() + ); + + return new Promise((resolve) => { + workloadAggregator.pipe(first()).subscribe((result) => { + expect(taskStore.aggregate.mock.calls[0][0]).toMatchObject({ + aggs: { + idleTasks: { + aggs: { + scheduleDensity: { + range: { + field: 'task.runAt', + ranges: [ + { + from: 'now', + // 50 buckets of 3s = 50 * 3 = 150s + to: 'now+150s', + }, + ], + }, + }, + }, + }, + }, + }); + resolve(); + }); + }); + }); + + test('recovers from errors fetching the workload', async () => { + const taskStore = taskStoreMock.create({}); + taskStore.aggregate + .mockResolvedValueOnce( + setTaskTypeCount(mockAggregatedResult(), 'alerting_telemetry', { + idle: 2, + }) + ) + .mockRejectedValueOnce(new Error('Elasticsearch has gone poof')) + .mockResolvedValueOnce( + setTaskTypeCount(mockAggregatedResult(), 'alerting_telemetry', { + idle: 1, + failed: 1, + }) + ); + const logger = loggingSystemMock.create().get(); + const workloadAggregator = createWorkloadAggregator(taskStore, 10, 3000, logger); + + return new Promise((resolve, reject) => { + workloadAggregator.pipe(take(2), bufferCount(2)).subscribe((results) => { + expect(results[0].key).toEqual('workload'); + expect(results[0].value).toMatchObject({ + count: 5, + task_types: { + actions_telemetry: { count: 2, status: { idle: 2 } }, + alerting_telemetry: { count: 2, status: { idle: 2 } }, + session_cleanup: { count: 1, status: { idle: 1 } }, + }, + }); + expect(results[1].key).toEqual('workload'); + expect(results[1].value).toMatchObject({ + count: 5, + task_types: { + actions_telemetry: { count: 2, status: { idle: 2 } }, + alerting_telemetry: { count: 2, status: { idle: 1, failed: 1 } }, + session_cleanup: { count: 1, status: { idle: 1 } }, + }, + }); + resolve(); + }, reject); + }); + }); +}); + +describe('estimateRecurringTaskScheduling', () => { + test('flattens out buckets with non recurring tasks', () => { + const now = Date.now(); + const schedule = times(10, (index) => ({ + key: index * 3000 + now, + nonRecurring: index, + })); + expect(estimateRecurringTaskScheduling(schedule, 3000)).toEqual([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); + }); + + test('estimates the buckets that recurring tasks might repeat in when recurring task interval equals the interval', () => { + const now = Date.now(); + const schedule: Array<{ + key: number; + nonRecurring: number; + recurring?: Array<[number, string]>; + }> = times(10, (index) => ({ + key: index * 3000 + now, + nonRecurring: 0, + })); + + schedule[0].nonRecurring = 1; + schedule[1].nonRecurring = 1; + schedule[4].recurring = [[1, '3s']]; + + expect(estimateRecurringTaskScheduling(schedule, 3000)).toEqual([1, 1, 0, 0, 1, 1, 1, 1, 1, 1]); + }); + + test('estimates the buckets that recurring tasks might repeat in when recurring task interval is larger than the interval', () => { + const now = Date.now(); + const schedule: Array<{ + key: number; + nonRecurring: number; + recurring?: Array<[number, string]>; + }> = times(10, (index) => ({ + key: index * 3000 + now, + nonRecurring: 0, + })); + + schedule[0].nonRecurring = 1; + schedule[1].nonRecurring = 1; + schedule[4].recurring = [[1, '6s']]; + + expect(estimateRecurringTaskScheduling(schedule, 3000)).toEqual([1, 1, 0, 0, 1, 0, 1, 0, 1, 0]); + }); + + test('estimates the buckets that recurring tasks might repeat in when recurring task interval doesnt divide by interval', () => { + const now = Date.now(); + const schedule: Array<{ + key: number; + nonRecurring: number; + recurring?: Array<[number, string]>; + }> = times(10, (index) => ({ + key: index * 3000 + now, + nonRecurring: 0, + })); + + schedule[0].nonRecurring = 1; + schedule[1].nonRecurring = 1; + schedule[4].recurring = [[1, '5s']]; + + expect(estimateRecurringTaskScheduling(schedule, 3000)).toEqual([1, 1, 0, 0, 1, 0, 1, 0, 1, 0]); + }); + + test('estimates the buckets that recurring tasks might repeat in when recurring tasks overlap', () => { + const now = Date.now(); + const schedule: Array<{ + key: number; + nonRecurring: number; + recurring?: Array<[number, string]>; + }> = times(20, (index) => ({ + key: index * 3000 + now, + nonRecurring: 0, + })); + + schedule[0].nonRecurring = 1; + schedule[1].nonRecurring = 1; + schedule[3].recurring = [[1, '3s']]; + schedule[4].recurring = [ + [2, '6s'], + [1, '8s'], + ]; + schedule[5].recurring = [[1, '5s']]; + schedule[6].nonRecurring = 3; + + expect(estimateRecurringTaskScheduling(schedule, 3000)).toEqual([ + 1, + 1, + 0, + 1, + 4, + 2, + 6, + 3, + 3, + 2, + 4, + 2, + 3, + 3, + 3, + 2, + 4, + 2, + 3, + 3, + ]); + }); +}); + +describe('padBuckets', () => { + test('returns zeroed out bucklets when there are no buckets in the histogram', async () => { + expect( + padBuckets(10, 3000, { + key: '2020-10-02T19:47:28.128Z-2020-10-02T19:48:28.128Z', + from: 1601668048128, + from_as_string: '2020-10-02T19:47:28.128Z', + to: 1601668108128, + to_as_string: '2020-10-02T19:48:28.128Z', + doc_count: 0, + histogram: { + buckets: [], + }, + }) + ).toEqual([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + }); + + test('pads buckets with zeros to fill out the entire period of time after detected buckets', async () => { + expect( + padBuckets(10, 3000, { + key: '2020-10-02T19:47:28.128Z-2020-10-02T19:48:28.128Z', + from: 1601668046000, + from_as_string: '2020-10-02T19:47:26.000Z', + to: 1601668076000, + to_as_string: '2020-10-02T19:47:56.000Z', + doc_count: 3, + histogram: { + buckets: [ + { + key_as_string: '2020-10-02T19:47:27.000Z', + key: 1601668047000, + doc_count: 1, + interval: { + sum_other_doc_count: 0, + buckets: [], + }, + }, + { + key_as_string: '2020-10-02T19:47:30.000Z', + key: 1601668050000, + doc_count: 1, + interval: { + sum_other_doc_count: 0, + buckets: [], + }, + }, + { + key_as_string: '2020-10-02T19:47:33.000Z', + key: 1601668053000, + doc_count: 0, + interval: { + sum_other_doc_count: 0, + buckets: [], + }, + }, + { + key_as_string: '2020-10-02T19:47:36.000Z', + key: 1601668056000, + doc_count: 0, + interval: { + sum_other_doc_count: 0, + buckets: [], + }, + }, + { + key_as_string: '2020-10-02T19:47:39.000Z', + key: 1601668059000, + doc_count: 0, + interval: { + sum_other_doc_count: 0, + buckets: [], + }, + }, + { + key_as_string: '2020-10-02T19:47:42.000Z', + key: 1601668062000, + doc_count: 1, + interval: { + sum_other_doc_count: 0, + buckets: [], + }, + }, + ], + }, + }) + ).toEqual([1, 1, 0, 0, 0, 1, 0, 0, 0, 0]); + }); + + test('pads buckets with zeros to fill out the entire period of time before detected buckets', async () => { + expect( + padBuckets(10, 3000, { + key: '2020-10-02T20:39:45.793Z-2020-10-02T20:40:14.793Z', + from: 1601671183000, + from_as_string: '2020-10-02T20:39:43.000Z', + to: 1601671213000, + to_as_string: '2020-10-02T20:40:13.000Z', + doc_count: 2, + histogram: { + buckets: [ + { + key_as_string: '2020-10-02T20:40:09.000Z', + key: 1601671209000, + doc_count: 1, + interval: { buckets: [] }, + }, + { + key_as_string: '2020-10-02T20:40:12.000Z', + key: 1601671212000, + doc_count: 1, + interval: { buckets: [] }, + }, + ], + }, + }) + ).toEqual([0, 0, 0, 0, 0, 0, 0, 0, 1, 1]); + }); + + test('pads buckets with zeros to fill out the entire period surounding the detected buckets', async () => { + expect( + padBuckets(20, 3000, { + key: '2020-10-02T20:39:45.793Z-2020-10-02T20:40:14.793Z', + from: 1601671185793, + from_as_string: '2020-10-02T20:39:45.793Z', + to: 1601671245793, + to_as_string: '2020-10-02T20:40:45.793Z', + doc_count: 2, + histogram: { + buckets: [ + { + key_as_string: '2020-10-02T20:40:09.000Z', + key: 1601671209000, + doc_count: 1, + interval: { buckets: [] }, + }, + { + key_as_string: '2020-10-02T20:40:12.000Z', + key: 1601671212000, + doc_count: 1, + interval: { buckets: [] }, + }, + ], + }, + }) + ).toEqual([0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + }); +}); + +function setTaskTypeCount( + { aggregations }: MockESResult, + taskType: string, + status: Record +) { + const taskTypes = aggregations!.taskType as AggregationResultOf< + WorkloadAggregation['aggs']['taskType'], + {} + >; + const buckets = [ + ...taskTypes.buckets.filter(({ key }) => key !== taskType), + { + key: taskType, + doc_count: Object.values(status).reduce((sum, count) => sum + count, 0), + status: { + sum_other_doc_count: 0, + buckets: Object.entries(status).map(([key, count]) => ({ + key, + doc_count: count, + })), + }, + }, + ]; + return ({ + hits: { + total: { value: buckets.reduce((sum, bucket) => sum + bucket.doc_count, 0) }, + }, + aggregations: { + ...aggregations, + taskType: { + sum_other_doc_count: 0, + buckets, + }, + }, + } as {}) as MockESResult; +} + +/** * + * This creates a mock histogram as returned by Elasticsearch + * + * @param from lower bound of query + * @param findFrom the timestamp (key) of the first bucket returned + * @param to upper bound of query + * @param interval the duration that each bucket coresponds to + * @param foundBuckets the buckets identified by ES, any buckets missing before or after which + * are still in the date range are assumed to have 0 results, ES only returns 0 for + * buckets that sit in between buckets which do have results + */ +function mockHistogram( + from: number, + findFrom: number, + to: number, + interval: number, + foundBuckets: Array +) { + const now = Date.now(); + const fromDate = new Date(now + from); + const toDate = new Date(now + to); + return { + key: `${fromDate.toISOString()}-${toDate.toISOString()}`, + from: now + from, + from_as_string: fromDate.toISOString(), + to: now + to, + to_as_string: toDate.toISOString(), + doc_count: foundBuckets.reduce((sum: number, count) => sum + (count ?? 0), 0), + histogram: { + buckets: foundBuckets.reduce( + (histogramBuckets, count, index) => { + if (typeof count === 'number') { + const key = new Date(now + findFrom + index * interval); + histogramBuckets.push({ + key_as_string: key.toISOString(), + key: key.getTime(), + doc_count: count, + interval: { buckets: [] }, + }); + } + return histogramBuckets; + }, + [] as Array<{ + key_as_string: string; + key: number; + doc_count: number; + interval: { + buckets: Array<{ + key: string; + doc_count: number; + }>; + }; + }> + ), + }, + }; +} diff --git a/x-pack/plugins/task_manager/server/monitoring/workload_statistics.ts b/x-pack/plugins/task_manager/server/monitoring/workload_statistics.ts new file mode 100644 index 0000000000000..fe70f24684ad9 --- /dev/null +++ b/x-pack/plugins/task_manager/server/monitoring/workload_statistics.ts @@ -0,0 +1,344 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { timer } from 'rxjs'; +import { mergeMap, map, catchError } from 'rxjs/operators'; +import { Logger } from 'src/core/server'; +import { JsonObject } from 'src/plugins/kibana_utils/common'; +import { keyBy, mapValues } from 'lodash'; +import { AggregatedStatProvider } from './runtime_statistics_aggregator'; +import { parseIntervalAsSecond, asInterval, parseIntervalAsMillisecond } from '../lib/intervals'; +import { AggregationResultOf } from '../../../apm/typings/elasticsearch/aggregations'; +import { HealthStatus } from './monitoring_stats_stream'; +import { TaskStore } from '../task_store'; + +interface StatusStat extends JsonObject { + [status: string]: number; +} +interface TaskTypeStat extends JsonObject { + [taskType: string]: { + count: number; + status: StatusStat; + }; +} + +export interface WorkloadStat extends JsonObject { + count: number; + task_types: TaskTypeStat; + schedule: Array<[string, number]>; + overdue: number; + estimated_schedule_density: number[]; +} + +export interface WorkloadAggregation { + aggs: { + taskType: { + terms: { field: string }; + aggs: { + status: { + terms: { field: string }; + }; + }; + }; + schedule: { + terms: { field: string }; + }; + idleTasks: { + filter: { + term: { 'task.status': string }; + }; + aggs: { + scheduleDensity: { + range: { + field: string; + ranges: [{ from: string; to: string }]; + }; + aggs: { + histogram: { + date_histogram: { + field: string; + fixed_interval: string; + }; + aggs: { + interval: { + terms: { field: string }; + }; + }; + }; + }; + }; + overdue: { + filter: { + range: { + 'task.runAt': { lt: string }; + }; + }; + }; + }; + }; + }; +} + +// The type of a bucket in the scheduleDensity range aggregation +type ScheduleDensityResult = AggregationResultOf< + WorkloadAggregation['aggs']['idleTasks']['aggs']['scheduleDensity'], + {} +>['buckets'][0]; +type ScheduledIntervals = ScheduleDensityResult['histogram']['buckets'][0]; + +// Set an upper bound just in case a customer sets a really high refresh rate +const MAX_SHCEDULE_DENSITY_BUCKETS = 50; + +export function createWorkloadAggregator( + taskStore: TaskStore, + refreshInterval: number, + pollInterval: number, + logger: Logger +): AggregatedStatProvider { + // calculate scheduleDensity going two refreshIntervals or 1 minute into into the future + // (the longer of the two) + const scheduleDensityBuckets = Math.min( + Math.max(Math.round(60000 / pollInterval), Math.round((refreshInterval * 2) / pollInterval)), + MAX_SHCEDULE_DENSITY_BUCKETS + ); + + return timer(0, refreshInterval).pipe( + mergeMap(() => + taskStore.aggregate({ + aggs: { + taskType: { + terms: { field: 'task.taskType' }, + aggs: { + status: { + terms: { field: 'task.status' }, + }, + }, + }, + schedule: { + terms: { field: 'task.schedule.interval' }, + }, + idleTasks: { + filter: { + term: { 'task.status': 'idle' }, + }, + aggs: { + scheduleDensity: { + // create a window of upcoming tasks + range: { + field: 'task.runAt', + ranges: [ + { + from: `now`, + to: `now+${asInterval(scheduleDensityBuckets * pollInterval)}`, + }, + ], + }, + aggs: { + // create histogram of scheduling in the window, with each bucket being a polling interval + histogram: { + date_histogram: { + field: 'task.runAt', + fixed_interval: asInterval(pollInterval), + }, + // break down each bucket in the historgram by schedule + aggs: { + interval: { + terms: { field: 'task.schedule.interval' }, + }, + }, + }, + }, + }, + overdue: { + filter: { + range: { + 'task.runAt': { lt: 'now' }, + }, + }, + }, + }, + }, + }, + }) + ), + map((result) => { + const { + aggregations, + hits: { + total: { value: count }, + }, + } = result; + + if ( + !( + aggregations?.taskType && + aggregations?.schedule && + aggregations?.idleTasks?.overdue && + aggregations?.idleTasks?.scheduleDensity + ) + ) { + throw new Error(`Invalid workload: ${JSON.stringify(result)}`); + } + + const taskTypes = aggregations.taskType.buckets; + const schedules = aggregations.schedule.buckets; + + const { + overdue: { doc_count: overdue }, + scheduleDensity: { buckets: [scheduleDensity] = [] } = {}, + } = aggregations.idleTasks; + + const summary: WorkloadStat = { + count, + task_types: mapValues(keyBy(taskTypes, 'key'), ({ doc_count: docCount, status }) => { + return { + count: docCount, + status: mapValues(keyBy(status.buckets, 'key'), 'doc_count'), + }; + }), + schedule: schedules + .sort( + (scheduleLeft, scheduleRight) => + parseIntervalAsSecond(scheduleLeft.key as string) - + parseIntervalAsSecond(scheduleRight.key as string) + ) + .map((schedule) => [schedule.key as string, schedule.doc_count]), + overdue, + estimated_schedule_density: padBuckets( + scheduleDensityBuckets, + pollInterval, + scheduleDensity + ), + }; + return { + key: 'workload', + value: summary, + }; + }), + catchError((ex: Error, caught) => { + logger.error(`[WorkloadAggregator]: ${ex}`); + // continue to pull values from the same observable + return caught; + }) + ); +} + +interface IntervalTaskCountTouple { + nonRecurring?: number; + recurring?: Array<[number, string]>; + key: number; +} + +export function padBuckets( + scheduleDensityBuckets: number, + pollInterval: number, + scheduleDensity: ScheduleDensityResult +): number[] { + if (scheduleDensity.from && scheduleDensity.to && scheduleDensity.histogram?.buckets?.length) { + const { histogram, from, to } = scheduleDensity; + const firstBucket = histogram.buckets[0].key; + const lastBucket = histogram.buckets[histogram.buckets.length - 1].key; + + const bucketsToPadBeforeFirstBucket = calculateBucketsBetween(firstBucket, from, pollInterval); + const bucketsToPadAfterLast = calculateBucketsBetween( + lastBucket + pollInterval, + to, + pollInterval + ); + + return estimateRecurringTaskScheduling( + [ + ...bucketsToPadBeforeFirstBucket, + ...histogram.buckets.map(countByIntervalInBucket), + ...bucketsToPadAfterLast, + ], + pollInterval + ); + } + return new Array(scheduleDensityBuckets).fill(0); +} + +function countByIntervalInBucket(bucket: ScheduledIntervals): IntervalTaskCountTouple { + if (bucket.doc_count === 0) { + return { nonRecurring: 0, key: bucket.key }; + } + const recurring: Array<[number, string]> = []; + let nonRecurring = bucket.doc_count; + for (const intervalBucket of bucket.interval.buckets) { + recurring.push([intervalBucket.doc_count, intervalBucket.key as string]); + nonRecurring -= intervalBucket.doc_count; + } + + return { nonRecurring, recurring, key: bucket.key }; +} + +function calculateBucketsBetween( + from: number, + to: number, + interval: number, + bucketInterval: number = interval +): Array<{ key: number }> { + const calcForwardInTime = from < to; + + // as task interval might not divide by the pollInterval (aka the bucket interval) + // we have to adjust for the "drift" that occurs when estimating when the next + // bucket the task might actually get scheduled in + const actualInterval = Math.ceil(interval / bucketInterval) * bucketInterval; + + const buckets: Array<{ key: number }> = []; + const toBound = calcForwardInTime ? to : -(to + actualInterval); + let fromBound = calcForwardInTime ? from : -from; + + while (fromBound < toBound) { + buckets.push({ key: fromBound }); + fromBound += actualInterval; + } + + return calcForwardInTime + ? buckets + : buckets.reverse().map((bucket) => { + bucket.key = Math.abs(bucket.key); + return bucket; + }); +} + +export function estimateRecurringTaskScheduling( + scheduleDensity: IntervalTaskCountTouple[], + pollInterval: number +) { + const lastKey = scheduleDensity[scheduleDensity.length - 1].key; + + return scheduleDensity.map((bucket, currentBucketIndex) => { + for (const [count, interval] of bucket.recurring ?? []) { + for (const recurrance of calculateBucketsBetween( + bucket.key, + // `calculateBucketsBetween` uses the `to` as a non-inclusive upper bound + // but lastKey is a bucket we wish to include + lastKey + pollInterval, + parseIntervalAsMillisecond(interval), + pollInterval + )) { + const recurranceBucketIndex = + currentBucketIndex + Math.ceil((recurrance.key - bucket.key) / pollInterval); + + if (recurranceBucketIndex < scheduleDensity.length) { + scheduleDensity[recurranceBucketIndex].nonRecurring = + count + (scheduleDensity[recurranceBucketIndex].nonRecurring ?? 0); + } + } + } + return bucket.nonRecurring ?? 0; + }); +} + +export function summarizeWorkloadStat( + workloadStats: WorkloadStat +): { value: WorkloadStat; status: HealthStatus } { + return { + value: workloadStats, + status: HealthStatus.OK, + }; +} diff --git a/x-pack/plugins/task_manager/server/plugin.test.ts b/x-pack/plugins/task_manager/server/plugin.test.ts index 50e7e9a7aa197..8388468164a4f 100644 --- a/x-pack/plugins/task_manager/server/plugin.test.ts +++ b/x-pack/plugins/task_manager/server/plugin.test.ts @@ -19,6 +19,16 @@ describe('TaskManagerPlugin', () => { poll_interval: 3000, max_poll_inactivity_cycles: 10, request_capacity: 1000, + monitored_aggregated_stats_refresh_rate: 5000, + monitored_stats_required_freshness: 5000, + monitored_stats_running_average_window: 50, + monitored_task_execution_thresholds: { + default: { + error_threshold: 90, + warn_threshold: 80, + }, + custom: {}, + }, }); pluginInitializerContext.env.instanceUuid = ''; @@ -38,6 +48,16 @@ describe('TaskManagerPlugin', () => { poll_interval: 3000, max_poll_inactivity_cycles: 10, request_capacity: 1000, + monitored_aggregated_stats_refresh_rate: 5000, + monitored_stats_required_freshness: 5000, + monitored_stats_running_average_window: 50, + monitored_task_execution_thresholds: { + default: { + error_threshold: 90, + warn_threshold: 80, + }, + custom: {}, + }, }); const taskManagerPlugin = new TaskManagerPlugin(pluginInitializerContext); diff --git a/x-pack/plugins/task_manager/server/plugin.ts b/x-pack/plugins/task_manager/server/plugin.ts index 92f492acf591e..8b723864bd234 100644 --- a/x-pack/plugins/task_manager/server/plugin.ts +++ b/x-pack/plugins/task_manager/server/plugin.ts @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { Observable } from 'rxjs'; +import { combineLatest, Observable, Subject } from 'rxjs'; import { first, map, distinctUntilChanged } from 'rxjs/operators'; import { PluginInitializerContext, @@ -22,6 +22,8 @@ import { TaskTypeDictionary } from './task_type_dictionary'; import { FetchResult, SearchOpts, TaskStore } from './task_store'; import { createManagedConfiguration } from './lib/create_managed_configuration'; import { TaskScheduling } from './task_scheduling'; +import { healthRoute } from './routes'; +import { createMonitoringStats, MonitoringStats } from './monitoring'; export type TaskManagerSetupContract = { addMiddleware: (middleware: Middleware) => void } & Pick< TaskTypeDictionary, @@ -43,6 +45,7 @@ export class TaskManagerPlugin private definitions: TaskTypeDictionary; private middleware: Middleware = createInitialMiddleware(); private elasticsearchAndSOAvailability$?: Observable; + private monitoringStats$ = new Subject(); constructor(private readonly initContext: PluginInitializerContext) { this.initContext = initContext; @@ -50,22 +53,22 @@ export class TaskManagerPlugin this.definitions = new TaskTypeDictionary(this.logger); } - public async setup({ savedObjects, status }: CoreSetup): Promise { + public async setup(core: CoreSetup): Promise { this.config = await this.initContext.config .create() .pipe(first()) .toPromise(); - this.elasticsearchAndSOAvailability$ = status.core$.pipe( + this.elasticsearchAndSOAvailability$ = core.status.core$.pipe( map( - (ev) => - ev.elasticsearch.level === ServiceStatusLevels.available && - ev.savedObjects.level === ServiceStatusLevels.available + ({ elasticsearch, savedObjects }) => + elasticsearch.level === ServiceStatusLevels.available && + savedObjects.level === ServiceStatusLevels.available ), distinctUntilChanged() ); - setupSavedObjects(savedObjects, this.config); + setupSavedObjects(core.savedObjects, this.config); this.taskManagerId = this.initContext.env.instanceUuid; if (!this.taskManagerId) { @@ -77,6 +80,26 @@ export class TaskManagerPlugin this.logger.info(`TaskManager is identified by the Kibana UUID: ${this.taskManagerId}`); } + // Routes + const router = core.http.createRouter(); + const serviceStatus$ = healthRoute( + router, + this.monitoringStats$, + this.logger, + this.taskManagerId, + this.config! + ); + + core.getStartServices().then(async () => { + core.status.set( + combineLatest([core.status.derivedStatus$, serviceStatus$]).pipe( + map(([derivedStatus, serviceStatus]) => + serviceStatus.level > derivedStatus.level ? serviceStatus : derivedStatus + ) + ) + ); + }); + return { addMiddleware: (middleware: Middleware) => { this.assertStillInSetup('add Middleware'); @@ -102,7 +125,7 @@ export class TaskManagerPlugin taskManagerId: `kibana:${this.taskManagerId!}`, }); - const { maxWorkersConfiguration$, pollIntervalConfiguration$ } = createManagedConfiguration({ + const managedConfiguration = createManagedConfiguration({ logger: this.logger, errors$: taskStore.errors$, startingMaxWorkers: this.config!.max_workers, @@ -115,11 +138,18 @@ export class TaskManagerPlugin logger: this.logger, taskStore, middleware: this.middleware, - maxWorkersConfiguration$, - pollIntervalConfiguration$, elasticsearchAndSOAvailability$: this.elasticsearchAndSOAvailability$!, + ...managedConfiguration, }); + createMonitoringStats( + this.taskPollingLifecycle, + taskStore, + this.config!, + managedConfiguration, + this.logger + ).subscribe((stat) => this.monitoringStats$.next(stat)); + const taskScheduling = new TaskScheduling({ logger: this.logger, taskStore, diff --git a/x-pack/plugins/task_manager/server/polling/task_poller.test.ts b/x-pack/plugins/task_manager/server/polling/task_poller.test.ts index 956c8b05f3860..f5f1667312d79 100644 --- a/x-pack/plugins/task_manager/server/polling/task_poller.test.ts +++ b/x-pack/plugins/task_manager/server/polling/task_poller.test.ts @@ -9,7 +9,8 @@ import { Subject, of, BehaviorSubject } from 'rxjs'; import { Option, none, some } from 'fp-ts/lib/Option'; import { createTaskPoller, PollingError, PollingErrorType } from './task_poller'; import { fakeSchedulers } from 'rxjs-marbles/jest'; -import { sleep, resolvable, Resolvable, mockLogger } from '../test_utils'; +import { sleep, resolvable, Resolvable } from '../test_utils'; +import { loggingSystemMock } from '../../../../../src/core/server/mocks'; import { asOk, asErr } from '../lib/result_type'; describe('TaskPoller', () => { @@ -24,7 +25,7 @@ describe('TaskPoller', () => { const work = jest.fn(async () => true); createTaskPoller({ - logger: mockLogger(), + logger: loggingSystemMock.create().get(), pollInterval$: of(pollInterval), bufferCapacity, getCapacity: () => 1, @@ -59,7 +60,7 @@ describe('TaskPoller', () => { const work = jest.fn(async () => true); createTaskPoller({ - logger: mockLogger(), + logger: loggingSystemMock.create().get(), pollInterval$, bufferCapacity, getCapacity: () => 1, @@ -101,7 +102,7 @@ describe('TaskPoller', () => { let hasCapacity = true; createTaskPoller({ - logger: mockLogger(), + logger: loggingSystemMock.create().get(), pollInterval$: of(pollInterval), bufferCapacity, work, @@ -160,7 +161,7 @@ describe('TaskPoller', () => { const work = jest.fn(async () => true); const pollRequests$ = new Subject>(); createTaskPoller({ - logger: mockLogger(), + logger: loggingSystemMock.create().get(), pollInterval$: of(pollInterval), bufferCapacity, work, @@ -206,7 +207,7 @@ describe('TaskPoller', () => { const work = jest.fn(async () => true); const pollRequests$ = new Subject>(); createTaskPoller({ - logger: mockLogger(), + logger: loggingSystemMock.create().get(), pollInterval$: of(pollInterval), bufferCapacity, work, @@ -251,7 +252,7 @@ describe('TaskPoller', () => { const work = jest.fn(async () => true); const pollRequests$ = new Subject>(); createTaskPoller({ - logger: mockLogger(), + logger: loggingSystemMock.create().get(), pollInterval$: of(pollInterval), bufferCapacity, work, @@ -288,7 +289,7 @@ describe('TaskPoller', () => { const handler = jest.fn(); const pollRequests$ = new Subject>(); createTaskPoller({ - logger: mockLogger(), + logger: loggingSystemMock.create().get(), pollInterval$: of(pollInterval), bufferCapacity, work: async (...args) => { @@ -339,7 +340,7 @@ describe('TaskPoller', () => { type ResolvableTupple = [string, PromiseLike & Resolvable]; const pollRequests$ = new Subject>(); createTaskPoller<[string, Resolvable], string[]>({ - logger: mockLogger(), + logger: loggingSystemMock.create().get(), pollInterval$: of(pollInterval), bufferCapacity, work: async (...resolvables) => { @@ -399,7 +400,7 @@ describe('TaskPoller', () => { const handler = jest.fn(); const pollRequests$ = new Subject>(); createTaskPoller({ - logger: mockLogger(), + logger: loggingSystemMock.create().get(), pollInterval$: of(pollInterval), bufferCapacity, work: async (...args) => { @@ -440,7 +441,7 @@ describe('TaskPoller', () => { return callCount; }); createTaskPoller({ - logger: mockLogger(), + logger: loggingSystemMock.create().get(), pollInterval$: of(pollInterval), bufferCapacity, work, @@ -483,7 +484,7 @@ describe('TaskPoller', () => { const work = jest.fn(async () => {}); const pollRequests$ = new Subject>(); createTaskPoller({ - logger: mockLogger(), + logger: loggingSystemMock.create().get(), pollInterval$: of(pollInterval), bufferCapacity, work, diff --git a/x-pack/plugins/task_manager/server/polling_lifecycle.test.ts b/x-pack/plugins/task_manager/server/polling_lifecycle.test.ts index 2e73c74a8b247..0f807976970cf 100644 --- a/x-pack/plugins/task_manager/server/polling_lifecycle.test.ts +++ b/x-pack/plugins/task_manager/server/polling_lifecycle.test.ts @@ -28,6 +28,16 @@ describe('TaskPollingLifecycle', () => { poll_interval: 6000000, max_poll_inactivity_cycles: 10, request_capacity: 1000, + monitored_aggregated_stats_refresh_rate: 5000, + monitored_stats_required_freshness: 5000, + monitored_stats_running_average_window: 50, + monitored_task_execution_thresholds: { + default: { + error_threshold: 90, + warn_threshold: 80, + }, + custom: {}, + }, }, taskStore: mockTaskStore, logger: taskManagerLogger, diff --git a/x-pack/plugins/task_manager/server/polling_lifecycle.ts b/x-pack/plugins/task_manager/server/polling_lifecycle.ts index 31fef6520ede5..73c2a897aaacb 100644 --- a/x-pack/plugins/task_manager/server/polling_lifecycle.ts +++ b/x-pack/plugins/task_manager/server/polling_lifecycle.ts @@ -9,6 +9,7 @@ import { performance } from 'perf_hooks'; import { pipe } from 'fp-ts/lib/pipeable'; import { Option, some, map as mapOptional } from 'fp-ts/lib/Option'; +import { tap } from 'rxjs/operators'; import { Logger } from '../../../../src/core/server'; import { Result, asErr, mapErr } from './lib/result_type'; @@ -21,6 +22,8 @@ import { TaskClaim, TaskRunRequest, asTaskRunRequestEvent, + TaskPollingCycle, + asTaskPollingCycleEvent, } from './task_events'; import { fillPool, FillPoolResult } from './lib/fill_pool'; import { Middleware } from './lib/middleware'; @@ -48,7 +51,12 @@ export type TaskPollingLifecycleOpts = { elasticsearchAndSOAvailability$: Observable; } & ManagedConfiguration; -export type TaskLifecycleEvent = TaskMarkRunning | TaskRun | TaskClaim | TaskRunRequest; +export type TaskLifecycleEvent = + | TaskMarkRunning + | TaskRun + | TaskClaim + | TaskRunRequest + | TaskPollingCycle; /** * The public interface into the task manager system. @@ -146,17 +154,23 @@ export class TaskPollingLifecycle { elasticsearchAndSOAvailability$.subscribe((areESAndSOAvailable) => { if (areESAndSOAvailable && !this.isStarted) { // start polling for work - this.pollingSubscription = poller$.subscribe( - mapErr((error: PollingError) => { - if (error.type === PollingErrorType.RequestCapacityReached) { - pipe( - error.data, - mapOptional((id) => this.emitEvent(asTaskRunRequestEvent(id, asErr(error)))) - ); - } - logger.error(error.message); - }) - ); + this.pollingSubscription = poller$ + .pipe( + tap( + mapErr((error: PollingError) => { + if (error.type === PollingErrorType.RequestCapacityReached) { + pipe( + error.data, + mapOptional((id) => this.emitEvent(asTaskRunRequestEvent(id, asErr(error)))) + ); + } + this.logger.error(error.message); + }) + ) + ) + .subscribe((event: Result>) => { + this.emitEvent(asTaskPollingCycleEvent(event)); + }); } else if (!areESAndSOAvailable && this.isStarted) { this.pollingSubscription.unsubscribe(); this.pool.cancelRunningTasks(); diff --git a/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.test.ts b/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.test.ts index 7cdbd8b11bb06..8a94ae4ed82f5 100644 --- a/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.test.ts +++ b/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.test.ts @@ -5,21 +5,12 @@ */ import _ from 'lodash'; -import { - asUpdateByQuery, - shouldBeOneOf, - mustBeAllOf, - ExistsFilter, - TermFilter, - RangeFilter, -} from './query_clauses'; +import { asUpdateByQuery, shouldBeOneOf, mustBeAllOf } from './query_clauses'; import { - updateFields, + updateFieldsAndMarkAsFailed, IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt, - TaskWithSchedule, - taskWithLessThanMaxAttempts, SortByRunAtAndRetryAt, } from './mark_available_tasks_as_claimed'; @@ -40,29 +31,29 @@ describe('mark_available_tasks_as_claimed', () => { createTaskRunner: () => ({ run: () => Promise.resolve() }), }, }); + const claimTasksById = undefined; const defaultMaxAttempts = 1; const taskManagerId = '3478fg6-82374f6-83467gf5-384g6f'; const claimOwnershipUntil = '2019-02-12T21:01:22.479Z'; + const fieldUpdates = { + ownerId: taskManagerId, + retryAt: claimOwnershipUntil, + }; expect( asUpdateByQuery({ query: mustBeAllOf( // Either a task with idle status and runAt <= now or // status running or claiming with a retryAt <= now. - shouldBeOneOf(IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt), - // Either task has an schedule or the attempts < the maximum configured - shouldBeOneOf( - TaskWithSchedule, - ...Array.from(definitions).map(([type, { maxAttempts }]) => - taskWithLessThanMaxAttempts(type, maxAttempts || defaultMaxAttempts) - ) - ) + shouldBeOneOf(IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt) + ), + update: updateFieldsAndMarkAsFailed( + fieldUpdates, + claimTasksById || [], + Array.from(definitions).reduce((accumulator, [type, { maxAttempts }]) => { + return { ...accumulator, [type]: maxAttempts || defaultMaxAttempts }; + }, {}) ), - update: updateFields({ - ownerId: taskManagerId, - status: 'claiming', - retryAt: claimOwnershipUntil, - }), sort: SortByRunAtAndRetryAt, }) ).toEqual({ @@ -100,42 +91,6 @@ describe('mark_available_tasks_as_claimed', () => { ], }, }, - // Either task has an recurring schedule or the attempts < the maximum configured - { - bool: { - should: [ - { exists: { field: 'task.schedule' } }, - { - bool: { - must: [ - { term: { 'task.taskType': 'sampleTask' } }, - { - range: { - 'task.attempts': { - lt: 5, - }, - }, - }, - ], - }, - }, - { - bool: { - must: [ - { term: { 'task.taskType': 'otherTask' } }, - { - range: { - 'task.attempts': { - lt: 1, - }, - }, - }, - ], - }, - }, - ], - }, - }, ], }, }, @@ -158,12 +113,26 @@ if (doc['task.runAt'].size()!=0) { }, seq_no_primary_term: true, script: { - source: `ctx._source.task.ownerId=params.ownerId; ctx._source.task.status=params.status; ctx._source.task.retryAt=params.retryAt;`, + source: ` + if (ctx._source.task.schedule != null || ctx._source.task.attempts < params.taskMaxAttempts[ctx._source.task.taskType] || params.claimTasksById.contains(ctx._id)) { + ctx._source.task.status = "claiming"; ${Object.keys(fieldUpdates) + .map((field) => `ctx._source.task.${field}=params.fieldUpdates.${field};`) + .join(' ')} + } else { + ctx._source.task.status = "failed"; + } + `, lang: 'painless', params: { - ownerId: taskManagerId, - retryAt: claimOwnershipUntil, - status: 'claiming', + fieldUpdates: { + ownerId: taskManagerId, + retryAt: claimOwnershipUntil, + }, + claimTasksById: [], + taskMaxAttempts: { + sampleTask: 5, + otherTask: 1, + }, }, }, }); diff --git a/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.ts b/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.ts index 699af3ed07d5d..072ec4648201a 100644 --- a/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.ts +++ b/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.ts @@ -100,12 +100,26 @@ if (doc['task.runAt'].size()!=0) { }, }; -export const updateFields = (fieldUpdates: { - [field: string]: string | number | Date; -}): ScriptClause => ({ - source: Object.keys(fieldUpdates) - .map((field) => `ctx._source.task.${field}=params.${field};`) - .join(' '), +export const updateFieldsAndMarkAsFailed = ( + fieldUpdates: { + [field: string]: string | number | Date; + }, + claimTasksById: string[], + taskMaxAttempts: { [field: string]: number } +): ScriptClause => ({ + source: ` + if (ctx._source.task.schedule != null || ctx._source.task.attempts < params.taskMaxAttempts[ctx._source.task.taskType] || params.claimTasksById.contains(ctx._id)) { + ctx._source.task.status = "claiming"; ${Object.keys(fieldUpdates) + .map((field) => `ctx._source.task.${field}=params.fieldUpdates.${field};`) + .join(' ')} + } else { + ctx._source.task.status = "failed"; + } + `, lang: 'painless', - params: fieldUpdates, + params: { + fieldUpdates, + claimTasksById, + taskMaxAttempts, + }, }); diff --git a/x-pack/plugins/task_manager/server/queries/query_clauses.ts b/x-pack/plugins/task_manager/server/queries/query_clauses.ts index f16ee302d6c83..5503b9cd94105 100644 --- a/x-pack/plugins/task_manager/server/queries/query_clauses.ts +++ b/x-pack/plugins/task_manager/server/queries/query_clauses.ts @@ -165,7 +165,12 @@ export interface ScriptClause { source: string; lang: string; params: { - [field: string]: string | number | Date; + [field: string]: + | string + | number + | Date + | string[] + | { [field: string]: string | number | Date }; }; } diff --git a/x-pack/plugins/task_manager/server/routes/_mock_handler_arguments.ts b/x-pack/plugins/task_manager/server/routes/_mock_handler_arguments.ts new file mode 100644 index 0000000000000..c9f4de25afaf7 --- /dev/null +++ b/x-pack/plugins/task_manager/server/routes/_mock_handler_arguments.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { RequestHandlerContext, KibanaRequest, KibanaResponseFactory } from 'kibana/server'; +import { identity } from 'lodash'; +import { httpServerMock } from '../../../../../src/core/server/mocks'; + +export function mockHandlerArguments( + {}: {}, + req: unknown, + res?: Array> +): [RequestHandlerContext, KibanaRequest, KibanaResponseFactory] { + return [ + ({} as unknown) as RequestHandlerContext, + req as KibanaRequest, + mockResponseFactory(res), + ]; +} + +export const mockResponseFactory = (resToMock: Array> = []) => { + const factory: jest.Mocked = httpServerMock.createResponseFactory(); + resToMock.forEach((key: string) => { + if (key in factory) { + Object.defineProperty(factory, key, { + value: jest.fn(identity), + }); + } + }); + return (factory as unknown) as KibanaResponseFactory; +}; diff --git a/x-pack/plugins/task_manager/server/routes/health.test.ts b/x-pack/plugins/task_manager/server/routes/health.test.ts new file mode 100644 index 0000000000000..5a0cef8eda94b --- /dev/null +++ b/x-pack/plugins/task_manager/server/routes/health.test.ts @@ -0,0 +1,393 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Observable, of, Subject } from 'rxjs'; +import { take } from 'rxjs/operators'; +import { merge } from 'lodash'; +import uuid from 'uuid'; +import { httpServiceMock } from 'src/core/server/mocks'; +import { healthRoute } from './health'; +import { mockHandlerArguments } from './_mock_handler_arguments'; +import { sleep } from '../test_utils'; +import { loggingSystemMock } from '../../../../../src/core/server/mocks'; +import { Logger } from '../../../../../src/core/server'; +import { MonitoringStats, summarizeMonitoringStats } from '../monitoring'; +import { ServiceStatusLevels } from 'src/core/server'; +import { configSchema, TaskManagerConfig } from '../config'; + +describe('healthRoute', () => { + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('registers the route', async () => { + const router = httpServiceMock.createRouter(); + + const logger = loggingSystemMock.create().get(); + healthRoute(router, of(), logger, uuid.v4(), getTaskManagerConfig()); + + const [config] = router.get.mock.calls[0]; + + expect(config.path).toMatchInlineSnapshot(`"/api/task_manager/_health"`); + }); + + it('logs the Task Manager stats at a fixed interval', async () => { + const router = httpServiceMock.createRouter(); + const logger = loggingSystemMock.create().get(); + + const mockStat = mockHealthStats(); + await sleep(10); + const skippedMockStat = mockHealthStats(); + await sleep(10); + const nextMockStat = mockHealthStats(); + + const stats$ = new Subject(); + + const id = uuid.v4(); + healthRoute( + router, + stats$, + logger, + id, + getTaskManagerConfig({ + monitored_stats_required_freshness: 1000, + monitored_aggregated_stats_refresh_rate: 60000, + }) + ); + + stats$.next(mockStat); + await sleep(500); + stats$.next(skippedMockStat); + await sleep(600); + stats$.next(nextMockStat); + + const firstDebug = JSON.parse( + (logger as jest.Mocked).debug.mock.calls[0][0].replace('Latest Monitored Stats: ', '') + ); + expect(firstDebug).toMatchObject({ + id, + timestamp: expect.any(String), + status: expect.any(String), + ...summarizeMonitoringStats(mockStat, getTaskManagerConfig({})), + }); + + const secondDebug = JSON.parse( + (logger as jest.Mocked).debug.mock.calls[1][0].replace('Latest Monitored Stats: ', '') + ); + expect(secondDebug).not.toMatchObject({ + id, + timestamp: expect.any(String), + status: expect.any(String), + ...summarizeMonitoringStats(skippedMockStat, getTaskManagerConfig({})), + }); + expect(secondDebug).toMatchObject({ + id, + timestamp: expect.any(String), + status: expect.any(String), + ...summarizeMonitoringStats(nextMockStat, getTaskManagerConfig({})), + }); + + expect(logger.debug).toHaveBeenCalledTimes(2); + }); + + it('returns a error status if the overall stats have not been updated within the required hot freshness', async () => { + const router = httpServiceMock.createRouter(); + + const stats$ = new Subject(); + + const serviceStatus$ = healthRoute( + router, + stats$, + loggingSystemMock.create().get(), + uuid.v4(), + getTaskManagerConfig({ + monitored_stats_required_freshness: 1000, + monitored_aggregated_stats_refresh_rate: 60000, + }) + ); + + const serviceStatus = getLatest(serviceStatus$); + + const [, handler] = router.get.mock.calls[0]; + + const [context, req, res] = mockHandlerArguments({}, {}, ['ok', 'internalError']); + + await sleep(0); + + stats$.next( + mockHealthStats({ + last_update: new Date(Date.now() - 1500).toISOString(), + }) + ); + + expect(await handler(context, req, res)).toMatchObject({ + body: { + status: 'error', + ...summarizeMonitoringStats( + mockHealthStats({ + last_update: expect.any(String), + stats: { + configuration: { + timestamp: expect.any(String), + }, + workload: { + timestamp: expect.any(String), + }, + runtime: { + timestamp: expect.any(String), + value: { + polling: { + last_successful_poll: expect.any(String), + }, + }, + }, + }, + }), + getTaskManagerConfig({}) + ), + }, + }); + + expect(await serviceStatus).toMatchObject({ + level: ServiceStatusLevels.unavailable, + summary: 'Task Manager is unavailable', + meta: { + status: 'error', + ...summarizeMonitoringStats( + mockHealthStats({ + last_update: expect.any(String), + stats: { + configuration: { + timestamp: expect.any(String), + }, + workload: { + timestamp: expect.any(String), + }, + runtime: { + timestamp: expect.any(String), + value: { + polling: { + last_successful_poll: expect.any(String), + }, + }, + }, + }, + }), + getTaskManagerConfig({}) + ), + }, + }); + }); + + it('returns a error status if the workload stats have not been updated within the required cold freshness', async () => { + const router = httpServiceMock.createRouter(); + + const stats$ = new Subject(); + + healthRoute( + router, + stats$, + loggingSystemMock.create().get(), + uuid.v4(), + getTaskManagerConfig({ + monitored_stats_required_freshness: 5000, + monitored_aggregated_stats_refresh_rate: 60000, + }) + ); + + await sleep(0); + + const lastUpdateOfWorkload = new Date(Date.now() - 120000).toISOString(); + stats$.next( + mockHealthStats({ + stats: { + workload: { + timestamp: lastUpdateOfWorkload, + }, + }, + }) + ); + + const [, handler] = router.get.mock.calls[0]; + + const [context, req, res] = mockHandlerArguments({}, {}, ['ok', 'internalError']); + + await sleep(2000); + + expect(await handler(context, req, res)).toMatchObject({ + body: { + status: 'error', + ...summarizeMonitoringStats( + mockHealthStats({ + last_update: expect.any(String), + stats: { + configuration: { + timestamp: expect.any(String), + }, + workload: { + timestamp: expect.any(String), + }, + runtime: { + timestamp: expect.any(String), + value: { + polling: { + last_successful_poll: expect.any(String), + }, + }, + }, + }, + }), + getTaskManagerConfig() + ), + }, + }); + }); + + it('returns a error status if the poller hasnt polled within the required hot freshness', async () => { + const router = httpServiceMock.createRouter(); + + const stats$ = new Subject(); + healthRoute( + router, + stats$, + loggingSystemMock.create().get(), + uuid.v4(), + getTaskManagerConfig({ + monitored_stats_required_freshness: 1000, + monitored_aggregated_stats_refresh_rate: 60000, + }) + ); + + await sleep(0); + + // eslint-disable-next-line @typescript-eslint/naming-convention + const last_successful_poll = new Date(Date.now() - 2000).toISOString(); + stats$.next( + mockHealthStats({ + stats: { + runtime: { + value: { + polling: { + last_successful_poll, + }, + }, + }, + }, + }) + ); + + const [, handler] = router.get.mock.calls[0]; + + const [context, req, res] = mockHandlerArguments({}, {}, ['ok', 'internalError']); + + expect(await handler(context, req, res)).toMatchObject({ + body: { + status: 'error', + ...summarizeMonitoringStats( + mockHealthStats({ + last_update: expect.any(String), + stats: { + configuration: { + timestamp: expect.any(String), + }, + workload: { + timestamp: expect.any(String), + }, + runtime: { + timestamp: expect.any(String), + value: { + polling: { + last_successful_poll, + }, + }, + }, + }, + }), + getTaskManagerConfig() + ), + }, + }); + }); +}); + +function mockHealthStats(overrides = {}) { + return (merge( + { + last_update: new Date().toISOString(), + stats: { + configuration: { + timestamp: new Date().toISOString(), + value: { + value: { + max_workers: 10, + poll_interval: 6000000, + max_poll_inactivity_cycles: 10, + request_capacity: 1000, + monitored_aggregated_stats_refresh_rate: 5000, + monitored_stats_running_average_window: 50, + monitored_task_execution_thresholds: { + default: { + error_threshold: 90, + warn_threshold: 80, + }, + custom: {}, + }, + }, + }, + }, + workload: { + timestamp: new Date().toISOString(), + value: { + count: 4, + taskTypes: { + actions_telemetry: { count: 2, status: { idle: 2 } }, + alerting_telemetry: { count: 1, status: { idle: 1 } }, + session_cleanup: { count: 1, status: { idle: 1 } }, + }, + schedule: {}, + overdue: 0, + estimatedScheduleDensity: [], + }, + }, + runtime: { + timestamp: new Date().toISOString(), + value: { + drift: [1000, 60000], + execution: { + duration: [], + result_frequency_percent_as_number: [], + }, + polling: { + last_successful_poll: new Date().toISOString(), + result_frequency_percent_as_number: [ + 'NoTasksClaimed', + 'NoTasksClaimed', + 'NoTasksClaimed', + ], + }, + }, + }, + }, + }, + overrides + ) as unknown) as MonitoringStats; +} + +async function getLatest(stream$: Observable) { + return new Promise((resolve) => stream$.pipe(take(1)).subscribe((stats) => resolve(stats))); +} + +const getTaskManagerConfig = (overrides: Partial = {}) => + configSchema.validate( + overrides.monitored_stats_required_freshness + ? { + // use `monitored_stats_required_freshness` as poll interval otherwise we might + // fail validation as it must be greather than the poll interval + poll_interval: overrides.monitored_stats_required_freshness, + ...overrides, + } + : overrides + ); diff --git a/x-pack/plugins/task_manager/server/routes/health.ts b/x-pack/plugins/task_manager/server/routes/health.ts new file mode 100644 index 0000000000000..8ddd728063d2a --- /dev/null +++ b/x-pack/plugins/task_manager/server/routes/health.ts @@ -0,0 +1,169 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + IRouter, + RequestHandlerContext, + KibanaRequest, + IKibanaResponse, + KibanaResponseFactory, +} from 'kibana/server'; +import { Observable, Subject } from 'rxjs'; +import { tap, map } from 'rxjs/operators'; +import { throttleTime } from 'rxjs/operators'; +import { isString } from 'lodash'; +import { JsonValue } from 'src/plugins/kibana_utils/common'; +import { Logger, ServiceStatus, ServiceStatusLevels } from '../../../../../src/core/server'; +import { + MonitoringStats, + summarizeMonitoringStats, + HealthStatus, + RawMonitoringStats, +} from '../monitoring'; +import { TaskManagerConfig } from '../config'; + +type MonitoredHealth = RawMonitoringStats & { id: string; status: HealthStatus; timestamp: string }; + +const LEVEL_SUMMARY = { + [ServiceStatusLevels.available.toString()]: 'Task Manager is healthy', + [ServiceStatusLevels.degraded.toString()]: 'Task Manager is unhealthy', + [ServiceStatusLevels.unavailable.toString()]: 'Task Manager is unavailable', +}; + +export function healthRoute( + router: IRouter, + monitoringStats$: Observable, + logger: Logger, + taskManagerId: string, + config: TaskManagerConfig +): Observable { + // if "hot" health stats are any more stale than monitored_stats_required_freshness (pollInterval +1s buffer by default) + // consider the system unhealthy + const requiredHotStatsFreshness: number = config.monitored_stats_required_freshness; + + // if "cold" health stats are any more stale than the configured refresh (+ a buffer), consider the system unhealthy + const requiredColdStatsFreshness: number = config.monitored_aggregated_stats_refresh_rate * 1.5; + + function calculateStatus(monitoredStats: MonitoringStats): MonitoredHealth { + const now = Date.now(); + const timestamp = new Date(now).toISOString(); + const summarizedStats = summarizeMonitoringStats(monitoredStats, config); + + /** + * If the monitored stats aren't fresh, return a red status + */ + const healthStatus = + hasStatus(summarizedStats.stats, HealthStatus.Error) || + hasExpiredHotTimestamps(summarizedStats, now, requiredHotStatsFreshness) || + hasExpiredColdTimestamps(summarizedStats, now, requiredColdStatsFreshness) + ? HealthStatus.Error + : hasStatus(summarizedStats.stats, HealthStatus.Warning) + ? HealthStatus.Warning + : HealthStatus.OK; + return { id: taskManagerId, timestamp, status: healthStatus, ...summarizedStats }; + } + + const serviceStatus$: Subject = new Subject(); + + /* keep track of last health summary, as we'll return that to the next call to _health */ + let lastMonitoredStats: MonitoringStats | null = null; + + /* Log Task Manager stats as a Debug log line at a fixed interval */ + monitoringStats$ + .pipe( + throttleTime(requiredHotStatsFreshness), + tap((stats) => { + lastMonitoredStats = stats; + }), + // Only calculate the summerized stats (calculates all runnign averages and evaluates state) + // when needed by throttling down to the requiredHotStatsFreshness + map((stats) => withServiceStatus(calculateStatus(stats))) + ) + .subscribe(([monitoredHealth, serviceStatus]) => { + serviceStatus$.next(serviceStatus); + logger.debug(`Latest Monitored Stats: ${JSON.stringify(monitoredHealth)}`); + }); + + router.get( + { + path: '/api/task_manager/_health', + validate: false, + }, + async function ( + context: RequestHandlerContext, + req: KibanaRequest, + res: KibanaResponseFactory + ): Promise { + return res.ok({ + body: lastMonitoredStats + ? calculateStatus(lastMonitoredStats) + : { id: taskManagerId, timestamp: new Date().toISOString(), status: HealthStatus.Error }, + }); + } + ); + return serviceStatus$; +} + +export function withServiceStatus( + monitoredHealth: MonitoredHealth +): [MonitoredHealth, ServiceStatus] { + const level = + monitoredHealth.status === HealthStatus.OK + ? ServiceStatusLevels.available + : monitoredHealth.status === HealthStatus.Warning + ? ServiceStatusLevels.degraded + : ServiceStatusLevels.unavailable; + return [ + monitoredHealth, + { + level, + summary: LEVEL_SUMMARY[level.toString()], + meta: monitoredHealth, + }, + ]; +} + +/** + * If certain "hot" stats are not fresh, then the _health api will should return a Red status + * @param monitoringStats The monitored stats + * @param now The time to compare against + * @param requiredFreshness How fresh should these stats be + */ +function hasExpiredHotTimestamps( + monitoringStats: RawMonitoringStats, + now: number, + requiredFreshness: number +): boolean { + return ( + now - + getOldestTimestamp( + monitoringStats.last_update, + monitoringStats.stats.runtime?.value.polling.last_successful_poll + ) > + requiredFreshness + ); +} + +function hasExpiredColdTimestamps( + monitoringStats: RawMonitoringStats, + now: number, + requiredFreshness: number +): boolean { + return now - getOldestTimestamp(monitoringStats.stats.workload?.timestamp) > requiredFreshness; +} + +function hasStatus(stats: RawMonitoringStats['stats'], status: HealthStatus): boolean { + return Object.values(stats) + .map((stat) => stat?.status === status) + .includes(true); +} + +function getOldestTimestamp(...timestamps: Array): number { + const validTimestamps = timestamps + .map((timestamp) => (isString(timestamp) ? Date.parse(timestamp) : NaN)) + .filter((timestamp) => !isNaN(timestamp)); + return validTimestamps.length ? Math.min(...validTimestamps) : 0; +} diff --git a/x-pack/plugins/task_manager/server/routes/index.ts b/x-pack/plugins/task_manager/server/routes/index.ts new file mode 100644 index 0000000000000..4fa1aa6cb7a9b --- /dev/null +++ b/x-pack/plugins/task_manager/server/routes/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { healthRoute } from './health'; diff --git a/x-pack/plugins/task_manager/server/task_events.ts b/x-pack/plugins/task_manager/server/task_events.ts index e1dd85f868cdd..b011d435e28dc 100644 --- a/x-pack/plugins/task_manager/server/task_events.ts +++ b/x-pack/plugins/task_manager/server/task_events.ts @@ -9,63 +9,109 @@ import { Option } from 'fp-ts/lib/Option'; import { ConcreteTaskInstance } from './task'; import { Result, Err } from './lib/result_type'; +import { FillPoolResult } from './lib/fill_pool'; +import { PollingError } from './polling'; +import { TaskRunResult } from './task_runner'; export enum TaskEventType { TASK_CLAIM = 'TASK_CLAIM', TASK_MARK_RUNNING = 'TASK_MARK_RUNNING', TASK_RUN = 'TASK_RUN', TASK_RUN_REQUEST = 'TASK_RUN_REQUEST', + TASK_POLLING_CYCLE = 'TASK_POLLING_CYCLE', +} + +export interface TaskTiming { + start: number; + stop: number; +} + +export function startTaskTimer(): () => TaskTiming { + const start = Date.now(); + return () => ({ start, stop: Date.now() }); } export interface TaskEvent { - id: string; + id?: string; + timing?: TaskTiming; type: TaskEventType; event: Result; } +export interface RanTask { + task: ConcreteTaskInstance; + result: TaskRunResult; +} +export type ErroredTask = RanTask & { + error: Error; +}; + export type TaskMarkRunning = TaskEvent; -export type TaskRun = TaskEvent; +export type TaskRun = TaskEvent; export type TaskClaim = TaskEvent>; export type TaskRunRequest = TaskEvent; +export type TaskPollingCycle = TaskEvent>; export function asTaskMarkRunningEvent( id: string, - event: Result + event: Result, + timing?: TaskTiming ): TaskMarkRunning { return { id, type: TaskEventType.TASK_MARK_RUNNING, event, + timing, }; } -export function asTaskRunEvent(id: string, event: Result): TaskRun { +export function asTaskRunEvent( + id: string, + event: Result, + timing?: TaskTiming +): TaskRun { return { id, type: TaskEventType.TASK_RUN, event, + timing, }; } export function asTaskClaimEvent( id: string, - event: Result> + event: Result>, + timing?: TaskTiming ): TaskClaim { return { id, type: TaskEventType.TASK_CLAIM, event, + timing, }; } export function asTaskRunRequestEvent( id: string, // we only emit a TaskRunRequest event when it fails - event: Err + event: Err, + timing?: TaskTiming ): TaskRunRequest { return { id, type: TaskEventType.TASK_RUN_REQUEST, event, + timing, + }; +} + +export function asTaskPollingCycleEvent( + event: Result>, + timing?: TaskTiming +): TaskPollingCycle { + return { + type: TaskEventType.TASK_POLLING_CYCLE, + event, + timing, }; } @@ -85,3 +131,8 @@ export function isTaskRunRequestEvent( ): taskEvent is TaskRunRequest { return taskEvent.type === TaskEventType.TASK_RUN_REQUEST; } +export function isTaskPollingCycleEvent( + taskEvent: TaskEvent +): taskEvent is TaskPollingCycle { + return taskEvent.type === TaskEventType.TASK_POLLING_CYCLE; +} diff --git a/x-pack/plugins/task_manager/server/task_pool.test.ts b/x-pack/plugins/task_manager/server/task_pool.test.ts index 12b731b2b78ae..a174af71ef18f 100644 --- a/x-pack/plugins/task_manager/server/task_pool.test.ts +++ b/x-pack/plugins/task_manager/server/task_pool.test.ts @@ -7,7 +7,9 @@ import sinon from 'sinon'; import { of, Subject } from 'rxjs'; import { TaskPool, TaskPoolRunResult } from './task_pool'; -import { mockLogger, resolvable, sleep } from './test_utils'; +import { resolvable, sleep } from './test_utils'; +import { loggingSystemMock } from '../../../../src/core/server/mocks'; +import { Logger } from '../../../../src/core/server'; import { asOk } from './lib/result_type'; import { SavedObjectsErrorHelpers } from '../../../../src/core/server'; import moment from 'moment'; @@ -16,7 +18,7 @@ describe('TaskPool', () => { test('occupiedWorkers are a sum of running tasks', async () => { const pool = new TaskPool({ maxWorkers$: of(200), - logger: mockLogger(), + logger: loggingSystemMock.create().get(), }); const result = await pool.run([{ ...mockTask() }, { ...mockTask() }, { ...mockTask() }]); @@ -28,7 +30,7 @@ describe('TaskPool', () => { test('availableWorkers are a function of total_capacity - occupiedWorkers', async () => { const pool = new TaskPool({ maxWorkers$: of(10), - logger: mockLogger(), + logger: loggingSystemMock.create().get(), }); const result = await pool.run([{ ...mockTask() }, { ...mockTask() }, { ...mockTask() }]); @@ -41,7 +43,7 @@ describe('TaskPool', () => { const maxWorkers$ = new Subject(); const pool = new TaskPool({ maxWorkers$, - logger: mockLogger(), + logger: loggingSystemMock.create().get(), }); expect(pool.availableWorkers).toEqual(0); @@ -52,7 +54,7 @@ describe('TaskPool', () => { test('does not run tasks that are beyond its available capacity', async () => { const pool = new TaskPool({ maxWorkers$: of(2), - logger: mockLogger(), + logger: loggingSystemMock.create().get(), }); const shouldRun = mockRun(); @@ -71,7 +73,7 @@ describe('TaskPool', () => { }); test('should log when marking a Task as running fails', async () => { - const logger = mockLogger(); + const logger = loggingSystemMock.create().get(); const pool = new TaskPool({ maxWorkers$: of(2), logger, @@ -84,7 +86,7 @@ describe('TaskPool', () => { const result = await pool.run([mockTask(), taskFailedToMarkAsRunning, mockTask()]); - expect(logger.error.mock.calls[0]).toMatchInlineSnapshot(` + expect((logger as jest.Mocked).error.mock.calls[0]).toMatchInlineSnapshot(` Array [ "Failed to mark Task TaskType \\"shooooo\\" as running: Mark Task as running has failed miserably", ] @@ -94,7 +96,7 @@ describe('TaskPool', () => { }); test('should log when running a Task fails', async () => { - const logger = mockLogger(); + const logger = loggingSystemMock.create().get(); const pool = new TaskPool({ maxWorkers$: of(3), logger, @@ -107,7 +109,7 @@ describe('TaskPool', () => { const result = await pool.run([mockTask(), taskFailedToRun, mockTask()]); - expect(logger.warn.mock.calls[0]).toMatchInlineSnapshot(` + expect((logger as jest.Mocked).warn.mock.calls[0]).toMatchInlineSnapshot(` Array [ "Task TaskType \\"shooooo\\" failed in attempt to run: Run Task has failed miserably", ] @@ -117,7 +119,7 @@ describe('TaskPool', () => { }); test('should not log when running a Task fails due to the Task SO having been deleted while in flight', async () => { - const logger = mockLogger(); + const logger = loggingSystemMock.create().get(); const pool = new TaskPool({ maxWorkers$: of(3), logger, @@ -139,7 +141,7 @@ describe('TaskPool', () => { }); test('Running a task which fails still takes up capacity', async () => { - const logger = mockLogger(); + const logger = loggingSystemMock.create().get(); const pool = new TaskPool({ maxWorkers$: of(1), logger, @@ -159,7 +161,7 @@ describe('TaskPool', () => { test('clears up capacity when a task completes', async () => { const pool = new TaskPool({ maxWorkers$: of(1), - logger: mockLogger(), + logger: loggingSystemMock.create().get(), }); const firstWork = resolvable(); @@ -202,7 +204,7 @@ describe('TaskPool', () => { }); test('run cancels expired tasks prior to running new tasks', async () => { - const logger = mockLogger(); + const logger = loggingSystemMock.create().get(); const pool = new TaskPool({ maxWorkers$: of(2), logger, @@ -259,7 +261,7 @@ describe('TaskPool', () => { }); test('logs if cancellation errors', async () => { - const logger = mockLogger(); + const logger = loggingSystemMock.create().get(); const pool = new TaskPool({ logger, maxWorkers$: of(20), @@ -290,7 +292,7 @@ describe('TaskPool', () => { // Allow the task to cancel... await cancelled; - expect(logger.error.mock.calls[0][0]).toMatchInlineSnapshot( + expect((logger as jest.Mocked).error.mock.calls[0][0]).toMatchInlineSnapshot( `"Failed to cancel task \\"shooooo!\\": Error: Dern!"` ); }); diff --git a/x-pack/plugins/task_manager/server/task_runner.test.ts b/x-pack/plugins/task_manager/server/task_runner.test.ts index 8fb1df444c603..676eeedf08f5a 100644 --- a/x-pack/plugins/task_manager/server/task_runner.test.ts +++ b/x-pack/plugins/task_manager/server/task_runner.test.ts @@ -6,16 +6,18 @@ import _ from 'lodash'; import sinon from 'sinon'; -import { minutesFromNow } from './lib/intervals'; +import { secondsFromNow } from './lib/intervals'; import { asOk, asErr } from './lib/result_type'; -import { TaskEvent, asTaskRunEvent, asTaskMarkRunningEvent } from './task_events'; +import { TaskManagerRunner, TaskRunResult } from './task_runner'; +import { TaskEvent, asTaskRunEvent, asTaskMarkRunningEvent, TaskRun } from './task_events'; import { ConcreteTaskInstance, TaskStatus, TaskDefinition, RunResult } from './task'; -import { TaskManagerRunner } from './task_runner'; import { SavedObjectsErrorHelpers } from '../../../../src/core/server'; import moment from 'moment'; import { TaskTypeDictionary } from './task_type_dictionary'; import { mockLogger } from './test_utils'; +const minutesFromNow = (mins: number): Date => secondsFromNow(mins * 60); + let fakeTimer: sinon.SinonFakeTimers; beforeAll(() => { @@ -812,7 +814,9 @@ describe('TaskManagerRunner', () => { await runner.run(); - expect(onTaskEvent).toHaveBeenCalledWith(asTaskRunEvent(id, asOk(instance))); + expect(onTaskEvent).toHaveBeenCalledWith( + withAnyTiming(asTaskRunEvent(id, asOk({ task: instance, result: TaskRunResult.Success }))) + ); }); test('emits TaskEvent when a recurring task is run successfully', async () => { @@ -839,14 +843,16 @@ describe('TaskManagerRunner', () => { await runner.run(); - expect(onTaskEvent).toHaveBeenCalledWith(asTaskRunEvent(id, asOk(instance))); + expect(onTaskEvent).toHaveBeenCalledWith( + withAnyTiming(asTaskRunEvent(id, asOk({ task: instance, result: TaskRunResult.Success }))) + ); }); test('emits TaskEvent when a task run throws an error', async () => { const id = _.random(1, 20).toString(); const error = new Error('Dangit!'); const onTaskEvent = jest.fn(); - const { runner } = testOpts({ + const { runner, instance } = testOpts({ onTaskEvent, instance: { id, @@ -864,7 +870,11 @@ describe('TaskManagerRunner', () => { }); await runner.run(); - expect(onTaskEvent).toHaveBeenCalledWith(asTaskRunEvent(id, asErr(error))); + expect(onTaskEvent).toHaveBeenCalledWith( + withAnyTiming( + asTaskRunEvent(id, asErr({ error, task: instance, result: TaskRunResult.RetryScheduled })) + ) + ); expect(onTaskEvent).toHaveBeenCalledTimes(1); }); @@ -872,7 +882,7 @@ describe('TaskManagerRunner', () => { const id = _.random(1, 20).toString(); const error = new Error('Dangit!'); const onTaskEvent = jest.fn(); - const { runner } = testOpts({ + const { runner, instance } = testOpts({ onTaskEvent, instance: { id, @@ -893,7 +903,11 @@ describe('TaskManagerRunner', () => { await runner.run(); - expect(onTaskEvent).toHaveBeenCalledWith(asTaskRunEvent(id, asErr(error))); + expect(onTaskEvent).toHaveBeenCalledWith( + withAnyTiming( + asTaskRunEvent(id, asErr({ error, task: instance, result: TaskRunResult.RetryScheduled })) + ) + ); expect(onTaskEvent).toHaveBeenCalledTimes(1); }); @@ -901,7 +915,7 @@ describe('TaskManagerRunner', () => { const id = _.random(1, 20).toString(); const error = new Error('Dangit!'); const onTaskEvent = jest.fn(); - const { runner, store } = testOpts({ + const { runner, store, instance: originalInstance } = testOpts({ onTaskEvent, instance: { id, @@ -925,7 +939,18 @@ describe('TaskManagerRunner', () => { const instance = store.update.args[0][0]; expect(instance.status).toBe('failed'); - expect(onTaskEvent).toHaveBeenCalledWith(asTaskRunEvent(id, asErr(error))); + expect(onTaskEvent).toHaveBeenCalledWith( + withAnyTiming( + asTaskRunEvent( + id, + asErr({ + error, + task: originalInstance, + result: TaskRunResult.Failed, + }) + ) + ) + ); expect(onTaskEvent).toHaveBeenCalledTimes(1); }); }); @@ -936,6 +961,13 @@ describe('TaskManagerRunner', () => { onTaskEvent?: (event: TaskEvent) => void; } + function withAnyTiming(taskRun: TaskRun) { + return { + ...taskRun, + timing: { start: expect.any(Number), stop: expect.any(Number) }, + }; + } + function testOpts(opts: TestOpts) { const callCluster = sinon.stub(); const createTaskRunner = sinon.stub(); diff --git a/x-pack/plugins/task_manager/server/task_runner.ts b/x-pack/plugins/task_manager/server/task_runner.ts index 24a487e366029..45e4cb3000570 100644 --- a/x-pack/plugins/task_manager/server/task_runner.ts +++ b/x-pack/plugins/task_manager/server/task_runner.ts @@ -10,16 +10,23 @@ * rescheduling, middleware application, etc. */ +import { Logger } from 'src/core/server'; import apm from 'elastic-apm-node'; import { performance } from 'perf_hooks'; import Joi from 'joi'; import { identity, defaults, flow } from 'lodash'; -import { Logger } from '../../../../src/core/server'; -import { asOk, asErr, mapErr, eitherAsync, unwrap, mapOk, Result } from './lib/result_type'; -import { TaskRun, TaskMarkRunning, asTaskRunEvent, asTaskMarkRunningEvent } from './task_events'; -import { intervalFromDate, intervalFromNow } from './lib/intervals'; import { Middleware } from './lib/middleware'; +import { asOk, asErr, mapErr, eitherAsync, unwrap, isOk, mapOk, Result } from './lib/result_type'; +import { + TaskRun, + TaskMarkRunning, + asTaskRunEvent, + asTaskMarkRunningEvent, + startTaskTimer, + TaskTiming, +} from './task_events'; +import { intervalFromDate, intervalFromNow } from './lib/intervals'; import { CancelFunction, CancellableTask, @@ -63,6 +70,21 @@ type Opts = { onTaskEvent?: (event: TaskRun | TaskMarkRunning) => void; } & Pick; +export enum TaskRunResult { + // Task completed successfully + Success = 'Success', + // Recurring Task completed successfully + SuccessRescheduled = 'Success', + // // Task completed successfully after a retry + // SuccessfulRetry = 'SuccessfulRetry', + // // Recurring Task completed successfully after a retry + // SuccessfulRetryRescheduled = 'SuccessfulRetry', + // Task has failed and a retry has been scheduled + RetryScheduled = 'RetryScheduled', + // Task has failed + Failed = 'Failed', +} + /** * Runs a background task, ensures that errors are properly handled, * allows for cancellation. @@ -172,6 +194,7 @@ export class TaskManagerRunner implements TaskRunner { taskInstance: this.instance, }); + const stopTaskTimer = startTaskTimer(); const apmTrans = apm.startTransaction( `taskManager run ${this.instance.taskType}`, 'taskManager' @@ -181,13 +204,16 @@ export class TaskManagerRunner implements TaskRunner { const result = await this.task.run(); const validatedResult = this.validateResult(result); if (apmTrans) apmTrans.end('success'); - return this.processResult(validatedResult); + return this.processResult(validatedResult, stopTaskTimer()); } catch (err) { this.logger.error(`Task ${this} failed: ${err}`); // in error scenario, we can not get the RunResult // re-use modifiedContext's state, which is correct as of beforeRun if (apmTrans) apmTrans.end('error'); - return this.processResult(asErr({ error: err, state: modifiedContext.taskInstance.state })); + return this.processResult( + asErr({ error: err, state: modifiedContext.taskInstance.state }), + stopTaskTimer() + ); } } @@ -337,8 +363,9 @@ export class TaskManagerRunner implements TaskRunner { private async processResultForRecurringTask( result: Result - ): Promise { - const fieldUpdates = flow( + ): Promise { + const hasTaskRunFailed = isOk(result); + const fieldUpdates: Partial & Pick = flow( // if running the task has failed ,try to correct by scheduling a retry in the near future mapErr(this.rescheduleFailedRun), // if retrying is possible (new runAt) or this is an recurring task - reschedule @@ -357,7 +384,7 @@ export class TaskManagerRunner implements TaskRunner { await this.bufferedTaskStore.update( defaults( { - ...(fieldUpdates as Partial), + ...fieldUpdates, // reset fields that track the lifecycle of the concluded `task run` startedAt: null, retryAt: null, @@ -366,9 +393,15 @@ export class TaskManagerRunner implements TaskRunner { this.instance ) ); + + return fieldUpdates.status === TaskStatus.Failed + ? TaskRunResult.Failed + : hasTaskRunFailed + ? TaskRunResult.SuccessRescheduled + : TaskRunResult.RetryScheduled; } - private async processResultWhenDone(): Promise { + private async processResultWhenDone(): Promise { // not a recurring task: clean up by removing the task instance from store try { await this.bufferedTaskStore.remove(this.instance.id); @@ -379,24 +412,38 @@ export class TaskManagerRunner implements TaskRunner { throw err; } } + return TaskRunResult.Success; } private async processResult( - result: Result + result: Result, + taskTiming: TaskTiming ): Promise> { + const task = this.instance; await eitherAsync( result, async ({ runAt }: SuccessfulRunResult) => { - if (runAt || this.instance.schedule) { - await this.processResultForRecurringTask(result); - } else { - await this.processResultWhenDone(); - } - this.onTaskEvent(asTaskRunEvent(this.id, asOk(this.instance))); + this.onTaskEvent( + asTaskRunEvent( + this.id, + asOk({ + task, + result: await (runAt || task.schedule + ? this.processResultForRecurringTask(result) + : this.processResultWhenDone()), + }), + taskTiming + ) + ); }, async ({ error }: FailedRunResult) => { - await this.processResultForRecurringTask(result); - this.onTaskEvent(asTaskRunEvent(this.id, asErr(error))); + this.onTaskEvent( + asTaskRunEvent( + this.id, + asErr({ task, result: await this.processResultForRecurringTask(result), error }), + taskTiming + ) + ); } ); return result; diff --git a/x-pack/plugins/task_manager/server/task_scheduling.test.ts b/x-pack/plugins/task_manager/server/task_scheduling.test.ts index 1f7f9250d9014..8d660f57ab875 100644 --- a/x-pack/plugins/task_manager/server/task_scheduling.test.ts +++ b/x-pack/plugins/task_manager/server/task_scheduling.test.ts @@ -21,6 +21,7 @@ import { asErr, asOk } from './lib/result_type'; import { ConcreteTaskInstance, TaskLifecycleResult, TaskStatus } from './task'; import { createInitialMiddleware } from './lib/middleware'; import { taskStoreMock } from './task_store.mock'; +import { TaskRunResult } from './task_runner'; import { mockLogger } from './test_utils'; describe('TaskScheduling', () => { @@ -113,7 +114,7 @@ describe('TaskScheduling', () => { const result = taskScheduling.runNow(id); const task = { id } as ConcreteTaskInstance; - events$.next(asTaskRunEvent(id, asOk(task))); + events$.next(asTaskRunEvent(id, asOk({ task, result: TaskRunResult.Success }))); return expect(result).resolves.toEqual({ id }); }); @@ -132,7 +133,16 @@ describe('TaskScheduling', () => { const task = { id } as ConcreteTaskInstance; events$.next(asTaskClaimEvent(id, asOk(task))); events$.next(asTaskMarkRunningEvent(id, asOk(task))); - events$.next(asTaskRunEvent(id, asErr(new Error('some thing gone wrong')))); + events$.next( + asTaskRunEvent( + id, + asErr({ + task, + error: new Error('some thing gone wrong'), + result: TaskRunResult.Failed, + }) + ) + ); return expect(result).rejects.toMatchInlineSnapshot( `[Error: Failed to run task "01ddff11-e88a-4d13-bc4e-256164e755e2": Error: some thing gone wrong]` @@ -306,10 +316,20 @@ describe('TaskScheduling', () => { const otherTask = { id: differentTask } as ConcreteTaskInstance; events$.next(asTaskClaimEvent(id, asOk(task))); events$.next(asTaskClaimEvent(differentTask, asOk(otherTask))); + events$.next( + asTaskRunEvent(differentTask, asOk({ task: otherTask, result: TaskRunResult.Success })) + ); - events$.next(asTaskRunEvent(differentTask, asOk(task))); - - events$.next(asTaskRunEvent(id, asErr(new Error('some thing gone wrong')))); + events$.next( + asTaskRunEvent( + id, + asErr({ + task, + error: new Error('some thing gone wrong'), + result: TaskRunResult.Failed, + }) + ) + ); return expect(result).rejects.toMatchInlineSnapshot( `[Error: Failed to run task "01ddff11-e88a-4d13-bc4e-256164e755e2": Error: some thing gone wrong]` diff --git a/x-pack/plugins/task_manager/server/task_scheduling.ts b/x-pack/plugins/task_manager/server/task_scheduling.ts index 00f7d853d7114..9806ada386e4a 100644 --- a/x-pack/plugins/task_manager/server/task_scheduling.ts +++ b/x-pack/plugins/task_manager/server/task_scheduling.ts @@ -10,7 +10,13 @@ import { Option, map as mapOptional, getOrElse } from 'fp-ts/lib/Option'; import { Logger } from '../../../../src/core/server'; import { asOk, either, map, mapErr, promiseResult } from './lib/result_type'; -import { isTaskRunEvent, isTaskClaimEvent, isTaskRunRequestEvent } from './task_events'; +import { + isTaskRunEvent, + isTaskClaimEvent, + isTaskRunRequestEvent, + RanTask, + ErroredTask, +} from './task_events'; import { Middleware } from './lib/middleware'; import { ConcreteTaskInstance, @@ -23,6 +29,7 @@ import { import { TaskStore } from './task_store'; import { ensureDeprecatedFieldsAreCorrected } from './lib/correct_deprecated_fields'; import { TaskLifecycleEvent, TaskPollingLifecycle } from './polling_lifecycle'; +import { FillPoolResult } from './lib/fill_pool'; const VERSION_CONFLICT_STATUS = 409; @@ -118,16 +125,19 @@ export class TaskScheduling { return reject(await this.identifyTaskFailureReason(taskId, error)); }, taskEvent.event); } else { - either>( + either< + RanTask | ConcreteTaskInstance | FillPoolResult, + Error | ErroredTask | Option + >( taskEvent.event, - (taskInstance: ConcreteTaskInstance) => { + (taskInstance: RanTask | ConcreteTaskInstance | FillPoolResult) => { // resolve if the task has run sucessfully if (isTaskRunEvent(taskEvent)) { subscription.unsubscribe(); - resolve({ id: taskInstance.id }); + resolve({ id: (taskInstance as RanTask).task.id }); } }, - async (error: Error | Option) => { + async (errorResult: Error | ErroredTask | Option) => { // reject if any error event takes place for the requested task subscription.unsubscribe(); return reject( @@ -135,7 +145,9 @@ export class TaskScheduling { `Failed to run task "${taskId}": ${ isTaskRunRequestEvent(taskEvent) ? `Task Manager is at capacity, please try again later` - : error + : isTaskRunEvent(taskEvent) + ? `${(errorResult as ErroredTask).error}` + : `${errorResult}` }` ) ); diff --git a/x-pack/plugins/task_manager/server/task_store.mock.ts b/x-pack/plugins/task_manager/server/task_store.mock.ts index 9b82a3e3ee7ab..7af1b9ef5f110 100644 --- a/x-pack/plugins/task_manager/server/task_store.mock.ts +++ b/x-pack/plugins/task_manager/server/task_store.mock.ts @@ -31,6 +31,7 @@ export const taskStoreMock = { get: jest.fn(), getLifecycle: jest.fn(), fetch: jest.fn(), + aggregate: jest.fn(), maxAttempts, index, taskManagerId, diff --git a/x-pack/plugins/task_manager/server/task_store.test.ts b/x-pack/plugins/task_manager/server/task_store.test.ts index 8d47d3dd30b82..46e55df4ee1e6 100644 --- a/x-pack/plugins/task_manager/server/task_store.test.ts +++ b/x-pack/plugins/task_manager/server/task_store.test.ts @@ -367,7 +367,7 @@ describe('TaskStore', () => { const { args: { - updateByQuery: { body: { query } = {} }, + updateByQuery: { body: { query, sort } = {} }, }, } = await testClaimAvailableTasks({ opts: { @@ -415,41 +415,6 @@ describe('TaskStore', () => { ], }, }, - { - bool: { - should: [ - { exists: { field: 'task.schedule' } }, - { - bool: { - must: [ - { term: { 'task.taskType': 'foo' } }, - { - range: { - 'task.attempts': { - lt: maxAttempts, - }, - }, - }, - ], - }, - }, - { - bool: { - must: [ - { term: { 'task.taskType': 'bar' } }, - { - range: { - 'task.attempts': { - lt: customMaxAttempts, - }, - }, - }, - ], - }, - }, - ], - }, - }, ], }, }, @@ -476,12 +441,36 @@ describe('TaskStore', () => { ], }, }); + expect(sort).toMatchObject([ + { + _script: { + type: 'number', + order: 'asc', + script: { + lang: 'painless', + source: ` +if (doc['task.retryAt'].size()!=0) { + return doc['task.retryAt'].value.toInstant().toEpochMilli(); +} +if (doc['task.runAt'].size()!=0) { + return doc['task.runAt'].value.toInstant().toEpochMilli(); +} + `, + }, + }, + }, + ]); }); test('it supports claiming specific tasks by id', async () => { const maxAttempts = _.random(2, 43); const customMaxAttempts = _.random(44, 100); const definitions = new TaskTypeDictionary(mockLogger()); + const taskManagerId = uuid.v1(); + const fieldUpdates = { + ownerId: taskManagerId, + retryAt: new Date(Date.now()), + }; definitions.registerTaskDefinitions({ foo: { title: 'foo', @@ -495,10 +484,11 @@ describe('TaskStore', () => { }); const { args: { - updateByQuery: { body: { query, sort } = {} }, + updateByQuery: { body: { query, script, sort } = {} }, }, } = await testClaimAvailableTasks({ opts: { + taskManagerId, maxAttempts, definitions, }, @@ -557,41 +547,6 @@ describe('TaskStore', () => { ], }, }, - { - bool: { - should: [ - { exists: { field: 'task.schedule' } }, - { - bool: { - must: [ - { term: { 'task.taskType': 'foo' } }, - { - range: { - 'task.attempts': { - lt: maxAttempts, - }, - }, - }, - ], - }, - }, - { - bool: { - must: [ - { term: { 'task.taskType': 'bar' } }, - { - range: { - 'task.attempts': { - lt: customMaxAttempts, - }, - }, - }, - ], - }, - }, - ], - }, - }, ], }, }, @@ -621,6 +576,30 @@ describe('TaskStore', () => { }, }); + expect(script).toMatchObject({ + source: ` + if (ctx._source.task.schedule != null || ctx._source.task.attempts < params.taskMaxAttempts[ctx._source.task.taskType] || params.claimTasksById.contains(ctx._id)) { + ctx._source.task.status = "claiming"; ${Object.keys(fieldUpdates) + .map((field) => `ctx._source.task.${field}=params.fieldUpdates.${field};`) + .join(' ')} + } else { + ctx._source.task.status = "failed"; + } + `, + lang: 'painless', + params: { + fieldUpdates, + claimTasksById: [ + 'task:33c6977a-ed6d-43bd-98d9-3f827f7b7cd8', + 'task:a208b22c-14ec-4fb4-995f-d2ff7a3b03b8', + ], + taskMaxAttempts: { + bar: customMaxAttempts, + foo: maxAttempts, + }, + }, + }); + expect(sort).toMatchObject([ '_score', { @@ -646,6 +625,10 @@ if (doc['task.runAt'].size()!=0) { test('it claims tasks by setting their ownerId, status and retryAt', async () => { const taskManagerId = uuid.v1(); const claimOwnershipUntil = new Date(Date.now()); + const fieldUpdates = { + ownerId: taskManagerId, + retryAt: claimOwnershipUntil, + }; const { args: { updateByQuery: { body: { script } = {} }, @@ -660,12 +643,24 @@ if (doc['task.runAt'].size()!=0) { }, }); expect(script).toMatchObject({ - source: `ctx._source.task.ownerId=params.ownerId; ctx._source.task.status=params.status; ctx._source.task.retryAt=params.retryAt;`, + source: ` + if (ctx._source.task.schedule != null || ctx._source.task.attempts < params.taskMaxAttempts[ctx._source.task.taskType] || params.claimTasksById.contains(ctx._id)) { + ctx._source.task.status = "claiming"; ${Object.keys(fieldUpdates) + .map((field) => `ctx._source.task.${field}=params.fieldUpdates.${field};`) + .join(' ')} + } else { + ctx._source.task.status = "failed"; + } + `, lang: 'painless', params: { - ownerId: taskManagerId, - retryAt: claimOwnershipUntil, - status: 'claiming', + fieldUpdates, + claimTasksById: [], + taskMaxAttempts: { + dernstraight: 2, + report: 2, + yawn: 2, + }, }, }); }); diff --git a/x-pack/plugins/task_manager/server/task_store.ts b/x-pack/plugins/task_manager/server/task_store.ts index 4c41be9577ad0..8c0d7764e009f 100644 --- a/x-pack/plugins/task_manager/server/task_store.ts +++ b/x-pack/plugins/task_manager/server/task_store.ts @@ -41,25 +41,23 @@ import { shouldBeOneOf, mustBeAllOf, filterDownBy, - ExistsFilter, - TermFilter, - RangeFilter, asPinnedQuery, matchesClauses, + SortOptions, } from './queries/query_clauses'; import { - updateFields, + updateFieldsAndMarkAsFailed, IdleTaskWithExpiredRunAt, InactiveTasks, RunningOrClaimingTaskWithExpiredRetryAt, - TaskWithSchedule, - taskWithLessThanMaxAttempts, SortByRunAtAndRetryAt, tasksClaimedByOwner, } from './queries/mark_available_tasks_as_claimed'; import { TaskTypeDictionary } from './task_type_dictionary'; +import { ESSearchResponse, ESSearchBody } from '../../apm/typings/elasticsearch'; + export interface StoreOpts { esClient: ElasticsearchClient; index: string; @@ -78,6 +76,9 @@ export interface SearchOpts { search_after?: unknown[]; } +export type AggregationOpts = Pick, 'aggs'> & + Pick; + export interface UpdateByQuerySearchOpts extends SearchOpts { script?: object; } @@ -258,20 +259,26 @@ export class TaskStore { claimTasksById: OwnershipClaimingOpts['claimTasksById'], size: OwnershipClaimingOpts['size'] ): Promise { - const tasksWithRemainingAttempts = [...this.definitions].map(([type, { maxAttempts }]) => - taskWithLessThanMaxAttempts(type, maxAttempts || this.maxAttempts) - ); + const taskMaxAttempts = [...this.definitions].reduce((accumulator, [type, { maxAttempts }]) => { + return { ...accumulator, [type]: maxAttempts || this.maxAttempts }; + }, {}); const queryForScheduledTasks = mustBeAllOf( // Either a task with idle status and runAt <= now or // status running or claiming with a retryAt <= now. - shouldBeOneOf(IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt), - // Either task has a schedule or the attempts < the maximum configured - shouldBeOneOf( - TaskWithSchedule, - ...tasksWithRemainingAttempts - ) + shouldBeOneOf(IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt) ); + // The documents should be sorted by runAt/retryAt, unless there are pinned + // tasks being queried, in which case we want to sort by score first, and then + // the runAt/retryAt. That way we'll get the pinned tasks first. Note that + // the score seems to favor newer documents rather than older documents, so + // if there are not pinned tasks being queried, we do NOT want to sort by score + // at all, just by runAt/retryAt. + const sort: SortOptions = [SortByRunAtAndRetryAt]; + if (claimTasksById && claimTasksById.length) { + sort.unshift('_score'); + } + const apmTrans = apm.startTransaction(`taskManager markAvailableTasksAsClaimed`, 'taskManager'); const { updated } = await this.updateByQuery( asUpdateByQuery({ @@ -283,17 +290,15 @@ export class TaskStore { ), filterDownBy(InactiveTasks) ), - update: updateFields({ - ownerId: this.taskManagerId, - status: 'claiming', - retryAt: claimOwnershipUntil, - }), - sort: [ - // sort by score first, so the "pinned" Tasks are first - '_score', - // the nsort by other fields - SortByRunAtAndRetryAt, - ], + update: updateFieldsAndMarkAsFailed( + { + ownerId: this.taskManagerId, + retryAt: claimOwnershipUntil, + }, + claimTasksById || [], + taskMaxAttempts + ), + sort, }), { max_docs: size, @@ -494,6 +499,25 @@ export class TaskStore { } } + public async aggregate({ + aggs, + query, + size = 0, + }: TSearchRequest): Promise> { + const { body } = await this.esClient.search< + ESSearchResponse + >({ + index: this.index, + ignore_unavailable: true, + body: ensureAggregationOnlyReturnsTaskObjects({ + query, + aggs, + size, + }), + }); + return body; + } + private async updateByQuery( opts: UpdateByQuerySearchOpts = {}, // eslint-disable-next-line @typescript-eslint/naming-convention @@ -579,6 +603,22 @@ function ensureQueryOnlyReturnsTaskObjects(opts: SearchOpts): SearchOpts { }; } +function ensureAggregationOnlyReturnsTaskObjects(opts: AggregationOpts): AggregationOpts { + const originalQuery = opts.query; + const filterToOnlyTasks = { + bool: { + filter: [{ term: { type: 'task' } }], + }, + }; + const query = originalQuery + ? { bool: { must: [filterToOnlyTasks, originalQuery] } } + : filterToOnlyTasks; + return { + ...opts, + query, + }; +} + function isSavedObjectsUpdateResponse( result: SavedObjectsUpdateResponse | Error ): result is SavedObjectsUpdateResponse { diff --git a/x-pack/plugins/telemetry_collection_xpack/tsconfig.json b/x-pack/plugins/telemetry_collection_xpack/tsconfig.json new file mode 100644 index 0000000000000..476f5926f757a --- /dev/null +++ b/x-pack/plugins/telemetry_collection_xpack/tsconfig.json @@ -0,0 +1,21 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "composite": true, + "outDir": "./target/types", + "emitDeclarationOnly": true, + "declaration": true, + "declarationMap": true + }, + "include": [ + "common/**/*", + "server/**/*", + "../../../typings/*" + ], + "references": [ + { "path": "../../../src/core/tsconfig.json" }, + { "path": "../../../src/plugins/telemetry_collection_manager/tsconfig.json" }, + { "path": "../../../src/plugins/telemetry/tsconfig.json" } + ] +} + diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_form.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_form.tsx index 8f19db6fc827d..404f9aeec7c68 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_form.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_form.tsx @@ -102,11 +102,6 @@ export const StepDefineForm: FC = React.memo((props) => { toastNotifications, }; - // TODO This should use the actual value of `indices.query.bool.max_clause_count` - const maxIndexFields = 1024; - const numIndexFields = indexPattern.fields.length; - const disabledQuery = numIndexFields > maxIndexFields; - const copyToClipboardSource = getIndexDevConsoleStatement(pivotQuery, indexPattern.title); const copyToClipboardSourceDescription = i18n.translate( 'xpack.transform.indexPreview.copyClipboardTooltip', @@ -181,18 +176,6 @@ export const StepDefineForm: FC = React.memo((props) => { label={i18n.translate('xpack.transform.stepDefineForm.indexPatternLabel', { defaultMessage: 'Index pattern', })} - helpText={ - disabledQuery - ? i18n.translate('xpack.transform.stepDefineForm.indexPatternHelpText', { - defaultMessage: - 'An optional query for this index pattern is not supported. The number of supported index fields is {maxIndexFields} whereas this index has {numIndexFields} fields.', - values: { - maxIndexFields, - numIndexFields, - }, - }) - : '' - } > {indexPattern.title} @@ -214,7 +197,7 @@ export const StepDefineForm: FC = React.memo((props) => { {/* Flex Column #1: Search Bar / Advanced Search Editor */} {searchItems.savedSearch === undefined && ( <> - {!disabledQuery && !isAdvancedSourceEditorEnabled && ( + {!isAdvancedSourceEditorEnabled && ( 阈值 0。", @@ -20294,7 +20287,6 @@ "xpack.triggersActionsUI.sections.addModalConnectorForm.saveButtonLabel": "保存", "xpack.triggersActionsUI.sections.addModalConnectorForm.updateSuccessNotificationText": "已创建“{connectorName}”", "xpack.triggersActionsUI.sections.alertAdd.betaBadgeTooltipContent": "{pluginName} 为公测版，可能会进行更改。设计和代码相对于正式发行版功能还不够成熟，将按原样提供，且不提供任何保证。公测版功能不受正式发行版功能支持 SLA 的约束。", - "xpack.triggersActionsUI.sections.alertAdd.cancelButtonLabel": "取消", "xpack.triggersActionsUI.sections.alertAdd.conditionPrompt": "定义条件", "xpack.triggersActionsUI.sections.alertAdd.errorLoadingAlertVisualizationTitle": "无法加载告警可视化", "xpack.triggersActionsUI.sections.alertAdd.flyoutTitle": "创建告警", @@ -20302,7 +20294,6 @@ "xpack.triggersActionsUI.sections.alertAdd.loadingAlertVisualizationDescription": "正在加载告警可视化……", "xpack.triggersActionsUI.sections.alertAdd.operationName": "创建", "xpack.triggersActionsUI.sections.alertAdd.previewAlertVisualizationDescription": "完成表达式以生成预览。", - "xpack.triggersActionsUI.sections.alertAdd.saveButtonLabel": "保存", "xpack.triggersActionsUI.sections.alertAdd.saveErrorNotificationText": "无法创建告警。", "xpack.triggersActionsUI.sections.alertAdd.saveSuccessNotificationText": "已保存“{alertName}”", "xpack.triggersActionsUI.sections.alertAdd.selectIndex": "选择索引", @@ -20396,7 +20387,6 @@ "xpack.triggersActionsUI.sections.alertsList.alertStatusOk": "确定", "xpack.triggersActionsUI.sections.alertsList.alertStatusPending": "待处理", "xpack.triggersActionsUI.sections.alertsList.alertStatusUnknown": "未知", - "xpack.triggersActionsUI.sections.alertsList.attentionBannerTitle": "在 {totalStausesError} 个 {totalStausesError, plural, one {{singleTitle}} other {# 个 {multipleTitle}}}中发现错误。", "xpack.triggersActionsUI.sections.alertsList.bulkActionPopover.buttonTitle": "管理告警", "xpack.triggersActionsUI.sections.alertsList.bulkActionPopover.deleteAllTitle": "删除", "xpack.triggersActionsUI.sections.alertsList.bulkActionPopover.disableAllTitle": "禁用", diff --git a/x-pack/plugins/triggers_actions_ui/public/application/boot.tsx b/x-pack/plugins/triggers_actions_ui/public/application/boot.tsx index c157f923e4447..80f9ac532d1c9 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/boot.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/boot.tsx @@ -28,5 +28,7 @@ export const boot = (bootDeps: BootDeps) => { , element ); - return () => unmountComponentAtNode(element); + return () => { + unmountComponentAtNode(element); + }; }; diff --git a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/types.ts b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/types.ts index 958d77a11c883..e22cd268f9bc5 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/types.ts +++ b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/types.ts @@ -110,6 +110,7 @@ export interface WebhookConfig { method: string; url: string; headers: Record; + hasAuth: boolean; } export interface WebhookSecrets { diff --git a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/webhook/webhook.test.tsx b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/webhook/webhook.test.tsx index 337c1f0f18a93..e4d9d3f009c7e 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/webhook/webhook.test.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/webhook/webhook.test.tsx @@ -28,7 +28,7 @@ describe('actionTypeRegistry.get() works', () => { }); describe('webhook connector validation', () => { - test('connector validation succeeds when connector config is valid', () => { + test('connector validation succeeds when hasAuth is true and connector config is valid', () => { const actionConnector = { secrets: { user: 'user', @@ -42,6 +42,35 @@ describe('webhook connector validation', () => { method: 'PUT', url: 'http://test.com', headers: { 'content-type': 'text' }, + hasAuth: true, + }, + } as WebhookActionConnector; + + expect(actionTypeModel.validateConnector(actionConnector)).toEqual({ + errors: { + url: [], + method: [], + user: [], + password: [], + }, + }); + }); + + test('connector validation succeeds when hasAuth is false and connector config is valid', () => { + const actionConnector = { + secrets: { + user: '', + password: '', + }, + id: 'test', + actionTypeId: '.webhook', + name: 'webhook', + isPreconfigured: false, + config: { + method: 'PUT', + url: 'http://test.com', + headers: { 'content-type': 'text' }, + hasAuth: false, }, } as WebhookActionConnector; @@ -65,6 +94,7 @@ describe('webhook connector validation', () => { name: 'webhook', config: { method: 'PUT', + hasAuth: true, }, } as WebhookActionConnector; @@ -73,7 +103,7 @@ describe('webhook connector validation', () => { url: ['URL is required.'], method: [], user: [], - password: ['Password is required.'], + password: ['Password is required when username is used.'], }, }); }); @@ -90,6 +120,7 @@ describe('webhook connector validation', () => { config: { method: 'PUT', url: 'invalid.url', + hasAuth: true, }, } as WebhookActionConnector; diff --git a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/webhook/webhook.tsx b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/webhook/webhook.tsx index 04077738e6015..db3ba9b78cee6 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/webhook/webhook.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/webhook/webhook.tsx @@ -74,26 +74,46 @@ export function getActionType(): ActionTypeModel< ) ); } - if (!action.secrets.user && action.secrets.password) { + if (action.config.hasAuth && !action.secrets.user && !action.secrets.password) { errors.user.push( i18n.translate( - 'xpack.triggersActionsUI.sections.addAction.webhookAction.error.requiredHostText', + 'xpack.triggersActionsUI.sections.addAction.webhookAction.error.requiredAuthUserNameText', { defaultMessage: 'Username is required.', } ) ); } - if (!action.secrets.password && action.secrets.user) { + if (action.config.hasAuth && !action.secrets.user && !action.secrets.password) { errors.password.push( i18n.translate( - 'xpack.triggersActionsUI.sections.addAction.webhookAction.error.requiredPasswordText', + 'xpack.triggersActionsUI.sections.addAction.webhookAction.error.requiredAuthPasswordText', { defaultMessage: 'Password is required.', } ) ); } + if (action.secrets.user && !action.secrets.password) { + errors.password.push( + i18n.translate( + 'xpack.triggersActionsUI.sections.addAction.webhookAction.error.requiredPasswordText', + { + defaultMessage: 'Password is required when username is used.', + } + ) + ); + } + if (!action.secrets.user && action.secrets.password) { + errors.user.push( + i18n.translate( + 'xpack.triggersActionsUI.sections.addAction.webhookAction.error.requiredUserText', + { + defaultMessage: 'Username is required when password is used.', + } + ) + ); + } return validationResult; }, validateParams: (actionParams: WebhookActionParams): ValidationResult => { diff --git a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/webhook/webhook_connectors.test.tsx b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/webhook/webhook_connectors.test.tsx index 45e4c566f7a27..4c5e78670f0c4 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/webhook/webhook_connectors.test.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/webhook/webhook_connectors.test.tsx @@ -24,6 +24,7 @@ describe('WebhookActionConnectorFields renders', () => { method: 'PUT', url: 'http:\\test', headers: { 'content-type': 'text' }, + hasAuth: true, }, } as WebhookActionConnector; const wrapper = mountWithIntl( @@ -50,7 +51,9 @@ describe('WebhookActionConnectorFields renders', () => { secrets: {}, actionTypeId: '.webhook', isPreconfigured: false, - config: {}, + config: { + hasAuth: true, + }, } as WebhookActionConnector; const wrapper = mountWithIntl( { method: 'PUT', url: 'http:\\test', headers: { 'content-type': 'text' }, + hasAuth: true, }, } as WebhookActionConnector; const wrapper = mountWithIntl( diff --git a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/webhook/webhook_connectors.tsx b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/webhook/webhook_connectors.tsx index e4f5ef023a529..15d4c6c30450e 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/webhook/webhook_connectors.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/webhook/webhook_connectors.tsx @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import React, { Fragment, useState } from 'react'; +import React, { Fragment, useEffect, useState } from 'react'; import { FormattedMessage } from '@kbn/i18n/react'; import { @@ -34,12 +34,19 @@ const WebhookActionConnectorFields: React.FunctionComponent> = ({ action, editActionConfig, editActionSecrets, errors, readOnly }) => { const { user, password } = action.secrets; - const { method, url, headers } = action.config; + const { method, url, headers, hasAuth } = action.config; const [httpHeaderKey, setHttpHeaderKey] = useState(''); const [httpHeaderValue, setHttpHeaderValue] = useState(''); const [hasHeaders, setHasHeaders] = useState(false); + useEffect(() => { + if (!action.id) { + editActionConfig('hasAuth', true); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + if (!method) { editActionConfig('method', 'post'); // set method to POST by default } @@ -268,9 +275,9 @@ const WebhookActionConnectorFields: React.FunctionComponent - +

- - - - - - {getEncryptedFieldNotifyLabel(!action.id)} - - - - - - 0 && user !== undefined} + + - 0 && user !== undefined} - name="user" - readOnly={readOnly} - value={user || ''} - data-test-subj="webhookUserInput" - onChange={(e) => { - editActionSecrets('user', e.target.value); - }} - onBlur={() => { - if (!user) { - editActionSecrets('user', ''); - } - }} - /> - - - - 0 && password !== undefined} - label={i18n.translate( - 'xpack.triggersActionsUI.components.builtinActionTypes.webhookAction.passwordTextFieldLabel', - { - defaultMessage: 'Password', + disabled={readOnly} + checked={hasAuth} + onChange={(e) => { + editActionConfig('hasAuth', e.target.checked); + if (!e.target.checked) { + editActionSecrets('user', null); + editActionSecrets('password', null); } - )} - > - 0 && password !== undefined} - value={password || ''} - data-test-subj="webhookPasswordInput" - onChange={(e) => { - editActionSecrets('password', e.target.value); - }} - onBlur={() => { - if (!password) { - editActionSecrets('password', ''); - } - }} - /> - + }} + /> - + {hasAuth ? ( + <> + {getEncryptedFieldNotifyLabel(!action.id)} + + + 0 && user !== undefined} + label={i18n.translate( + 'xpack.triggersActionsUI.components.builtinActionTypes.webhookAction.userTextFieldLabel', + { + defaultMessage: 'Username', + } + )} + > + 0 && user !== undefined} + name="user" + readOnly={readOnly} + value={user || ''} + data-test-subj="webhookUserInput" + onChange={(e) => { + editActionSecrets('user', e.target.value); + }} + onBlur={() => { + if (!user) { + editActionSecrets('user', ''); + } + }} + /> + + + + 0 && password !== undefined} + label={i18n.translate( + 'xpack.triggersActionsUI.components.builtinActionTypes.webhookAction.passwordTextFieldLabel', + { + defaultMessage: 'Password', + } + )} + > + 0 && password !== undefined} + value={password || ''} + data-test-subj="webhookPasswordInput" + onChange={(e) => { + editActionSecrets('password', e.target.value); + }} + onBlur={() => { + if (!password) { + editActionSecrets('password', ''); + } + }} + /> + + + + + ) : null} - - + + + + + + + ); } return ( - + + + + + ); } diff --git a/x-pack/plugins/triggers_actions_ui/public/application/components/health_check.test.tsx b/x-pack/plugins/triggers_actions_ui/public/application/components/health_check.test.tsx index 1d908920db8b0..a7de73c9aab29 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/components/health_check.test.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/components/health_check.test.tsx @@ -10,6 +10,7 @@ import { HealthCheck } from './health_check'; import { act } from 'react-dom/test-utils'; import { httpServiceMock } from '../../../../../../src/core/public/mocks'; +import { HealthContextProvider } from '../context/health_context'; const docLinks = { ELASTIC_WEBSITE_URL: 'elastic.co/', DOC_LINK_VERSION: 'current' }; @@ -20,9 +21,11 @@ describe('health check', () => { http.get.mockImplementationOnce(() => new Promise(() => {})); const { queryByText, container } = render( - -

{'shouldnt render'}

- + + +

{'shouldnt render'}

+ + ); await act(async () => { // wait for useEffect to run @@ -32,13 +35,33 @@ describe('health check', () => { expect(queryByText('shouldnt render')).not.toBeInTheDocument(); }); + it('renders children immediately if waitForCheck is false', async () => { + http.get.mockImplementationOnce(() => new Promise(() => {})); + + const { queryByText, container } = render( + + +

{'should render'}

+ + + ); + await act(async () => { + // wait for useEffect to run + }); + + expect(container.getElementsByClassName('euiLoadingSpinner').length).toBe(0); + expect(queryByText('should render')).toBeInTheDocument(); + }); + it('renders children if keys are enabled', async () => { http.get.mockResolvedValue({ isSufficientlySecure: true, hasPermanentEncryptionKey: true }); const { queryByText } = render( - -

{'should render'}

- + + +

{'should render'}

+ + ); await act(async () => { // wait for useEffect to run @@ -53,9 +76,11 @@ describe('health check', () => { })); const { queryAllByText } = render( - -

{'should render'}

- + + +

{'should render'}

+ + ); await act(async () => { // wait for useEffect to run @@ -81,9 +106,11 @@ describe('health check', () => { })); const { queryByText, queryByRole } = render( - -

{'should render'}

- + + +

{'should render'}

+ + ); await act(async () => { // wait for useEffect to run @@ -108,9 +135,11 @@ describe('health check', () => { })); const { queryByText } = render( - -

{'should render'}

- + + +

{'should render'}

+ + ); await act(async () => { // wait for useEffect to run diff --git a/x-pack/plugins/triggers_actions_ui/public/application/components/health_check.tsx b/x-pack/plugins/triggers_actions_ui/public/application/components/health_check.tsx index 009f582424765..c4d0b4976266e 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/components/health_check.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/components/health_check.tsx @@ -18,33 +18,39 @@ import { EuiEmptyPrompt, EuiCode } from '@elastic/eui'; import { AlertingFrameworkHealth } from '../../types'; import { health } from '../lib/alert_api'; import './health_check.scss'; +import { useHealthContext } from '../context/health_context'; interface Props { docLinks: Pick; http: HttpSetup; inFlyout?: boolean; + waitForCheck: boolean; } export const HealthCheck: React.FunctionComponent = ({ docLinks, http, children, + waitForCheck, inFlyout = false, }) => { + const { setLoadingHealthCheck } = useHealthContext(); const [alertingHealth, setAlertingHealth] = React.useState>(none); React.useEffect(() => { (async function () { + setLoadingHealthCheck(true); setAlertingHealth(some(await health({ http }))); + setLoadingHealthCheck(false); })(); - }, [http]); + }, [http, setLoadingHealthCheck]); const className = inFlyout ? 'alertingFlyoutHealthCheck' : 'alertingHealthCheck'; return pipe( alertingHealth, fold( - () => , + () => (waitForCheck ? : {children}), (healthCheck) => { return healthCheck?.isSufficientlySecure && healthCheck?.hasPermanentEncryptionKey ? ( {children} diff --git a/x-pack/plugins/triggers_actions_ui/public/application/context/health_context.tsx b/x-pack/plugins/triggers_actions_ui/public/application/context/health_context.tsx new file mode 100644 index 0000000000000..de27f6db761e8 --- /dev/null +++ b/x-pack/plugins/triggers_actions_ui/public/application/context/health_context.tsx @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { createContext, useCallback, useContext, useMemo, useState } from 'react'; + +export interface HealthContextValue { + loadingHealthCheck: boolean; + setLoadingHealthCheck: (loading: boolean) => void; +} + +const defaultHealthContext: HealthContextValue = { + loadingHealthCheck: false, + setLoadingHealthCheck: (loading: boolean) => { + throw new Error( + 'setLoadingHealthCheck was not initialized, set it when you invoke the context' + ); + }, +}; + +const HealthContext = createContext(defaultHealthContext); + +export const HealthContextProvider = ({ children }: { children: React.ReactNode }) => { + const [loading, setLoading] = useState(false); + + const setLoadingHealthCheck = useCallback((isLoading: boolean) => { + setLoading(isLoading); + }, []); + + const value = useMemo(() => { + return { loadingHealthCheck: loading, setLoadingHealthCheck }; + }, [loading, setLoadingHealthCheck]); + + return {children}; +}; + +export const useHealthContext = () => { + const ctx = useContext(HealthContext); + if (!ctx) { + throw new Error('HealthContext has not been set.'); + } + return ctx; +}; diff --git a/x-pack/plugins/triggers_actions_ui/public/application/home.tsx b/x-pack/plugins/triggers_actions_ui/public/application/home.tsx index eb6b1ada3ba93..f009a04d40978 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/home.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/home.tsx @@ -31,6 +31,7 @@ import { ActionsConnectorsList } from './sections/actions_connectors_list/compon import { AlertsList } from './sections/alerts_list/components/alerts_list'; import { PLUGIN } from './constants/plugin'; import { HealthCheck } from './components/health_check'; +import { HealthContextProvider } from './context/health_context'; interface MatchParams { section: Section; @@ -139,9 +140,11 @@ export const TriggersActionsUIHome: React.FunctionComponent ( - - - + + + + + )} /> )} @@ -149,9 +152,11 @@ export const TriggersActionsUIHome: React.FunctionComponent ( - - - + + + + + )} /> diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_form/alert_add.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_form/alert_add.tsx index 7be7e60c2e19c..763462ba6ebf4 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_form/alert_add.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_form/alert_add.tsx @@ -10,11 +10,6 @@ import { EuiTitle, EuiFlyoutHeader, EuiFlyout, - EuiFlyoutFooter, - EuiFlexGroup, - EuiFlexItem, - EuiButtonEmpty, - EuiButton, EuiFlyoutBody, EuiPortal, EuiBetaBadge, @@ -29,6 +24,8 @@ import { HealthCheck } from '../../components/health_check'; import { PLUGIN } from '../../constants/plugin'; import { ConfirmAlertSave } from './confirm_alert_save'; import { hasShowActionsCapability } from '../../lib/capabilities'; +import AlertAddFooter from './alert_add_footer'; +import { HealthContextProvider } from '../../context/health_context'; interface AlertAddProps { consumer: string; @@ -183,54 +180,37 @@ export const AlertAdd = ({ - - - + + + + + { + setIsSaving(true); + if (shouldConfirmSave) { + setIsConfirmAlertSaveModalOpen(true); + } else { + await saveAlertAndCloseFlyout(); + } + }} + onCancel={closeFlyout} /> - - - - - - {i18n.translate('xpack.triggersActionsUI.sections.alertAdd.cancelButtonLabel', { - defaultMessage: 'Cancel', - })} - - - - { - setIsSaving(true); - if (shouldConfirmSave) { - setIsConfirmAlertSaveModalOpen(true); - } else { - await saveAlertAndCloseFlyout(); - } - }} - > - - - - - - + + {isConfirmAlertSaveModalOpen && ( { diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_form/alert_add_footer.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_form/alert_add_footer.tsx new file mode 100644 index 0000000000000..92e1198de8440 --- /dev/null +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_form/alert_add_footer.tsx @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import React from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { + EuiFlyoutFooter, + EuiFlexGroup, + EuiFlexItem, + EuiButtonEmpty, + EuiButton, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { useHealthContext } from '../../context/health_context'; + +interface AlertAddFooterProps { + isSaving: boolean; + hasErrors: boolean; + onSave: () => void; + onCancel: () => void; +} + +export const AlertAddFooter = ({ isSaving, hasErrors, onSave, onCancel }: AlertAddFooterProps) => { + const { loadingHealthCheck } = useHealthContext(); + + return ( + + + + + {i18n.translate('xpack.triggersActionsUI.sections.alertAddFooter.cancelButtonLabel', { + defaultMessage: 'Cancel', + })} + + + + + + + + + + ); +}; + +// eslint-disable-next-line import/no-default-export +export { AlertAddFooter as default }; diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_form/alert_edit.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_form/alert_edit.tsx index b60aa04ee9f27..0435a4cc33cb8 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_form/alert_edit.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_form/alert_edit.tsx @@ -28,6 +28,7 @@ import { alertReducer } from './alert_reducer'; import { updateAlert } from '../../lib/alert_api'; import { HealthCheck } from '../../components/health_check'; import { PLUGIN } from '../../constants/plugin'; +import { HealthContextProvider } from '../../context/health_context'; interface AlertEditProps { initialAlert: Alert; @@ -135,74 +136,82 @@ export const AlertEdit = ({ initialAlert, onClose }: AlertEditProps) => { - - - {hasActionsDisabled && ( - - - - - )} - - - - - - - {i18n.translate('xpack.triggersActionsUI.sections.alertEdit.cancelButtonLabel', { - defaultMessage: 'Cancel', - })} - - - - { - setIsSaving(true); - const savedAlert = await onSaveAlert(); - setIsSaving(false); - if (savedAlert) { - closeFlyout(); - if (reloadAlerts) { - reloadAlerts(); - } - } - }} - > - + + + {hasActionsDisabled && ( + + - - - - - + + + )} + + + + + + + {i18n.translate( + 'xpack.triggersActionsUI.sections.alertEdit.cancelButtonLabel', + { + defaultMessage: 'Cancel', + } + )} + + + + { + setIsSaving(true); + const savedAlert = await onSaveAlert(); + setIsSaving(false); + if (savedAlert) { + closeFlyout(); + if (reloadAlerts) { + reloadAlerts(); + } + } + }} + > + + + + + + + ); diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_form/alert_form.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_form/alert_form.tsx index 8800f149c033b..d2ca0abe566ad 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_form/alert_form.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_form/alert_form.tsx @@ -105,9 +105,7 @@ export const AlertForm = ({ } = alertsContext; const canShowActions = hasShowActionsCapability(capabilities); - const [alertTypeModel, setAlertTypeModel] = useState( - alert.alertTypeId ? alertTypeRegistry.get(alert.alertTypeId) : null - ); + const [alertTypeModel, setAlertTypeModel] = useState(null); const [alertTypesIndex, setAlertTypesIndex] = useState(undefined); const [alertInterval, setAlertInterval] = useState( @@ -149,6 +147,10 @@ export const AlertForm = ({ // eslint-disable-next-line react-hooks/exhaustive-deps }, []); + useEffect(() => { + setAlertTypeModel(alert.alertTypeId ? alertTypeRegistry.get(alert.alertTypeId) : null); + }, [alert, alertTypeRegistry]); + const setAlertProperty = (key: string, value: any) => { dispatch({ command: { type: 'setProperty' }, payload: { key, value } }); }; diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/action_type_filter.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/action_type_filter.tsx index 76c13a46db1a9..5a444cd3e9e60 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/action_type_filter.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/action_type_filter.tsx @@ -40,6 +40,7 @@ export const ActionTypeFilter: React.FunctionComponent = numActiveFilters={selectedValues.length} numFilters={selectedValues.length} onClick={() => setIsPopoverOpen(!isPopoverOpen)} + data-test-subj="actionTypeFilterButton" > = } }} checked={selectedValues.includes(item.id) ? 'on' : undefined} + data-test-subj={`actionType${item.id}FilterOption`} > {item.name} diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alert_status_filter.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alert_status_filter.tsx index 87e7a82cd8f23..1bae0b885dbfa 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alert_status_filter.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alert_status_filter.tsx @@ -54,6 +54,7 @@ export const AlertStatusFilter: React.FunctionComponent numActiveFilters={selectedValues.length} numFilters={selectedValues.length} onClick={() => setIsPopoverOpen(!isPopoverOpen)} + data-test-subj="alertStatusFilterButton" > } }} checked={selectedValues.includes(item) ? 'on' : undefined} + data-test-subj={`alertStatus${item}FilerOption`} > {alertsStatusesTranslationsMapping[item]} diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alerts_list.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alerts_list.tsx index 1ef695f3ed112..9eb1149cf3905 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alerts_list.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alerts_list.tsx @@ -437,15 +437,14 @@ export const AlertsList: React.FunctionComponent = () => { title={ } iconType="alert" + data-test-subj="alertsErrorBanner" > { - + { - + { - + - + - + { - + = ({ numActiveFilters={selectedValues.length} numFilters={selectedValues.length} onClick={() => setIsPopoverOpen(!isPopoverOpen)} + data-test-subj="alertTypeFilterButton" > = ({ } }} checked={selectedValues.includes(item.value) ? 'on' : undefined} + data-test-subj={`alertType${item.value}FilterOption`} > {item.name} diff --git a/x-pack/plugins/triggers_actions_ui/public/plugin.ts b/x-pack/plugins/triggers_actions_ui/public/plugin.ts index ef585eca1d67c..874a380f56b5f 100644 --- a/x-pack/plugins/triggers_actions_ui/public/plugin.ts +++ b/x-pack/plugins/triggers_actions_ui/public/plugin.ts @@ -5,10 +5,10 @@ */ import { - CoreStart, CoreSetup, - PluginInitializerContext, + CoreStart, Plugin as CorePlugin, + PluginInitializerContext, } from 'src/core/public'; import { i18n } from '@kbn/i18n'; @@ -17,10 +17,9 @@ import { registerBuiltInAlertTypes } from './application/components/builtin_aler import { ActionTypeModel, AlertTypeModel } from './types'; import { TypeRegistry } from './application/type_registry'; import { - ManagementSetup, ManagementAppMountParams, + ManagementSetup, } from '../../../../src/plugins/management/public'; -import { boot } from './application/boot'; import { ChartsPluginStart } from '../../../../src/plugins/charts/public'; import { PluginStartContract as AlertingStart } from '../../alerts/public'; import { DataPublicPluginStart } from '../../../../src/plugins/data/public'; @@ -63,11 +62,9 @@ export class Plugin private initializerContext: PluginInitializerContext; constructor(initializerContext: PluginInitializerContext) { - const actionTypeRegistry = new TypeRegistry(); - this.actionTypeRegistry = actionTypeRegistry; + this.actionTypeRegistry = new TypeRegistry(); - const alertTypeRegistry = new TypeRegistry(); - this.alertTypeRegistry = alertTypeRegistry; + this.alertTypeRegistry = new TypeRegistry(); this.initializerContext = initializerContext; } @@ -88,7 +85,10 @@ export class Plugin PluginsStart, unknown ]; - boot({ + + const { boot } = await import('./application/boot'); + + return boot({ dataPlugin: pluginsStart.data, charts: pluginsStart.charts, alerts: pluginsStart.alerts, @@ -107,7 +107,6 @@ export class Plugin actionTypeRegistry, alertTypeRegistry, }); - return () => {}; }, }); diff --git a/x-pack/plugins/uptime/public/components/common/__tests__/uptime_date_picker.test.tsx b/x-pack/plugins/uptime/public/components/common/__tests__/uptime_date_picker.test.tsx index 16853211433ca..18e058e305606 100644 --- a/x-pack/plugins/uptime/public/components/common/__tests__/uptime_date_picker.test.tsx +++ b/x-pack/plugins/uptime/public/components/common/__tests__/uptime_date_picker.test.tsx @@ -6,7 +6,16 @@ import React from 'react'; import { UptimeDatePicker } from '../uptime_date_picker'; -import { renderWithRouter, shallowWithRouter, MountWithReduxProvider } from '../../../lib'; +import { + renderWithRouter, + shallowWithRouter, + MountWithReduxProvider, + mountWithRouterRedux, +} from '../../../lib'; +import { UptimeStartupPluginsContextProvider } from '../../../contexts'; +import { startPlugins } from '../../../lib/__mocks__/uptime_plugin_start_mock'; +import { ClientPluginsStart } from '../../../apps/plugin'; +import { createMemoryHistory } from 'history'; describe('UptimeDatePicker component', () => { it('validates props with shallow render', () => { @@ -22,4 +31,59 @@ describe('UptimeDatePicker component', () => { ); expect(component).toMatchSnapshot(); }); + + it('uses shared date range state when there is no url date range state', () => { + const customHistory = createMemoryHistory(); + jest.spyOn(customHistory, 'push'); + + const component = mountWithRouterRedux( + )} + > + + , + { customHistory } + ); + + const startBtn = component.find('[data-test-subj="superDatePickerstartDatePopoverButton"]'); + + expect(startBtn.text()).toBe('~ 30 minutes ago'); + + const endBtn = component.find('[data-test-subj="superDatePickerendDatePopoverButton"]'); + + expect(endBtn.text()).toBe('~ 15 minutes ago'); + + expect(customHistory.push).toHaveBeenCalledWith({ + pathname: '/', + search: 'dateRangeStart=now-30m&dateRangeEnd=now-15m', + }); + }); + + it('should use url date range even if shared date range is present', () => { + const customHistory = createMemoryHistory({ + initialEntries: ['/?g=%22%22&dateRangeStart=now-10m&dateRangeEnd=now'], + }); + + jest.spyOn(customHistory, 'push'); + + const component = mountWithRouterRedux( + )} + > + + , + { customHistory } + ); + + const showDateBtn = component.find('[data-test-subj="superDatePickerShowDatesButton"]'); + + expect(showDateBtn.childAt(0).text()).toBe('Last 10 minutes'); + + // it should update shared state + + expect(startPlugins.data.query.timefilter.timefilter.setTime).toHaveBeenCalledWith({ + from: 'now-10m', + to: 'now', + }); + }); }); diff --git a/x-pack/plugins/uptime/public/components/common/uptime_date_picker.tsx b/x-pack/plugins/uptime/public/components/common/uptime_date_picker.tsx index 1d0dcad73795b..cc8d6271abd73 100644 --- a/x-pack/plugins/uptime/public/components/common/uptime_date_picker.tsx +++ b/x-pack/plugins/uptime/public/components/common/uptime_date_picker.tsx @@ -4,11 +4,15 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { useContext } from 'react'; +import React, { useContext, useEffect } from 'react'; import { EuiSuperDatePicker } from '@elastic/eui'; import { useUrlParams } from '../../hooks'; import { CLIENT_DEFAULTS } from '../../../common/constants'; -import { UptimeRefreshContext, UptimeSettingsContext } from '../../contexts'; +import { + UptimeRefreshContext, + UptimeSettingsContext, + UptimeStartupPluginsContext, +} from '../../contexts'; export interface CommonlyUsedRange { from: string; @@ -16,12 +20,43 @@ export interface CommonlyUsedRange { display: string; } +const isUptimeDefaultDateRange = (dateRangeStart: string, dateRangeEnd: string) => { + const { DATE_RANGE_START, DATE_RANGE_END } = CLIENT_DEFAULTS; + + return dateRangeStart === DATE_RANGE_START && dateRangeEnd === DATE_RANGE_END; +}; + export const UptimeDatePicker = () => { const [getUrlParams, updateUrl] = useUrlParams(); - const { autorefreshInterval, autorefreshIsPaused, dateRangeStart, dateRangeEnd } = getUrlParams(); const { commonlyUsedRanges } = useContext(UptimeSettingsContext); const { refreshApp } = useContext(UptimeRefreshContext); + const { data } = useContext(UptimeStartupPluginsContext); + + // read time from state and update the url + const sharedTimeState = data?.query.timefilter.timefilter.getTime(); + + const { + autorefreshInterval, + autorefreshIsPaused, + dateRangeStart: start, + dateRangeEnd: end, + } = getUrlParams(); + + useEffect(() => { + const { from, to } = sharedTimeState ?? {}; + // if it's uptime default range, and we have shared state from kibana, let's use that + if (isUptimeDefaultDateRange(start, end) && (from !== start || to !== end)) { + updateUrl({ dateRangeStart: from, dateRangeEnd: to }); + } else if (from !== start || to !== end) { + // if it's coming url. let's update shared state + data?.query.timefilter.timefilter.setTime({ from: start, to: end }); + } + + // only need at start, rest date picker on change fucn will take care off + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + const euiCommonlyUsedRanges = commonlyUsedRanges ? commonlyUsedRanges.map( ({ from, to, display }: { from: string; to: string; display: string }) => { @@ -36,13 +71,17 @@ export const UptimeDatePicker = () => { return ( { - updateUrl({ dateRangeStart: start, dateRangeEnd: end }); + onTimeChange={({ start: startN, end: endN }) => { + if (data?.query?.timefilter?.timefilter) { + data?.query.timefilter.timefilter.setTime({ from: startN, to: endN }); + } + + updateUrl({ dateRangeStart: startN, dateRangeEnd: endN }); refreshApp(); }} onRefresh={refreshApp} diff --git a/x-pack/plugins/uptime/public/lib/__mocks__/uptime_plugin_start_mock.ts b/x-pack/plugins/uptime/public/lib/__mocks__/uptime_plugin_start_mock.ts new file mode 100644 index 0000000000000..6d2ea80a3b6f2 --- /dev/null +++ b/x-pack/plugins/uptime/public/lib/__mocks__/uptime_plugin_start_mock.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +interface InputTimeRange { + from: string; + to: string; +} + +export const startPlugins = { + data: { + query: { + timefilter: { + timefilter: { + getTime: () => ({ to: 'now-15m', from: 'now-30m' }), + setTime: jest.fn(({ from, to }: InputTimeRange) => {}), + }, + }, + }, + }, +}; diff --git a/x-pack/plugins/uptime/public/lib/helper/helper_with_router.tsx b/x-pack/plugins/uptime/public/lib/helper/helper_with_router.tsx index 7da570e909425..5219fb3242539 100644 --- a/x-pack/plugins/uptime/public/lib/helper/helper_with_router.tsx +++ b/x-pack/plugins/uptime/public/lib/helper/helper_with_router.tsx @@ -20,22 +20,19 @@ const helperWithRouter: ( wrapReduxStore?: boolean, storeState?: AppState ) => R = (helper, component, customHistory, wrapReduxStore, storeState) => { - if (customHistory) { - customHistory.location.key = 'TestKeyForTesting'; - return helper({component}); - } - const history = createMemoryHistory(); + const history = customHistory ?? createMemoryHistory(); + history.location.key = 'TestKeyForTesting'; + const routerWrapper = {component}; + if (wrapReduxStore) { return helper( - - {component} - + {routerWrapper} ); } - return helper({component}); + return helper(routerWrapper); }; export const renderWithRouter = (component: ReactElement, customHistory?: MemoryHistory) => { diff --git a/x-pack/plugins/uptime/server/lib/alerts/status_check.ts b/x-pack/plugins/uptime/server/lib/alerts/status_check.ts index 3fc26811d46eb..7feb916046e3a 100644 --- a/x-pack/plugins/uptime/server/lib/alerts/status_check.ts +++ b/x-pack/plugins/uptime/server/lib/alerts/status_check.ts @@ -7,6 +7,7 @@ import { schema } from '@kbn/config-schema'; import { i18n } from '@kbn/i18n'; import Mustache from 'mustache'; +import { ElasticsearchClient } from 'kibana/server'; import { UptimeAlertTypeFactory } from './types'; import { esKuery } from '../../../../../../src/plugins/data/server'; import { JsonObject } from '../../../../../../src/plugins/kibana_utils/common'; @@ -81,6 +82,7 @@ export const generateFilterDSL = async ( export const formatFilterString = async ( dynamicSettings: DynamicSettings, callES: ESAPICaller, + esClient: ElasticsearchClient, filters: StatusCheckFilters, search: string, libs?: UMServerLibs @@ -88,9 +90,10 @@ export const formatFilterString = async ( await generateFilterDSL( () => libs?.requests?.getIndexPattern - ? libs?.requests?.getIndexPattern({ callES, dynamicSettings }) + ? libs?.requests?.getIndexPattern({ callES, esClient, dynamicSettings }) : getUptimeIndexPattern({ callES, + esClient, dynamicSettings, }), filters, @@ -237,6 +240,7 @@ export const statusCheckAlertFactory: UptimeAlertTypeFactory = (_server, libs) = async executor( { params: rawParams, state, services: { alertInstanceFactory } }, callES, + esClient, dynamicSettings ) { const { @@ -252,7 +256,14 @@ export const statusCheckAlertFactory: UptimeAlertTypeFactory = (_server, libs) = timerange: oldVersionTimeRange, } = rawParams; - const filterString = await formatFilterString(dynamicSettings, callES, filters, search, libs); + const filterString = await formatFilterString( + dynamicSettings, + callES, + esClient, + filters, + search, + libs + ); const timerange = oldVersionTimeRange || { from: isAutoGenerated diff --git a/x-pack/plugins/uptime/server/lib/alerts/uptime_alert_wrapper.ts b/x-pack/plugins/uptime/server/lib/alerts/uptime_alert_wrapper.ts index b8a56405ca160..390b6d347996c 100644 --- a/x-pack/plugins/uptime/server/lib/alerts/uptime_alert_wrapper.ts +++ b/x-pack/plugins/uptime/server/lib/alerts/uptime_alert_wrapper.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { ILegacyScopedClusterClient } from 'kibana/server'; +import { ILegacyScopedClusterClient, ElasticsearchClient } from 'kibana/server'; import { AlertExecutorOptions, AlertType, AlertTypeState } from '../../../../alerts/server'; import { savedObjectsAdapter } from '../saved_objects'; import { DynamicSettings } from '../../../common/runtime_types'; @@ -13,6 +13,7 @@ export interface UptimeAlertType extends Omit Promise; } @@ -22,13 +23,13 @@ export const uptimeAlertWrapper = (uptimeAlert: UptimeAlertType) => ({ producer: 'uptime', executor: async (options: AlertExecutorOptions) => { const { - services: { callCluster: callES }, + services: { callCluster: callES, scopedClusterClient }, } = options; const dynamicSettings = await savedObjectsAdapter.getUptimeDynamicSettings( options.services.savedObjectsClient ); - return uptimeAlert.executor(options, callES, dynamicSettings); + return uptimeAlert.executor(options, callES, scopedClusterClient, dynamicSettings); }, }); diff --git a/x-pack/plugins/uptime/server/lib/requests/get_index_pattern.ts b/x-pack/plugins/uptime/server/lib/requests/get_index_pattern.ts index 1d284143a1ab0..06846a73ed3d7 100644 --- a/x-pack/plugins/uptime/server/lib/requests/get_index_pattern.ts +++ b/x-pack/plugins/uptime/server/lib/requests/get_index_pattern.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { LegacyAPICaller, LegacyCallAPIOptions } from 'src/core/server'; +import { ElasticsearchClient } from 'kibana/server'; import { UMElasticsearchQueryFn } from '../adapters'; import { IndexPatternsFetcher, FieldDescriptor } from '../../../../../../src/plugins/data/server'; @@ -14,15 +14,10 @@ export interface IndexPatternTitleAndFields { } export const getUptimeIndexPattern: UMElasticsearchQueryFn< - {}, + { esClient: ElasticsearchClient }, IndexPatternTitleAndFields | undefined -> = async ({ callES, dynamicSettings }) => { - const callAsCurrentUser: LegacyAPICaller = async ( - endpoint: string, - clientParams: Record = {}, - options?: LegacyCallAPIOptions - ) => callES(endpoint, clientParams, options); - const indexPatternsFetcher = new IndexPatternsFetcher(callAsCurrentUser); +> = async ({ esClient, dynamicSettings }) => { + const indexPatternsFetcher = new IndexPatternsFetcher(esClient); // Since `getDynamicIndexPattern` is called in setup_request (and thus by every endpoint) // and since `getFieldsForWildcard` will throw if the specified indices don't exist, diff --git a/x-pack/plugins/uptime/server/lib/requests/get_monitor_details.ts b/x-pack/plugins/uptime/server/lib/requests/get_monitor_details.ts index fbcbc37ae0cc2..ec750f92656b2 100644 --- a/x-pack/plugins/uptime/server/lib/requests/get_monitor_details.ts +++ b/x-pack/plugins/uptime/server/lib/requests/get_monitor_details.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { ElasticsearchClient } from 'kibana/server'; import { ESAPICaller, UMElasticsearchQueryFn } from '../adapters'; import { MonitorDetails, MonitorError } from '../../../common/runtime_types'; import { formatFilterString } from '../alerts/status_check'; @@ -13,10 +14,12 @@ export interface GetMonitorDetailsParams { dateStart: string; dateEnd: string; alertsClient: any; + esClient: ElasticsearchClient; } const getMonitorAlerts = async ( callES: ESAPICaller, + esClient: ElasticsearchClient, dynamicSettings: any, alertsClient: any, monitorId: string @@ -67,6 +70,7 @@ const getMonitorAlerts = async ( const parsedFilters = await formatFilterString( dynamicSettings, callES, + esClient, currAlert.params.filters, currAlert.params.search ); @@ -84,7 +88,7 @@ const getMonitorAlerts = async ( export const getMonitorDetails: UMElasticsearchQueryFn< GetMonitorDetailsParams, MonitorDetails -> = async ({ callES, dynamicSettings, monitorId, dateStart, dateEnd, alertsClient }) => { +> = async ({ callES, esClient, dynamicSettings, monitorId, dateStart, dateEnd, alertsClient }) => { const queryFilters: any = [ { range: { @@ -134,7 +138,13 @@ export const getMonitorDetails: UMElasticsearchQueryFn< const monitorError: MonitorError | undefined = data?.error; const errorTimestamp: string | undefined = data?.['@timestamp']; - const monAlerts = await getMonitorAlerts(callES, dynamicSettings, alertsClient, monitorId); + const monAlerts = await getMonitorAlerts( + callES, + esClient, + dynamicSettings, + alertsClient, + monitorId + ); return { monitorId, error: monitorError, diff --git a/x-pack/plugins/uptime/server/rest_api/index_state/get_index_pattern.ts b/x-pack/plugins/uptime/server/rest_api/index_state/get_index_pattern.ts index 26715f0ff37b6..baf999158a29e 100644 --- a/x-pack/plugins/uptime/server/rest_api/index_state/get_index_pattern.ts +++ b/x-pack/plugins/uptime/server/rest_api/index_state/get_index_pattern.ts @@ -16,7 +16,11 @@ export const createGetIndexPatternRoute: UMRestApiRouteFactory = (libs: UMServer try { return response.ok({ body: { - ...(await libs.requests.getIndexPattern({ callES, dynamicSettings })), + ...(await libs.requests.getIndexPattern({ + callES, + esClient: _context.core.elasticsearch.client.asCurrentUser, + dynamicSettings, + })), }, }); } catch (e) { diff --git a/x-pack/plugins/uptime/server/rest_api/monitors/monitors_details.ts b/x-pack/plugins/uptime/server/rest_api/monitors/monitors_details.ts index bb54effc0d57e..8bbb4fcb5575c 100644 --- a/x-pack/plugins/uptime/server/rest_api/monitors/monitors_details.ts +++ b/x-pack/plugins/uptime/server/rest_api/monitors/monitors_details.ts @@ -28,6 +28,7 @@ export const createGetMonitorDetailsRoute: UMRestApiRouteFactory = (libs: UMServ body: { ...(await libs.requests.getMonitorDetails({ callES, + esClient: context.core.elasticsearch.client.asCurrentUser, dynamicSettings, monitorId, dateStart, diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/tests/actions/builtin_action_types/webhook.ts b/x-pack/test/alerting_api_integration/security_and_spaces/tests/actions/builtin_action_types/webhook.ts index ef14dd9ec2eff..64d9711730c7b 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/tests/actions/builtin_action_types/webhook.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/tests/actions/builtin_action_types/webhook.ts @@ -20,6 +20,7 @@ import { const defaultValues: Record = { headers: null, method: 'post', + hasAuth: true, }; function parsePort(url: Record): Record { diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/actions/migrations.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/actions/migrations.ts index 1dd88832e38b7..d46d60905da1c 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/actions/migrations.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/actions/migrations.ts @@ -30,25 +30,47 @@ export default function createGetTests({ getService }: FtrProviderContext) { expect(response.status).to.eql(200); expect(response.body.config).key('incidentConfiguration'); expect(response.body.config).not.key('casesConfiguration'); - expect(response.body.config.incidentConfiguration).to.eql({ - mapping: [ - { - actionType: 'overwrite', - source: 'title', - target: 'summary', - }, - { - actionType: 'overwrite', - source: 'description', - target: 'description', - }, - { - actionType: 'append', - source: 'comments', - target: 'comments', - }, - ], + expect(response.body.config).to.eql({ + apiUrl: + 'http://elastic:changeme@localhost:5620/api/_actions-FTS-external-service-simulators/jira', + incidentConfiguration: { + mapping: [ + { + actionType: 'overwrite', + source: 'title', + target: 'summary', + }, + { + actionType: 'overwrite', + source: 'description', + target: 'description', + }, + { + actionType: 'append', + source: 'comments', + target: 'comments', + }, + ], + }, + projectKey: 'CK', }); }); + + it('7.11.0 migrates webhook connector configurations to have `hasAuth` property', async () => { + const responseWithAuth = await supertest.get( + `${getUrlPrefix(``)}/api/actions/action/949f909b-20a0-46e3-aadb-6a4d117bb592` + ); + + expect(responseWithAuth.status).to.eql(200); + expect(responseWithAuth.body.config).key('hasAuth'); + expect(responseWithAuth.body.config.hasAuth).to.eql(true); + + const responseNoAuth = await supertest.get( + `${getUrlPrefix(``)}/api/actions/action/7434121e-045a-47d6-a0a6-0b6da752397a` + ); + expect(responseNoAuth.status).to.eql(200); + expect(responseNoAuth.body.config).key('hasAuth'); + expect(responseNoAuth.body.config.hasAuth).to.eql(false); + }); }); } diff --git a/x-pack/test/api_integration/apis/maps/index.js b/x-pack/test/api_integration/apis/maps/index.js index 9f1fe96c5637b..36a8fa2ec045e 100644 --- a/x-pack/test/api_integration/apis/maps/index.js +++ b/x-pack/test/api_integration/apis/maps/index.js @@ -18,6 +18,7 @@ export default function ({ loadTestFile, getService }) { loadTestFile(require.resolve('./migrations')); loadTestFile(require.resolve('./get_tile')); loadTestFile(require.resolve('./get_grid_tile')); + loadTestFile(require.resolve('./proxy_api')); }); }); } diff --git a/x-pack/test/api_integration/apis/maps/proxy_api.js b/x-pack/test/api_integration/apis/maps/proxy_api.js new file mode 100644 index 0000000000000..f85de9dc1670a --- /dev/null +++ b/x-pack/test/api_integration/apis/maps/proxy_api.js @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; + +export default function ({ getService }) { + const supertest = getService('supertest'); + + describe('EMS proxy', () => { + it('should correctly rewrite url and format', async () => { + const resp = await supertest + .get(`/api/maps/ems/files/v7.10/manifest`) + .set('kbn-xsrf', 'kibana') + .expect(200); + + expect(resp.body.layers.length).to.be.greaterThan(0); + + //Check world-layer + const worldLayer = resp.body.layers.find((layer) => layer.layer_id === 'world_countries'); + expect(worldLayer.formats.length).to.be.greaterThan(0); + expect(worldLayer.formats[0].type).to.be('geojson'); + expect(worldLayer.formats[0].url).to.be('file?id=world_countries'); + }); + }); +} diff --git a/x-pack/test/api_integration/config.ts b/x-pack/test/api_integration/config.ts index 97fd968ce7992..fc021ffa0ca5b 100644 --- a/x-pack/test/api_integration/config.ts +++ b/x-pack/test/api_integration/config.ts @@ -25,6 +25,7 @@ export async function getApiIntegrationConfig({ readConfigFile }: FtrConfigProvi ...xPackFunctionalTestsConfig.get('kbnTestServer'), serverArgs: [ ...xPackFunctionalTestsConfig.get('kbnTestServer.serverArgs'), + '--map.proxyElasticMapsServiceInMaps=true', '--xpack.security.session.idleTimeout=3600000', // 1 hour '--telemetry.optIn=true', '--xpack.fleet.enabled=true', diff --git a/x-pack/test/apm_api_integration/basic/tests/correlations/slow_durations.ts b/x-pack/test/apm_api_integration/basic/tests/correlations/slow_durations.ts index ada67bbec070b..085a81c5f1bf4 100644 --- a/x-pack/test/apm_api_integration/basic/tests/correlations/slow_durations.ts +++ b/x-pack/test/apm_api_integration/basic/tests/correlations/slow_durations.ts @@ -25,7 +25,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { 'user.username,user.id,host.ip,user_agent.name,kubernetes.pod.uuid,url.domain,container.id,service.node.name'; // Failing: See https://github.com/elastic/kibana/issues/81264 - describe.skip('Slow durations', () => { + describe('Slow durations', () => { const url = format({ pathname: `/api/apm/correlations/slow_durations`, query: { start, end, durationPercentile, fieldNames }, @@ -40,21 +40,21 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); }); - describe('with default scoring', () => { - let response: PromiseReturnType; - before(async () => { - await esArchiver.load(archiveName); - response = await supertest.get(url); - }); - + describe('when data is loaded', () => { + before(() => esArchiver.load(archiveName)); after(() => esArchiver.unload(archiveName)); - it('returns successfully', () => { - expect(response.status).to.eql(200); - }); + describe('making request with default args', () => { + let response: PromiseReturnType; + before(async () => { + response = await supertest.get(url); + + it('returns successfully', () => { + expect(response.status).to.eql(200); + }); - it('returns fields in response', () => { - expectSnapshot(Object.keys(response.body.response)).toMatchInline(` + it('returns fields in response', () => { + expectSnapshot(Object.keys(response.body.response)).toMatchInline(` Array [ "service.node.name", "host.ip", @@ -64,14 +64,14 @@ export default function ApiTest({ getService }: FtrProviderContext) { "url.domain", ] `); - }); + }); - it('returns cardinality for each field', () => { - const cardinalitys = Object.values(response.body.response).map( - (field: any) => field.cardinality - ); + it('returns cardinality for each field', () => { + const cardinalitys = Object.values(response.body.response).map( + (field: any) => field.cardinality + ); - expectSnapshot(cardinalitys).toMatchInline(` + expectSnapshot(cardinalitys).toMatchInline(` Array [ 5, 6, @@ -81,11 +81,11 @@ export default function ApiTest({ getService }: FtrProviderContext) { 4, ] `); - }); + }); - it('returns buckets', () => { - const { buckets } = response.body.response['user.id'].value; - expectSnapshot(buckets[0]).toMatchInline(` + it('returns buckets', () => { + const { buckets } = response.body.response['user.id'].value; + expectSnapshot(buckets[0]).toMatchInline(` Object { "bg_count": 32, "doc_count": 6, @@ -93,46 +93,23 @@ export default function ApiTest({ getService }: FtrProviderContext) { "score": 0.1875, } `); + }); + }); }); - }); - describe('with different scoring', () => { - before(async () => esArchiver.load(archiveName)); - after(() => esArchiver.unload(archiveName)); - - it(`returns buckets for each score`, async () => { - const promises = ['percentage', 'jlh', 'chi_square', 'gnd'].map(async (scoring) => { - const response = await supertest.get( - format({ - pathname: `/api/apm/correlations/slow_durations`, - query: { start, end, durationPercentile, fieldNames, scoring }, - }) - ); - - return { name: scoring, value: response.body.response['user.id'].value.buckets[0].score }; + describe('making a request for each "scoring"', () => { + ['percentage', 'jlh', 'chi_square', 'gnd'].map(async (scoring) => { + it(`returns response for scoring "${scoring}"`, async () => { + const response = await supertest.get( + format({ + pathname: `/api/apm/correlations/slow_durations`, + query: { start, end, durationPercentile, fieldNames, scoring }, + }) + ); + + expect(response.status).to.be(200); + }); }); - - const res = await Promise.all(promises); - expectSnapshot(res).toMatchInline(` - Array [ - Object { - "name": "percentage", - "value": 0.1875, - }, - Object { - "name": "jlh", - "value": 3.33506905769659, - }, - Object { - "name": "chi_square", - "value": 219.192006524483, - }, - Object { - "name": "gnd", - "value": 0.671406580688819, - }, - ] - `); }); }); }); diff --git a/x-pack/test/functional/apps/dashboard/async_search/async_search.ts b/x-pack/test/functional/apps/dashboard/async_search/async_search.ts index 004e51e459caa..6932a88635a67 100644 --- a/x-pack/test/functional/apps/dashboard/async_search/async_search.ts +++ b/x-pack/test/functional/apps/dashboard/async_search/async_search.ts @@ -8,10 +8,20 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../ftr_provider_context'; export default function ({ getService, getPageObjects }: FtrProviderContext) { + const es = getService('es'); const testSubjects = getService('testSubjects'); + const log = getService('log'); const PageObjects = getPageObjects(['common', 'header', 'dashboard', 'visChart']); describe('dashboard with async search', () => { + before(async function () { + const { body } = await es.info(); + if (!body.version.number.includes('SNAPSHOT')) { + log.debug('Skipping because this build does not have the required shard_delay agg'); + this.skip(); + } + }); + it('not delayed should load', async () => { await PageObjects.common.navigateToApp('dashboard'); await PageObjects.dashboard.gotoDashboardEditMode('Not Delayed'); diff --git a/x-pack/test/functional/apps/lens/persistent_context.ts b/x-pack/test/functional/apps/lens/persistent_context.ts index 15515df92a89f..8d536aac3f795 100644 --- a/x-pack/test/functional/apps/lens/persistent_context.ts +++ b/x-pack/test/functional/apps/lens/persistent_context.ts @@ -66,5 +66,28 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(timeRange.end).to.equal('Sep 19, 2025 @ 06:31:44.000'); await filterBar.hasFilter('ip', '97.220.3.248', false, true); }); + + it('keeps time range and pinned filters after refreshing directly after saving', async () => { + // restore defaults so visualization becomes saveable + await security.testUser.restoreDefaults(); + await PageObjects.lens.configureDimension({ + dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', + operation: 'date_histogram', + field: '@timestamp', + }); + + await PageObjects.lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', + operation: 'avg', + field: 'bytes', + }); + await PageObjects.lens.save('persistentcontext'); + await browser.refresh(); + await PageObjects.header.waitUntilLoadingHasFinished(); + const timeRange = await PageObjects.timePicker.getTimeConfig(); + expect(timeRange.start).to.equal('Sep 7, 2015 @ 06:31:44.000'); + expect(timeRange.end).to.equal('Sep 19, 2025 @ 06:31:44.000'); + await filterBar.hasFilter('ip', '97.220.3.248', false, true); + }); }); } diff --git a/x-pack/test/functional/apps/maps/discover.js b/x-pack/test/functional/apps/maps/discover.js index 8dbd98ed3af2f..6a2c1f8437698 100644 --- a/x-pack/test/functional/apps/maps/discover.js +++ b/x-pack/test/functional/apps/maps/discover.js @@ -36,6 +36,7 @@ export default function ({ getService, getPageObjects }) { expect(doesLayerExist).to.equal(true); const hits = await PageObjects.maps.getHits(); expect(hits).to.equal('4'); + await PageObjects.maps.refreshAndClearUnsavedChangesWarning(); }); it('should link geo_point fields to Maps application with time and query context', async () => { @@ -55,6 +56,7 @@ export default function ({ getService, getPageObjects }) { expect(doesLayerExist).to.equal(true); const hits = await PageObjects.maps.getHits(); expect(hits).to.equal('7'); + await PageObjects.maps.refreshAndClearUnsavedChangesWarning(); }); }); } diff --git a/x-pack/test/functional/apps/maps/embeddable/dashboard.js b/x-pack/test/functional/apps/maps/embeddable/dashboard.js index 3c935b385cb39..0c8a208e92ece 100644 --- a/x-pack/test/functional/apps/maps/embeddable/dashboard.js +++ b/x-pack/test/functional/apps/maps/embeddable/dashboard.js @@ -15,6 +15,7 @@ export default function ({ getPageObjects, getService }) { const inspector = getService('inspector'); const testSubjects = getService('testSubjects'); const browser = getService('browser'); + const retry = getService('retry'); describe('embed in dashboard', () => { before(async () => { @@ -50,9 +51,11 @@ export default function ({ getPageObjects, getService }) { it('should populate inspector with requests for map embeddable', async () => { await dashboardPanelActions.openInspectorByTitle('join example'); - const joinExampleRequestNames = await inspector.getRequestNames(); + await retry.try(async () => { + const joinExampleRequestNames = await inspector.getRequestNames(); + expect(joinExampleRequestNames).to.equal('geo_shapes*,meta_for_geo_shapes*.shape_name'); + }); await inspector.close(); - expect(joinExampleRequestNames).to.equal('geo_shapes*,meta_for_geo_shapes*.shape_name'); await dashboardPanelActions.openInspectorByTitle('geo grid vector grid example'); const gridExampleRequestNames = await inspector.getRequestNames(); diff --git a/x-pack/test/functional/apps/maps/joins.js b/x-pack/test/functional/apps/maps/joins.js index bd5ecfe2a2504..0e2850dafbccc 100644 --- a/x-pack/test/functional/apps/maps/joins.js +++ b/x-pack/test/functional/apps/maps/joins.js @@ -38,6 +38,7 @@ export default function ({ getPageObjects, getService }) { after(async () => { await inspector.close(); + await PageObjects.maps.refreshAndClearUnsavedChangesWarning(); await security.testUser.restoreDefaults(); }); diff --git a/x-pack/test/functional/apps/maps/layer_visibility.js b/x-pack/test/functional/apps/maps/layer_visibility.js index dd9b93c995695..75a0e7da0f256 100644 --- a/x-pack/test/functional/apps/maps/layer_visibility.js +++ b/x-pack/test/functional/apps/maps/layer_visibility.js @@ -19,6 +19,7 @@ export default function ({ getPageObjects, getService }) { afterEach(async () => { await inspector.close(); + await PageObjects.maps.refreshAndClearUnsavedChangesWarning(); await security.testUser.restoreDefaults(); }); diff --git a/x-pack/test/functional/apps/maps/vector_styling.js b/x-pack/test/functional/apps/maps/vector_styling.js index 1def542982dd8..e4c5eaf892c76 100644 --- a/x-pack/test/functional/apps/maps/vector_styling.js +++ b/x-pack/test/functional/apps/maps/vector_styling.js @@ -16,6 +16,7 @@ export default function ({ getService, getPageObjects }) { await PageObjects.maps.loadSavedMap('document example'); }); after(async () => { + await PageObjects.maps.refreshAndClearUnsavedChangesWarning(); await security.testUser.restoreDefaults(); }); diff --git a/x-pack/test/functional/apps/monitoring/elasticsearch/node_detail.js b/x-pack/test/functional/apps/monitoring/elasticsearch/node_detail.js index c304b177cb04b..a58f66214e772 100644 --- a/x-pack/test/functional/apps/monitoring/elasticsearch/node_detail.js +++ b/x-pack/test/functional/apps/monitoring/elasticsearch/node_detail.js @@ -13,8 +13,7 @@ export default function ({ getService, getPageObjects }) { const nodesList = getService('monitoringElasticsearchNodes'); const nodeDetail = getService('monitoringElasticsearchNodeDetail'); - // Failing: See https://github.com/elastic/kibana/issues/81166 - describe.skip('Elasticsearch node detail', () => { + describe('Elasticsearch node detail', () => { describe('Active Nodes', () => { const { setup, tearDown } = getLifecycleMethods(getService, getPageObjects); diff --git a/x-pack/test/functional/es_archives/actions/data.json b/x-pack/test/functional/es_archives/actions/data.json index aeeca87deb9ff..18d67da1752bc 100644 --- a/x-pack/test/functional/es_archives/actions/data.json +++ b/x-pack/test/functional/es_archives/actions/data.json @@ -56,3 +56,57 @@ "type": "_doc" } } + +{ + "type": "doc", + "value": { + "id": "action:949f909b-20a0-46e3-aadb-6a4d117bb592", + "index": ".kibana_1", + "source": { + "action": { + "actionTypeId": ".webhook", + "config": { + "headers": null, + "method": "post", + "url": "http://localhost" + }, + "name": "A webhook with auth", + "secrets": "LUqlrITACjqPmcWGlbl+H4RsGGOlw8LM0Urq8r7y6jNT7Igv3J7FjKJ2NXfNTaghVBO7e9x3wZOtiycwyoAdviTyYm1pspni24vH+OT70xaSuXcDoxfGwiLEcaG04INDnUJX4dtmRerxqR9ChktC70LNtOU3sqjYI2tWt2vOqGeq" + }, + "migrationVersion": { + "action": "7.10.0" + }, + "references": [ + ], + "type": "action", + "updated_at": "2020-10-26T21:29:47.380Z" + } + } +} + +{ + "type": "doc", + "value": { + "id": "action:7434121e-045a-47d6-a0a6-0b6da752397a", + "index": ".kibana_1", + "source": { + "action": { + "actionTypeId": ".webhook", + "config": { + "headers": null, + "method": "post", + "url": "http://localhost" + }, + "name": "A webhook with no auth", + "secrets": "tOwFq20hbUrcp3FX7stKB5aJaQQdLNQwomSNym8BgnFaBBafPOASv5T0tGdGsTr/CA7VK+N/wYBHQPzt0apF8Z/UYl63ZXqck5tSoFDnQW77zv1VVQ5wEwN1qkAQQcfrXTXU2wYVAYZNSuHkbeRjcasfG0ty1K+J7A==" + }, + "migrationVersion": { + "action": "7.10.0" + }, + "references": [ + ], + "type": "action", + "updated_at": "2020-10-26T21:30:35.146Z" + } + } +} diff --git a/x-pack/test/functional/page_objects/gis_page.ts b/x-pack/test/functional/page_objects/gis_page.ts index 7be0aa425509e..c4f1bd7dc2a6b 100644 --- a/x-pack/test/functional/page_objects/gis_page.ts +++ b/x-pack/test/functional/page_objects/gis_page.ts @@ -19,6 +19,14 @@ export function GisPageProvider({ getService, getPageObjects }: FtrProviderConte const queryBar = getService('queryBar'); const comboBox = getService('comboBox'); const renderable = getService('renderable'); + const browser = getService('browser'); + const MenuToggle = getService('MenuToggle'); + + const setViewPopoverToggle = new MenuToggle({ + name: 'SetView Popover', + menuTestSubject: 'mapSetViewForm', + toggleButtonTestSubject: 'toggleSetViewVisibilityButton', + }); function escapeLayerName(layerName: string) { return layerName.split(' ').join('_'); @@ -233,41 +241,11 @@ export function GisPageProvider({ getService, getPageObjects }: FtrProviderConte return buttons.length; } - async isSetViewPopoverOpen() { - return await testSubjects.exists('mapSetViewForm', { timeout: 500 }); - } - - async openSetViewPopover() { - const isOpen = await this.isSetViewPopoverOpen(); - if (!isOpen) { - await retry.try(async () => { - await testSubjects.click('toggleSetViewVisibilityButton'); - const isOpenAfterClick = await this.isSetViewPopoverOpen(); - if (!isOpenAfterClick) { - throw new Error('set view popover not opened'); - } - }); - } - } - - async closeSetViewPopover() { - const isOpen = await this.isSetViewPopoverOpen(); - if (isOpen) { - await retry.try(async () => { - await testSubjects.click('toggleSetViewVisibilityButton'); - const isOpenAfterClick = await this.isSetViewPopoverOpen(); - if (isOpenAfterClick) { - throw new Error('set view popover not closed'); - } - }); - } - } - async setView(lat: number, lon: number, zoom: number) { log.debug( `Set view lat: ${lat.toString()}, lon: ${lon.toString()}, zoom: ${zoom.toString()}` ); - await this.openSetViewPopover(); + await setViewPopoverToggle.open(); await testSubjects.setValue('latitudeInput', lat.toString()); await testSubjects.setValue('longitudeInput', lon.toString()); await testSubjects.setValue('zoomInput', zoom.toString()); @@ -277,11 +255,20 @@ export function GisPageProvider({ getService, getPageObjects }: FtrProviderConte async getView() { log.debug('Get view'); - await this.openSetViewPopover(); - const lat = await testSubjects.getAttribute('latitudeInput', 'value'); - const lon = await testSubjects.getAttribute('longitudeInput', 'value'); - const zoom = await testSubjects.getAttribute('zoomInput', 'value'); - await this.closeSetViewPopover(); + await setViewPopoverToggle.open(); + // this method is regularly called within a retry, so we need to reduce the timeouts + // of the retries done within the getAttribute method in order to ensure that they fail + // early enough to retry getView() + const getAttributeOptions = { + tryTimeout: 5000, + findTimeout: 1000, + }; + + const lat = await testSubjects.getAttribute('latitudeInput', 'value', getAttributeOptions); + const lon = await testSubjects.getAttribute('longitudeInput', 'value', getAttributeOptions); + const zoom = await testSubjects.getAttribute('zoomInput', 'value', getAttributeOptions); + + await setViewPopoverToggle.close(); return { lat: parseFloat(lat), lon: parseFloat(lon), @@ -692,6 +679,13 @@ export function GisPageProvider({ getService, getPageObjects }: FtrProviderConte } await testSubjects.click('mapSettingSubmitButton'); } + + async refreshAndClearUnsavedChangesWarning() { + await browser.refresh(); + // accept alert if it pops up + const alert = await browser.getAlert(); + await alert?.accept(); + } } return new GisPage(); } diff --git a/x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/alert_create_flyout.ts b/x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/alert_create_flyout.ts new file mode 100644 index 0000000000000..7d99d3635106d --- /dev/null +++ b/x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/alert_create_flyout.ts @@ -0,0 +1,142 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import uuid from 'uuid'; +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../ftr_provider_context'; + +function generateUniqueKey() { + return uuid.v4().replace(/-/g, ''); +} + +export default ({ getPageObjects, getService }: FtrProviderContext) => { + const testSubjects = getService('testSubjects'); + const pageObjects = getPageObjects(['common', 'triggersActionsUI', 'header']); + const supertest = getService('supertest'); + const find = getService('find'); + const retry = getService('retry'); + + async function getAlertsByName(name: string) { + const { + body: { data: alerts }, + } = await supertest.get(`/api/alerts/_find?search=${name}&search_fields=name`).expect(200); + + return alerts; + } + + async function deleteAlerts(alertIds: string[]) { + alertIds.forEach(async (alertId: string) => { + await supertest.delete(`/api/alerts/alert/${alertId}`).set('kbn-xsrf', 'foo').expect(204, ''); + }); + } + + async function defineAlert(alertName: string) { + await pageObjects.triggersActionsUI.clickCreateAlertButton(); + await testSubjects.setValue('alertNameInput', alertName); + await testSubjects.click('.index-threshold-SelectOption'); + await testSubjects.click('selectIndexExpression'); + const comboBox = await find.byCssSelector('#indexSelectSearchBox'); + await comboBox.click(); + await comboBox.type('k'); + const filterSelectItem = await find.byCssSelector(`.euiFilterSelectItem`); + await filterSelectItem.click(); + await testSubjects.click('thresholdAlertTimeFieldSelect'); + await retry.try(async () => { + const fieldOptions = await find.allByCssSelector('#thresholdTimeField option'); + expect(fieldOptions[1]).not.to.be(undefined); + await fieldOptions[1].click(); + }); + await testSubjects.click('closePopover'); + // need this two out of popup clicks to close them + const nameInput = await testSubjects.find('alertNameInput'); + await nameInput.click(); + } + + describe('create alert', function () { + before(async () => { + await pageObjects.common.navigateToApp('triggersActions'); + await testSubjects.click('alertsTab'); + }); + + it('should create an alert', async () => { + const alertName = generateUniqueKey(); + await defineAlert(alertName); + + await testSubjects.click('.slack-ActionTypeSelectOption'); + await testSubjects.click('addNewActionConnectorButton-.slack'); + const slackConnectorName = generateUniqueKey(); + await testSubjects.setValue('nameInput', slackConnectorName); + await testSubjects.setValue('slackWebhookUrlInput', 'https://test'); + await find.clickByCssSelector('[data-test-subj="saveActionButtonModal"]:not(disabled)'); + const createdConnectorToastTitle = await pageObjects.common.closeToast(); + expect(createdConnectorToastTitle).to.eql(`Created '${slackConnectorName}'`); + await testSubjects.setValue('messageTextArea', 'test message '); + await testSubjects.click('messageAddVariableButton'); + await testSubjects.click('variableMenuButton-0'); + const messageTextArea = await find.byCssSelector('[data-test-subj="messageTextArea"]'); + expect(await messageTextArea.getAttribute('value')).to.eql('test message {{alertId}}'); + await messageTextArea.type(' some additional text '); + + await testSubjects.click('messageAddVariableButton'); + await testSubjects.click('variableMenuButton-1'); + + expect(await messageTextArea.getAttribute('value')).to.eql( + 'test message {{alertId}} some additional text {{alertInstanceId}}' + ); + + await testSubjects.click('saveAlertButton'); + const toastTitle = await pageObjects.common.closeToast(); + expect(toastTitle).to.eql(`Created alert "${alertName}"`); + await pageObjects.triggersActionsUI.searchAlerts(alertName); + const searchResultsAfterSave = await pageObjects.triggersActionsUI.getAlertsList(); + expect(searchResultsAfterSave).to.eql([ + { + name: alertName, + tagsText: '', + alertType: 'Index threshold', + interval: '1m', + }, + ]); + + // clean up created alert + const alertsToDelete = await getAlertsByName(alertName); + await deleteAlerts(alertsToDelete.map((alertItem: { id: string }) => alertItem.id)); + }); + + it('should show save confirmation before creating alert with no actions', async () => { + const alertName = generateUniqueKey(); + await defineAlert(alertName); + + await testSubjects.click('saveAlertButton'); + await testSubjects.existOrFail('confirmAlertSaveModal'); + await testSubjects.click('confirmAlertSaveModal > confirmModalCancelButton'); + await testSubjects.missingOrFail('confirmAlertSaveModal'); + await find.existsByCssSelector('[data-test-subj="saveAlertButton"]:not(disabled)'); + + await testSubjects.click('saveAlertButton'); + await testSubjects.existOrFail('confirmAlertSaveModal'); + await testSubjects.click('confirmAlertSaveModal > confirmModalConfirmButton'); + await testSubjects.missingOrFail('confirmAlertSaveModal'); + + const toastTitle = await pageObjects.common.closeToast(); + expect(toastTitle).to.eql(`Created alert "${alertName}"`); + await pageObjects.triggersActionsUI.searchAlerts(alertName); + const searchResultsAfterSave = await pageObjects.triggersActionsUI.getAlertsList(); + expect(searchResultsAfterSave).to.eql([ + { + name: alertName, + tagsText: '', + alertType: 'Index threshold', + interval: '1m', + }, + ]); + + // clean up created alert + const alertsToDelete = await getAlertsByName(alertName); + await deleteAlerts(alertsToDelete.map((alertItem: { id: string }) => alertItem.id)); + }); + }); +}; diff --git a/x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/alerts.ts b/x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/alerts_list.ts similarity index 66% rename from x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/alerts.ts rename to x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/alerts_list.ts index ce821b56d3a8a..a69db68c7d35e 100644 --- a/x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/alerts.ts +++ b/x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/alerts_list.ts @@ -14,12 +14,19 @@ function generateUniqueKey() { } export default ({ getPageObjects, getService }: FtrProviderContext) => { + const alerting = getService('alerting'); const testSubjects = getService('testSubjects'); + const find = getService('find'); const pageObjects = getPageObjects(['common', 'triggersActionsUI', 'header']); const supertest = getService('supertest'); - const find = getService('find'); const retry = getService('retry'); + async function deleteAlerts(alertIds: string[]) { + alertIds.forEach(async (alertId: string) => { + await supertest.delete(`/api/alerts/alert/${alertId}`).set('kbn-xsrf', 'foo').expect(204, ''); + }); + } + async function createAlert(overwrites: Record = {}) { const { body: createdAlert } = await supertest .post(`/api/alerts/alert`) @@ -40,132 +47,42 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { return createdAlert; } - async function getAlertsByName(name: string) { - const { - body: { data: alerts }, - } = await supertest.get(`/api/alerts/_find?search=${name}&search_fields=name`).expect(200); - - return alerts; - } - - async function deleteAlerts(alertIds: string[]) { - alertIds.forEach(async (alertId: string) => { - await supertest.delete(`/api/alerts/alert/${alertId}`).set('kbn-xsrf', 'foo').expect(204, ''); - }); + async function createFailingAlert(overwrites: Record = {}) { + const { body: createdAlert } = await supertest + .post(`/api/alerts/alert`) + .set('kbn-xsrf', 'foo') + .send({ + enabled: true, + name: generateUniqueKey(), + tags: ['foo', 'bar'], + alertTypeId: 'test.failing', + consumer: 'alerts', + schedule: { interval: '30s' }, + throttle: '1m', + actions: [], + params: {}, + ...overwrites, + }) + .expect(200); + return createdAlert; } - async function defineAlert(alertName: string) { - await pageObjects.triggersActionsUI.clickCreateAlertButton(); - await testSubjects.setValue('alertNameInput', alertName); - await testSubjects.click('.index-threshold-SelectOption'); - await testSubjects.click('selectIndexExpression'); - const comboBox = await find.byCssSelector('#indexSelectSearchBox'); - await comboBox.click(); - await comboBox.type('k'); - const filterSelectItem = await find.byCssSelector(`.euiFilterSelectItem`); - await filterSelectItem.click(); - await testSubjects.click('thresholdAlertTimeFieldSelect'); - await retry.try(async () => { - const fieldOptions = await find.allByCssSelector('#thresholdTimeField option'); - expect(fieldOptions[1]).not.to.be(undefined); - await fieldOptions[1].click(); - }); - await testSubjects.click('closePopover'); - // need this two out of popup clicks to close them - const nameInput = await testSubjects.find('alertNameInput'); - await nameInput.click(); + async function refreshAlertsList() { + await testSubjects.click('alertsTab'); } - describe('alerts', function () { + describe('alerts list', function () { before(async () => { await pageObjects.common.navigateToApp('triggersActions'); await testSubjects.click('alertsTab'); }); - it('should create an alert', async () => { - const alertName = generateUniqueKey(); - await defineAlert(alertName); - - await testSubjects.click('.slack-ActionTypeSelectOption'); - await testSubjects.click('addNewActionConnectorButton-.slack'); - const slackConnectorName = generateUniqueKey(); - await testSubjects.setValue('nameInput', slackConnectorName); - await testSubjects.setValue('slackWebhookUrlInput', 'https://test'); - await find.clickByCssSelector('[data-test-subj="saveActionButtonModal"]:not(disabled)'); - const createdConnectorToastTitle = await pageObjects.common.closeToast(); - expect(createdConnectorToastTitle).to.eql(`Created '${slackConnectorName}'`); - await testSubjects.setValue('messageTextArea', 'test message '); - await testSubjects.click('messageAddVariableButton'); - await testSubjects.click('variableMenuButton-0'); - const messageTextArea = await find.byCssSelector('[data-test-subj="messageTextArea"]'); - expect(await messageTextArea.getAttribute('value')).to.eql('test message {{alertId}}'); - await messageTextArea.type(' some additional text '); - - await testSubjects.click('messageAddVariableButton'); - await testSubjects.click('variableMenuButton-1'); - - expect(await messageTextArea.getAttribute('value')).to.eql( - 'test message {{alertId}} some additional text {{alertInstanceId}}' - ); - - await testSubjects.click('saveAlertButton'); - const toastTitle = await pageObjects.common.closeToast(); - expect(toastTitle).to.eql(`Created alert "${alertName}"`); - await pageObjects.triggersActionsUI.searchAlerts(alertName); - const searchResultsAfterSave = await pageObjects.triggersActionsUI.getAlertsList(); - expect(searchResultsAfterSave).to.eql([ - { - name: alertName, - tagsText: '', - alertType: 'Index threshold', - interval: '1m', - }, - ]); - - // clean up created alert - const alertsToDelete = await getAlertsByName(alertName); - await deleteAlerts(alertsToDelete.map((alertItem: { id: string }) => alertItem.id)); - }); - - it('should show save confirmation before creating alert with no actions', async () => { - const alertName = generateUniqueKey(); - await defineAlert(alertName); - - await testSubjects.click('saveAlertButton'); - await testSubjects.existOrFail('confirmAlertSaveModal'); - await testSubjects.click('confirmAlertSaveModal > confirmModalCancelButton'); - await testSubjects.missingOrFail('confirmAlertSaveModal'); - await find.existsByCssSelector('[data-test-subj="saveAlertButton"]:not(disabled)'); - - await testSubjects.click('saveAlertButton'); - await testSubjects.existOrFail('confirmAlertSaveModal'); - await testSubjects.click('confirmAlertSaveModal > confirmModalConfirmButton'); - await testSubjects.missingOrFail('confirmAlertSaveModal'); - - const toastTitle = await pageObjects.common.closeToast(); - expect(toastTitle).to.eql(`Created alert "${alertName}"`); - await pageObjects.triggersActionsUI.searchAlerts(alertName); - const searchResultsAfterSave = await pageObjects.triggersActionsUI.getAlertsList(); - expect(searchResultsAfterSave).to.eql([ - { - name: alertName, - tagsText: '', - alertType: 'Index threshold', - interval: '1m', - }, - ]); - - // clean up created alert - const alertsToDelete = await getAlertsByName(alertName); - await deleteAlerts(alertsToDelete.map((alertItem: { id: string }) => alertItem.id)); - }); - it('should display alerts in alphabetical order', async () => { const uniqueKey = generateUniqueKey(); const a = await createAlert({ name: 'b', tags: [uniqueKey] }); const b = await createAlert({ name: 'c', tags: [uniqueKey] }); const c = await createAlert({ name: 'a', tags: [uniqueKey] }); - + await refreshAlertsList(); await pageObjects.triggersActionsUI.searchAlerts(uniqueKey); const searchResults = await pageObjects.triggersActionsUI.getAlertsList(); @@ -179,6 +96,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { it('should search for alert', async () => { const createdAlert = await createAlert(); + await refreshAlertsList(); await pageObjects.triggersActionsUI.searchAlerts(createdAlert.name); const searchResults = await pageObjects.triggersActionsUI.getAlertsList(); @@ -195,6 +113,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { it('should search for tags', async () => { const createdAlert = await createAlert(); + await refreshAlertsList(); await pageObjects.triggersActionsUI.searchAlerts(`${createdAlert.name} foo`); const searchResults = await pageObjects.triggersActionsUI.getAlertsList(); @@ -210,13 +129,17 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { }); it('should display an empty list when search did not return any alerts', async () => { + const createdAlert = await createAlert(); + await refreshAlertsList(); await pageObjects.triggersActionsUI.searchAlerts(`An Alert That For Sure Doesn't Exist!`); expect(await pageObjects.triggersActionsUI.isAlertsListDisplayed()).to.eql(true); + await deleteAlerts([createdAlert.id]); }); it('should disable single alert', async () => { const createdAlert = await createAlert(); + await refreshAlertsList(); await pageObjects.triggersActionsUI.searchAlerts(createdAlert.name); await testSubjects.click('collapsedItemActions'); @@ -235,6 +158,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { it('should re-enable single alert', async () => { const createdAlert = await createAlert(); + await refreshAlertsList(); await pageObjects.triggersActionsUI.searchAlerts(createdAlert.name); await testSubjects.click('collapsedItemActions'); @@ -259,6 +183,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { it('should mute single alert', async () => { const createdAlert = await createAlert(); + await refreshAlertsList(); await pageObjects.triggersActionsUI.searchAlerts(createdAlert.name); await testSubjects.click('collapsedItemActions'); @@ -277,6 +202,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { it('should unmute single alert', async () => { const createdAlert = await createAlert(); + await refreshAlertsList(); await pageObjects.triggersActionsUI.searchAlerts(createdAlert.name); await testSubjects.click('collapsedItemActions'); @@ -302,6 +228,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { it('should delete single alert', async () => { const firstAlert = await createAlert(); const secondAlert = await createAlert(); + await refreshAlertsList(); await pageObjects.triggersActionsUI.searchAlerts(secondAlert.name); await testSubjects.click('collapsedItemActions'); @@ -324,6 +251,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { it('should mute all selection', async () => { const createdAlert = await createAlert(); + await refreshAlertsList(); await pageObjects.triggersActionsUI.searchAlerts(createdAlert.name); await testSubjects.click(`checkboxSelectRow-${createdAlert.id}`); @@ -347,6 +275,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { it('should unmute all selection', async () => { const createdAlert = await createAlert(); + await refreshAlertsList(); await pageObjects.triggersActionsUI.searchAlerts(createdAlert.name); await testSubjects.click(`checkboxSelectRow-${createdAlert.id}`); @@ -372,6 +301,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { it('should disable all selection', async () => { const createdAlert = await createAlert(); + await refreshAlertsList(); await pageObjects.triggersActionsUI.searchAlerts(createdAlert.name); await testSubjects.click(`checkboxSelectRow-${createdAlert.id}`); @@ -395,6 +325,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { it('should enable all selection', async () => { const createdAlert = await createAlert(); + await refreshAlertsList(); await pageObjects.triggersActionsUI.searchAlerts(createdAlert.name); await testSubjects.click(`checkboxSelectRow-${createdAlert.id}`); @@ -424,7 +355,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { const createdAlertsFirstPage = await Promise.all( times(2, () => createAlert({ name: `${namePrefix}-0${count++}` })) ); - + await refreshAlertsList(); await pageObjects.triggersActionsUI.searchAlerts(namePrefix); for (const createdAlert of createdAlertsFirstPage) { @@ -447,5 +378,140 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { const searchResultsAfterDelete = await pageObjects.triggersActionsUI.getAlertsList(); expect(searchResultsAfterDelete).to.have.length(0); }); + + it('should filter alerts by the status', async () => { + const createdAlert = await createAlert(); + const failinfAlert = await createFailingAlert(); + // initialy alert get Pending status, so we need to retry refresh list logic to get the post execution statuses + await retry.try(async () => { + await refreshAlertsList(); + const refreshResults = await pageObjects.triggersActionsUI.getAlertsListWithStatus(); + expect(refreshResults.map((item) => item.status).sort()).to.eql(['Error', 'Ok']); + }); + await testSubjects.click('alertStatusFilterButton'); + await testSubjects.click('alertStatuserrorFilerOption'); // select Error status filter + await retry.try(async () => { + const filterErrorOnlyResults = await pageObjects.triggersActionsUI.getAlertsListWithStatus(); + expect(filterErrorOnlyResults).to.eql([ + { + name: failinfAlert.name, + tagsText: 'foo, bar', + alertType: 'Test: Failing', + interval: '30s', + status: 'Error', + }, + ]); + }); + + await deleteAlerts([createdAlert.id, failinfAlert.id]); + }); + + it('should display total alerts by status and error banner only when exists alerts with status error', async () => { + const createdAlert = await createAlert(); + await retry.try(async () => { + await refreshAlertsList(); + const refreshResults = await pageObjects.triggersActionsUI.getAlertsListWithStatus(); + expect(refreshResults).to.eql([ + { + name: createdAlert.name, + tagsText: 'foo, bar', + alertType: 'Test: Noop', + interval: '1m', + status: 'Ok', + }, + ]); + }); + + const alertsErrorBannerWhenNoErrors = await find.allByCssSelector( + '[data-test-subj="alertsErrorBanner"]' + ); + expect(alertsErrorBannerWhenNoErrors).to.have.length(0); + + const failingAlert = await createFailingAlert(); + await retry.try(async () => { + await refreshAlertsList(); + const alertsErrorBannerExistErrors = await find.allByCssSelector( + '[data-test-subj="alertsErrorBanner"]' + ); + expect(alertsErrorBannerExistErrors).to.have.length(1); + expect( + await ( + await alertsErrorBannerExistErrors[0].findByCssSelector('.euiCallOutHeader') + ).getVisibleText() + ).to.equal('Error found in 1 alert.'); + }); + + expect(await testSubjects.getVisibleText('totalAlertsCount')).to.be( + 'Showing: 2 of 2 alerts.' + ); + expect(await testSubjects.getVisibleText('totalActiveAlertsCount')).to.be('Active: 0'); + expect(await testSubjects.getVisibleText('totalOkAlertsCount')).to.be('Ok: 1'); + expect(await testSubjects.getVisibleText('totalErrorAlertsCount')).to.be('Error: 1'); + expect(await testSubjects.getVisibleText('totalPendingAlertsCount')).to.be('Pending: 0'); + expect(await testSubjects.getVisibleText('totalUnknownAlertsCount')).to.be('Unknown: 0'); + + await deleteAlerts([createdAlert.id, failingAlert.id]); + }); + + it('should filter alerts by the alert type', async () => { + const noopAlert = await createAlert(); + const failinfAlert = await createFailingAlert(); + await refreshAlertsList(); + await testSubjects.click('alertTypeFilterButton'); + await testSubjects.click('alertTypetest.failingFilterOption'); + + await retry.try(async () => { + const filterFailingAlertOnlyResults = await pageObjects.triggersActionsUI.getAlertsList(); + expect(filterFailingAlertOnlyResults).to.eql([ + { + name: failinfAlert.name, + tagsText: 'foo, bar', + alertType: 'Test: Failing', + interval: '30s', + }, + ]); + }); + + await deleteAlerts([noopAlert.id, failinfAlert.id]); + }); + + it('should filter alerts by the action type', async () => { + const noopAlert = await createAlert(); + const action = await alerting.actions.createAction({ + name: `slack-${Date.now()}`, + actionTypeId: '.slack', + config: {}, + secrets: { + webhookUrl: 'https://test', + }, + }); + const noopAlertWithAction = await createAlert({ + actions: [ + { + id: action.id, + actionTypeId: '.slack', + group: 'default', + params: { level: 'info', message: 'gfghfhg' }, + }, + ], + }); + await refreshAlertsList(); + await testSubjects.click('actionTypeFilterButton'); + await testSubjects.click('actionType.slackFilterOption'); + + await retry.try(async () => { + const filterWithSlackOnlyResults = await pageObjects.triggersActionsUI.getAlertsList(); + expect(filterWithSlackOnlyResults).to.eql([ + { + name: noopAlertWithAction.name, + tagsText: 'foo, bar', + alertType: 'Test: Noop', + interval: '1m', + }, + ]); + }); + + await deleteAlerts([noopAlertWithAction.id, noopAlert.id]); + }); }); }; diff --git a/x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/index.ts b/x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/index.ts index 3292184b36056..8600cb6c852f5 100644 --- a/x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/index.ts +++ b/x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/index.ts @@ -11,7 +11,8 @@ export default ({ loadTestFile, getService }: FtrProviderContext) => { this.tags('ciGroup10'); loadTestFile(require.resolve('./home_page')); loadTestFile(require.resolve('./connectors')); - loadTestFile(require.resolve('./alerts')); + loadTestFile(require.resolve('./alerts_list')); + loadTestFile(require.resolve('./alert_create_flyout')); loadTestFile(require.resolve('./details')); }); }; diff --git a/x-pack/test/functional_with_es_ssl/fixtures/plugins/alerts/server/plugin.ts b/x-pack/test/functional_with_es_ssl/fixtures/plugins/alerts/server/plugin.ts index fd7869eac918f..e3927f6bfffb9 100644 --- a/x-pack/test/functional_with_es_ssl/fixtures/plugins/alerts/server/plugin.ts +++ b/x-pack/test/functional_with_es_ssl/fixtures/plugins/alerts/server/plugin.ts @@ -21,16 +21,17 @@ export class AlertingFixturePlugin implements Plugin { + return getRowItemData(row, $); + }); + }, + async getAlertsListWithStatus() { + const table = await find.byCssSelector('[data-test-subj="alertsList"] table'); + const $ = await table.parseDomContent(); + return $.findTestSubjects('alert-row') + .toArray() + .map((row) => { + const rowItem = getRowItemData(row, $); return { - name: $(row) - .findTestSubject('alertsTableCell-name') - .find('.euiTableCellContent') - .text(), - tagsText: $(row) - .findTestSubject('alertsTableCell-tagsText') - .find('.euiTableCellContent') - .text(), - alertType: $(row) - .findTestSubject('alertsTableCell-alertType') - .find('.euiTableCellContent') - .text(), - interval: $(row) - .findTestSubject('alertsTableCell-interval') + ...rowItem, + status: $(row) + .findTestSubject('alertsTableCell-status') .find('.euiTableCellContent') .text(), }; diff --git a/x-pack/test/plugin_api_integration/config.ts b/x-pack/test/plugin_api_integration/config.ts index b89ed6ad550a3..30a361ea2a379 100644 --- a/x-pack/test/plugin_api_integration/config.ts +++ b/x-pack/test/plugin_api_integration/config.ts @@ -43,6 +43,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { '--xpack.eventLog.enabled=true', '--xpack.eventLog.logEntries=true', '--xpack.eventLog.indexEntries=true', + '--xpack.task_manager.monitored_aggregated_stats_refresh_rate=5000', ...plugins.map( (pluginDir) => `--plugin-path=${path.resolve(__dirname, 'plugins', pluginDir)}` ), diff --git a/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/plugin.ts b/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/plugin.ts index 803df6a66ea58..b5d2c98d8cbcd 100644 --- a/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/plugin.ts +++ b/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/plugin.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import _ from 'lodash'; import { Plugin, CoreSetup, CoreStart } from 'src/core/server'; import { EventEmitter } from 'events'; import { Subject } from 'rxjs'; @@ -103,6 +104,29 @@ export class SampleTaskManagerFixturePlugin // fail after the first failed run maxAttempts: 1, }, + sampleRecurringTaskTimingOut: { + title: 'Sample Recurring Task that Times Out', + description: 'A sample task that times out each run.', + maxAttempts: 3, + timeout: '1s', + createTaskRunner: () => ({ + async run() { + return await new Promise((resolve) => {}); + }, + }), + }, + sampleOneTimeTaskTimingOut: { + title: 'Sample One-Time Task that Times Out', + description: 'A sample task that times out each run.', + maxAttempts: 3, + timeout: '1s', + getRetry: (attempts: number, error: object) => new Date(Date.now() + _.random(2, 5) * 1000), + createTaskRunner: () => ({ + async run() { + return await new Promise((resolve) => {}); + }, + }), + }, }); taskManager.addMiddleware({ diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/health_route.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/health_route.ts new file mode 100644 index 0000000000000..9b02b58573673 --- /dev/null +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/health_route.ts @@ -0,0 +1,203 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; +import url from 'url'; +import { keyBy, mapValues } from 'lodash'; +import supertestAsPromised from 'supertest-as-promised'; +import { FtrProviderContext } from '../../ftr_provider_context'; +import { ConcreteTaskInstance } from '../../../../plugins/task_manager/server'; + +interface MonitoringStats { + last_update: string; + status: string; + stats: { + configuration: { + timestamp: string; + value: Record; + }; + workload: { + timestamp: string; + value: { + count: number; + task_types: Record; + schedule: Array<[string, number]>; + overdue: number; + estimated_schedule_density: number[]; + }; + }; + runtime: { + timestamp: string; + value: { + drift: Record; + execution: { + duration: Record>; + result_frequency_percent_as_number: Record>; + }; + polling: { + last_successful_poll: string; + result_frequency_percent_as_number: Record; + }; + }; + }; + }; +} + +export default function ({ getService }: FtrProviderContext) { + const config = getService('config'); + const retry = getService('retry'); + const supertest = supertestAsPromised(url.format(config.get('servers.kibana'))); + + function getHealthRequest() { + return supertest.get('/api/task_manager/_health').set('kbn-xsrf', 'foo'); + } + + function getHealth(): Promise { + return getHealthRequest() + .expect(200) + .then((response) => response.body); + } + + function scheduleTask(task: Partial): Promise { + return supertest + .post('/api/sample_tasks/schedule') + .set('kbn-xsrf', 'xxx') + .send({ task }) + .expect(200) + .then((response: { body: ConcreteTaskInstance }) => response.body); + } + + const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + + const monitoredAggregatedStatsRefreshRate = 5000; + + describe('health', () => { + it('should return basic configuration of task manager', async () => { + const health = await getHealth(); + expect(health.status).to.eql('OK'); + expect(health.stats.configuration.value).to.eql({ + poll_interval: 3000, + max_poll_inactivity_cycles: 10, + monitored_aggregated_stats_refresh_rate: monitoredAggregatedStatsRefreshRate, + monitored_stats_running_average_window: 50, + monitored_task_execution_thresholds: { + custom: {}, + default: { + error_threshold: 90, + warn_threshold: 80, + }, + }, + request_capacity: 1000, + max_workers: 10, + }); + }); + + it('should return the task manager workload', async () => { + const health = await getHealth(); + const { + status, + stats: { workload }, + } = health; + + expect(status).to.eql('OK'); + + const sumSampleTaskInWorkload = + (workload.value.task_types as { + sampleTask?: { count: number }; + }).sampleTask?.count ?? 0; + const scheduledWorkload = (mapValues( + keyBy(workload.value.schedule as Array<[string, number]>, ([interval, count]) => interval), + ([, count]) => count + ) as unknown) as { '37m': number | undefined; '37s': number | undefined }; + + await scheduleTask({ + taskType: 'sampleTask', + schedule: { interval: '37s' }, + }); + + await scheduleTask({ + taskType: 'sampleTask', + schedule: { interval: '37m' }, + }); + + await retry.try(async () => { + // workload is configured to refresh every 5s in FTs + await delay(monitoredAggregatedStatsRefreshRate); + + const workloadAfterScheduling = (await getHealth()).stats.workload.value; + + expect( + (workloadAfterScheduling.task_types as { sampleTask: { count: number } }).sampleTask.count + ).to.eql(sumSampleTaskInWorkload + 2); + + const schedulesWorkloadAfterScheduling = (mapValues( + keyBy( + workloadAfterScheduling.schedule as Array<[string, number]>, + ([interval]) => interval + ), + ([, count]) => count + ) as unknown) as { + '37m': number; + '37s': number; + }; + expect(schedulesWorkloadAfterScheduling['37s']).to.eql(1 + (scheduledWorkload['37s'] ?? 0)); + expect(schedulesWorkloadAfterScheduling['37m']).to.eql(1 + (scheduledWorkload['37m'] ?? 0)); + }); + }); + + it('should return a breakdown of idleTasks in the task manager workload', async () => { + const { + workload: { value: workload }, + } = (await getHealth()).stats; + + expect(typeof workload.overdue).to.eql('number'); + + expect(Array.isArray(workload.estimated_schedule_density)).to.eql(true); + + // test run with the default poll_interval of 3s and a monitored_aggregated_stats_refresh_rate of 5s, + // so we expect the estimated_schedule_density to span a minute (which means 20 buckets, as 60s / 3s = 20) + expect(workload.estimated_schedule_density.length).to.eql(20); + }); + + it('should return the task manager runtime stats', async () => { + await scheduleTask({ + taskType: 'sampleTask', + schedule: { interval: '5s' }, + }); + + const { + runtime: { + value: { drift, polling, execution }, + }, + } = (await getHealth()).stats; + + expect(isNaN(Date.parse(polling.last_successful_poll as string))).to.eql(false); + expect(typeof polling.result_frequency_percent_as_number.NoTasksClaimed).to.eql('number'); + expect(typeof polling.result_frequency_percent_as_number.RanOutOfCapacity).to.eql('number'); + expect(typeof polling.result_frequency_percent_as_number.PoolFilled).to.eql('number'); + + expect(typeof drift.p50).to.eql('number'); + expect(typeof drift.p90).to.eql('number'); + expect(typeof drift.p95).to.eql('number'); + expect(typeof drift.p99).to.eql('number'); + + expect(typeof execution.duration.sampleTask.p50).to.eql('number'); + expect(typeof execution.duration.sampleTask.p90).to.eql('number'); + expect(typeof execution.duration.sampleTask.p95).to.eql('number'); + expect(typeof execution.duration.sampleTask.p99).to.eql('number'); + + expect(typeof execution.result_frequency_percent_as_number.sampleTask.Success).to.eql( + 'number' + ); + expect(typeof execution.result_frequency_percent_as_number.sampleTask.RetryScheduled).to.eql( + 'number' + ); + expect(typeof execution.result_frequency_percent_as_number.sampleTask.Failed).to.eql( + 'number' + ); + }); + }); +} diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/index.js b/x-pack/test/plugin_api_integration/test_suites/task_manager/index.ts similarity index 58% rename from x-pack/test/plugin_api_integration/test_suites/task_manager/index.js rename to x-pack/test/plugin_api_integration/test_suites/task_manager/index.ts index 93350ad4d2c80..b542bff3a4aa9 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/index.js +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/index.ts @@ -4,9 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -export default function ({ loadTestFile }) { +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function ({ loadTestFile }: FtrProviderContext) { describe('task_manager', function taskManagerSuite() { this.tags('ciGroup2'); - loadTestFile(require.resolve('./task_manager_integration')); + loadTestFile(require.resolve('./health_route')); + loadTestFile(require.resolve('./task_management')); }); } diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts similarity index 78% rename from x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js rename to x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts index c87a5039360b8..1fd313c1ac437 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts @@ -8,19 +8,48 @@ import _ from 'lodash'; import expect from '@kbn/expect'; import url from 'url'; import supertestAsPromised from 'supertest-as-promised'; +import { FtrProviderContext } from '../../ftr_provider_context'; +import TaskManagerMapping from '../../../../plugins/task_manager/server/saved_objects/mappings.json'; +import { + DEFAULT_MAX_WORKERS, + DEFAULT_POLL_INTERVAL, +} from '../../../../plugins/task_manager/server/config'; +import { ConcreteTaskInstance } from '../../../../plugins/task_manager/server'; const { task: { properties: taskManagerIndexMapping }, -} = require('../../../../plugins/task_manager/server/saved_objects/mappings.json'); +} = TaskManagerMapping; -const { - DEFAULT_MAX_WORKERS, - DEFAULT_POLL_INTERVAL, -} = require('../../../../plugins/task_manager/server/config.ts'); +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); -const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +export interface RawDoc { + _id: string; + _source: any; + _type?: string; +} +export interface SearchResults { + hits: { + hits: RawDoc[]; + }; +} -export default function ({ getService }) { +type DeprecatedConcreteTaskInstance = Omit & { + interval: string; +}; + +type SerializedConcreteTaskInstance = Omit< + ConcreteTaskInstance, + 'state' | 'params' | 'scheduledAt' | 'startedAt' | 'retryAt' | 'runAt' +> & { + state: State; + params: Params; + scheduledAt: string; + startedAt: string | null; + retryAt: string | null; + runAt: string; +}; + +export default function ({ getService }: FtrProviderContext) { const es = getService('legacyEs'); const log = getService('log'); const retry = getService('retry'); @@ -53,14 +82,18 @@ export default function ({ getService }) { } }); - function currentTasks() { + function currentTasks(): Promise<{ + docs: Array>; + }> { return supertest .get('/api/sample_tasks') .expect(200) .then((response) => response.body); } - function currentTask(task) { + function currentTask( + task: string + ): Promise> { return supertest .get(`/api/sample_tasks/task/${task}`) .send({ task }) @@ -69,32 +102,30 @@ export default function ({ getService }) { } function ensureTasksIndexRefreshed() { - return supertest - .get(`/api/ensure_tasks_index_refreshed`) - .send({}) - .expect(200) - .then((response) => response.body); + return supertest.get(`/api/ensure_tasks_index_refreshed`).send({}).expect(200); } - function historyDocs(taskId) { + function historyDocs(taskId?: string): Promise { return es .search({ index: testHistoryIndex, q: taskId ? `taskId:${taskId}` : 'type:task', }) - .then((result) => result.hits.hits); + .then((result: SearchResults) => result.hits.hits); } - function scheduleTask(task) { + function scheduleTask( + task: Partial + ): Promise { return supertest .post('/api/sample_tasks/schedule') .set('kbn-xsrf', 'xxx') .send({ task }) .expect(200) - .then((response) => response.body); + .then((response: { body: SerializedConcreteTaskInstance }) => response.body); } - function runTaskNow(task) { + function runTaskNow(task: { id: string }) { return supertest .post('/api/sample_tasks/run_now') .set('kbn-xsrf', 'xxx') @@ -103,16 +134,16 @@ export default function ({ getService }) { .then((response) => response.body); } - function scheduleTaskIfNotExists(task) { + function scheduleTaskIfNotExists(task: Partial) { return supertest .post('/api/sample_tasks/ensure_scheduled') .set('kbn-xsrf', 'xxx') .send({ task }) .expect(200) - .then((response) => response.body); + .then((response: { body: ConcreteTaskInstance }) => response.body); } - function releaseTasksWaitingForEventToComplete(event) { + function releaseTasksWaitingForEventToComplete(event: string) { return supertest .post('/api/sample_tasks/event') .set('kbn-xsrf', 'xxx') @@ -120,11 +151,17 @@ export default function ({ getService }) { .expect(200); } - function getTaskById(tasks, id) { + function getTaskById( + tasks: Array>, + id: string + ) { return tasks.filter((task) => task.id === id)[0]; } - async function provideParamsToTasksWaitingForParams(taskId, data = {}) { + async function provideParamsToTasksWaitingForParams( + taskId: string, + data: Record = {} + ) { // wait for task to start running and stall on waitForParams await retry.try(async () => { const tasks = (await currentTasks()).docs; @@ -151,7 +188,7 @@ export default function ({ getService }) { await retry.try(async () => { expect((await historyDocs()).length).to.eql(1); - const [task] = (await currentTasks()).docs; + const [task] = (await currentTasks<{ count: number }>()).docs; log.debug(`Task found: ${task.id}`); log.debug(`Task status: ${task.status}`); log.debug(`Task state: ${JSON.stringify(task.state, null, 2)}`); @@ -236,7 +273,7 @@ export default function ({ getService }) { await retry.try(async () => { expect((await historyDocs(originalTask.id)).length).to.eql(1); - const [task] = (await currentTasks()).docs; + const [task] = (await currentTasks<{ count: number }>()).docs; expect(task.attempts).to.eql(0); expect(task.state.count).to.eql(count + 1); @@ -257,7 +294,7 @@ export default function ({ getService }) { await retry.try(async () => { expect((await historyDocs()).length).to.eql(1); - const [task] = (await currentTasks()).docs; + const [task] = (await currentTasks<{ count: number }>()).docs; expect(task.attempts).to.eql(0); expect(task.state.count).to.eql(1); @@ -278,7 +315,7 @@ export default function ({ getService }) { await retry.try(async () => { expect((await historyDocs()).length).to.eql(1); - const [task] = (await currentTasks()).docs; + const [task] = (await currentTasks<{ count: number }>()).docs; expect(task.attempts).to.eql(0); expect(task.state.count).to.eql(1); @@ -299,7 +336,7 @@ export default function ({ getService }) { 1 ); - const [task] = (await currentTasks()).docs.filter( + const [task] = (await currentTasks<{ count: number }>()).docs.filter( (taskDoc) => taskDoc.id === originalTask.id ); @@ -322,7 +359,7 @@ export default function ({ getService }) { .length ).to.eql(2); - const [task] = (await currentTasks()).docs.filter( + const [task] = (await currentTasks<{ count: number }>()).docs.filter( (taskDoc) => taskDoc.id === originalTask.id ); expect(task.state.count).to.eql(2); @@ -343,7 +380,7 @@ export default function ({ getService }) { const docs = await historyDocs(originalTask.id); expect(docs.length).to.eql(1); - const task = await currentTask(originalTask.id); + const task = await currentTask<{ count: number }>(originalTask.id); expect(task.state.count).to.eql(1); @@ -393,16 +430,16 @@ export default function ({ getService }) { expect(await runNowResult).to.eql({ id: originalTask.id }); await retry.try(async () => { - const task = await currentTask(originalTask.id); + const task = await currentTask<{ count: number }>(originalTask.id); expect(task.state.count).to.eql(2); }); // drain tasks, othrwise they'll keep Task Manager stalled await retry.try(async () => { await releaseTasksWaitingForEventToComplete('releaseTheOthers'); - const tasks = (await currentTasks()).docs.filter( - (task) => task.params.originalParams.waitForEvent === 'releaseTheOthers' - ); + const tasks = ( + await currentTasks<{}, { originalParams: { waitForEvent: string } }>() + ).docs.filter((task) => task.params.originalParams.waitForEvent === 'releaseTheOthers'); expect(tasks.length).to.eql(0); }); }); @@ -420,7 +457,7 @@ export default function ({ getService }) { 1 ); - const task = await currentTask(originalTask.id); + const task = await currentTask<{ count: number }>(originalTask.id); expect(task.state.count).to.eql(1); expect(task.status).to.eql('idle'); @@ -437,7 +474,7 @@ export default function ({ getService }) { expect(successfulRunNowResult).to.eql({ id: originalTask.id }); await retry.try(async () => { - const task = await currentTask(originalTask.id); + const task = await currentTask<{ count: number }>(originalTask.id); expect(task.state.count).to.eql(2); expect(task.status).to.eql('idle'); }); @@ -515,7 +552,7 @@ export default function ({ getService }) { // finish first run by emitting 'runNowHasBeenAttempted' event await releaseTasksWaitingForEventToComplete('runNowHasBeenAttempted'); await retry.try(async () => { - const tasks = (await currentTasks()).docs; + const tasks = (await currentTasks<{ count: number }>()).docs; expect(getTaskById(tasks, longRunningTask.id).state.count).to.eql(1); const task = await currentTask(longRunningTask.id); @@ -564,12 +601,14 @@ export default function ({ getService }) { expect(await runNowResultWithExpectedFailure).to.eql({ id: taskThatFailsBeforeRunNow.id }); }); - async function expectReschedule(originalRunAt, currentTask, expectedDiff) { + async function expectReschedule( + originalRunAt: number, + task: SerializedConcreteTaskInstance, + expectedDiff: number + ) { const buffer = 10000; - expect(Date.parse(currentTask.runAt) - originalRunAt).to.be.greaterThan( - expectedDiff - buffer - ); - expect(Date.parse(currentTask.runAt) - originalRunAt).to.be.lessThan(expectedDiff + buffer); + expect(Date.parse(task.runAt) - originalRunAt).to.be.greaterThan(expectedDiff - buffer); + expect(Date.parse(task.runAt) - originalRunAt).to.be.lessThan(expectedDiff + buffer); } it('should run tasks in parallel, allowing for long running tasks along side faster tasks', async () => { @@ -594,18 +633,47 @@ export default function ({ getService }) { }); await retry.try(async () => { - const tasks = (await currentTasks()).docs; + const tasks = (await currentTasks<{ count: number }>()).docs; expect(getTaskById(tasks, fastTask.id).state.count).to.eql(2); }); await releaseTasksWaitingForEventToComplete('rescheduleHasHappened'); await retry.try(async () => { - const tasks = (await currentTasks()).docs; + const tasks = (await currentTasks<{ count: number }>()).docs; expect(getTaskById(tasks, fastTask.id).state.count).to.greaterThan(2); expect(getTaskById(tasks, longRunningTask.id).state.count).to.eql(1); }); }); + + it('should mark non-recurring task as failed if task is still running but maxAttempts has been reached', async () => { + const task = await scheduleTask({ + taskType: 'sampleOneTimeTaskTimingOut', + params: {}, + }); + + await retry.try(async () => { + const [scheduledTask] = (await currentTasks()).docs; + expect(scheduledTask.id).to.eql(task.id); + expect(scheduledTask.status).to.eql('failed'); + expect(scheduledTask.attempts).to.eql(3); + }); + }); + + it('should continue claiming recurring task even if maxAttempts has been reached', async () => { + const task = await scheduleTask({ + taskType: 'sampleRecurringTaskTimingOut', + schedule: { interval: '1s' }, + params: {}, + }); + + await retry.try(async () => { + const [scheduledTask] = (await currentTasks()).docs; + expect(scheduledTask.id).to.eql(task.id); + expect(scheduledTask.status).to.eql('claiming'); + expect(scheduledTask.attempts).to.eql(4); + }); + }); }); } diff --git a/x-pack/test/tsconfig.json b/x-pack/test/tsconfig.json index 297eb2e9b4540..7bd38ea4afab7 100644 --- a/x-pack/test/tsconfig.json +++ b/x-pack/test/tsconfig.json @@ -26,6 +26,7 @@ { "path": "../../src/plugins/telemetry_collection_manager/tsconfig.json" }, { "path": "../../src/plugins/telemetry/tsconfig.json" }, { "path": "../../src/plugins/kibana_usage_collection/tsconfig.json" }, + { "path": "../plugins/telemetry_collection_xpack/tsconfig.json" }, { "path": "../../src/plugins/newsfeed/tsconfig.json" } ] } diff --git a/x-pack/tsconfig.json b/x-pack/tsconfig.json index 79309369386cf..382d4c073d41e 100644 --- a/x-pack/tsconfig.json +++ b/x-pack/tsconfig.json @@ -13,7 +13,8 @@ "plugins/apm/e2e/cypress/**/*", "plugins/apm/scripts/**/*", "plugins/licensing/**/*", - "plugins/global_search/**/*" + "plugins/global_search/**/*", + "plugins/telemetry_collection_xpack/**/*" ], "compilerOptions": { "paths": { @@ -35,6 +36,7 @@ { "path": "../src/plugins/telemetry_collection_manager/tsconfig.json" }, { "path": "../src/plugins/telemetry/tsconfig.json" }, { "path": "../src/plugins/kibana_usage_collection/tsconfig.json" }, + { "path": "./plugins/telemetry_collection_xpack/tsconfig.json" }, { "path": "../src/plugins/newsfeed/tsconfig.json" }, ] } diff --git a/x-pack/tsconfig.refs.json b/x-pack/tsconfig.refs.json index a389bbcf0272b..a4dfa15a3d8ab 100644 --- a/x-pack/tsconfig.refs.json +++ b/x-pack/tsconfig.refs.json @@ -3,5 +3,6 @@ "references": [ { "path": "./plugins/licensing/tsconfig.json" }, { "path": "./plugins/global_search/tsconfig.json" }, + { "path": "./plugins/telemetry_collection_xpack/tsconfig.json" }, ] } diff --git a/yarn.lock b/yarn.lock index e2f5ed412a14a..18f868440f508 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1218,10 +1218,10 @@ dependencies: "@elastic/apm-rum-core" "^5.7.0" -"@elastic/charts@23.2.1": - version "23.2.1" - resolved "https://registry.yarnpkg.com/@elastic/charts/-/charts-23.2.1.tgz#1f48629fe4597655a7f119fd019c4d5a2cbaf252" - integrity sha512-L2jUPAWwE0xLry6DcqcngVLCa9R32pfz5jW1fyOJRWSq1Fay2swOw4joBe8PmHpvl2s8EwWi9qWBORR1z3hUeQ== +"@elastic/charts@24.0.0": + version "24.0.0" + resolved "https://registry.yarnpkg.com/@elastic/charts/-/charts-24.0.0.tgz#7b97b00a3dc873f46f764de0f28573e236b76aa7" + integrity sha512-ZFIdHcU48Wes7eb1R+48L7xLH4p7D9oSdkoX/iuwt+znD353UhiYK9u+dbrpMXeOMtFYt7dktzVAbouHcJCZPA== dependencies: "@popperjs/core" "^2.4.0" chroma-js "^2.1.0" @@ -9820,10 +9820,10 @@ cypress-promise@^1.1.0: resolved "https://registry.yarnpkg.com/cypress-promise/-/cypress-promise-1.1.0.tgz#f2d66965945fe198431aaf692d5157cea9d47b25" integrity sha512-DhIf5PJ/a0iY+Yii6n7Rbwq+9TJxU4pupXYzf9mZd8nPG0AzQrj9i+pqINv4xbI2EV1p+PKW3maCkR7oPG4GrA== -cypress@^5.0.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/cypress/-/cypress-5.2.0.tgz#6902efd90703242a2539f0623c6e1118aff01f95" - integrity sha512-9S2spcrpIXrQ+CQIKHsjRoLQyRc2ehB06clJXPXXp1zyOL/uZMM3Qc20ipNki4CcNwY0nBTQZffPbRpODeGYQg== +cypress@5.4.0: + version "5.4.0" + resolved "https://registry.yarnpkg.com/cypress/-/cypress-5.4.0.tgz#8833a76e91129add601f823d43c53eb512d162c5" + integrity sha512-BJR+u3DRSYMqaBS1a3l1rbh5AkMRHugbxcYYzkl+xYlO6dzcJVE8uAhghzVI/hxijCyBg1iuSe4TRp/g1PUg8Q== dependencies: "@cypress/listr-verbose-renderer" "^0.4.1" "@cypress/request" "^2.88.5"