From 0d486aec785765c4557887f17379c348c0ab4136 Mon Sep 17 00:00:00 2001 From: Liza Katz Date: Wed, 24 Feb 2021 19:57:27 +0200 Subject: [PATCH 01/13] [Search] Dev docs (#90979) * dev docs * sessions tutorial * title * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Code review Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> --- dev_docs/tutorials/data/search.mdx | 496 +++++++++++++++++++++++++++++ 1 file changed, 496 insertions(+) create mode 100644 dev_docs/tutorials/data/search.mdx diff --git a/dev_docs/tutorials/data/search.mdx b/dev_docs/tutorials/data/search.mdx new file mode 100644 index 0000000000000..69b4d5dab58b5 --- /dev/null +++ b/dev_docs/tutorials/data/search.mdx @@ -0,0 +1,496 @@ +--- +id: kibDevTutorialDataSearchAndSessions +slug: /kibana-dev-docs/tutorials/data/search-and-sessions +title: Kibana data.search Services +summary: Kibana Search Services +date: 2021-02-10 +tags: ['kibana', 'onboarding', 'dev', 'tutorials', 'search', 'sessions', 'search-sessions'] +--- + +## Search service + +### Low level search + +Searching data stored in Elasticsearch can be done in various ways, for example using the Elasticsearch REST API or using an `Elasticsearch Client` for low level access. + +However, the recommended and easist way to search Elasticsearch is by using the low level search service. The service is exposed from the `data` plugin, and by using it, you not only gain access to the data you stored, but also to capabilities, such as Custom Search Strategies, Asynchronous Search, Partial Results, Search Sessions, and more. + +Here is a basic example for using the `data.search` service from a custom plugin: + +```ts +import { CoreStart, Plugin } from 'kibana/public'; +import { DataPublicPluginStart, isCompleteResponse, isErrorResponse } from import { DataPublicPluginStart, isCompleteResponse, isErrorResponse } from '../../src/plugins/data'; + +export interface MyPluginStartDependencies { + data: DataPublicPluginStart; +} + +export class MyPlugin implements Plugin { + public start(core: CoreStart, { data }: MyPluginStartDependencies) { + const query = { + filter: [{ + match_all: {} + }], + }; + const req = { + params: { + index: 'my-index-*', + body: { + query, + aggs: {}, + }, + } + }; + data.search.search(req).subscribe({ + next: (result) => { + if (isCompleteResponse(res)) { + // handle search result + } else if (isErrorResponse(res)) { + // handle error, this means that some results were returned, but the search has failed to complete. + } else { + // handle partial results if you want. + } + }, + error: (e) => { + // handle error thrown, for example a server hangup + }, + }) + } +} +``` + +Note: The `data` plugin contains services to help you generate the `query` and `aggs` portions, as well as managing indices using the `data.indexPatterns` service. + + + The `data.search` service is available on both server and client, with similar APIs. + + +#### Error handling + +The `search` method can throw several types of errors, for example: + + - `EsError` for errors originating in Elasticsearch errors + - `PainlessError` for errors originating from a Painless script + - `AbortError` if the search was aborted via an `AbortController` + - `HttpError` in case of a network error + +To display the errors in the context of an application, use the helper method provided on the `data.search` service. These errors are shown in a toast message, using the `core.notifications` service. + +```ts +data.search.search(req).subscribe({ + next: (result) => {}, + error: (e) => { + data.search.showError(e); + }, +}) +``` + +If you decide to handle errors by yourself, watch for errors coming from `Elasticsearch`. They have an additional `attributes` property that holds the raw error from `Elasticsearch`. + +```ts +data.search.search(req).subscribe({ + next: (result) => {}, + error: (e) => { + if (e instanceof IEsError) { + showErrorReason(e.attributes); + } + }, +}) +``` + +#### Stop a running search + +The search service `search` method supports a second argument called `options`. One of these options provides an `abortSignal` to stop searches from running to completion, if the result is no longer needed. + +```ts +import { AbortError } from '../../src/data/public'; + +const abortController = new AbortController(); +data.search.search(req, { + abortSignal: abortController.signal, +}).subscribe({ + next: (result) => { + // handle result + }, + error: (e) => { + if (e instanceof AbortError) { + // you can ignore this error + return; + } + // handle error, for example a server hangup + }, +}); + +// Abort the search request after a second +setTimeout(() => { + abortController.abort(); +}, 1000); +``` + +#### Search strategies + +By default, the search service uses the DSL query and aggregation syntax and returns the response from Elasticsearch as is. It also provides several additional basic strategies, such as Async DSL (`x-pack` default) and EQL. + +For example, to run an EQL query using the `data.search` service, you should to specify the strategy name using the options parameter: + +```ts +const req = getEqlRequest(); +data.search.search(req, { + strategy: EQL_SEARCH_STRATEGY, +}).subscribe({ + next: (result) => { + // handle EQL result + }, +}); +``` + +##### Custom search strategies + +To use a different query syntax, preprocess the request, or process the response before returning it to the client, you can create and register a custom search strategy to encapsulate your custom logic. + +The following example shows how to define, register, and use a search strategy that preprocesses the request before sending it to the default DSL search strategy, and then processes the response before returning. + +```ts +// ./myPlugin/server/myStrategy.ts + +/** + * Your custom search strategy should implement the ISearchStrategy interface, requiring at minimum a `search` function. + */ +export const mySearchStrategyProvider = ( + data: PluginStart +): ISearchStrategy => { + const preprocessRequest = (request: IMyStrategyRequest) => { + // Custom preprocessing + } + + const formatResponse = (response: IMyStrategyResponse) => { + // Custom post-processing + } + + // Get the default search strategy + const es = data.search.getSearchStrategy(ES_SEARCH_STRATEGY); + return { + search: (request, options, deps) => { + return formatResponse(es.search(preprocessRequest(request), options, deps)); + }, + }; +}; +``` + +```ts +// ./myPlugin/server/plugin.ts +import type { + CoreSetup, + CoreStart, + Plugin, +} from 'kibana/server'; + +import { mySearchStrategyProvider } from './my_strategy'; + +/** + * Your plugin will receive the `data` plugin contact in both the setup and start lifecycle hooks. + */ +export interface MyPluginSetupDeps { + data: PluginSetup; +} + +export interface MyPluginStartDeps { + data: PluginStart; +} + +/** + * In your custom server side plugin, register the strategy from the setup contract + */ +export class MyPlugin implements Plugin { + public setup( + core: CoreSetup, + deps: MyPluginSetupDeps + ) { + core.getStartServices().then(([_, depsStart]) => { + const myStrategy = mySearchStrategyProvider(depsStart.data); + deps.data.search.registerSearchStrategy('myCustomStrategy', myStrategy); + }); + } +} +``` + +```ts +// ./myPlugin/public/plugin.ts +const req = getRequest(); +data.search.search(req, { + strategy: 'myCustomStrategy', +}).subscribe({ + next: (result) => { + // handle result + }, +}); +``` + +##### Async search and custom async search strategies + +The open source default search strategy (`ES_SEARCH_STRATEGY`), run searches synchronously, keeping an open connection to Elasticsearch while the query executes. The duration of these queries is restricted by the `elasticsearch.requestTimeout` setting in `kibana.yml`, which is 30 seconds by default. + +This synchronous execution works great in most cases. However, with the introduction of features such as `data tiers` and `runtime fields`, the need to allow slower-running queries, where holding an open connection might be inefficient, has increased. In 7.7, `Elasticsearch` introduced the [async_search API](https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html), allowing a query to run longer without keeping an open connection. Instead, the initial search request returns an ID that identifies the search running in `Elasticsearch`. This ID can then be used to retrieve, cancel, or manage the search result. + +The [async_search API](https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html) is what drives more advanced `Kibana` `search` features, such as `partial results` and `search sessions`. [When available](https://www.elastic.co/subscriptions), the default search strategy of `Kibana` is automatically set to the **async** default search strategy (`ENHANCED_ES_SEARCH_STRATEGY`), empowering Kibana to run longer queries, with an **optional** duration restriction defined by the UI setting `search:timeout`. + +If you are implementing your own async custom search strategy, make sure to implement `cancel` and `extend`, as shown in the following example: + +```ts +// ./myPlugin/server/myEnhancedStrategy.ts +export const myEnhancedSearchStrategyProvider = ( + data: PluginStart +): ISearchStrategy => { + // Get the default search strategy + const ese = data.search.getSearchStrategy(ENHANCED_ES_SEARCH_STRATEGY); + return { + search: (request, options, deps) => { + // search will be called multiple times, + // be sure your response formatting is capable of handling partial results, as well as the final result. + return formatResponse(ese.search(request, options, deps)); + }, + cancel: async (id, options, deps) => { + // call the cancel method of the async strategy you are using or implement your own cancellation function. + await ese.cancel(id, options, deps); + }, + extend: async (id, keepAlive, options, deps) => { + // async search results are not stored indefinitely. By default, they expire after 7 days (or as defined by xpack.data_enhanced.search.sessions.defaultExpiration setting in kibana.yml). + // call the extend method of the async strategy you are using or implement your own extend function. + await ese.extend(id, options, deps); + }, + }; +}; +``` + +### High level search + +The high level search service is a simplified way to create and run search requests, without writing custom DSL queries. + +#### Search source + +```ts +function searchWithSearchSource() { + const indexPattern = data.indexPatterns.getDefault(); + const query = data.query.queryString.getQuery(); + const filters = data.query.filterManager.getFilters(); + const timefilter = data.query.timefilter.timefilter.createFilter(indexPattern); + if (timefilter) { + filters.push(timefilter); + } + + const searchSource = await data.search.searchSource.create(); + + searchSource + .setField('index', indexPattern) + .setField('filter', filters) + .setField('query', query) + .setField('fields', selectedFields.length ? selectedFields.map((f) => f.name) : ['*']) + .setField('aggs', getAggsDsl()); + + searchSource.fetch$().subscribe({ + next: () => {}, + error: () => {}, + }); +} +``` + +### Partial results + +When searching using an `async` strategy (such as async DSL and async EQL), the search service will stream back partial results. + +Although you can ignore the partial results and wait for the final result before rendering, you can also use the partial results to create a more interactive experience for your users. It is highly advised, however, to make sure users are aware that the results they are seeing are partial. + +```ts +// Handling partial results +data.search.search(req).subscribe({ + next: (result) => { + if (isCompleteResponse(res)) { + renderFinalResult(res); + } else if (isPartialResponse(res)) { + renderPartialResult(res); + } + }, +}) + +// Skipping partial results +const finalResult = await data.search.search(req).toPromise(); +``` + +### Search sessions + +A search session is a higher level concept than search. A search session describes a grouping of one or more async search requests with additional context. + +Search sessions are handy when you want to enable a user to run something asynchronously (for example, a dashboard over a long period of time), and then quickly restore the results at a later time. The `Search Service` transparently fetches results from the `.async-search` index, instead of running each request again. + +Internally, any search run within a search session is saved into an object, allowing Kibana to manage their lifecycle. Most saved objects are deleted automatically after a short period of time, but if a user chooses to save the search session, the saved object is persisted, so that results can be restored in a later time. + +Stored search sessions are listed in the *Management* application, under *Kibana > Search Sessions*, making it easy to find, manage, and restore them. + +As a developer, you might encounter these two common, use cases: + + * Running a search inside an existing search session + * Supporting search sessions in your application + +#### Running a search inside an existing search session + +For this example, assume you are implementing a new type of `Embeddable` that will be shown on dashboards. The same principle applies, however, to any search requests that you are running, as long as the application you are running inside is managing an active session. + +Because the Dashboard application is already managing a search session, all you need to do is pass down the `searchSessionId` argument to any `search` call. This applies to both the low and high level search APIs. + +The search information will be added to the saved object for the search session. + +```ts +export class SearchEmbeddable + extends Embeddable { + + private async fetchData() { + // Every embeddable receives an optional `searchSessionId` input parameter. + const { searchSessionId } = this.input; + + // Setup your search source + this.configureSearchSource(); + + try { + // Mark the embeddable as loading + this.updateOutput({ loading: true, error: undefined }); + + // Make the request, wait for the final result + const resp = await searchSource.fetch$({ + sessionId: searchSessionId, + }).toPromise(); + + this.useSearchResult(resp); + + this.updateOutput({ loading: false, error: undefined }); + } catch (error) { + // handle search errors + this.updateOutput({ loading: false, error }); + } + } +} + +``` + +You can also retrieve the active `Search Session ID` from the `Search Service` directly: + +```ts +async function fetchData(data: DataPublicPluginStart) { + try { + return await searchSource.fetch$({ + sessionId: data.search.sessions.getSessionId(), + }).toPromise(); + } catch (e) { + // handle search errors + } +} + +``` + + + Search sessions are initiated by the client. If you are using a route that runs server side searches, you can send the `searchSessionId` to the server, and then pass it down to the server side `data.search` function call. + + +#### Supporting search sessions in your application + +Before implementing the ability to create and restore search sessions in your application, ask yourself the following questions: + +1. **Does your application normally run long operations?** For example, it makes sense for a user to generate a Dashboard or a Canvas report from data stored in cold storage. However, when editing a single visualization, it is best to work with a shorter timeframe of hot or warm data. +2. **Does it make sense for your application to restore a search session?** For example, you might want to restore an interesting configuration of filters of older documents you found in Discover. However, a single Lens or Map visualization might not be as helpful, outside the context of a specific dashboard. +3. **What is a search session in the context of your application?** Although Discover and Dashboard start a new search session every time the time range or filters change, or when the user clicks **Refresh**, you can manage your sessions differently. For example, if your application has tabs, you might group searches from multiple tabs into a single search session. You must be able to clearly define the **state** used to create the search session`. The **state** refers to any setting that might change the queries being set to `Elasticsearch`. + +Once you answer those questions, proceed to implement the following bits of code in your application. + +##### Provide storage configuration + +In your plugin's `start` lifecycle method, call the `enableStorage` method. This method helps the `Session Service` gather the information required to save the search sessions upon a user's request and construct the restore state: + +```ts +export class MyPlugin implements Plugin { + public start(core: CoreStart, { data }: MyPluginStartDependencies) { + const sessionRestorationDataProvider: SearchSessionInfoProvider = { + data, + getDashboard + } + + data.search.session.enableStorage({ + getName: async () => { + // return the name you want to give the saved Search Session + return `MyApp_${Math.random()}`; + }, + getUrlGeneratorData: async () => { + return { + urlGeneratorId: MY_URL_GENERATOR, + initialState: getUrlGeneratorState({ ...deps, shouldRestoreSearchSession: false }), + restoreState: getUrlGeneratorState({ ...deps, shouldRestoreSearchSession: true }), + }; + }, + }); + } +} +``` + + + The restore state of a search session may be different from the initial state used to create it. For example, where the initial state may contain relative dates, in the restore state, those must be converted to absolute dates. Read more about the [NowProvider](). + + + + Calling `enableStorage` will also enable the `Search Session Indicator` component in the chrome component of your solution. The `Search Session Indicator` is a small button, used by default to engage users and save new search sessions. To implement your own UI, contact the Kibana application services team to decouple this behavior. + + +##### Start a new search session + +Make sure to call `start` when the **state** you previously defined changes. + +```ts + +function onSearchSessionConfigChange() { + this.searchSessionId = data.search.sessions.start(); +} + +``` + +Pass the `searchSessionId` to every `search` call inside your application. If you're using `Embeddables`, pass down the `searchSessionId` as `input`. + +If you can't pass the `searchSessionId` directly, you can retrieve it from the service. + +```ts +const currentSearchSessionId = data.search.sessions.getSessionId(); + +``` + +##### Clear search sessions + +Creating a new search session clears the previous one. You must explicitly `clear` the search session when your application is being destroyed: + +```ts +function onDestroy() { + data.search.session.clear(); +} + +``` + +If you don't call `clear`, you will see a warning in the console while developing. However, when running in production, you will get a fatal error. This is done to avoid leakage of unrelated search requests into an existing search session left open by mistake. + +##### Restore search sessions + +The last step of the integration is restoring an existing search session. The `searchSessionId` parameter and the rest of the restore state are passed into the application via the URL. Non-URL support is planned for future releases. + +If you detect the presense of a `searchSessionId` parameter in the URL, call the `restore` method **instead** of calling `start`. The previous example would now become: + +```ts + +function onSearchSessionConfigChange(searchSessionIdFromUrl?: string) { + if (searchSessionIdFromUrl) { + data.search.sessions.restore(searchSessionIdFromUrl); + } else { + data.search.sessions.start(); + } +} + +``` + +Once you `restore` the session, as long as all `search` requests run with the same `searchSessionId`, the search session should be seamlessly restored. + +##### Customize the user experience + +TBD From 96e34d96b1f670edd99ee2a7ce95022a7f00d5f5 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen <43350163+qn895@users.noreply.github.com> Date: Wed, 24 Feb 2021 12:10:04 -0600 Subject: [PATCH 02/13] [ML] Add functional tests for Transform start/delete (#92314) --- .../action_delete/delete_action_modal.tsx | 1 + .../action_start/start_action_modal.tsx | 1 + .../test/functional/apps/transform/cloning.ts | 2 +- .../functional/apps/transform/deleting.ts | 136 ++++++++++++++++ .../test/functional/apps/transform/editing.ts | 2 +- .../test/functional/apps/transform/index.ts | 39 ++++- .../functional/apps/transform/starting.ts | 106 ++++++++++++ .../services/transform/management.ts | 3 + .../services/transform/transform_table.ts | 152 ++++++++++++++++-- 9 files changed, 426 insertions(+), 16 deletions(-) create mode 100644 x-pack/test/functional/apps/transform/deleting.ts create mode 100644 x-pack/test/functional/apps/transform/starting.ts diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/delete_action_modal.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/delete_action_modal.tsx index d82f0769c8b74..fb846d041bd17 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/delete_action_modal.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/delete_action_modal.tsx @@ -123,6 +123,7 @@ export const DeleteActionModal: FC = ({ return ( = ({ closeModal, items, startAndC return ( { + await esArchiver.loadIfNeeded('ml/ecommerce'); + await transform.testResources.createIndexPatternIfNeeded('ft_ecommerce', 'order_date'); + + for (const testData of testDataList) { + await transform.api.createAndRunTransform( + testData.originalConfig.id, + testData.originalConfig + ); + } + + await transform.testResources.setKibanaTimeZoneToUTC(); + await transform.securityUI.loginAsTransformPowerUser(); + }); + + after(async () => { + for (const testData of testDataList) { + await transform.testResources.deleteIndexPatternByTitle(testData.originalConfig.dest.index); + await transform.api.deleteIndices(testData.originalConfig.dest.index); + } + await transform.api.cleanTransformIndices(); + }); + + for (const testData of testDataList) { + describe(`${testData.suiteTitle}`, function () { + it('delete transform', async () => { + await transform.testExecution.logTestStep('should load the home page'); + await transform.navigation.navigateTo(); + await transform.management.assertTransformListPageExists(); + + await transform.testExecution.logTestStep('should display the transforms table'); + await transform.management.assertTransformsTableExists(); + + if (testData.expected.row.mode === 'continuous') { + await transform.testExecution.logTestStep('should have the delete action disabled'); + await transform.table.assertTransformRowActionEnabled( + testData.originalConfig.id, + 'Delete', + false + ); + + await transform.testExecution.logTestStep('should stop the transform'); + await transform.table.clickTransformRowActionWithRetry( + testData.originalConfig.id, + 'Stop' + ); + } + + await transform.testExecution.logTestStep('should display the stopped transform'); + await transform.table.assertTransformRowFields(testData.originalConfig.id, { + id: testData.originalConfig.id, + description: testData.originalConfig.description, + status: testData.expected.row.status, + mode: testData.expected.row.mode, + progress: testData.expected.row.progress, + }); + + await transform.testExecution.logTestStep('should show the delete modal'); + await transform.table.assertTransformRowActionEnabled( + testData.originalConfig.id, + 'Delete', + true + ); + await transform.table.clickTransformRowActionWithRetry( + testData.originalConfig.id, + 'Delete' + ); + await transform.table.assertTransformDeleteModalExists(); + + await transform.testExecution.logTestStep('should delete the transform'); + await transform.table.confirmDeleteTransform(); + await transform.table.assertTransformRowNotExists(testData.originalConfig.id); + }); + }); + } + }); +} diff --git a/x-pack/test/functional/apps/transform/editing.ts b/x-pack/test/functional/apps/transform/editing.ts index 71a7cf02df1fd..1f0bb058bdc38 100644 --- a/x-pack/test/functional/apps/transform/editing.ts +++ b/x-pack/test/functional/apps/transform/editing.ts @@ -109,7 +109,7 @@ export default function ({ getService }: FtrProviderContext) { await transform.table.filterWithSearchString(testData.originalConfig.id, 1); await transform.testExecution.logTestStep('should show the actions popover'); - await transform.table.assertTransformRowActions(false); + await transform.table.assertTransformRowActions(testData.originalConfig.id, false); await transform.testExecution.logTestStep('should show the edit flyout'); await transform.table.clickTransformRowAction('Edit'); diff --git a/x-pack/test/functional/apps/transform/index.ts b/x-pack/test/functional/apps/transform/index.ts index 63d8d0b51bc8c..1440f0a3f9a09 100644 --- a/x-pack/test/functional/apps/transform/index.ts +++ b/x-pack/test/functional/apps/transform/index.ts @@ -6,7 +6,10 @@ */ import { FtrProviderContext } from '../../ftr_provider_context'; -import { TransformLatestConfig } from '../../../../plugins/transform/common/types/transform'; +import { + TransformLatestConfig, + TransformPivotConfig, +} from '../../../../plugins/transform/common/types/transform'; export default function ({ getService, loadTestFile }: FtrProviderContext) { const esArchiver = getService('esArchiver'); @@ -41,6 +44,8 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./cloning')); loadTestFile(require.resolve('./editing')); loadTestFile(require.resolve('./feature_controls')); + loadTestFile(require.resolve('./deleting')); + loadTestFile(require.resolve('./starting')); }); } export interface ComboboxOption { @@ -80,20 +85,46 @@ export function isLatestTransformTestData(arg: any): arg is LatestTransformTestD return arg.type === 'latest'; } -export function getLatestTransformConfig(): TransformLatestConfig { +export function getPivotTransformConfig( + prefix: string, + continuous?: boolean +): TransformPivotConfig { const timestamp = Date.now(); return { - id: `ec_cloning_2_${timestamp}`, + id: `ec_${prefix}_pivot_${timestamp}_${continuous ? 'cont' : 'batch'}`, + source: { index: ['ft_ecommerce'] }, + pivot: { + group_by: { category: { terms: { field: 'category.keyword' } } }, + aggregations: { 'products.base_price.avg': { avg: { field: 'products.base_price' } } }, + }, + description: `ecommerce ${ + continuous ? 'continuous' : 'batch' + } transform with avg(products.base_price) grouped by terms(category.keyword)`, + dest: { index: `user-ec_2_${timestamp}` }, + ...(continuous ? { sync: { time: { field: 'order_date', delay: '60s' } } } : {}), + }; +} + +export function getLatestTransformConfig( + prefix: string, + continuous?: boolean +): TransformLatestConfig { + const timestamp = Date.now(); + return { + id: `ec_${prefix}_latest_${timestamp}_${continuous ? 'cont' : 'batch'}`, source: { index: ['ft_ecommerce'] }, latest: { unique_key: ['category.keyword'], sort: 'order_date', }, - description: 'ecommerce batch transform with category unique key and sorted by order date', + description: `ecommerce ${ + continuous ? 'continuous' : 'batch' + } transform with category unique key and sorted by order date`, frequency: '3s', settings: { max_page_search_size: 250, }, dest: { index: `user-ec_3_${timestamp}` }, + ...(continuous ? { sync: { time: { field: 'order_date', delay: '60s' } } } : {}), }; } diff --git a/x-pack/test/functional/apps/transform/starting.ts b/x-pack/test/functional/apps/transform/starting.ts new file mode 100644 index 0000000000000..4b0b6f8dade66 --- /dev/null +++ b/x-pack/test/functional/apps/transform/starting.ts @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../ftr_provider_context'; +import { TRANSFORM_STATE } from '../../../../plugins/transform/common/constants'; +import { getLatestTransformConfig, getPivotTransformConfig } from './index'; + +export default function ({ getService }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const transform = getService('transform'); + + describe('starting', function () { + const PREFIX = 'starting'; + const testDataList = [ + { + suiteTitle: 'batch transform with pivot configuration', + originalConfig: getPivotTransformConfig(PREFIX, false), + mode: 'batch', + }, + { + suiteTitle: 'continuous transform with pivot configuration', + originalConfig: getPivotTransformConfig(PREFIX, true), + mode: 'continuous', + }, + { + suiteTitle: 'batch transform with latest configuration', + originalConfig: getLatestTransformConfig(PREFIX, false), + mode: 'batch', + }, + { + suiteTitle: 'continuous transform with latest configuration', + originalConfig: getLatestTransformConfig(PREFIX, true), + mode: 'continuous', + }, + ]; + + before(async () => { + await esArchiver.loadIfNeeded('ml/ecommerce'); + await transform.testResources.createIndexPatternIfNeeded('ft_ecommerce', 'order_date'); + + for (const testData of testDataList) { + await transform.api.createTransform(testData.originalConfig.id, testData.originalConfig); + } + await transform.testResources.setKibanaTimeZoneToUTC(); + await transform.securityUI.loginAsTransformPowerUser(); + }); + + after(async () => { + for (const testData of testDataList) { + await transform.testResources.deleteIndexPatternByTitle(testData.originalConfig.dest.index); + await transform.api.deleteIndices(testData.originalConfig.dest.index); + } + + await transform.api.cleanTransformIndices(); + }); + + for (const testData of testDataList) { + const transformId = testData.originalConfig.id; + + describe(`${testData.suiteTitle}`, function () { + it('start transform', async () => { + await transform.testExecution.logTestStep('should load the home page'); + await transform.navigation.navigateTo(); + await transform.management.assertTransformListPageExists(); + + await transform.testExecution.logTestStep('should display the transforms table'); + await transform.management.assertTransformsTableExists(); + + await transform.testExecution.logTestStep( + 'should display the original transform in the transform list' + ); + await transform.table.filterWithSearchString(transformId, 1); + + await transform.testExecution.logTestStep('should start the transform'); + await transform.table.assertTransformRowActionEnabled(transformId, 'Start', true); + await transform.table.clickTransformRowActionWithRetry(transformId, 'Start'); + await transform.table.confirmStartTransform(); + await transform.table.clearSearchString(testDataList.length); + + if (testData.mode === 'continuous') { + await transform.testExecution.logTestStep('should display the started transform'); + await transform.table.assertTransformRowStatusNotEql( + testData.originalConfig.id, + TRANSFORM_STATE.STOPPED + ); + } else { + await transform.table.assertTransformRowProgressGreaterThan(transformId, 0); + } + + await transform.table.assertTransformRowStatusNotEql( + testData.originalConfig.id, + TRANSFORM_STATE.FAILED + ); + await transform.table.assertTransformRowStatusNotEql( + testData.originalConfig.id, + TRANSFORM_STATE.ABORTING + ); + }); + }); + } + }); +} diff --git a/x-pack/test/functional/services/transform/management.ts b/x-pack/test/functional/services/transform/management.ts index fdfd1d1d9b40f..807c3d49e344c 100644 --- a/x-pack/test/functional/services/transform/management.ts +++ b/x-pack/test/functional/services/transform/management.ts @@ -5,8 +5,11 @@ * 2.0. */ +import { ProvidedType } from '@kbn/test/types/ftr'; import { FtrProviderContext } from '../../ftr_provider_context'; +export type TransformManagement = ProvidedType; + export function TransformManagementProvider({ getService }: FtrProviderContext) { const testSubjects = getService('testSubjects'); diff --git a/x-pack/test/functional/services/transform/transform_table.ts b/x-pack/test/functional/services/transform/transform_table.ts index 72626580e9461..ce2625677e479 100644 --- a/x-pack/test/functional/services/transform/transform_table.ts +++ b/x-pack/test/functional/services/transform/transform_table.ts @@ -12,6 +12,7 @@ import { FtrProviderContext } from '../../ftr_provider_context'; export function TransformTableProvider({ getService }: FtrProviderContext) { const retry = getService('retry'); const testSubjects = getService('testSubjects'); + const browser = getService('browser'); return new (class TransformTable { public async parseTransformTable() { @@ -129,21 +130,63 @@ export function TransformTableProvider({ getService }: FtrProviderContext) { const filteredRows = rows.filter((row) => row.id === filter); expect(filteredRows).to.have.length( expectedRowCount, - `Filtered DFA job table should have ${expectedRowCount} row(s) for filter '${filter}' (got matching items '${filteredRows}')` + `Filtered Transform table should have ${expectedRowCount} row(s) for filter '${filter}' (got matching items '${filteredRows}')` ); } - public async assertTransformRowFields(transformId: string, expectedRow: object) { + public async clearSearchString(expectedRowCount: number = 1) { + await this.waitForTransformsToLoad(); + const tableListContainer = await testSubjects.find('transformListTableContainer'); + const searchBarInput = await tableListContainer.findByClassName('euiFieldSearch'); + await searchBarInput.clearValueWithKeyboard(); const rows = await this.parseTransformTable(); - const transformRow = rows.filter((row) => row.id === transformId)[0]; - expect(transformRow).to.eql( - expectedRow, - `Expected transform row to be '${JSON.stringify(expectedRow)}' (got '${JSON.stringify( - transformRow - )}')` + expect(rows).to.have.length( + expectedRowCount, + `Transform table should have ${expectedRowCount} row(s) after clearing search' (got '${rows.length}')` ); } + public async assertTransformRowFields(transformId: string, expectedRow: object) { + await retry.tryForTime(30 * 1000, async () => { + await this.refreshTransformList(); + const rows = await this.parseTransformTable(); + const transformRow = rows.filter((row) => row.id === transformId)[0]; + expect(transformRow).to.eql( + expectedRow, + `Expected transform row to be '${JSON.stringify(expectedRow)}' (got '${JSON.stringify( + transformRow + )}')` + ); + }); + } + + public async assertTransformRowProgressGreaterThan( + transformId: string, + expectedProgress: number + ) { + await retry.tryForTime(30 * 1000, async () => { + await this.refreshTransformList(); + const rows = await this.parseTransformTable(); + const transformRow = rows.filter((row) => row.id === transformId)[0]; + expect(transformRow.progress).to.greaterThan( + 0, + `Expected transform row progress to be greater than '${expectedProgress}' (got '${transformRow.progress}')` + ); + }); + } + + public async assertTransformRowStatusNotEql(transformId: string, status: string) { + await retry.tryForTime(30 * 1000, async () => { + await this.refreshTransformList(); + const rows = await this.parseTransformTable(); + const transformRow = rows.filter((row) => row.id === transformId)[0]; + expect(transformRow.status).to.not.eql( + status, + `Expected transform row status to not be '${status}' (got '${transformRow.status}')` + ); + }); + } + public async assertTransformExpandedRow() { await testSubjects.click('transformListRowDetailsToggle'); @@ -185,8 +228,13 @@ export function TransformTableProvider({ getService }: FtrProviderContext) { }); } - public async assertTransformRowActions(isTransformRunning = false) { - await testSubjects.click('euiCollapsedItemActionsButton'); + public rowSelector(transformId: string, subSelector?: string) { + const row = `~transformListTable > ~row-${transformId}`; + return !subSelector ? row : `${row} > ${subSelector}`; + } + + public async assertTransformRowActions(transformId: string, isTransformRunning = false) { + await testSubjects.click(this.rowSelector(transformId, 'euiCollapsedItemActionsButton')); await testSubjects.existOrFail('transformActionClone'); await testSubjects.existOrFail('transformActionDelete'); @@ -201,6 +249,42 @@ export function TransformTableProvider({ getService }: FtrProviderContext) { } } + public async assertTransformRowActionEnabled( + transformId: string, + action: 'Delete' | 'Start' | 'Stop' | 'Clone' | 'Edit', + expectedValue: boolean + ) { + const selector = `transformAction${action}`; + await retry.tryForTime(60 * 1000, async () => { + await this.refreshTransformList(); + + await browser.pressKeys(browser.keys.ESCAPE); + await testSubjects.click(this.rowSelector(transformId, 'euiCollapsedItemActionsButton')); + + await testSubjects.existOrFail(selector); + const isEnabled = await testSubjects.isEnabled(selector); + expect(isEnabled).to.eql( + expectedValue, + `Expected '${action}' button to be '${expectedValue ? 'enabled' : 'disabled'}' (got '${ + isEnabled ? 'enabled' : 'disabled' + }')` + ); + }); + } + + public async clickTransformRowActionWithRetry( + transformId: string, + action: 'Delete' | 'Start' | 'Stop' | 'Clone' | 'Edit' + ) { + await retry.tryForTime(30 * 1000, async () => { + await browser.pressKeys(browser.keys.ESCAPE); + await testSubjects.click(this.rowSelector(transformId, 'euiCollapsedItemActionsButton')); + await testSubjects.existOrFail(`transformAction${action}`); + await testSubjects.click(`transformAction${action}`); + await testSubjects.missingOrFail(`transformAction${action}`); + }); + } + public async clickTransformRowAction(action: string) { await testSubjects.click(`transformAction${action}`); } @@ -214,5 +298,53 @@ export function TransformTableProvider({ getService }: FtrProviderContext) { await this.waitForTransformsExpandedRowPreviewTabToLoad(); await this.assertEuiDataGridColumnValues('transformPivotPreview', column, values); } + + public async assertTransformDeleteModalExists() { + await testSubjects.existOrFail('transformDeleteModal', { timeout: 60 * 1000 }); + } + + public async assertTransformDeleteModalNotExists() { + await testSubjects.missingOrFail('transformDeleteModal', { timeout: 60 * 1000 }); + } + + public async assertTransformStartModalExists() { + await testSubjects.existOrFail('transformStartModal', { timeout: 60 * 1000 }); + } + + public async assertTransformStartModalNotExists() { + await testSubjects.missingOrFail('transformStartModal', { timeout: 60 * 1000 }); + } + + public async confirmDeleteTransform() { + await retry.tryForTime(30 * 1000, async () => { + await this.assertTransformDeleteModalExists(); + await testSubjects.click('transformDeleteModal > confirmModalConfirmButton'); + await this.assertTransformDeleteModalNotExists(); + }); + } + + public async assertTransformRowNotExists(transformId: string) { + await retry.tryForTime(30 * 1000, async () => { + // If after deletion, and there's no transform left + const noTransformsFoundMessageExists = await testSubjects.exists( + 'transformNoTransformsFound' + ); + + if (noTransformsFoundMessageExists) { + return true; + } else { + // Checks that the tranform was deleted + await this.filterWithSearchString(transformId, 0); + } + }); + } + + public async confirmStartTransform() { + await retry.tryForTime(30 * 1000, async () => { + await this.assertTransformStartModalExists(); + await testSubjects.click('transformStartModal > confirmModalConfirmButton'); + await this.assertTransformStartModalNotExists(); + }); + } })(); } From b303c9df70b802cd4c27f914c4314c8a7ed52080 Mon Sep 17 00:00:00 2001 From: ymao1 Date: Wed, 24 Feb 2021 13:25:27 -0500 Subject: [PATCH 03/13] =?UTF-8?q?[Alerting]=20Fixing=20Failing=20test:=20X?= =?UTF-8?q?-Pack=20Alerting=20API=20Integration=20Tests.x-pack/test/alerti?= =?UTF-8?q?ng=5Fapi=5Fintegration/security=5Fand=5Fspaces/tests/alerting/r?= =?UTF-8?q?bac=5Flegacy=C2=B7ts=20-=20alerting=20api=20integration=20secur?= =?UTF-8?q?ity=20and=20spaces=20enabled=20Alerts=20legacy=20alerts=20alert?= =?UTF-8?q?s=20superuser=20at=20space1=20should=20schedule=20actions=20on?= =?UTF-8?q?=20legacy=20alerts=20(#92549)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Unskipping test * Increasing runAt time * Increasing runAt time * Logging * Increasing wait time even more * Removing logs * Resetting task status * Re-enabling all tests * Re-enabling all tests * Adding comment --- .../fixtures/plugins/alerts/server/routes.ts | 39 ++++++++++++++++++- .../tests/alerting/index.ts | 3 +- .../tests/alerting/rbac_legacy.ts | 18 +++++++++ 3 files changed, 57 insertions(+), 3 deletions(-) diff --git a/x-pack/test/alerting_api_integration/common/fixtures/plugins/alerts/server/routes.ts b/x-pack/test/alerting_api_integration/common/fixtures/plugins/alerts/server/routes.ts index d9e362a99e648..1b92b11eacdcc 100644 --- a/x-pack/test/alerting_api_integration/common/fixtures/plugins/alerts/server/routes.ts +++ b/x-pack/test/alerting_api_integration/common/fixtures/plugins/alerts/server/routes.ts @@ -15,7 +15,10 @@ import { import { schema } from '@kbn/config-schema'; import { InvalidatePendingApiKey } from '../../../../../../../plugins/alerts/server/types'; import { RawAlert } from '../../../../../../../plugins/alerts/server/types'; -import { TaskInstance } from '../../../../../../../plugins/task_manager/server'; +import { + ConcreteTaskInstance, + TaskInstance, +} from '../../../../../../../plugins/task_manager/server'; import { FixtureStartDeps } from './plugin'; export function defineRoutes(core: CoreSetup) { @@ -188,6 +191,40 @@ export function defineRoutes(core: CoreSetup) { } ); + router.put( + { + path: '/api/alerts_fixture/{id}/reset_task_status', + validate: { + params: schema.object({ + id: schema.string(), + }), + body: schema.object({ + status: schema.string(), + }), + }, + }, + async ( + context: RequestHandlerContext, + req: KibanaRequest, + res: KibanaResponseFactory + ): Promise> => { + const { id } = req.params; + const { status } = req.body; + + const [{ savedObjects }] = await core.getStartServices(); + const savedObjectsWithTasksAndAlerts = await savedObjects.getScopedClient(req, { + includedHiddenTypes: ['task', 'alert'], + }); + const alert = await savedObjectsWithTasksAndAlerts.get('alert', id); + const result = await savedObjectsWithTasksAndAlerts.update( + 'task', + alert.attributes.scheduledTaskId!, + { status } + ); + return res.ok({ body: result }); + } + ); + router.get( { path: '/api/alerts_fixture/api_keys_pending_invalidation', diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/index.ts b/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/index.ts index c1f65fab3669e..e8cc8ea699e17 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/index.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/index.ts @@ -11,8 +11,7 @@ import { setupSpacesAndUsers, tearDown } from '..'; // eslint-disable-next-line import/no-default-export export default function alertingTests({ loadTestFile, getService }: FtrProviderContext) { describe('Alerts', () => { - // FLAKY: https://github.com/elastic/kibana/issues/86952 - describe.skip('legacy alerts', () => { + describe('legacy alerts', () => { before(async () => { await setupSpacesAndUsers(getService); }); diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/rbac_legacy.ts b/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/rbac_legacy.ts index ef5914965ddce..3db3565374740 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/rbac_legacy.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/rbac_legacy.ts @@ -77,6 +77,7 @@ export default function alertTests({ getService }: FtrProviderContext) { case 'space_1_all at space1': case 'superuser at space1': case 'space_1_all_with_restricted_fixture at space1': + await resetTaskStatus(migratedAlertId); await ensureLegacyAlertHasBeenMigrated(migratedAlertId); await updateMigratedAlertToUseApiKeyOfCurrentUser(migratedAlertId); @@ -92,6 +93,7 @@ export default function alertTests({ getService }: FtrProviderContext) { await ensureAlertIsRunning(); break; case 'global_read at space1': + await resetTaskStatus(migratedAlertId); await ensureLegacyAlertHasBeenMigrated(migratedAlertId); await updateMigratedAlertToUseApiKeyOfCurrentUser(migratedAlertId); @@ -115,6 +117,7 @@ export default function alertTests({ getService }: FtrProviderContext) { }); break; case 'space_1_all_alerts_none_actions at space1': + await resetTaskStatus(migratedAlertId); await ensureLegacyAlertHasBeenMigrated(migratedAlertId); await updateMigratedAlertToUseApiKeyOfCurrentUser(migratedAlertId); @@ -140,6 +143,21 @@ export default function alertTests({ getService }: FtrProviderContext) { throw new Error(`Scenario untested: ${JSON.stringify(scenario)}`); } + async function resetTaskStatus(alertId: string) { + // occasionally when the task manager starts running while the alert saved objects + // are mid-migration, the task will fail and set its status to "failed". this prevents + // the alert from running ever again and downstream tasks that depend on successful alert + // execution will fail. this ensures the task status is set to "idle" so the + // task manager will continue claiming and executing it. + await supertest + .put(`${getUrlPrefix(space.id)}/api/alerts_fixture/${alertId}/reset_task_status`) + .set('kbn-xsrf', 'foo') + .send({ + status: 'idle', + }) + .expect(200); + } + async function ensureLegacyAlertHasBeenMigrated(alertId: string) { const getResponse = await supertestWithoutAuth .get(`${getUrlPrefix(space.id)}/api/alerts/alert/${alertId}`) From 8957ed8eef07731857d206641a1b02d062ce9469 Mon Sep 17 00:00:00 2001 From: Shahzad Date: Wed, 24 Feb 2021 19:28:24 +0100 Subject: [PATCH 04/13] [Uptime] waterfall improve legend spacing (#92158) --- .../synthetics/step_detail/step_detail.tsx | 8 ++++---- .../waterfall/waterfall_chart_wrapper.tsx | 6 +++++- .../synthetics/waterfall/components/legend.tsx | 15 +++++++++++---- .../waterfall/components/waterfall_chart.tsx | 8 ++++++-- 4 files changed, 26 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/uptime/public/components/monitor/synthetics/step_detail/step_detail.tsx b/x-pack/plugins/uptime/public/components/monitor/synthetics/step_detail/step_detail.tsx index 9fcd946df2f84..befe53219a449 100644 --- a/x-pack/plugins/uptime/public/components/monitor/synthetics/step_detail/step_detail.tsx +++ b/x-pack/plugins/uptime/public/components/monitor/synthetics/step_detail/step_detail.tsx @@ -68,16 +68,16 @@ export const StepDetail: React.FC = ({ }) => { return ( <> - + - +

{stepName}

 - + = ({ - + = (item) => { - return {item.name}; + return ( + + {item.name} + + ); }; interface Props { diff --git a/x-pack/plugins/uptime/public/components/monitor/synthetics/waterfall/components/legend.tsx b/x-pack/plugins/uptime/public/components/monitor/synthetics/waterfall/components/legend.tsx index c746a5cc63a9b..9a66b586d1d56 100644 --- a/x-pack/plugins/uptime/public/components/monitor/synthetics/waterfall/components/legend.tsx +++ b/x-pack/plugins/uptime/public/components/monitor/synthetics/waterfall/components/legend.tsx @@ -9,18 +9,25 @@ import React from 'react'; import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; import { IWaterfallContext } from '../context/waterfall_chart'; import { WaterfallChartProps } from './waterfall_chart'; +import { euiStyled } from '../../../../../../../../../src/plugins/kibana_react/common'; interface LegendProps { items: Required['legendItems']; render: Required['renderLegendItem']; } +const StyledFlexItem = euiStyled(EuiFlexItem)` + margin-right: ${(props) => props.theme.eui.paddingSizes.m}; + max-width: 7%; + min-width: 160px; +`; + export const Legend: React.FC = ({ items, render }) => { return ( - - {items.map((item, index) => { - return {render(item, index)}; - })} + + {items.map((item, index) => ( + {render(item, index)} + ))} ); }; diff --git a/x-pack/plugins/uptime/public/components/monitor/synthetics/waterfall/components/waterfall_chart.tsx b/x-pack/plugins/uptime/public/components/monitor/synthetics/waterfall/components/waterfall_chart.tsx index 59990b29db5db..119c907f76ca1 100644 --- a/x-pack/plugins/uptime/public/components/monitor/synthetics/waterfall/components/waterfall_chart.tsx +++ b/x-pack/plugins/uptime/public/components/monitor/synthetics/waterfall/components/waterfall_chart.tsx @@ -120,8 +120,12 @@ export const WaterfallChart = ({ - - + + {shouldRenderSidebar && } Date: Wed, 24 Feb 2021 11:40:04 -0700 Subject: [PATCH 05/13] [dev/build_ts_refs] enable caching by default (#92513) Co-authored-by: spalger Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- src/dev/ci_setup/setup.sh | 4 ++++ src/dev/typescript/build_ts_refs_cli.ts | 5 ++++- src/dev/typescript/ref_output_cache/ref_output_cache.ts | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/dev/ci_setup/setup.sh b/src/dev/ci_setup/setup.sh index c4559029e5607..f9c1e67c0540d 100755 --- a/src/dev/ci_setup/setup.sh +++ b/src/dev/ci_setup/setup.sh @@ -21,6 +21,10 @@ cp "src/dev/ci_setup/.bazelrc-ci" "$HOME/.bazelrc"; echo "# Appended by src/dev/ci_setup/setup.sh" >> "$HOME/.bazelrc" echo "build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY" >> "$HOME/.bazelrc" +if [[ "$BUILD_TS_REFS_CACHE_ENABLE" != "true" ]]; then + export BUILD_TS_REFS_CACHE_ENABLE=false +fi + ### ### install dependencies ### diff --git a/src/dev/typescript/build_ts_refs_cli.ts b/src/dev/typescript/build_ts_refs_cli.ts index fc8911a251773..a073e58623278 100644 --- a/src/dev/typescript/build_ts_refs_cli.ts +++ b/src/dev/typescript/build_ts_refs_cli.ts @@ -23,7 +23,7 @@ export async function runBuildRefsCli() { async ({ log, flags }) => { const outDirs = getOutputsDeep(REF_CONFIG_PATHS); - const cacheEnabled = process.env.BUILD_TS_REFS_CACHE_ENABLE === 'true' || !!flags.cache; + const cacheEnabled = process.env.BUILD_TS_REFS_CACHE_ENABLE !== 'false' && !!flags.cache; const doCapture = process.env.BUILD_TS_REFS_CACHE_CAPTURE === 'true'; const doClean = !!flags.clean || doCapture; const doInitCache = cacheEnabled && !doClean; @@ -62,6 +62,9 @@ export async function runBuildRefsCli() { description: 'Build TypeScript projects', flags: { boolean: ['clean', 'cache'], + default: { + cache: true, + }, }, log: { defaultLevel: 'debug', diff --git a/src/dev/typescript/ref_output_cache/ref_output_cache.ts b/src/dev/typescript/ref_output_cache/ref_output_cache.ts index 342470ce0c6e3..6f51243e47555 100644 --- a/src/dev/typescript/ref_output_cache/ref_output_cache.ts +++ b/src/dev/typescript/ref_output_cache/ref_output_cache.ts @@ -132,7 +132,7 @@ export class RefOutputCache { this.log.debug(`[${relative}] clearing outDir and replacing with cache`); await del(outDir); await unzip(Path.resolve(tmpDir, cacheName), outDir); - await Fs.writeFile(Path.resolve(outDir, OUTDIR_MERGE_BASE_FILENAME), archive.sha); + await Fs.writeFile(Path.resolve(outDir, OUTDIR_MERGE_BASE_FILENAME), this.mergeBase); }); } From 78bed9e56d891b065779dc5ab4bfe593f2061238 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Wed, 24 Feb 2021 14:04:46 -0500 Subject: [PATCH 06/13] [Lens] Fix bug in Safari and Firefox form rendering (#92542) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../editor_frame/config_panel/layer_panel.tsx | 509 +++++++++--------- 1 file changed, 256 insertions(+), 253 deletions(-) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx index 59b64de369745..1d75e873f9b18 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx @@ -187,272 +187,275 @@ export function LayerPanel( ]); return ( -
- - - - - - - {layerDatasource && ( - - { - const newState = - typeof updater === 'function' ? updater(layerDatasourceState) : updater; - // Look for removed columns - const nextPublicAPI = layerDatasource.getPublicAPI({ - state: newState, - layerId, - }); - const nextTable = new Set( - nextPublicAPI.getTableSpec().map(({ columnId }) => columnId) - ); - const removed = datasourcePublicAPI - .getTableSpec() - .map(({ columnId }) => columnId) - .filter((columnId) => !nextTable.has(columnId)); - let nextVisState = props.visualizationState; - removed.forEach((columnId) => { - nextVisState = activeVisualization.removeDimension({ - layerId, - columnId, - prevState: nextVisState, - }); - }); - - props.updateAll(datasourceId, newState, nextVisState); - }, + <> +
+ + + + - )} - - - - - {groups.map((group, groupIndex) => { - const isMissing = !isEmptyLayer && group.required && group.accessors.length === 0; - return ( - {group.groupLabel}} - labelType="legend" - key={group.groupId} - isInvalid={isMissing} - error={ - isMissing ? ( -
- {i18n.translate('xpack.lens.editorFrame.requiredDimensionWarningLabel', { - defaultMessage: 'Required dimension', - })} -
- ) : ( - [] - ) - } - > - <> - - {group.accessors.map((accessorConfig, accessorIndex) => { - const { columnId } = accessorConfig; - return ( - -
- { - setActiveDimension({ - isNew: false, - activeGroup: group, - activeId: id, - }); - }} - onRemoveClick={(id: string) => { - trackUiEvent('indexpattern_dimension_removed'); - props.updateAll( - datasourceId, - layerDatasource.removeColumn({ - layerId, - columnId: id, - prevState: layerDatasourceState, - }), - activeVisualization.removeDimension({ - layerId, - columnId: id, - prevState: props.visualizationState, - }) - ); - removeButtonRef(id); - }} - > - - -
- - ); - })} - - {group.supportsMoreColumns ? ( - { - setActiveDimension({ - activeGroup: group, - activeId: id, - isNew: true, - }); - }} - onDrop={onDrop} - /> - ) : null} - - - ); - })} - { - if (layerDatasource.updateStateOnCloseDimension) { - const newState = layerDatasource.updateStateOnCloseDimension({ - state: layerDatasourceState, - layerId, - columnId: activeId!, - }); - if (newState) { - props.updateDatasource(datasourceId, newState); - } - } - setActiveDimension(initialActiveDimensionState); - }} - panel={ - <> - {activeGroup && activeId && ( + {layerDatasource && ( + { - if (shouldReplaceDimension || shouldRemoveDimension) { - props.updateAll( - datasourceId, - newState, - shouldRemoveDimension - ? activeVisualization.removeDimension({ - layerId, - columnId: activeId, - prevState: props.visualizationState, - }) - : activeVisualization.setDimension({ - layerId, - groupId: activeGroup.groupId, - columnId: activeId, - prevState: props.visualizationState, - }) - ); - } else { - props.updateDatasource(datasourceId, newState); - } - setActiveDimension({ - ...activeDimension, - isNew: false, + layerId, + state: layerDatasourceState, + activeData: props.framePublicAPI.activeData, + setState: (updater: unknown) => { + const newState = + typeof updater === 'function' ? updater(layerDatasourceState) : updater; + // Look for removed columns + const nextPublicAPI = layerDatasource.getPublicAPI({ + state: newState, + layerId, + }); + const nextTable = new Set( + nextPublicAPI.getTableSpec().map(({ columnId }) => columnId) + ); + const removed = datasourcePublicAPI + .getTableSpec() + .map(({ columnId }) => columnId) + .filter((columnId) => !nextTable.has(columnId)); + let nextVisState = props.visualizationState; + removed.forEach((columnId) => { + nextVisState = activeVisualization.removeDimension({ + layerId, + columnId, + prevState: nextVisState, + }); }); + + props.updateAll(datasourceId, newState, nextVisState); }, }} /> - )} - {activeGroup && - activeId && - !activeDimension.isNew && - activeVisualization.renderDimensionEditor && - activeGroup?.enableDimensionEditor && ( -
- + )} + + + + + {groups.map((group, groupIndex) => { + const isMissing = !isEmptyLayer && group.required && group.accessors.length === 0; + return ( + {group.groupLabel}
} + labelType="legend" + key={group.groupId} + isInvalid={isMissing} + error={ + isMissing ? ( +
+ {i18n.translate('xpack.lens.editorFrame.requiredDimensionWarningLabel', { + defaultMessage: 'Required dimension', + })} +
+ ) : ( + [] + ) + } + > + <> + + {group.accessors.map((accessorConfig, accessorIndex) => { + const { columnId } = accessorConfig; + + return ( + +
+ { + setActiveDimension({ + isNew: false, + activeGroup: group, + activeId: id, + }); + }} + onRemoveClick={(id: string) => { + trackUiEvent('indexpattern_dimension_removed'); + props.updateAll( + datasourceId, + layerDatasource.removeColumn({ + layerId, + columnId: id, + prevState: layerDatasourceState, + }), + activeVisualization.removeDimension({ + layerId, + columnId: id, + prevState: props.visualizationState, + }) + ); + removeButtonRef(id); + }} + > + + +
+ + ); + })} + + {group.supportsMoreColumns ? ( + { + setActiveDimension({ + activeGroup: group, + activeId: id, + isNew: true, + }); }} + onDrop={onDrop} /> - - )} - - } - /> + ) : null} + + + ); + })} - + - - - - - - -
+ + + + + + +
+ + { + if (layerDatasource.updateStateOnCloseDimension) { + const newState = layerDatasource.updateStateOnCloseDimension({ + state: layerDatasourceState, + layerId, + columnId: activeId!, + }); + if (newState) { + props.updateDatasource(datasourceId, newState); + } + } + setActiveDimension(initialActiveDimensionState); + }} + panel={ + <> + {activeGroup && activeId && ( + { + if (shouldReplaceDimension || shouldRemoveDimension) { + props.updateAll( + datasourceId, + newState, + shouldRemoveDimension + ? activeVisualization.removeDimension({ + layerId, + columnId: activeId, + prevState: props.visualizationState, + }) + : activeVisualization.setDimension({ + layerId, + groupId: activeGroup.groupId, + columnId: activeId, + prevState: props.visualizationState, + }) + ); + } else { + props.updateDatasource(datasourceId, newState); + } + setActiveDimension({ + ...activeDimension, + isNew: false, + }); + }, + }} + /> + )} + {activeGroup && + activeId && + !activeDimension.isNew && + activeVisualization.renderDimensionEditor && + activeGroup?.enableDimensionEditor && ( +
+ +
+ )} + + } + /> + ); } From 2950852cf1e295b30b5865f0c4afcbaf7a49d8c8 Mon Sep 17 00:00:00 2001 From: Larry Gregory Date: Wed, 24 Feb 2021 14:45:55 -0500 Subject: [PATCH 07/13] Upgrade lodash to 4.17.21 (#92666) --- package.json | 7 +++---- yarn.lock | 8 ++++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index 70918f02dcd41..dd483ba53034e 100644 --- a/package.json +++ b/package.json @@ -74,11 +74,11 @@ "**/cross-fetch/node-fetch": "^2.6.1", "**/deepmerge": "^4.2.2", "**/fast-deep-equal": "^3.1.1", - "**/graphql-toolkit/lodash": "^4.17.15", + "**/graphql-toolkit/lodash": "^4.17.21", "**/hoist-non-react-statics": "^3.3.2", "**/isomorphic-fetch/node-fetch": "^2.6.1", "**/istanbul-instrumenter-loader/schema-utils": "1.0.0", - "**/load-grunt-config/lodash": "^4.17.20", + "**/load-grunt-config/lodash": "^4.17.21", "**/minimist": "^1.2.5", "**/node-jose/node-forge": "^0.10.0", "**/prismjs": "1.22.0", @@ -233,7 +233,7 @@ "json-stringify-safe": "5.0.1", "jsonwebtoken": "^8.5.1", "load-json-file": "^6.2.0", - "lodash": "^4.17.20", + "lodash": "^4.17.21", "lru-cache": "^4.1.5", "markdown-it": "^10.0.0", "md5": "^2.1.0", @@ -390,7 +390,6 @@ "@storybook/addon-essentials": "^6.0.26", "@storybook/addon-knobs": "^6.0.26", "@storybook/addon-storyshots": "^6.0.26", - "@storybook/addons": "^6.0.16", "@storybook/components": "^6.0.26", "@storybook/core": "^6.0.26", "@storybook/core-events": "^6.0.26", diff --git a/yarn.lock b/yarn.lock index 4a3399ece1fd0..7c93a37dffe6f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -19991,10 +19991,10 @@ lodash.uniq@4.5.0, lodash.uniq@^4.5.0: resolved "https://registry.yarnpkg.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773" integrity sha1-0CJTc662Uq3BvILklFM5qEJ1R3M= -lodash@4.17.11, lodash@4.17.15, lodash@>4.17.4, lodash@^4, lodash@^4.0.0, lodash@^4.0.1, lodash@^4.10.0, lodash@^4.15.0, lodash@^4.17.10, lodash@^4.17.11, lodash@^4.17.13, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.4, lodash@^4.2.0, lodash@~4.17.10, lodash@~4.17.15, lodash@~4.17.19, lodash@~4.17.20: - version "4.17.20" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.20.tgz#b44a9b6297bcb698f1c51a3545a2b3b368d59c52" - integrity sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA== +lodash@4.17.11, lodash@4.17.15, lodash@>4.17.4, lodash@^4, lodash@^4.0.0, lodash@^4.0.1, lodash@^4.10.0, lodash@^4.15.0, lodash@^4.17.10, lodash@^4.17.11, lodash@^4.17.13, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.17.4, lodash@^4.2.0, lodash@~4.17.10, lodash@~4.17.15, lodash@~4.17.19, lodash@~4.17.20: + version "4.17.21" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" + integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== log-ok@^0.1.1: version "0.1.1" From 32ffc80768bf8987485c22ccec23f1c6868b7e9a Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Wed, 24 Feb 2021 21:48:40 +0200 Subject: [PATCH 08/13] [Security Solution][Case] Fix alerts push (#91638) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../plugins/case/server/client/cases/mock.ts | 15 +++++ .../plugins/case/server/client/cases/types.ts | 2 +- .../case/server/client/cases/utils.test.ts | 60 +++++++++++++------ .../plugins/case/server/client/cases/utils.ts | 52 ++++++++++++---- .../server/routes/api/cases/push_case.test.ts | 2 +- .../connectors/servicenow/translations.ts | 2 +- 6 files changed, 99 insertions(+), 34 deletions(-) diff --git a/x-pack/plugins/case/server/client/cases/mock.ts b/x-pack/plugins/case/server/client/cases/mock.ts index 809c4ad1ea1bd..490519187f49e 100644 --- a/x-pack/plugins/case/server/client/cases/mock.ts +++ b/x-pack/plugins/case/server/client/cases/mock.ts @@ -11,6 +11,7 @@ import { ConnectorMappingsAttributes, CaseUserActionsResponse, AssociationType, + CommentResponseAlertsType, } from '../../../common/api'; import { BasicParams } from './types'; @@ -76,6 +77,20 @@ export const commentAlert: CommentResponse = { version: 'WzEsMV0=', }; +export const commentAlertMultipleIds: CommentResponseAlertsType = { + ...commentAlert, + id: 'mock-comment-2', + alertId: ['alert-id-1', 'alert-id-2'], + index: 'alert-index-1', + type: CommentType.alert as const, +}; + +export const commentGeneratedAlert: CommentResponseAlertsType = { + ...commentAlertMultipleIds, + id: 'mock-comment-3', + type: CommentType.generatedAlert as const, +}; + export const defaultPipes = ['informationCreated']; export const basicParams: BasicParams = { description: 'a description', diff --git a/x-pack/plugins/case/server/client/cases/types.ts b/x-pack/plugins/case/server/client/cases/types.ts index f1d56e7132bd1..2dd2caf9fe73a 100644 --- a/x-pack/plugins/case/server/client/cases/types.ts +++ b/x-pack/plugins/case/server/client/cases/types.ts @@ -72,7 +72,7 @@ export interface TransformFieldsArgs { export interface ExternalServiceComment { comment: string; - commentId: string; + commentId?: string; } export interface MapIncident { diff --git a/x-pack/plugins/case/server/client/cases/utils.test.ts b/x-pack/plugins/case/server/client/cases/utils.test.ts index 361d0fb561afd..44e7a682aa7ed 100644 --- a/x-pack/plugins/case/server/client/cases/utils.test.ts +++ b/x-pack/plugins/case/server/client/cases/utils.test.ts @@ -17,6 +17,8 @@ import { basicParams, userActions, commentAlert, + commentAlertMultipleIds, + commentGeneratedAlert, } from './mock'; import { @@ -48,7 +50,7 @@ describe('utils', () => { { actionType: 'overwrite', key: 'short_description', - pipes: ['informationCreated'], + pipes: [], value: 'a title', }, { @@ -71,7 +73,7 @@ describe('utils', () => { { actionType: 'overwrite', key: 'short_description', - pipes: ['myTestPipe'], + pipes: [], value: 'a title', }, { @@ -98,7 +100,7 @@ describe('utils', () => { }); expect(res).toEqual({ - short_description: 'a title (created at 2020-03-13T08:34:53.450Z by Elastic User)', + short_description: 'a title', description: 'a description (created at 2020-03-13T08:34:53.450Z by Elastic User)', }); }); @@ -122,13 +124,13 @@ describe('utils', () => { }, fields, currentIncident: { - short_description: 'first title (created at 2020-03-13T08:34:53.450Z by Elastic User)', + short_description: 'first title', description: 'first description (created at 2020-03-13T08:34:53.450Z by Elastic User)', }, }); expect(res).toEqual({ - short_description: 'a title (updated at 2020-03-15T08:34:53.450Z by Another User)', + short_description: 'a title', description: 'first description (created at 2020-03-13T08:34:53.450Z by Elastic User) \r\na description (updated at 2020-03-15T08:34:53.450Z by Another User)', }); @@ -168,7 +170,7 @@ describe('utils', () => { }); expect(res).toEqual({ - short_description: 'a title (created at 2020-03-13T08:34:53.450Z by elastic)', + short_description: 'a title', description: 'a description (created at 2020-03-13T08:34:53.450Z by elastic)', }); }); @@ -190,7 +192,7 @@ describe('utils', () => { }); expect(res).toEqual({ - short_description: 'a title (updated at 2020-03-15T08:34:53.450Z by anotherUser)', + short_description: 'a title', description: 'a description (updated at 2020-03-15T08:34:53.450Z by anotherUser)', }); }); @@ -448,8 +450,7 @@ describe('utils', () => { labels: ['defacement'], issueType: null, parent: null, - short_description: - 'Super Bad Security Issue (created at 2019-11-25T21:54:48.952Z by elastic)', + short_description: 'Super Bad Security Issue', description: 'This is a brand new case of a bad meanie defacing data (created at 2019-11-25T21:54:48.952Z by elastic)', externalId: null, @@ -504,7 +505,7 @@ describe('utils', () => { expect(res.comments).toEqual([]); }); - it('it creates comments of type alert correctly', async () => { + it('it adds the total alert comments correctly', async () => { const res = await createIncident({ actionsClient: actionsMock, theCase: { @@ -512,7 +513,9 @@ describe('utils', () => { comments: [ { ...commentObj, id: 'comment-user-1' }, { ...commentAlert, id: 'comment-alert-1' }, - { ...commentAlert, id: 'comment-alert-2' }, + { + ...commentAlertMultipleIds, + }, ], }, // Remove second push @@ -536,14 +539,36 @@ describe('utils', () => { commentId: 'comment-user-1', }, { - comment: - 'Alert with ids alert-id-1 added to case (added at 2019-11-25T21:55:00.177Z by elastic)', - commentId: 'comment-alert-1', + comment: 'Elastic Security Alerts attached to the case: 3', }, + ]); + }); + + it('it removes alerts correctly', async () => { + const res = await createIncident({ + actionsClient: actionsMock, + theCase: { + ...theCase, + comments: [ + { ...commentObj, id: 'comment-user-1' }, + commentAlertMultipleIds, + commentGeneratedAlert, + ], + }, + userActions, + connector, + mappings, + alerts: [], + }); + + expect(res.comments).toEqual([ { comment: - 'Alert with ids alert-id-1 added to case (added at 2019-11-25T21:55:00.177Z by elastic)', - commentId: 'comment-alert-2', + 'Wow, good luck catching that bad meanie! (added at 2019-11-25T21:55:00.177Z by elastic)', + commentId: 'comment-user-1', + }, + { + comment: 'Elastic Security Alerts attached to the case: 4', }, ]); }); @@ -578,8 +603,7 @@ describe('utils', () => { description: 'fun description \r\nThis is a brand new case of a bad meanie defacing data (updated at 2019-11-25T21:54:48.952Z by elastic)', externalId: 'external-id', - short_description: - 'Super Bad Security Issue (updated at 2019-11-25T21:54:48.952Z by elastic)', + short_description: 'Super Bad Security Issue', }, comments: [], }); diff --git a/x-pack/plugins/case/server/client/cases/utils.ts b/x-pack/plugins/case/server/client/cases/utils.ts index fda4142bf77c7..a5013d9b93982 100644 --- a/x-pack/plugins/case/server/client/cases/utils.ts +++ b/x-pack/plugins/case/server/client/cases/utils.ts @@ -40,6 +40,15 @@ import { } from './types'; import { getAlertIds } from '../../routes/api/utils'; +interface CreateIncidentArgs { + actionsClient: ActionsClient; + theCase: CaseResponse; + userActions: CaseUserActionsResponse; + connector: ActionConnector; + mappings: ConnectorMappingsAttributes[]; + alerts: CaseClientGetAlertsResponse; +} + export const getLatestPushInfo = ( connectorId: string, userActions: CaseUserActionsResponse @@ -75,14 +84,13 @@ const getCommentContent = (comment: CommentResponse): string => { return ''; }; -interface CreateIncidentArgs { - actionsClient: ActionsClient; - theCase: CaseResponse; - userActions: CaseUserActionsResponse; - connector: ActionConnector; - mappings: ConnectorMappingsAttributes[]; - alerts: CaseClientGetAlertsResponse; -} +const countAlerts = (comments: CaseResponse['comments']): number => + comments?.reduce((total, comment) => { + if (comment.type === CommentType.alert || comment.type === CommentType.generatedAlert) { + return total + (Array.isArray(comment.alertId) ? comment.alertId.length : 1); + } + return total; + }, 0) ?? 0; export const createIncident = async ({ actionsClient, @@ -152,22 +160,34 @@ export const createIncident = async ({ userActions .slice(latestPushInfo?.index ?? 0) .filter( - (action, index) => - Array.isArray(action.action_field) && action.action_field[0] === 'comment' + (action) => Array.isArray(action.action_field) && action.action_field[0] === 'comment' ) .map((action) => action.comment_id) ); - const commentsToBeUpdated = caseComments?.filter((comment) => - commentsIdsToBeUpdated.has(comment.id) + + const commentsToBeUpdated = caseComments?.filter( + (comment) => + // We push only user's comments + comment.type === CommentType.user && commentsIdsToBeUpdated.has(comment.id) ); + const totalAlerts = countAlerts(caseComments); + let comments: ExternalServiceComment[] = []; + if (commentsToBeUpdated && Array.isArray(commentsToBeUpdated) && commentsToBeUpdated.length > 0) { const commentsMapping = mappings.find((m) => m.source === 'comments'); if (commentsMapping?.action_type !== 'nothing') { comments = transformComments(commentsToBeUpdated, ['informationAdded']); } } + + if (totalAlerts > 0) { + comments.push({ + comment: `Elastic Security Alerts attached to the case: ${totalAlerts}`, + }); + } + return { incident, comments }; }; @@ -247,7 +267,13 @@ export const prepareFieldsForTransformation = ({ key: mapping.target, value: params[mapping.source] ?? '', actionType: mapping.action_type, - pipes: mapping.action_type === 'append' ? [...defaultPipes, 'append'] : defaultPipes, + pipes: + // Do not transform titles + mapping.source !== 'title' + ? mapping.action_type === 'append' + ? [...defaultPipes, 'append'] + : defaultPipes + : [], }, ] : acc, diff --git a/x-pack/plugins/case/server/routes/api/cases/push_case.test.ts b/x-pack/plugins/case/server/routes/api/cases/push_case.test.ts index bf398d1ffcf40..c8501130493ba 100644 --- a/x-pack/plugins/case/server/routes/api/cases/push_case.test.ts +++ b/x-pack/plugins/case/server/routes/api/cases/push_case.test.ts @@ -170,7 +170,7 @@ describe('Push case', () => { parent: null, priority: 'High', labels: ['LOLBins'], - summary: 'Another bad one (created at 2019-11-25T22:32:17.947Z by elastic)', + summary: 'Another bad one', description: 'Oh no, a bad meanie going LOLBins all over the place! (created at 2019-11-25T22:32:17.947Z by elastic)', externalId: null, diff --git a/x-pack/plugins/security_solution/public/cases/components/connectors/servicenow/translations.ts b/x-pack/plugins/security_solution/public/cases/components/connectors/servicenow/translations.ts index 0867dc41eeb78..77c263385df0a 100644 --- a/x-pack/plugins/security_solution/public/cases/components/connectors/servicenow/translations.ts +++ b/x-pack/plugins/security_solution/public/cases/components/connectors/servicenow/translations.ts @@ -87,7 +87,7 @@ export const PRIORITY = i18n.translate( export const ALERT_FIELDS_LABEL = i18n.translate( 'xpack.securitySolution.components.connectors.serviceNow.alertFieldsTitle', { - defaultMessage: 'Fields associated with alerts', + defaultMessage: 'Select Observables to push', } ); From fe6bd2ecbb47cfcd0d1a03c9f1d63d82a686db86 Mon Sep 17 00:00:00 2001 From: Alison Goryachev Date: Wed, 24 Feb 2021 16:15:06 -0500 Subject: [PATCH 09/13] [Upgrade Assistant] Align code between branches (#91862) --- .../translations/translations/ja-JP.json | 2 - .../translations/translations/zh-CN.json | 2 - .../upgrade_assistant/common/constants.ts | 15 +++++++ .../plugins/upgrade_assistant/common/types.ts | 2 +- .../application/components/tabs.test.tsx | 9 ++--- .../tabs/checkup/checkup_tab.test.tsx | 10 ++--- .../tabs/checkup/deprecations/index_table.tsx | 6 ++- .../__snapshots__/warning_step.test.tsx.snap | 24 ----------- .../reindex/flyout/checklist_step.test.tsx | 2 +- .../reindex/flyout/warning_step.test.tsx | 40 +++++++++++-------- .../reindex/flyout/warnings_step.tsx | 29 +++++++++----- .../server/lib/__fixtures__/version.ts | 11 ++--- .../server/lib/es_migration_apis.test.ts | 1 + .../server/lib/es_version_precheck.test.ts | 5 ++- .../lib/reindexing/index_settings.test.ts | 30 ++++++++++++-- .../server/lib/reindexing/index_settings.ts | 18 +++++++-- .../lib/reindexing/reindex_actions.test.ts | 5 ++- .../server/lib/reindexing/reindex_actions.ts | 36 ++++++++++++----- .../lib/reindexing/reindex_service.test.ts | 12 ++++-- .../server/lib/reindexing/reindex_service.ts | 7 +++- .../server/lib/reindexing/types.ts | 13 ++++++ .../reindex_indices/reindex_indices.test.ts | 4 +- 22 files changed, 180 insertions(+), 103 deletions(-) create mode 100644 x-pack/plugins/upgrade_assistant/common/constants.ts diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index d68e6c375a592..15333f71861b8 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -21526,8 +21526,6 @@ "xpack.upgradeAssistant.checkupTab.reindexing.flyout.indexClosedCallout.calloutDetails.reindexingTakesLongerEmphasis": "再インデックスには通常よりも時間がかかることがあります", "xpack.upgradeAssistant.checkupTab.reindexing.flyout.indexClosedCallout.calloutTitle": "インデックスが閉じました", "xpack.upgradeAssistant.checkupTab.reindexing.flyout.openAndCloseDocumentation": "ドキュメンテーション", - "xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.apmReindexWarningDetail": "バージョン 7.0.0 以降、APM データは Elastic Common Schema で表示されます。過去の APM データは再インデックスされるまで表示されません。", - "xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.apmReindexWarningTitle": "このインデックスは ECS 形式に変換されます", "xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.destructiveCallout.calloutDetail": "インデックスをバックアップして、互換性を破るそれぞれの変更に同意することで再インデックスしてください。", "xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.destructiveCallout.calloutTitle": "このインデックスには元に戻すことのできない破壊的な変更が含まれています", "xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.documentationLinkLabel": "ドキュメント", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index dc39b7b03634b..8051b24bf9c03 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -21576,8 +21576,6 @@ "xpack.upgradeAssistant.checkupTab.reindexing.flyout.indexClosedCallout.calloutDetails.reindexingTakesLongerEmphasis": "重新索引可能比通常花费更多的时间", "xpack.upgradeAssistant.checkupTab.reindexing.flyout.indexClosedCallout.calloutTitle": "索引已关闭", "xpack.upgradeAssistant.checkupTab.reindexing.flyout.openAndCloseDocumentation": "文档", - "xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.apmReindexWarningDetail": "从版本 7.0.0 开始，将以 Elastic Common Schema 格式表示 APM 数据。只有重新索引历史 APM 数据后，其才可见。", - "xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.apmReindexWarningTitle": "此索引将转换成 ECS 格式", "xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.destructiveCallout.calloutDetail": "备份您的索引，然后通过接受每个重大更改来继续重新索引。", "xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.destructiveCallout.calloutTitle": "此索引需要无法撤消的破坏性更改", "xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.documentationLinkLabel": "文档", diff --git a/x-pack/plugins/upgrade_assistant/common/constants.ts b/x-pack/plugins/upgrade_assistant/common/constants.ts new file mode 100644 index 0000000000000..654a1a3331733 --- /dev/null +++ b/x-pack/plugins/upgrade_assistant/common/constants.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import SemVer from 'semver/classes/semver'; + +/* + * These constants are used only in tests to add conditional logic based on Kibana version + * On master, the version should represent the next major version (e.g., master --> 8.0.0) + * The release branch should match the release version (e.g., 7.x --> 7.0.0) + */ +export const mockKibanaVersion = '8.0.0'; +export const mockKibanaSemverVersion = new SemVer(mockKibanaVersion); diff --git a/x-pack/plugins/upgrade_assistant/common/types.ts b/x-pack/plugins/upgrade_assistant/common/types.ts index 91a19bfec3e81..6d83bdc5f36e9 100644 --- a/x-pack/plugins/upgrade_assistant/common/types.ts +++ b/x-pack/plugins/upgrade_assistant/common/types.ts @@ -94,7 +94,7 @@ export type ReindexSavedObject = SavedObject; export enum ReindexWarning { // 7.0 -> 8.0 warnings - apmReindex, + customTypeName, // 8.0 -> 9.0 warnings } diff --git a/x-pack/plugins/upgrade_assistant/public/application/components/tabs.test.tsx b/x-pack/plugins/upgrade_assistant/public/application/components/tabs.test.tsx index ee722a3937216..b732f6806a388 100644 --- a/x-pack/plugins/upgrade_assistant/public/application/components/tabs.test.tsx +++ b/x-pack/plugins/upgrade_assistant/public/application/components/tabs.test.tsx @@ -6,9 +6,9 @@ */ import React from 'react'; -import SemVer from 'semver/classes/semver'; import { mountWithIntl } from '@kbn/test/jest'; import { httpServiceMock } from 'src/core/public/mocks'; +import { mockKibanaSemverVersion } from '../../../common/constants'; import { UpgradeAssistantTabs } from './tabs'; import { LoadingState } from './types'; @@ -18,7 +18,6 @@ import { OverviewTab } from './tabs/overview'; const promisesToResolve = () => new Promise((resolve) => setTimeout(resolve, 0)); const mockHttp = httpServiceMock.createSetupContract(); -const mockKibanaVersion = new SemVer('8.0.0'); jest.mock('../app_context', () => { return { @@ -29,9 +28,9 @@ jest.mock('../app_context', () => { ELASTIC_WEBSITE_URL: 'https://www.elastic.co/', }, kibanaVersionInfo: { - currentMajor: mockKibanaVersion.major, - prevMajor: mockKibanaVersion.major - 1, - nextMajor: mockKibanaVersion.major + 1, + currentMajor: mockKibanaSemverVersion.major, + prevMajor: mockKibanaSemverVersion.major - 1, + nextMajor: mockKibanaSemverVersion.major + 1, }, }; }, diff --git a/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/checkup_tab.test.tsx b/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/checkup_tab.test.tsx index 1ed1e0b01f65b..bf890c856239e 100644 --- a/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/checkup_tab.test.tsx +++ b/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/checkup_tab.test.tsx @@ -7,7 +7,7 @@ import { shallow } from 'enzyme'; import React from 'react'; -import SemVer from 'semver/classes/semver'; +import { mockKibanaSemverVersion } from '../../../../../common/constants'; import { LoadingState } from '../../types'; import AssistanceData from '../__fixtures__/checkup_api_response.json'; @@ -22,8 +22,6 @@ const defaultProps = { setSelectedTabIndex: jest.fn(), }; -const mockKibanaVersion = new SemVer('8.0.0'); - jest.mock('../../../app_context', () => { return { useAppContext: () => { @@ -33,9 +31,9 @@ jest.mock('../../../app_context', () => { ELASTIC_WEBSITE_URL: 'https://www.elastic.co/', }, kibanaVersionInfo: { - currentMajor: mockKibanaVersion.major, - prevMajor: mockKibanaVersion.major - 1, - nextMajor: mockKibanaVersion.major + 1, + currentMajor: mockKibanaSemverVersion.major, + prevMajor: mockKibanaSemverVersion.major - 1, + nextMajor: mockKibanaSemverVersion.major + 1, }, }; }, diff --git a/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/index_table.tsx b/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/index_table.tsx index 67aa5d8b9d7de..292887853e4b3 100644 --- a/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/index_table.tsx +++ b/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/index_table.tsx @@ -143,9 +143,11 @@ export class IndexDeprecationTable extends React.Component< private generateActionsColumn() { // NOTE: this naive implementation assumes all indices in the table are - // should show the reindex button. This should work for known usecases. + // should show the reindex button. This should work for known use cases. const { indices } = this.props; - if (!indices.find((i) => i.reindex === true)) { + const hasActionsColumn = Boolean(indices.find((i) => i.reindex === true)); + + if (hasActionsColumn === false) { return null; } diff --git a/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/reindex/flyout/__snapshots__/warning_step.test.tsx.snap b/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/reindex/flyout/__snapshots__/warning_step.test.tsx.snap index d92db98ae40cb..dba019550f2a1 100644 --- a/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/reindex/flyout/__snapshots__/warning_step.test.tsx.snap +++ b/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/reindex/flyout/__snapshots__/warning_step.test.tsx.snap @@ -23,30 +23,6 @@ exports[`WarningsFlyoutStep renders 1`] = `

- - } - documentationUrl="https://www.elastic.co/guide/en/observability/master/whats-new.html" - label={ - - } - onChange={[Function]} - warning={0} - /> { status: undefined, reindexTaskPercComplete: null, errorMessage: null, - reindexWarnings: [ReindexWarning.apmReindex], + reindexWarnings: [ReindexWarning.customTypeName], hasRequiredPrivileges: true, } as ReindexState, }; diff --git a/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/reindex/flyout/warning_step.test.tsx b/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/reindex/flyout/warning_step.test.tsx index 9f76ef0aa78ba..d365cd82ba86c 100644 --- a/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/reindex/flyout/warning_step.test.tsx +++ b/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/reindex/flyout/warning_step.test.tsx @@ -8,6 +8,7 @@ import { I18nProvider } from '@kbn/i18n/react'; import { mount, shallow } from 'enzyme'; import React from 'react'; +import { mockKibanaSemverVersion } from '../../../../../../../../common/constants'; import { ReindexWarning } from '../../../../../../../../common/types'; import { idForWarning, WarningsFlyoutStep } from './warnings_step'; @@ -20,6 +21,11 @@ jest.mock('../../../../../../app_context', () => { DOC_LINK_VERSION: 'current', ELASTIC_WEBSITE_URL: 'https://www.elastic.co/', }, + kibanaVersionInfo: { + currentMajor: mockKibanaSemverVersion.major, + prevMajor: mockKibanaSemverVersion.major - 1, + nextMajor: mockKibanaSemverVersion.major + 1, + }, }; }, }; @@ -28,7 +34,7 @@ jest.mock('../../../../../../app_context', () => { describe('WarningsFlyoutStep', () => { const defaultProps = { advanceNextStep: jest.fn(), - warnings: [ReindexWarning.apmReindex], + warnings: [ReindexWarning.customTypeName], closeFlyout: jest.fn(), renderGlobalCallouts: jest.fn(), }; @@ -37,19 +43,21 @@ describe('WarningsFlyoutStep', () => { expect(shallow()).toMatchSnapshot(); }); - it('does not allow proceeding until all are checked', () => { - const wrapper = mount( - - - - ); - const button = wrapper.find('EuiButton'); - - button.simulate('click'); - expect(defaultProps.advanceNextStep).not.toHaveBeenCalled(); - - wrapper.find(`input#${idForWarning(ReindexWarning.apmReindex)}`).simulate('change'); - button.simulate('click'); - expect(defaultProps.advanceNextStep).toHaveBeenCalled(); - }); + if (mockKibanaSemverVersion.major === 7) { + it('does not allow proceeding until all are checked', () => { + const wrapper = mount( + + + + ); + const button = wrapper.find('EuiButton'); + + button.simulate('click'); + expect(defaultProps.advanceNextStep).not.toHaveBeenCalled(); + + wrapper.find(`input#${idForWarning(ReindexWarning.customTypeName)}`).simulate('change'); + button.simulate('click'); + expect(defaultProps.advanceNextStep).toHaveBeenCalled(); + }); + } }); diff --git a/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/reindex/flyout/warnings_step.tsx b/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/reindex/flyout/warnings_step.tsx index 2e6b039a2fe76..f6620e4125c9a 100644 --- a/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/reindex/flyout/warnings_step.tsx +++ b/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/reindex/flyout/warnings_step.tsx @@ -10,6 +10,7 @@ import React, { useState } from 'react'; import { EuiButton, EuiButtonEmpty, + EuiCode, EuiCallOut, EuiCheckbox, EuiFlexGroup, @@ -100,9 +101,9 @@ export const WarningsFlyoutStep: React.FunctionComponent @@ -128,25 +129,31 @@ export const WarningsFlyoutStep: React.FunctionComponent - {warnings.includes(ReindexWarning.apmReindex) && ( + {kibanaVersionInfo.currentMajor === 7 && warnings.includes(ReindexWarning.customTypeName) && ( _doc, + }} /> } description={ _doc, + }} /> } - documentationUrl={`${observabilityDocBasePath}/master/whats-new.html`} + documentationUrl={`${esDocBasePath}/${DOC_LINK_VERSION}/removal-of-types.html`} /> )} diff --git a/x-pack/plugins/upgrade_assistant/server/lib/__fixtures__/version.ts b/x-pack/plugins/upgrade_assistant/server/lib/__fixtures__/version.ts index 6caad4f5050fc..d93fe7920f1d7 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/__fixtures__/version.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/__fixtures__/version.ts @@ -5,16 +5,13 @@ * 2.0. */ -import { SemVer } from 'semver'; +import { mockKibanaSemverVersion } from '../../../common/constants'; -export const MOCK_VERSION_STRING = '8.0.0'; - -export const getMockVersionInfo = (versionString = MOCK_VERSION_STRING) => { - const currentVersion = new SemVer(versionString); - const currentMajor = currentVersion.major; +export const getMockVersionInfo = () => { + const currentMajor = mockKibanaSemverVersion.major; return { - currentVersion, + currentVersion: mockKibanaSemverVersion, currentMajor, prevMajor: currentMajor - 1, nextMajor: currentMajor + 1, diff --git a/x-pack/plugins/upgrade_assistant/server/lib/es_migration_apis.test.ts b/x-pack/plugins/upgrade_assistant/server/lib/es_migration_apis.test.ts index 479a7475efd68..9ab8d0aa7cffb 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/es_migration_apis.test.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/es_migration_apis.test.ts @@ -24,6 +24,7 @@ describe('getUpgradeAssistantStatus', () => { const resolvedIndices = { indices: fakeIndexNames.map((f) => ({ name: f, attributes: ['open'] })), }; + // @ts-expect-error mock data is too loosely typed const deprecationsResponse: DeprecationAPIResponse = _.cloneDeep(fakeDeprecations); diff --git a/x-pack/plugins/upgrade_assistant/server/lib/es_version_precheck.test.ts b/x-pack/plugins/upgrade_assistant/server/lib/es_version_precheck.test.ts index 25dcd2521525d..f4631f3ba459d 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/es_version_precheck.test.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/es_version_precheck.test.ts @@ -9,7 +9,8 @@ import { SemVer } from 'semver'; import { IScopedClusterClient, kibanaResponseFactory } from 'src/core/server'; import { coreMock } from 'src/core/server/mocks'; import { licensingMock } from '../../../../plugins/licensing/server/mocks'; -import { MOCK_VERSION_STRING, getMockVersionInfo } from './__fixtures__/version'; +import { mockKibanaVersion } from '../../common/constants'; +import { getMockVersionInfo } from './__fixtures__/version'; import { esVersionCheck, @@ -97,7 +98,7 @@ describe('verifyAllMatchKibanaVersion', () => { describe('EsVersionPrecheck', () => { beforeEach(() => { - versionService.setup(MOCK_VERSION_STRING); + versionService.setup(mockKibanaVersion); }); it('returns a 403 when callCluster fails with a 403', async () => { diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/index_settings.test.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/index_settings.test.ts index 609f36c25619e..f778981b95054 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/index_settings.test.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/index_settings.test.ts @@ -5,8 +5,10 @@ * 2.0. */ +import { mockKibanaSemverVersion, mockKibanaVersion } from '../../../common/constants'; +import { ReindexWarning } from '../../../common/types'; import { versionService } from '../version'; -import { MOCK_VERSION_STRING, getMockVersionInfo } from '../__fixtures__/version'; +import { getMockVersionInfo } from '../__fixtures__/version'; import { generateNewIndexName, @@ -123,7 +125,7 @@ describe('transformFlatSettings', () => { describe('sourceNameForIndex', () => { beforeEach(() => { - versionService.setup(MOCK_VERSION_STRING); + versionService.setup(mockKibanaVersion); }); it('parses internal indices', () => { @@ -144,7 +146,7 @@ describe('sourceNameForIndex', () => { describe('generateNewIndexName', () => { beforeEach(() => { - versionService.setup(MOCK_VERSION_STRING); + versionService.setup(mockKibanaVersion); }); it('parses internal indices', () => { @@ -177,4 +179,26 @@ describe('getReindexWarnings', () => { }) ).toEqual([]); }); + + if (mockKibanaSemverVersion.major === 7) { + describe('customTypeName warning', () => { + it('returns customTypeName for non-_doc mapping types', () => { + expect( + getReindexWarnings({ + settings: {}, + mappings: { doc: {} }, + }) + ).toEqual([ReindexWarning.customTypeName]); + }); + + it('does not return customTypeName for _doc mapping types', () => { + expect( + getReindexWarnings({ + settings: {}, + mappings: { _doc: {} }, + }) + ).toEqual([]); + }); + }); + } }); diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/index_settings.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/index_settings.ts index 11cc01b69d3a5..70e1992d5b3e9 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/index_settings.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/index_settings.ts @@ -8,8 +8,7 @@ import { flow, omit } from 'lodash'; import { ReindexWarning } from '../../../common/types'; import { versionService } from '../version'; -import { FlatSettings } from './types'; - +import { FlatSettings, FlatSettingsWithTypeName } from './types'; export interface ParsedIndexName { cleanIndexName: string; baseName: string; @@ -69,11 +68,24 @@ export const generateNewIndexName = (indexName: string): string => { * Returns an array of warnings that should be displayed to user before reindexing begins. * @param flatSettings */ -export const getReindexWarnings = (flatSettings: FlatSettings): ReindexWarning[] => { +export const getReindexWarnings = ( + flatSettings: FlatSettingsWithTypeName | FlatSettings +): ReindexWarning[] => { const warnings = [ // No warnings yet for 8.0 -> 9.0 ] as Array<[ReindexWarning, boolean]>; + if (versionService.getMajorVersion() === 7) { + const DEFAULT_TYPE_NAME = '_doc'; + // In 7+ it's not possible to have more than one type anyways, so always grab the first + // (and only) key. + const typeName = Object.getOwnPropertyNames(flatSettings.mappings)[0]; + + const typeNameWarning = Boolean(typeName && typeName !== DEFAULT_TYPE_NAME); + + warnings.push([ReindexWarning.customTypeName, typeNameWarning]); + } + return warnings.filter(([_, applies]) => applies).map(([warning, _]) => warning); }; diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.test.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.test.ts index 59c83a05aa551..592c2d15b9c0c 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.test.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.test.ts @@ -19,9 +19,10 @@ import { ReindexStatus, ReindexStep, } from '../../../common/types'; +import { mockKibanaVersion } from '../../../common/constants'; import { versionService } from '../version'; import { LOCK_WINDOW, ReindexActions, reindexActionsFactory } from './reindex_actions'; -import { MOCK_VERSION_STRING, getMockVersionInfo } from '../__fixtures__/version'; +import { getMockVersionInfo } from '../__fixtures__/version'; const { currentMajor, prevMajor } = getMockVersionInfo(); @@ -53,7 +54,7 @@ describe('ReindexActions', () => { describe('createReindexOp', () => { beforeEach(() => { - versionService.setup(MOCK_VERSION_STRING); + versionService.setup(mockKibanaVersion); client.create.mockResolvedValue(); }); diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.ts index 738d54c6f6d4f..fe8844b28e37a 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.ts @@ -21,8 +21,9 @@ import { ReindexStatus, ReindexStep, } from '../../../common/types'; +import { versionService } from '../version'; import { generateNewIndexName } from './index_settings'; -import { FlatSettings } from './types'; +import { FlatSettings, FlatSettingsWithTypeName } from './types'; // TODO: base on elasticsearch.requestTimeout? export const LOCK_WINDOW = moment.duration(90, 'seconds'); @@ -85,7 +86,7 @@ export interface ReindexActions { * Retrieve index settings (in flat, dot-notation style) and mappings. * @param indexName */ - getFlatSettings(indexName: string): Promise; + getFlatSettings(indexName: string): Promise; // ----- Functions below are for enforcing locks around groups of indices like ML or Watcher @@ -237,18 +238,33 @@ export const reindexActionsFactory = ( }, async getFlatSettings(indexName: string) { - const { body: flatSettings } = await esClient.indices.get<{ - [indexName: string]: FlatSettings; - }>({ - index: indexName, - flat_settings: true, - }); + let flatSettings; + + if (versionService.getMajorVersion() === 7) { + // On 7.x, we need to get index settings with mapping type + flatSettings = await esClient.indices.get<{ + [indexName: string]: FlatSettingsWithTypeName; + }>({ + index: indexName, + flat_settings: true, + // This @ts-ignore is needed on master since the flag is deprecated on >7.x + // @ts-ignore + include_type_name: true, + }); + } else { + flatSettings = await esClient.indices.get<{ + [indexName: string]: FlatSettings; + }>({ + index: indexName, + flat_settings: true, + }); + } - if (!flatSettings[indexName]) { + if (!flatSettings.body[indexName]) { return null; } - return flatSettings[indexName]; + return flatSettings.body[indexName]; }, async _fetchAndLockIndexGroupDoc(indexGroup) { diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts index 69105465a04f0..a91cf8ddeada9 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts @@ -20,10 +20,11 @@ import { ReindexStatus, ReindexStep, } from '../../../common/types'; +import { mockKibanaVersion } from '../../../common/constants'; import { licensingMock } from '../../../../licensing/server/mocks'; import { LicensingPluginSetup } from '../../../../licensing/server'; -import { MOCK_VERSION_STRING, getMockVersionInfo } from '../__fixtures__/version'; +import { getMockVersionInfo } from '../__fixtures__/version'; import { esIndicesStateCheck } from '../es_indices_state_check'; import { versionService } from '../version'; @@ -88,7 +89,7 @@ describe('reindexService', () => { licensingPluginSetup ); - versionService.setup(MOCK_VERSION_STRING); + versionService.setup(mockKibanaVersion); }); describe('hasRequiredPrivileges', () => { @@ -215,7 +216,7 @@ describe('reindexService', () => { 'index.provided_name': indexName, }, mappings: { - properties: { https: { type: 'boolean' } }, + _doc: { properties: { https: { type: 'boolean' } } }, }, }); @@ -571,7 +572,10 @@ describe('reindexService', () => { const mlReindexedOp = { id: '2', - attributes: { ...reindexOp.attributes, indexName: '.reindexed-v7-ml-anomalies' }, + attributes: { + ...reindexOp.attributes, + indexName: `.reindexed-v${prevMajor}-ml-anomalies`, + }, } as ReindexSavedObject; const updatedOp = await service.processNextStep(mlReindexedOp); diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts index 72bcb5330f819..1b5f91e0c53b8 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts @@ -219,7 +219,7 @@ export const reindexServiceFactory = ( .cancel({ task_id: reindexOp.attributes.reindexTaskId ?? undefined, }) - .catch((e) => undefined); // Ignore any exceptions trying to cancel (it may have already completed). + .catch(() => undefined); // Ignore any exceptions trying to cancel (it may have already completed). } // Set index back to writable if we ever got past this point. @@ -347,6 +347,11 @@ export const reindexServiceFactory = ( await esClient.indices.open({ index: indexName }); } + const flatSettings = await actions.getFlatSettings(indexName); + if (!flatSettings) { + throw error.indexNotFound(`Index ${indexName} does not exist.`); + } + const { body: startReindexResponse } = await esClient.reindex({ refresh: true, wait_for_completion: false, diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/types.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/types.ts index b24625a8c2a9d..569316e276e43 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/types.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/types.ts @@ -27,3 +27,16 @@ export interface FlatSettings { _meta?: MetaProperties; }; } + +// Specific to 7.x-8 upgrade +export interface FlatSettingsWithTypeName { + settings: { + [key: string]: string; + }; + mappings: { + [typeName: string]: { + properties?: MappingProperties; + _meta?: MetaProperties; + }; + }; +} diff --git a/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices/reindex_indices.test.ts b/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices/reindex_indices.test.ts index 82d039ab9413a..21dded346bbd3 100644 --- a/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices/reindex_indices.test.ts +++ b/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices/reindex_indices.test.ts @@ -89,7 +89,9 @@ describe('reindex API', () => { mockReindexService.findReindexOperation.mockResolvedValueOnce({ attributes: { indexName: 'wowIndex', status: ReindexStatus.inProgress }, }); - mockReindexService.detectReindexWarnings.mockResolvedValueOnce([ReindexWarning.apmReindex]); + mockReindexService.detectReindexWarnings.mockResolvedValueOnce([ + ReindexWarning.customTypeName, + ]); const resp = await routeDependencies.router.getHandler({ method: 'get', From 510bc698ffeb3ec704c379c8aeb16ca6112fc59c Mon Sep 17 00:00:00 2001 From: Corey Robertson Date: Wed, 24 Feb 2021 16:23:44 -0500 Subject: [PATCH 10/13] [Dashboard] Export appropriate references from byValue panels (#91567) * Adds references from byValue panels when saving dashboard * Remove extra spaces * Rework a type check * Fix type check Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../visualize_embeddable_factory.tsx | 40 +++++++++++++++++++ .../embeddable/embeddable_factory.ts | 14 ++++++- .../embeddable/map_embeddable_factory.ts | 17 +++++++- 3 files changed, 69 insertions(+), 2 deletions(-) diff --git a/src/plugins/visualizations/public/embeddable/visualize_embeddable_factory.tsx b/src/plugins/visualizations/public/embeddable/visualize_embeddable_factory.tsx index a7a5b8626914a..349e024f31c31 100644 --- a/src/plugins/visualizations/public/embeddable/visualize_embeddable_factory.tsx +++ b/src/plugins/visualizations/public/embeddable/visualize_embeddable_factory.tsx @@ -9,7 +9,9 @@ import { i18n } from '@kbn/i18n'; import { SavedObjectMetaData, OnSaveProps } from 'src/plugins/saved_objects/public'; import { first } from 'rxjs/operators'; +import { EmbeddableStateWithType } from 'src/plugins/embeddable/common'; import { SavedObjectAttributes } from '../../../../core/public'; +import { extractSearchSourceReferences } from '../../../data/public'; import { EmbeddableFactoryDefinition, EmbeddableOutput, @@ -236,4 +238,42 @@ export class VisualizeEmbeddableFactory } ); } + + public extract(_state: EmbeddableStateWithType) { + const state = (_state as unknown) as VisualizeInput; + const references = []; + + if (state.savedVis?.data.searchSource) { + const [, searchSourceReferences] = extractSearchSourceReferences( + state.savedVis.data.searchSource + ); + + references.push(...searchSourceReferences); + } + + if (state.savedVis?.data.savedSearchId) { + references.push({ + name: 'search_0', + type: 'search', + id: String(state.savedVis.data.savedSearchId), + }); + } + + if (state.savedVis?.params.controls) { + const controls = state.savedVis.params.controls; + controls.forEach((control: Record, i: number) => { + if (!control.indexPattern) { + return; + } + control.indexPatternRefName = `control_${i}_index_pattern`; + references.push({ + name: control.indexPatternRefName, + type: 'index-pattern', + id: control.indexPattern, + }); + }); + } + + return { state: _state, references }; + } } diff --git a/x-pack/plugins/lens/public/editor_frame_service/embeddable/embeddable_factory.ts b/x-pack/plugins/lens/public/editor_frame_service/embeddable/embeddable_factory.ts index 4c40282012d6d..a676b7283671c 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/embeddable/embeddable_factory.ts +++ b/x-pack/plugins/lens/public/editor_frame_service/embeddable/embeddable_factory.ts @@ -5,10 +5,11 @@ * 2.0. */ -import { Capabilities, HttpSetup } from 'kibana/public'; +import { Capabilities, HttpSetup, SavedObjectReference } from 'kibana/public'; import { i18n } from '@kbn/i18n'; import { RecursiveReadonly } from '@kbn/utility-types'; import { Ast } from '@kbn/interpreter/target/common'; +import { EmbeddableStateWithType } from 'src/plugins/embeddable/common'; import { IndexPatternsContract, TimefilterContract, @@ -105,4 +106,15 @@ export class EmbeddableFactory implements EmbeddableFactoryDefinition { parent ); } + + extract(state: EmbeddableStateWithType) { + let references: SavedObjectReference[] = []; + const typedState = (state as unknown) as LensEmbeddableInput; + + if ('attributes' in typedState && typedState.attributes !== undefined) { + references = typedState.attributes.references; + } + + return { state, references }; + } } diff --git a/x-pack/plugins/maps/public/embeddable/map_embeddable_factory.ts b/x-pack/plugins/maps/public/embeddable/map_embeddable_factory.ts index b039076305498..7e15bfa9a340e 100644 --- a/x-pack/plugins/maps/public/embeddable/map_embeddable_factory.ts +++ b/x-pack/plugins/maps/public/embeddable/map_embeddable_factory.ts @@ -6,6 +6,7 @@ */ import { i18n } from '@kbn/i18n'; +import { EmbeddableStateWithType } from 'src/plugins/embeddable/common'; import { EmbeddableFactoryDefinition, IContainer, @@ -13,8 +14,10 @@ import { import '../index.scss'; import { MAP_SAVED_OBJECT_TYPE, APP_ICON } from '../../common/constants'; import { getMapEmbeddableDisplayName } from '../../common/i18n_getters'; -import { MapByReferenceInput, MapEmbeddableInput } from './types'; +import { MapByReferenceInput, MapEmbeddableInput, MapByValueInput } from './types'; import { lazyLoadMapModules } from '../lazy_load_bundle'; +// @ts-expect-error +import { extractReferences } from '../../common/migrations/references'; export class MapEmbeddableFactory implements EmbeddableFactoryDefinition { type = MAP_SAVED_OBJECT_TYPE; @@ -61,4 +64,16 @@ export class MapEmbeddableFactory implements EmbeddableFactoryDefinition { parent ); }; + + extract(state: EmbeddableStateWithType) { + const maybeMapByValueInput = state as EmbeddableStateWithType | MapByValueInput; + + if ((maybeMapByValueInput as MapByValueInput).attributes !== undefined) { + const { references } = extractReferences(maybeMapByValueInput); + + return { state, references }; + } + + return { state, references: [] }; + } } From ebb04354018e5a400a7beb89f71cf188028f1c20 Mon Sep 17 00:00:00 2001 From: Frank Hassanabad Date: Wed, 24 Feb 2021 14:31:55 -0700 Subject: [PATCH 11/13] [Security Solutions][Detection Engine] Fixes bug with not being able to duplicate indicator matches (#92565) ## Summary Fixes an unreleased regression bug where indicator rules could not be be duplicated. https://github.com/elastic/kibana/issues/90356 - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../indicator_match_rule.spec.ts | 32 ++++++++++++++- .../cypress/screens/alerts_detection_rules.ts | 6 +++ .../cypress/tasks/alerts_detection_rules.ts | 29 ++++++++++++++ .../cypress/tasks/api_calls/rules.ts | 40 ++++++++++++++++++- .../detection_engine/rules/all/actions.tsx | 8 +++- .../detection_engine/rules/all/columns.tsx | 1 + 6 files changed, 113 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/cypress/integration/detection_rules/indicator_match_rule.spec.ts b/x-pack/plugins/security_solution/cypress/integration/detection_rules/indicator_match_rule.spec.ts index 966ce3098d6a7..ef9c7f49cb371 100644 --- a/x-pack/plugins/security_solution/cypress/integration/detection_rules/indicator_match_rule.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/detection_rules/indicator_match_rule.spec.ts @@ -60,12 +60,15 @@ import { } from '../../tasks/alerts'; import { changeRowsPerPageTo300, + duplicateFirstRule, + duplicateRuleFromMenu, filterByCustomRules, goToCreateNewRule, goToRuleDetails, waitForRulesTableToBeLoaded, } from '../../tasks/alerts_detection_rules'; -import { cleanKibana } from '../../tasks/common'; +import { createCustomIndicatorRule } from '../../tasks/api_calls/rules'; +import { cleanKibana, reload } from '../../tasks/common'; import { createAndActivateRule, fillAboutRuleAndContinue, @@ -92,8 +95,10 @@ import { waitForAlertsToPopulate, waitForTheRuleToBeExecuted, } from '../../tasks/create_new_rule'; +import { waitForKibana } from '../../tasks/edit_rule'; import { esArchiverLoad, esArchiverUnload } from '../../tasks/es_archiver'; import { loginAndWaitForPageWithoutDateRange } from '../../tasks/login'; +import { goBackToAllRulesTable } from '../../tasks/rule_details'; import { DETECTIONS_URL, RULE_CREATION } from '../../urls/navigation'; @@ -465,5 +470,30 @@ describe('indicator match', () => { cy.get(ALERT_RULE_RISK_SCORE).first().should('have.text', newThreatIndicatorRule.riskScore); }); }); + + describe('Duplicates the indicator rule', () => { + beforeEach(() => { + cleanKibana(); + loginAndWaitForPageWithoutDateRange(DETECTIONS_URL); + goToManageAlertsDetectionRules(); + createCustomIndicatorRule(newThreatIndicatorRule); + reload(); + }); + + it('Allows the rule to be duplicated from the table', () => { + waitForKibana(); + duplicateFirstRule(); + cy.contains(RULE_NAME, `${newThreatIndicatorRule.name} [Duplicate]`); + }); + + it('Allows the rule to be duplicated from the edit screen', () => { + waitForKibana(); + goToRuleDetails(); + duplicateRuleFromMenu(); + goBackToAllRulesTable(); + reload(); + cy.contains(RULE_NAME, `${newThreatIndicatorRule.name} [Duplicate]`); + }); + }); }); }); diff --git a/x-pack/plugins/security_solution/cypress/screens/alerts_detection_rules.ts b/x-pack/plugins/security_solution/cypress/screens/alerts_detection_rules.ts index 68baad7d3d259..30365c9bd4c70 100644 --- a/x-pack/plugins/security_solution/cypress/screens/alerts_detection_rules.ts +++ b/x-pack/plugins/security_solution/cypress/screens/alerts_detection_rules.ts @@ -17,6 +17,12 @@ export const DELETE_RULE_ACTION_BTN = '[data-test-subj="deleteRuleAction"]'; export const EDIT_RULE_ACTION_BTN = '[data-test-subj="editRuleAction"]'; +export const DUPLICATE_RULE_ACTION_BTN = '[data-test-subj="duplicateRuleAction"]'; + +export const DUPLICATE_RULE_MENU_PANEL_BTN = '[data-test-subj="rules-details-duplicate-rule"]'; + +export const REFRESH_BTN = '[data-test-subj="refreshRulesAction"] button'; + export const DELETE_RULE_BULK_BTN = '[data-test-subj="deleteRuleBulk"]'; export const ELASTIC_RULES_BTN = '[data-test-subj="showElasticRulesFilterButton"]'; diff --git a/x-pack/plugins/security_solution/cypress/tasks/alerts_detection_rules.ts b/x-pack/plugins/security_solution/cypress/tasks/alerts_detection_rules.ts index 3553889449e6d..529ef4afdfa63 100644 --- a/x-pack/plugins/security_solution/cypress/tasks/alerts_detection_rules.ts +++ b/x-pack/plugins/security_solution/cypress/tasks/alerts_detection_rules.ts @@ -31,6 +31,8 @@ import { RULE_AUTO_REFRESH_IDLE_MODAL_CONTINUE, rowsPerPageSelector, pageSelector, + DUPLICATE_RULE_ACTION_BTN, + DUPLICATE_RULE_MENU_PANEL_BTN, } from '../screens/alerts_detection_rules'; import { ALL_ACTIONS, DELETE_RULE } from '../screens/rule_details'; @@ -45,6 +47,33 @@ export const editFirstRule = () => { cy.get(EDIT_RULE_ACTION_BTN).click(); }; +export const duplicateFirstRule = () => { + cy.get(COLLAPSED_ACTION_BTN).should('be.visible'); + cy.get(COLLAPSED_ACTION_BTN).first().click({ force: true }); + cy.get(DUPLICATE_RULE_ACTION_BTN).should('be.visible'); + cy.get(DUPLICATE_RULE_ACTION_BTN).click(); +}; + +/** + * Duplicates the rule from the menu and does additional + * pipes and checking that the elements are present on the + * page as well as removed when doing the clicks to help reduce + * flake. + */ +export const duplicateRuleFromMenu = () => { + cy.get(ALL_ACTIONS).should('be.visible'); + cy.root() + .pipe(($el) => { + $el.find(ALL_ACTIONS).trigger('click'); + return $el.find(DUPLICATE_RULE_MENU_PANEL_BTN); + }) + .should(($el) => expect($el).to.be.visible); + // Because of a fade effect and fast clicking this can produce more than one click + cy.get(DUPLICATE_RULE_MENU_PANEL_BTN) + .pipe(($el) => $el.trigger('click')) + .should('not.be.visible'); +}; + export const deleteFirstRule = () => { cy.get(COLLAPSED_ACTION_BTN).first().click({ force: true }); cy.get(DELETE_RULE_ACTION_BTN).click(); diff --git a/x-pack/plugins/security_solution/cypress/tasks/api_calls/rules.ts b/x-pack/plugins/security_solution/cypress/tasks/api_calls/rules.ts index ab6063f5809c4..99f5bd9c20230 100644 --- a/x-pack/plugins/security_solution/cypress/tasks/api_calls/rules.ts +++ b/x-pack/plugins/security_solution/cypress/tasks/api_calls/rules.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { CustomRule } from '../../objects/rule'; +import { CustomRule, ThreatIndicatorRule } from '../../objects/rule'; export const createCustomRule = (rule: CustomRule, ruleId = 'rule_testing') => cy.request({ @@ -29,6 +29,44 @@ export const createCustomRule = (rule: CustomRule, ruleId = 'rule_testing') => failOnStatusCode: false, }); +export const createCustomIndicatorRule = (rule: ThreatIndicatorRule, ruleId = 'rule_testing') => + cy.request({ + method: 'POST', + url: 'api/detection_engine/rules', + body: { + rule_id: ruleId, + risk_score: parseInt(rule.riskScore, 10), + description: rule.description, + interval: '10s', + name: rule.name, + severity: rule.severity.toLocaleLowerCase(), + type: 'threat_match', + threat_mapping: [ + { + entries: [ + { + field: rule.indicatorMapping, + type: 'mapping', + value: rule.indicatorMapping, + }, + ], + }, + ], + threat_query: '*:*', + threat_language: 'kuery', + threat_filters: [], + threat_index: ['mock*'], + threat_indicator_path: '', + from: 'now-17520h', + index: ['exceptions-*'], + query: rule.customQuery || '*:*', + language: 'kuery', + enabled: false, + }, + headers: { 'kbn-xsrf': 'cypress-creds' }, + failOnStatusCode: false, + }); + export const createCustomRuleActivated = (rule: CustomRule, ruleId = '1') => cy.request({ method: 'POST', diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/actions.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/actions.tsx index 3b1f9e620127d..6cc75a3fda03c 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/actions.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/actions.tsx @@ -8,6 +8,7 @@ import * as H from 'history'; import React, { Dispatch } from 'react'; +import { CreateRulesSchema } from '../../../../../../common/detection_engine/schemas/request'; import { deleteRules, duplicateRules, @@ -28,6 +29,7 @@ import { track, METRIC_TYPE, TELEMETRY_EVENT } from '../../../../../common/lib/t import * as i18n from '../translations'; import { bucketRulesResponse } from './helpers'; +import { transformOutput } from '../../../../containers/detection_engine/rules/transforms'; export const editRuleAction = (rule: Rule, history: H.History) => { history.push(getEditRuleUrl(rule.id)); @@ -41,7 +43,11 @@ export const duplicateRulesAction = async ( ) => { try { dispatch({ type: 'loadingRuleIds', ids: ruleIds, actionType: 'duplicate' }); - const response = await duplicateRules({ rules }); + const response = await duplicateRules({ + // We cast this back and forth here as the front end types are not really the right io-ts ones + // and the two types conflict with each other. + rules: rules.map((rule) => transformOutput(rule as CreateRulesSchema) as Rule), + }); const { errors } = bucketRulesResponse(response); if (errors.length > 0) { displayErrorToast( diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/columns.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/columns.tsx index d2488bd3d043c..d2eadef48d9c7 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/columns.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/columns.tsx @@ -67,6 +67,7 @@ export const getActions = ( enabled: (rowItem: Rule) => canEditRuleWithActions(rowItem, actionsPrivileges), }, { + 'data-test-subj': 'duplicateRuleAction', description: i18n.DUPLICATE_RULE, icon: 'copy', name: !actionsPrivileges ? ( From 3471eaa481d72c0971c2268e79eda6d155687800 Mon Sep 17 00:00:00 2001 From: Davis Plumlee <56367316+dplumlee@users.noreply.github.com> Date: Wed, 24 Feb 2021 17:16:50 -0500 Subject: [PATCH 12/13] [Security Solution][Detection Rules] Update prebuilt rule threats to match schema (#92281) --- .../schemas/common/schemas.ts | 20 +++-- .../add_prepackged_rules_schema.test.ts | 21 +++-- .../request/import_rules_schema.test.ts | 21 +++-- .../request/patch_rules_schema.test.ts | 8 +- .../schemas/request/rule_schemas.test.ts | 8 +- .../rules/description_step/helpers.tsx | 89 ++++++++++--------- .../components/rules/mitre/helpers.test.tsx | 2 +- .../rules/mitre/subtechnique_fields.tsx | 51 ++++++----- .../rules/mitre/technique_fields.tsx | 13 +-- .../detection_engine/rules/create/helpers.ts | 2 +- 10 files changed, 137 insertions(+), 98 deletions(-) diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts index d97820f010a80..bfe450d240b08 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts @@ -444,13 +444,19 @@ export const threat_technique = t.intersection([ ]); export type ThreatTechnique = t.TypeOf; export const threat_techniques = t.array(threat_technique); -export const threat = t.exact( - t.type({ - framework: threat_framework, - tactic: threat_tactic, - technique: threat_techniques, - }) -); +export const threat = t.intersection([ + t.exact( + t.type({ + framework: threat_framework, + tactic: threat_tactic, + }) + ), + t.exact( + t.partial({ + technique: threat_techniques, + }) + ), +]); export type Threat = t.TypeOf; export const threats = t.array(threat); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackged_rules_schema.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackged_rules_schema.test.ts index 93094e3445488..f3bef5ad7445f 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackged_rules_schema.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackged_rules_schema.test.ts @@ -924,7 +924,7 @@ describe('add prepackaged rules schema', () => { expect(message.schema).toEqual({}); }); - test('You cannot send in an array of threat that are missing "technique"', () => { + test('You can send in an array of threat that are missing "technique"', () => { const payload: Omit & { threat: Array>>; } = { @@ -944,10 +944,21 @@ describe('add prepackaged rules schema', () => { const decoded = addPrepackagedRulesSchema.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); - expect(getPaths(left(message.errors))).toEqual([ - 'Invalid value "undefined" supplied to "threat,technique"', - ]); - expect(message.schema).toEqual({}); + expect(getPaths(left(message.errors))).toEqual([]); + const expected: AddPrepackagedRulesSchemaDecoded = { + ...getAddPrepackagedRulesSchemaDecodedMock(), + threat: [ + { + framework: 'fake', + tactic: { + id: 'fakeId', + name: 'fakeName', + reference: 'fakeRef', + }, + }, + ], + }; + expect(message.schema).toEqual(expected); }); test('You can optionally send in an array of false positives', () => { diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.test.ts index a59c873658411..2caedd2e01193 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.test.ts @@ -926,7 +926,7 @@ describe('import rules schema', () => { expect(message.schema).toEqual({}); }); - test('You cannot send in an array of threat that are missing "technique"', () => { + test('You can send in an array of threat that are missing "technique"', () => { const payload: Omit & { threat: Array>>; } = { @@ -946,10 +946,21 @@ describe('import rules schema', () => { const decoded = importRulesSchema.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); - expect(getPaths(left(message.errors))).toEqual([ - 'Invalid value "undefined" supplied to "threat,technique"', - ]); - expect(message.schema).toEqual({}); + expect(getPaths(left(message.errors))).toEqual([]); + const expected: ImportRulesSchemaDecoded = { + ...getImportRulesSchemaDecodedMock(), + threat: [ + { + framework: 'fake', + tactic: { + id: 'fakeId', + name: 'fakeName', + reference: 'fakeRef', + }, + }, + ], + }; + expect(message.schema).toEqual(expected); }); test('You can optionally send in an array of false positives', () => { diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.test.ts index 8cdb85a555451..3dfa12acc29d5 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.test.ts @@ -973,7 +973,7 @@ describe('patch_rules_schema', () => { expect(message.schema).toEqual({}); }); - test('threat is invalid when updated with missing technique', () => { + test('threat is valid when updated with missing technique', () => { const threat: Omit = [ { framework: 'fake', @@ -993,10 +993,8 @@ describe('patch_rules_schema', () => { const decoded = patchRulesSchema.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); - expect(getPaths(left(message.errors))).toEqual([ - 'Invalid value "undefined" supplied to "threat,technique"', - ]); - expect(message.schema).toEqual({}); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); }); test('validates with timeline_id and timeline_title', () => { diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.test.ts index 6b8211b23088c..70ff921d3b334 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.test.ts @@ -618,7 +618,7 @@ describe('create rules schema', () => { expect(message.schema).toEqual({}); }); - test('You cannot send in an array of threat that are missing "technique"', () => { + test('You can send in an array of threat that are missing "technique"', () => { const payload = { ...getCreateRulesSchemaMock(), threat: [ @@ -636,10 +636,8 @@ describe('create rules schema', () => { const decoded = createRulesSchema.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); - expect(getPaths(left(message.errors))).toEqual([ - 'Invalid value "undefined" supplied to "threat,technique"', - ]); - expect(message.schema).toEqual({}); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); }); test('You can optionally send in an array of false positives', () => { diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx index 7e2da88a58f18..af3e427056867 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx @@ -157,49 +157,54 @@ export const buildThreatDescription = ({ label, threat }: BuildThreatDescription : `${singleThreat.tactic.name} (${singleThreat.tactic.id})`} - {singleThreat.technique.map((technique, techniqueIndex) => { - const myTechnique = techniquesOptions.find((t) => t.id === technique.id); - return ( - - - {myTechnique != null - ? myTechnique.label - : `${technique.name} (${technique.id})`} - - - {technique.subtechnique != null && - technique.subtechnique.map((subtechnique, subtechniqueIndex) => { - const mySubtechnique = subtechniquesOptions.find( - (t) => t.id === subtechnique.id - ); - return ( - - { + const myTechnique = techniquesOptions.find((t) => t.id === technique.id); + return ( + + + {myTechnique != null + ? myTechnique.label + : `${technique.name} (${technique.id})`} + + + {technique.subtechnique != null && + technique.subtechnique.map((subtechnique, subtechniqueIndex) => { + const mySubtechnique = subtechniquesOptions.find( + (t) => t.id === subtechnique.id + ); + return ( + - {mySubtechnique != null - ? mySubtechnique.label - : `${subtechnique.name} (${subtechnique.id})`} - - - ); - })} - - - ); - })} + + {mySubtechnique != null + ? mySubtechnique.label + : `${subtechnique.name} (${subtechnique.id})`} + + + ); + })} + + + ); + })} ); diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/mitre/helpers.test.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/mitre/helpers.test.tsx index da18f28257452..2a083ef89ab19 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/mitre/helpers.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/mitre/helpers.test.tsx @@ -8,7 +8,7 @@ import { getValidThreat } from '../../../mitre/valid_threat_mock'; import { hasSubtechniqueOptions } from './helpers'; -const mockTechniques = getValidThreat()[0].technique; +const mockTechniques = getValidThreat()[0].technique ?? []; describe('helpers', () => { describe('hasSubtechniqueOptions', () => { diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/mitre/subtechnique_fields.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/mitre/subtechnique_fields.tsx index e3c771534beda..d283c19bd13da 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/mitre/subtechnique_fields.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/mitre/subtechnique_fields.tsx @@ -51,45 +51,46 @@ export const MitreAttackSubtechniqueFields: React.FC = ({ const values = field.value as Threats; const technique = useMemo(() => { - return values[threatIndex].technique[techniqueIndex]; - }, [values, threatIndex, techniqueIndex]); + return [...(values[threatIndex].technique ?? [])]; + }, [values, threatIndex]); const removeSubtechnique = useCallback( (index: number) => { const threats = [...(field.value as Threats)]; - const subtechniques = threats[threatIndex].technique[techniqueIndex].subtechnique; + const subtechniques = technique[techniqueIndex].subtechnique ?? []; if (subtechniques != null) { subtechniques.splice(index, 1); - threats[threatIndex].technique[techniqueIndex] = { - ...threats[threatIndex].technique[techniqueIndex], + technique[techniqueIndex] = { + ...technique[techniqueIndex], subtechnique: subtechniques, }; + threats[threatIndex].technique = technique; onFieldChange(threats); } }, - [field, threatIndex, onFieldChange, techniqueIndex] + [field, onFieldChange, techniqueIndex, technique, threatIndex] ); const addMitreAttackSubtechnique = useCallback(() => { const threats = [...(field.value as Threats)]; - const subtechniques = threats[threatIndex].technique[techniqueIndex].subtechnique; + const subtechniques = technique[techniqueIndex].subtechnique; if (subtechniques != null) { - threats[threatIndex].technique[techniqueIndex] = { - ...threats[threatIndex].technique[techniqueIndex], + technique[techniqueIndex] = { + ...technique[techniqueIndex], subtechnique: [...subtechniques, { id: 'none', name: 'none', reference: 'none' }], }; } else { - threats[threatIndex].technique[techniqueIndex] = { - ...threats[threatIndex].technique[techniqueIndex], + technique[techniqueIndex] = { + ...technique[techniqueIndex], subtechnique: [{ id: 'none', name: 'none', reference: 'none' }], }; } - + threats[threatIndex].technique = technique; onFieldChange(threats); - }, [field, threatIndex, onFieldChange, techniqueIndex]); + }, [field, onFieldChange, techniqueIndex, technique, threatIndex]); const updateSubtechnique = useCallback( (index: number, value: string) => { @@ -99,7 +100,7 @@ export const MitreAttackSubtechniqueFields: React.FC = ({ name: '', reference: '', }; - const subtechniques = threats[threatIndex].technique[techniqueIndex].subtechnique; + const subtechniques = technique[techniqueIndex].subtechnique; if (subtechniques != null) { onFieldChange([ @@ -107,9 +108,9 @@ export const MitreAttackSubtechniqueFields: React.FC = ({ { ...threats[threatIndex], technique: [ - ...threats[threatIndex].technique.slice(0, techniqueIndex), + ...technique.slice(0, techniqueIndex), { - ...threats[threatIndex].technique[techniqueIndex], + ...technique[techniqueIndex], subtechnique: [ ...subtechniques.slice(0, index), { @@ -120,19 +121,21 @@ export const MitreAttackSubtechniqueFields: React.FC = ({ ...subtechniques.slice(index + 1), ], }, - ...threats[threatIndex].technique.slice(techniqueIndex + 1), + ...technique.slice(techniqueIndex + 1), ], }, ...threats.slice(threatIndex + 1), ]); } }, - [threatIndex, techniqueIndex, onFieldChange, field] + [threatIndex, techniqueIndex, onFieldChange, field, technique] ); const getSelectSubtechnique = useCallback( (index: number, disabled: boolean, subtechnique: ThreatSubtechnique) => { - const options = subtechniquesOptions.filter((t) => t.techniqueId === technique.id); + const options = subtechniquesOptions.filter( + (t) => t.techniqueId === technique[techniqueIndex].id + ); return ( <> @@ -166,13 +169,17 @@ export const MitreAttackSubtechniqueFields: React.FC = ({ ); }, - [field, updateSubtechnique, technique] + [field, updateSubtechnique, technique, techniqueIndex] ); + const subtechniques = useMemo(() => { + return technique[techniqueIndex].subtechnique; + }, [technique, techniqueIndex]); + return ( - {technique.subtechnique != null && - technique.subtechnique.map((subtechnique, index) => ( + {subtechniques != null && + subtechniques.map((subtechnique, index) => (
= ({ const removeTechnique = useCallback( (index: number) => { const threats = [...(field.value as Threats)]; - const techniques = threats[threatIndex].technique; + const techniques = threats[threatIndex].technique ?? []; techniques.splice(index, 1); threats[threatIndex] = { ...threats[threatIndex], @@ -73,7 +73,7 @@ export const MitreAttackTechniqueFields: React.FC = ({ threats[threatIndex] = { ...threats[threatIndex], technique: [ - ...threats[threatIndex].technique, + ...(threats[threatIndex].technique ?? []), { id: 'none', name: 'none', reference: 'none', subtechnique: [] }, ], }; @@ -88,19 +88,20 @@ export const MitreAttackTechniqueFields: React.FC = ({ name: '', reference: '', }; + const technique = threats[threatIndex].technique ?? []; onFieldChange([ ...threats.slice(0, threatIndex), { ...threats[threatIndex], technique: [ - ...threats[threatIndex].technique.slice(0, index), + ...technique.slice(0, index), { id, reference, name, subtechnique: [], }, - ...threats[threatIndex].technique.slice(index + 1), + ...technique.slice(index + 1), ], }, ...threats.slice(threatIndex + 1), @@ -147,9 +148,11 @@ export const MitreAttackTechniqueFields: React.FC = ({ [field, updateTechnique] ); + const techniques = values[threatIndex].technique ?? []; + return ( - {values[threatIndex].technique.map((technique, index) => ( + {techniques.map((technique, index) => (
{ .map((threat) => { return { ...threat, - technique: trimThreatsWithNoName(threat.technique).map((technique) => { + technique: trimThreatsWithNoName(threat.technique ?? []).map((technique) => { return { ...technique, subtechnique: From 0280d5a92bc9f7197176599f78894123dc740dd5 Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Wed, 24 Feb 2021 15:25:34 -0700 Subject: [PATCH 13/13] [core.logging] Add RewriteAppender for filtering LogMeta. (#91492) --- ...a-plugin-core-server.appenderconfigtype.md | 2 +- packages/kbn-logging/src/appenders.ts | 18 ++ .../http/integration_tests/logging.test.ts | 136 +++++++++++++++- .../http/logging/get_response_log.test.ts | 47 ++++++ .../server/http/logging/get_response_log.ts | 28 +++- src/core/server/logging/README.mdx | 118 ++++++++++++++ .../server/logging/appenders/appenders.ts | 5 + .../server/logging/appenders/rewrite/mocks.ts | 20 +++ .../appenders/rewrite/policies/index.ts | 30 ++++ .../appenders/rewrite/policies/meta/index.ts | 13 ++ .../rewrite/policies/meta/meta_policy.test.ts | 154 ++++++++++++++++++ .../rewrite/policies/meta/meta_policy.ts | 90 ++++++++++ .../appenders/rewrite/policies/policy.ts | 16 ++ .../rewrite/rewrite_appender.test.mocks.ts | 19 +++ .../rewrite/rewrite_appender.test.ts | 137 ++++++++++++++++ .../appenders/rewrite/rewrite_appender.ts | 100 ++++++++++++ .../server/logging/logging_config.test.ts | 1 - .../server/logging/logging_system.test.ts | 70 ++++++++ src/core/server/logging/logging_system.ts | 37 +++++ src/core/server/server.api.md | 3 +- 20 files changed, 1031 insertions(+), 13 deletions(-) create mode 100644 src/core/server/logging/appenders/rewrite/mocks.ts create mode 100644 src/core/server/logging/appenders/rewrite/policies/index.ts create mode 100644 src/core/server/logging/appenders/rewrite/policies/meta/index.ts create mode 100644 src/core/server/logging/appenders/rewrite/policies/meta/meta_policy.test.ts create mode 100644 src/core/server/logging/appenders/rewrite/policies/meta/meta_policy.ts create mode 100644 src/core/server/logging/appenders/rewrite/policies/policy.ts create mode 100644 src/core/server/logging/appenders/rewrite/rewrite_appender.test.mocks.ts create mode 100644 src/core/server/logging/appenders/rewrite/rewrite_appender.test.ts create mode 100644 src/core/server/logging/appenders/rewrite/rewrite_appender.ts diff --git a/docs/development/core/server/kibana-plugin-core-server.appenderconfigtype.md b/docs/development/core/server/kibana-plugin-core-server.appenderconfigtype.md index a50df950628b3..f6de959589eca 100644 --- a/docs/development/core/server/kibana-plugin-core-server.appenderconfigtype.md +++ b/docs/development/core/server/kibana-plugin-core-server.appenderconfigtype.md @@ -8,5 +8,5 @@ Signature: ```typescript -export declare type AppenderConfigType = ConsoleAppenderConfig | FileAppenderConfig | LegacyAppenderConfig | RollingFileAppenderConfig; +export declare type AppenderConfigType = ConsoleAppenderConfig | FileAppenderConfig | LegacyAppenderConfig | RewriteAppenderConfig | RollingFileAppenderConfig; ``` diff --git a/packages/kbn-logging/src/appenders.ts b/packages/kbn-logging/src/appenders.ts index 1b128c0f29201..48422db34b336 100644 --- a/packages/kbn-logging/src/appenders.ts +++ b/packages/kbn-logging/src/appenders.ts @@ -15,6 +15,24 @@ import { LogRecord } from './log_record'; */ export interface Appender { append(record: LogRecord): void; + /** + * Appenders can be "attached" to one another so that they are able to act + * as a sort of middleware by calling `append` on a different appender. + * + * As appenders cannot be attached to each other until they are configured, + * the `addAppender` method can be used to pass in a newly configured appender + * to attach. + */ + addAppender?(appenderRef: string, appender: Appender): void; + /** + * For appenders which implement `addAppender`, they should declare a list of + * `appenderRefs`, which specify the names of the appenders that their configuration + * depends on. + * + * Note that these are the appender key names that the user specifies in their + * config, _not_ the names of the appender types themselves. + */ + appenderRefs?: string[]; } /** diff --git a/src/core/server/http/integration_tests/logging.test.ts b/src/core/server/http/integration_tests/logging.test.ts index fcf2cd2ba3372..62cb699bc49f6 100644 --- a/src/core/server/http/integration_tests/logging.test.ts +++ b/src/core/server/http/integration_tests/logging.test.ts @@ -251,7 +251,7 @@ describe('request logging', () => { expect(JSON.parse(meta).http.response.headers.bar).toBe('world'); }); - it('filters sensitive request headers', async () => { + it('filters sensitive request headers by default', async () => { const { http } = await root.setup(); http.createRouter('/').post( @@ -283,7 +283,139 @@ describe('request logging', () => { expect(JSON.parse(meta).http.request.headers.authorization).toBe('[REDACTED]'); }); - it('filters sensitive response headers', async () => { + it('filters sensitive request headers when RewriteAppender is configured', async () => { + root = kbnTestServer.createRoot({ + logging: { + silent: true, + appenders: { + 'test-console': { + type: 'console', + layout: { + type: 'pattern', + pattern: '%level|%logger|%message|%meta', + }, + }, + rewrite: { + type: 'rewrite', + appenders: ['test-console'], + policy: { + type: 'meta', + mode: 'update', + properties: [ + { path: 'http.request.headers.authorization', value: '[REDACTED]' }, + ], + }, + }, + }, + loggers: [ + { + name: 'http.server.response', + appenders: ['rewrite'], + level: 'debug', + }, + ], + }, + plugins: { + initialize: false, + }, + }); + const { http } = await root.setup(); + + http.createRouter('/').post( + { + path: '/ping', + validate: { + body: schema.object({ message: schema.string() }), + }, + options: { + authRequired: 'optional', + body: { + accepts: ['application/json'], + }, + timeout: { payload: 100 }, + }, + }, + (context, req, res) => res.ok({ body: { message: req.body.message } }) + ); + await root.start(); + + await kbnTestServer.request + .post(root, '/ping') + .set('content-type', 'application/json') + .set('authorization', 'abc') + .send({ message: 'hi' }) + .expect(200); + expect(mockConsoleLog).toHaveBeenCalledTimes(1); + const [, , , meta] = mockConsoleLog.mock.calls[0][0].split('|'); + expect(JSON.parse(meta).http.request.headers.authorization).toBe('[REDACTED]'); + }); + + it('filters sensitive response headers by defaut', async () => { + const { http } = await root.setup(); + + http.createRouter('/').post( + { + path: '/ping', + validate: { + body: schema.object({ message: schema.string() }), + }, + options: { + authRequired: 'optional', + body: { + accepts: ['application/json'], + }, + timeout: { payload: 100 }, + }, + }, + (context, req, res) => + res.ok({ headers: { 'set-cookie': ['123'] }, body: { message: req.body.message } }) + ); + await root.start(); + + await kbnTestServer.request + .post(root, '/ping') + .set('Content-Type', 'application/json') + .send({ message: 'hi' }) + .expect(200); + expect(mockConsoleLog).toHaveBeenCalledTimes(1); + const [, , , meta] = mockConsoleLog.mock.calls[0][0].split('|'); + expect(JSON.parse(meta).http.response.headers['set-cookie']).toBe('[REDACTED]'); + }); + + it('filters sensitive response headers when RewriteAppender is configured', async () => { + root = kbnTestServer.createRoot({ + logging: { + silent: true, + appenders: { + 'test-console': { + type: 'console', + layout: { + type: 'pattern', + pattern: '%level|%logger|%message|%meta', + }, + }, + rewrite: { + type: 'rewrite', + appenders: ['test-console'], + policy: { + type: 'meta', + mode: 'update', + properties: [{ path: 'http.response.headers.set-cookie', value: '[REDACTED]' }], + }, + }, + }, + loggers: [ + { + name: 'http.server.response', + appenders: ['rewrite'], + level: 'debug', + }, + ], + }, + plugins: { + initialize: false, + }, + }); const { http } = await root.setup(); http.createRouter('/').post( diff --git a/src/core/server/http/logging/get_response_log.test.ts b/src/core/server/http/logging/get_response_log.test.ts index 46c4f1d95e3be..64241ff44fc6b 100644 --- a/src/core/server/http/logging/get_response_log.test.ts +++ b/src/core/server/http/logging/get_response_log.test.ts @@ -171,6 +171,53 @@ describe('getEcsResponseLog', () => { }); test('does not mutate original headers', () => { + const reqHeaders = { a: 'foo', b: ['hello', 'world'] }; + const resHeaders = { headers: { c: 'bar' } }; + const req = createMockHapiRequest({ + headers: reqHeaders, + response: { headers: resHeaders }, + }); + + const responseLog = getEcsResponseLog(req, logger); + expect(reqHeaders).toMatchInlineSnapshot(` + Object { + "a": "foo", + "b": Array [ + "hello", + "world", + ], + } + `); + expect(resHeaders).toMatchInlineSnapshot(` + Object { + "headers": Object { + "c": "bar", + }, + } + `); + + responseLog.http.request.headers.a = 'testA'; + responseLog.http.request.headers.b[1] = 'testB'; + responseLog.http.request.headers.c = 'testC'; + expect(reqHeaders).toMatchInlineSnapshot(` + Object { + "a": "foo", + "b": Array [ + "hello", + "world", + ], + } + `); + expect(resHeaders).toMatchInlineSnapshot(` + Object { + "headers": Object { + "c": "bar", + }, + } + `); + }); + + test('does not mutate original headers when redacting sensitive data', () => { const reqHeaders = { authorization: 'a', cookie: 'b', 'user-agent': 'hi' }; const resHeaders = { headers: { 'content-length': 123, 'set-cookie': 'c' } }; const req = createMockHapiRequest({ diff --git a/src/core/server/http/logging/get_response_log.ts b/src/core/server/http/logging/get_response_log.ts index f75acde93bf40..57c02e05bebff 100644 --- a/src/core/server/http/logging/get_response_log.ts +++ b/src/core/server/http/logging/get_response_log.ts @@ -18,14 +18,22 @@ const ECS_VERSION = '1.7.0'; const FORBIDDEN_HEADERS = ['authorization', 'cookie', 'set-cookie']; const REDACTED_HEADER_TEXT = '[REDACTED]'; +type HapiHeaders = Record; + // We are excluding sensitive headers by default, until we have a log filtering mechanism. -function redactSensitiveHeaders( - headers?: Record -): Record { - const result = {} as Record; +function redactSensitiveHeaders(key: string, value: string | string[]): string | string[] { + return FORBIDDEN_HEADERS.includes(key) ? REDACTED_HEADER_TEXT : value; +} + +// Shallow clone the headers so they are not mutated if filtered by a RewriteAppender. +function cloneAndFilterHeaders(headers?: HapiHeaders) { + const result = {} as HapiHeaders; if (headers) { for (const key of Object.keys(headers)) { - result[key] = FORBIDDEN_HEADERS.includes(key) ? REDACTED_HEADER_TEXT : headers[key]; + result[key] = redactSensitiveHeaders( + key, + Array.isArray(headers[key]) ? [...headers[key]] : headers[key] + ); } } return result; @@ -45,7 +53,11 @@ export function getEcsResponseLog(request: Request, log: Logger): LogMeta { // eslint-disable-next-line @typescript-eslint/naming-convention const status_code = isBoom(response) ? response.output.statusCode : response.statusCode; - const responseHeaders = isBoom(response) ? response.output.headers : response.headers; + + const requestHeaders = cloneAndFilterHeaders(request.headers); + const responseHeaders = cloneAndFilterHeaders( + isBoom(response) ? (response.output.headers as HapiHeaders) : response.headers + ); // borrowed from the hapi/good implementation const responseTime = (request.info.completed || request.info.responded) - request.info.received; @@ -66,7 +78,7 @@ export function getEcsResponseLog(request: Request, log: Logger): LogMeta { mime_type: request.mime, referrer: request.info.referrer, // @ts-expect-error Headers are not yet part of ECS: https://github.com/elastic/ecs/issues/232. - headers: redactSensitiveHeaders(request.headers), + headers: requestHeaders, }, response: { body: { @@ -74,7 +86,7 @@ export function getEcsResponseLog(request: Request, log: Logger): LogMeta { }, status_code, // @ts-expect-error Headers are not yet part of ECS: https://github.com/elastic/ecs/issues/232. - headers: redactSensitiveHeaders(responseHeaders), + headers: responseHeaders, // responseTime is a custom non-ECS field responseTime: !isNaN(responseTime) ? responseTime : undefined, }, diff --git a/src/core/server/logging/README.mdx b/src/core/server/logging/README.mdx index 8c093d0231585..1575e67d7b8ee 100644 --- a/src/core/server/logging/README.mdx +++ b/src/core/server/logging/README.mdx @@ -278,6 +278,124 @@ The maximum number of files to keep. Once this number is reached, oldest files w The default value is `7` +### Rewrite Appender + +*This appender is currently considered experimental and is not intended +for public consumption. The API is subject to change at any time.* + +Similar to log4j's `RewriteAppender`, this appender serves as a sort of middleware, +modifying the provided log events before passing them along to another +appender. + +```yaml +logging: + appenders: + my-rewrite-appender: + type: rewrite + appenders: [console, file] # name of "destination" appender(s) + policy: + # ... +``` + +The most common use case for the `RewriteAppender` is when you want to +filter or censor sensitive data that may be contained in a log entry. +In fact, with a default configuration, Kibana will automatically redact +any `authorization`, `cookie`, or `set-cookie` headers when logging http +requests & responses. + +To configure additional rewrite rules, you'll need to specify a `RewritePolicy`. + +#### Rewrite Policies + +Rewrite policies exist to indicate which parts of a log record can be +modified within the rewrite appender. + +**Meta** + +The `meta` rewrite policy can read and modify any data contained in the +`LogMeta` before passing it along to a destination appender. + +Meta policies must specify one of three modes, which indicate which action +to perform on the configured properties: +- `update` updates an existing property at the provided `path`. +- `remove` removes an existing property at the provided `path`. + +The `properties` are listed as a `path` and `value` pair, where `path` is +the dot-delimited path to the target property in the `LogMeta` object, and +`value` is the value to add or update in that target property. When using +the `remove` mode, a `value` is not necessary. + +Here's an example of how you would replace any `cookie` header values with `[REDACTED]`: + +```yaml +logging: + appenders: + my-rewrite-appender: + type: rewrite + appenders: [console] + policy: + type: meta # indicates that we want to rewrite the LogMeta + mode: update # will update an existing property only + properties: + - path: "http.request.headers.cookie" # path to property + value: "[REDACTED]" # value to replace at path +``` + +Rewrite appenders can even be passed to other rewrite appenders to apply +multiple filter policies/modes, as long as it doesn't create a circular +reference. Each rewrite appender is applied sequentially (one after the other). +```yaml +logging: + appenders: + remove-request-headers: + type: rewrite + appenders: [censor-response-headers] # redirect to the next rewrite appender + policy: + type: meta + mode: remove + properties: + - path: "http.request.headers" # remove all request headers + censor-response-headers: + type: rewrite + appenders: [console] # output to console + policy: + type: meta + mode: update + properties: + - path: "http.response.headers.set-cookie" + value: "[REDACTED]" +``` + +#### Complete Example +```yaml +logging: + appenders: + console: + type: console + layout: + type: pattern + highlight: true + pattern: "[%date][%level][%logger] %message %meta" + file: + type: file + fileName: ./kibana.log + layout: + type: json + censor: + type: rewrite + appenders: [console, file] + policy: + type: meta + mode: update + properties: + - path: "http.request.headers.cookie" + value: "[REDACTED]" + loggers: + - name: http.server.response + appenders: [censor] # pass these logs to our rewrite appender + level: debug +``` + ## Configuration As any configuration in the platform, logging configuration is validated against the predefined schema and if there are diff --git a/src/core/server/logging/appenders/appenders.ts b/src/core/server/logging/appenders/appenders.ts index a41a6a2f68fa1..88df355bd5ebe 100644 --- a/src/core/server/logging/appenders/appenders.ts +++ b/src/core/server/logging/appenders/appenders.ts @@ -17,6 +17,7 @@ import { import { Layouts } from '../layouts/layouts'; import { ConsoleAppender, ConsoleAppenderConfig } from './console/console_appender'; import { FileAppender, FileAppenderConfig } from './file/file_appender'; +import { RewriteAppender, RewriteAppenderConfig } from './rewrite/rewrite_appender'; import { RollingFileAppender, RollingFileAppenderConfig, @@ -32,6 +33,7 @@ export const appendersSchema = schema.oneOf([ ConsoleAppender.configSchema, FileAppender.configSchema, LegacyAppender.configSchema, + RewriteAppender.configSchema, RollingFileAppender.configSchema, ]); @@ -40,6 +42,7 @@ export type AppenderConfigType = | ConsoleAppenderConfig | FileAppenderConfig | LegacyAppenderConfig + | RewriteAppenderConfig | RollingFileAppenderConfig; /** @internal */ @@ -57,6 +60,8 @@ export class Appenders { return new ConsoleAppender(Layouts.create(config.layout)); case 'file': return new FileAppender(Layouts.create(config.layout), config.fileName); + case 'rewrite': + return new RewriteAppender(config); case 'rolling-file': return new RollingFileAppender(config); case 'legacy-appender': diff --git a/src/core/server/logging/appenders/rewrite/mocks.ts b/src/core/server/logging/appenders/rewrite/mocks.ts new file mode 100644 index 0000000000000..a19756e25bf8e --- /dev/null +++ b/src/core/server/logging/appenders/rewrite/mocks.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { RewritePolicy } from './policies/policy'; + +const createPolicyMock = () => { + const mock: jest.Mocked = { + rewrite: jest.fn((x) => x), + }; + return mock; +}; + +export const rewriteAppenderMocks = { + createPolicy: createPolicyMock, +}; diff --git a/src/core/server/logging/appenders/rewrite/policies/index.ts b/src/core/server/logging/appenders/rewrite/policies/index.ts new file mode 100644 index 0000000000000..ae3be1e4de916 --- /dev/null +++ b/src/core/server/logging/appenders/rewrite/policies/index.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { assertNever } from '@kbn/std'; +import { RewritePolicy } from './policy'; +import { MetaRewritePolicy, MetaRewritePolicyConfig, metaRewritePolicyConfigSchema } from './meta'; + +export { RewritePolicy }; + +/** + * Available rewrite policies which specify what part of a {@link LogRecord} + * can be modified. + */ +export type RewritePolicyConfig = MetaRewritePolicyConfig; + +export const rewritePolicyConfigSchema = metaRewritePolicyConfigSchema; + +export const createRewritePolicy = (config: RewritePolicyConfig): RewritePolicy => { + switch (config.type) { + case 'meta': + return new MetaRewritePolicy(config); + default: + return assertNever(config.type); + } +}; diff --git a/src/core/server/logging/appenders/rewrite/policies/meta/index.ts b/src/core/server/logging/appenders/rewrite/policies/meta/index.ts new file mode 100644 index 0000000000000..afdfd6fb709d3 --- /dev/null +++ b/src/core/server/logging/appenders/rewrite/policies/meta/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export { + MetaRewritePolicy, + MetaRewritePolicyConfig, + metaRewritePolicyConfigSchema, +} from './meta_policy'; diff --git a/src/core/server/logging/appenders/rewrite/policies/meta/meta_policy.test.ts b/src/core/server/logging/appenders/rewrite/policies/meta/meta_policy.test.ts new file mode 100644 index 0000000000000..52b88331a75be --- /dev/null +++ b/src/core/server/logging/appenders/rewrite/policies/meta/meta_policy.test.ts @@ -0,0 +1,154 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { LogRecord, LogLevel, LogMeta } from '@kbn/logging'; +import { MetaRewritePolicy, MetaRewritePolicyConfig } from './meta_policy'; + +describe('MetaRewritePolicy', () => { + const createPolicy = ( + mode: MetaRewritePolicyConfig['mode'], + properties: MetaRewritePolicyConfig['properties'] + ) => new MetaRewritePolicy({ type: 'meta', mode, properties }); + + const createLogRecord = (meta: LogMeta = {}): LogRecord => ({ + timestamp: new Date(Date.UTC(2012, 1, 1, 14, 30, 22, 11)), + level: LogLevel.Info, + context: 'context', + message: 'just a log', + pid: 42, + meta, + }); + + describe('mode: update', () => { + it('updates existing properties in LogMeta', () => { + const log = createLogRecord({ a: 'before' }); + const policy = createPolicy('update', [{ path: 'a', value: 'after' }]); + expect(policy.rewrite(log).meta!.a).toBe('after'); + }); + + it('updates nested properties in LogMeta', () => { + const log = createLogRecord({ a: 'before a', b: { c: 'before b.c' }, d: [0, 1] }); + const policy = createPolicy('update', [ + { path: 'a', value: 'after a' }, + { path: 'b.c', value: 'after b.c' }, + { path: 'd[1]', value: 2 }, + ]); + expect(policy.rewrite(log).meta).toMatchInlineSnapshot(` + Object { + "a": "after a", + "b": Object { + "c": "after b.c", + }, + "d": Array [ + 0, + 2, + ], + } + `); + }); + + it('handles string, number, boolean, null', () => { + const policy = createPolicy('update', [ + { path: 'a', value: false }, + { path: 'b', value: null }, + { path: 'c', value: 123 }, + { path: 'd', value: 'hi' }, + ]); + const log = createLogRecord({ + a: 'a', + b: 'b', + c: 'c', + d: 'd', + }); + expect(policy.rewrite(log).meta).toMatchInlineSnapshot(` + Object { + "a": false, + "b": null, + "c": 123, + "d": "hi", + } + `); + }); + + it(`does not add properties which don't exist yet`, () => { + const policy = createPolicy('update', [ + { path: 'a.b', value: 'foo' }, + { path: 'a.c', value: 'bar' }, + ]); + const log = createLogRecord({ a: { b: 'existing meta' } }); + const { meta } = policy.rewrite(log); + expect(meta!.a.b).toBe('foo'); + expect(meta!.a.c).toBeUndefined(); + }); + + it('does not touch anything outside of LogMeta', () => { + const policy = createPolicy('update', [{ path: 'a', value: 'bar' }]); + const message = Symbol(); + expect( + policy.rewrite(({ message, meta: { a: 'foo' } } as unknown) as LogRecord).message + ).toBe(message); + expect(policy.rewrite(({ message, meta: { a: 'foo' } } as unknown) as LogRecord)) + .toMatchInlineSnapshot(` + Object { + "message": Symbol(), + "meta": Object { + "a": "bar", + }, + } + `); + }); + }); + + describe('mode: remove', () => { + it('removes existing properties in LogMeta', () => { + const log = createLogRecord({ a: 'goodbye' }); + const policy = createPolicy('remove', [{ path: 'a' }]); + expect(policy.rewrite(log).meta!.a).toBeUndefined(); + }); + + it('removes nested properties in LogMeta', () => { + const log = createLogRecord({ a: 'a', b: { c: 'b.c' }, d: [0, 1] }); + const policy = createPolicy('remove', [{ path: 'b.c' }, { path: 'd[1]' }]); + expect(policy.rewrite(log).meta).toMatchInlineSnapshot(` + Object { + "a": "a", + "b": Object {}, + "d": Array [ + 0, + undefined, + ], + } + `); + }); + + it('has no effect if property does not exist', () => { + const log = createLogRecord({ a: 'a' }); + const policy = createPolicy('remove', [{ path: 'b' }]); + expect(policy.rewrite(log).meta).toMatchInlineSnapshot(` + Object { + "a": "a", + } + `); + }); + + it('does not touch anything outside of LogMeta', () => { + const policy = createPolicy('remove', [{ path: 'message' }]); + const message = Symbol(); + expect( + policy.rewrite(({ message, meta: { message: 'foo' } } as unknown) as LogRecord).message + ).toBe(message); + expect(policy.rewrite(({ message, meta: { message: 'foo' } } as unknown) as LogRecord)) + .toMatchInlineSnapshot(` + Object { + "message": Symbol(), + "meta": Object {}, + } + `); + }); + }); +}); diff --git a/src/core/server/logging/appenders/rewrite/policies/meta/meta_policy.ts b/src/core/server/logging/appenders/rewrite/policies/meta/meta_policy.ts new file mode 100644 index 0000000000000..2215b3489539f --- /dev/null +++ b/src/core/server/logging/appenders/rewrite/policies/meta/meta_policy.ts @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { schema } from '@kbn/config-schema'; +import { LogRecord } from '@kbn/logging'; +import { set } from '@elastic/safer-lodash-set'; +import { has, unset } from 'lodash'; +import { assertNever } from '@kbn/std'; +import { RewritePolicy } from '../policy'; + +type MetaRewritePolicyConfigProperties = Array<{ + path: string; + value?: string | number | boolean | null; +}>; + +export interface MetaRewritePolicyConfig { + type: 'meta'; + + /** + * The 'mode' specifies what action to perform on the specified properties. + * - 'update' updates an existing property at the provided 'path'. + * - 'remove' removes an existing property at the provided 'path'. + */ + mode: 'remove' | 'update'; + + /** + * The properties to modify. + * + * @remarks + * Each provided 'path' is relative to the record's {@link LogMeta}. + * For the 'remove' mode, no 'value' is provided. + */ + properties: MetaRewritePolicyConfigProperties; +} + +export const metaRewritePolicyConfigSchema = schema.object({ + type: schema.literal('meta'), + mode: schema.oneOf([schema.literal('update'), schema.literal('remove')], { + defaultValue: 'update', + }), + properties: schema.arrayOf( + schema.object({ + path: schema.string(), + value: schema.maybe( + schema.nullable(schema.oneOf([schema.string(), schema.number(), schema.boolean()])) + ), + }) + ), +}); + +/** + * A rewrite policy which can add, remove, or update properties + * from a record's {@link LogMeta}. + */ +export class MetaRewritePolicy implements RewritePolicy { + constructor(private readonly config: MetaRewritePolicyConfig) {} + + rewrite(record: LogRecord): LogRecord { + switch (this.config.mode) { + case 'update': + return this.update(record); + case 'remove': + return this.remove(record); + default: + return assertNever(this.config.mode); + } + } + + private update(record: LogRecord) { + for (const { path, value } of this.config.properties) { + if (!has(record, `meta.${path}`)) { + continue; // don't add properties which don't already exist + } + set(record, `meta.${path}`, value); + } + return record; + } + + private remove(record: LogRecord) { + for (const { path } of this.config.properties) { + unset(record, `meta.${path}`); + } + return record; + } +} diff --git a/src/core/server/logging/appenders/rewrite/policies/policy.ts b/src/core/server/logging/appenders/rewrite/policies/policy.ts new file mode 100644 index 0000000000000..f8aef887965fd --- /dev/null +++ b/src/core/server/logging/appenders/rewrite/policies/policy.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { LogRecord } from '@kbn/logging'; + +/** + * Rewrites a {@link LogRecord} based on the policy's configuration. + **/ +export interface RewritePolicy { + rewrite(record: LogRecord): LogRecord; +} diff --git a/src/core/server/logging/appenders/rewrite/rewrite_appender.test.mocks.ts b/src/core/server/logging/appenders/rewrite/rewrite_appender.test.mocks.ts new file mode 100644 index 0000000000000..9d29a68305792 --- /dev/null +++ b/src/core/server/logging/appenders/rewrite/rewrite_appender.test.mocks.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { schema } from '@kbn/config-schema'; + +export const createRewritePolicyMock = jest.fn(); +jest.doMock('./policies', () => ({ + rewritePolicyConfigSchema: schema.any(), + createRewritePolicy: createRewritePolicyMock, +})); + +export const resetAllMocks = () => { + createRewritePolicyMock.mockReset(); +}; diff --git a/src/core/server/logging/appenders/rewrite/rewrite_appender.test.ts b/src/core/server/logging/appenders/rewrite/rewrite_appender.test.ts new file mode 100644 index 0000000000000..72a54b5012ce5 --- /dev/null +++ b/src/core/server/logging/appenders/rewrite/rewrite_appender.test.ts @@ -0,0 +1,137 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { MockedKeys } from '@kbn/utility-types/jest'; +import { createRewritePolicyMock, resetAllMocks } from './rewrite_appender.test.mocks'; +import { rewriteAppenderMocks } from './mocks'; +import { LogLevel, LogRecord, LogMeta, DisposableAppender } from '@kbn/logging'; +import { RewriteAppender, RewriteAppenderConfig } from './rewrite_appender'; + +// Helper to ensure tuple is typed [A, B] instead of Array +const toTuple = (a: A, b: B): [A, B] => [a, b]; + +const createAppenderMock = (name: string) => { + const appenderMock: MockedKeys = { + append: jest.fn(), + dispose: jest.fn(), + }; + + return toTuple(name, appenderMock); +}; + +const createConfig = (appenderNames: string[]): RewriteAppenderConfig => ({ + type: 'rewrite', + appenders: appenderNames, + policy: { + type: 'meta', + mode: 'update', + properties: [{ path: 'foo', value: 'bar' }], + }, +}); + +const createLogRecord = (meta: LogMeta = {}): LogRecord => ({ + timestamp: new Date(), + level: LogLevel.Info, + context: 'context', + message: 'just a log', + pid: 42, + meta, +}); + +describe('RewriteAppender', () => { + let policy: ReturnType; + + beforeEach(() => { + policy = rewriteAppenderMocks.createPolicy(); + createRewritePolicyMock.mockReturnValue(policy); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + afterAll(() => { + resetAllMocks(); + }); + + it('creates a rewrite policy with the provided config', () => { + const config = createConfig([]); + new RewriteAppender(config); + expect(createRewritePolicyMock).toHaveBeenCalledTimes(1); + expect(createRewritePolicyMock).toHaveBeenCalledWith(config.policy); + }); + + describe('#addAppender', () => { + it('updates the map of available appenders', () => { + const config = createConfig(['mock1']); + const appender = new RewriteAppender(config); + appender.addAppender(...createAppenderMock('mock1')); + expect(() => { + appender.append(createLogRecord()); + }).not.toThrowError(); + }); + }); + + describe('#append', () => { + it('calls the configured appenders with the provided LogRecord', () => { + const config = createConfig(['mock1', 'mock2']); + const appenderMocks = [createAppenderMock('mock1'), createAppenderMock('mock2')]; + + const appender = new RewriteAppender(config); + appenderMocks.forEach((mock) => appender.addAppender(...mock)); + + const log1 = createLogRecord({ a: 'b' }); + const log2 = createLogRecord({ c: 'd' }); + + appender.append(log1); + + expect(appenderMocks[0][1].append).toHaveBeenCalledTimes(1); + expect(appenderMocks[1][1].append).toHaveBeenCalledTimes(1); + expect(appenderMocks[0][1].append).toHaveBeenCalledWith(log1); + expect(appenderMocks[1][1].append).toHaveBeenCalledWith(log1); + + appender.append(log2); + + expect(appenderMocks[0][1].append).toHaveBeenCalledTimes(2); + expect(appenderMocks[1][1].append).toHaveBeenCalledTimes(2); + expect(appenderMocks[0][1].append).toHaveBeenCalledWith(log2); + expect(appenderMocks[1][1].append).toHaveBeenCalledWith(log2); + }); + + it('calls `rewrite` on the configured policy', () => { + const config = createConfig(['mock1']); + + const appender = new RewriteAppender(config); + appender.addAppender(...createAppenderMock('mock1')); + + const log1 = createLogRecord({ a: 'b' }); + const log2 = createLogRecord({ c: 'd' }); + + appender.append(log1); + + expect(policy.rewrite).toHaveBeenCalledTimes(1); + expect(policy.rewrite.mock.calls).toEqual([[log1]]); + + appender.append(log2); + + expect(policy.rewrite).toHaveBeenCalledTimes(2); + expect(policy.rewrite.mock.calls).toEqual([[log1], [log2]]); + }); + + it('throws if an appender key cannot be found', () => { + const config = createConfig(['oops']); + const appender = new RewriteAppender(config); + + expect(() => { + appender.append(createLogRecord()); + }).toThrowErrorMatchingInlineSnapshot( + `"Rewrite Appender could not find appender key \\"oops\\". Be sure \`appender.addAppender()\` was called before \`appender.append()\`."` + ); + }); + }); +}); diff --git a/src/core/server/logging/appenders/rewrite/rewrite_appender.ts b/src/core/server/logging/appenders/rewrite/rewrite_appender.ts new file mode 100644 index 0000000000000..e54d8ba40ebfc --- /dev/null +++ b/src/core/server/logging/appenders/rewrite/rewrite_appender.ts @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { schema } from '@kbn/config-schema'; +import { LogRecord, Appender, DisposableAppender } from '@kbn/logging'; +import { + createRewritePolicy, + rewritePolicyConfigSchema, + RewritePolicy, + RewritePolicyConfig, +} from './policies'; + +export interface RewriteAppenderConfig { + type: 'rewrite'; + /** + * The {@link Appender | appender(s)} to pass the log event to after + * implementing the specified rewrite policy. + */ + appenders: string[]; + /** + * The {@link RewritePolicy | policy} to use to manipulate the provided data. + */ + policy: RewritePolicyConfig; +} + +/** + * Appender that can modify the `LogRecord` instances it receives before passing + * them along to another {@link Appender}. + * @internal + */ +export class RewriteAppender implements DisposableAppender { + public static configSchema = schema.object({ + type: schema.literal('rewrite'), + appenders: schema.arrayOf(schema.string(), { defaultValue: [] }), + policy: rewritePolicyConfigSchema, + }); + + private appenders: Map = new Map(); + private readonly policy: RewritePolicy; + + constructor(private readonly config: RewriteAppenderConfig) { + this.policy = createRewritePolicy(config.policy); + } + + /** + * List of appenders that are dependencies of this appender. + * + * `addAppender` will throw an error when called with an appender + * reference that isn't in this list. + */ + public get appenderRefs() { + return this.config.appenders; + } + + /** + * Appenders can be "attached" to this one so that the RewriteAppender + * is able to act as a sort of middleware by calling `append` on other appenders. + * + * As appenders cannot be attached to each other until they are created, + * the `addAppender` method is used to pass in a configured appender. + */ + public addAppender(appenderRef: string, appender: Appender) { + if (!this.appenderRefs.includes(appenderRef)) { + throw new Error( + `addAppender was called with an appender key that is missing from the appenderRefs: "${appenderRef}".` + ); + } + + this.appenders.set(appenderRef, appender); + } + + /** + * Modifies the `record` and passes it to the specified appender. + */ + public append(record: LogRecord) { + const rewrittenRecord = this.policy.rewrite(record); + for (const appenderRef of this.appenderRefs) { + const appender = this.appenders.get(appenderRef); + if (!appender) { + throw new Error( + `Rewrite Appender could not find appender key "${appenderRef}". ` + + 'Be sure `appender.addAppender()` was called before `appender.append()`.' + ); + } + appender.append(rewrittenRecord); + } + } + + /** + * Disposes `RewriteAppender`. + */ + public dispose() { + this.appenders.clear(); + } +} diff --git a/src/core/server/logging/logging_config.test.ts b/src/core/server/logging/logging_config.test.ts index 2cb5831a8fb4c..83f3c139e371a 100644 --- a/src/core/server/logging/logging_config.test.ts +++ b/src/core/server/logging/logging_config.test.ts @@ -78,7 +78,6 @@ test('correctly fills in custom `appenders` config.', () => { type: 'console', layout: { type: 'pattern', highlight: true }, }); - expect(configValue.appenders.get('console')).toEqual({ type: 'console', layout: { type: 'pattern' }, diff --git a/src/core/server/logging/logging_system.test.ts b/src/core/server/logging/logging_system.test.ts index f68d6c6a97fbc..8a6fe71bc6222 100644 --- a/src/core/server/logging/logging_system.test.ts +++ b/src/core/server/logging/logging_system.test.ts @@ -134,6 +134,76 @@ test('uses `root` logger if context name is not specified.', async () => { expect(mockConsoleLog.mock.calls).toMatchSnapshot(); }); +test('attaches appenders to appenders that declare refs', async () => { + await system.upgrade( + config.schema.validate({ + appenders: { + console: { + type: 'console', + layout: { type: 'pattern', pattern: '[%logger] %message %meta' }, + }, + file: { + type: 'file', + layout: { type: 'pattern', pattern: '[%logger] %message %meta' }, + fileName: 'path', + }, + rewrite: { + type: 'rewrite', + appenders: ['console', 'file'], + policy: { type: 'meta', mode: 'remove', properties: [{ path: 'b' }] }, + }, + }, + loggers: [{ name: 'tests', level: 'warn', appenders: ['rewrite'] }], + }) + ); + + const testLogger = system.get('tests'); + testLogger.warn('This message goes to a test context.', { a: 'hi', b: 'remove me' }); + + expect(mockConsoleLog).toHaveBeenCalledTimes(1); + expect(mockConsoleLog.mock.calls[0][0]).toMatchInlineSnapshot( + `"[tests] This message goes to a test context. {\\"a\\":\\"hi\\"}"` + ); + + expect(mockStreamWrite).toHaveBeenCalledTimes(1); + expect(mockStreamWrite.mock.calls[0][0]).toMatchInlineSnapshot(` + "[tests] This message goes to a test context. {\\"a\\":\\"hi\\"} + " + `); +}); + +test('throws if a circular appender reference is detected', async () => { + expect(async () => { + await system.upgrade( + config.schema.validate({ + appenders: { + console: { type: 'console', layout: { type: 'pattern' } }, + a: { + type: 'rewrite', + appenders: ['b'], + policy: { type: 'meta', mode: 'remove', properties: [{ path: 'b' }] }, + }, + b: { + type: 'rewrite', + appenders: ['c'], + policy: { type: 'meta', mode: 'remove', properties: [{ path: 'b' }] }, + }, + c: { + type: 'rewrite', + appenders: ['console', 'a'], + policy: { type: 'meta', mode: 'remove', properties: [{ path: 'b' }] }, + }, + }, + loggers: [{ name: 'tests', level: 'warn', appenders: ['a'] }], + }) + ); + }).rejects.toThrowErrorMatchingInlineSnapshot( + `"Circular appender reference detected: [b -> c -> a -> b]"` + ); + + expect(mockConsoleLog).toHaveBeenCalledTimes(0); +}); + test('`stop()` disposes all appenders.', async () => { await system.upgrade( config.schema.validate({ diff --git a/src/core/server/logging/logging_system.ts b/src/core/server/logging/logging_system.ts index 9ae434aff41d3..d7c34b48c4101 100644 --- a/src/core/server/logging/logging_system.ts +++ b/src/core/server/logging/logging_system.ts @@ -146,6 +146,26 @@ export class LoggingSystem implements LoggerFactory { return this.getLoggerConfigByContext(config, LoggingConfig.getParentLoggerContext(context)); } + /** + * Retrieves an appender by the provided key, after first checking that no circular + * dependencies exist between appender refs. + */ + private getAppenderByRef(appenderRef: string) { + const checkCircularRefs = (key: string, stack: string[]) => { + if (stack.includes(key)) { + throw new Error(`Circular appender reference detected: [${stack.join(' -> ')} -> ${key}]`); + } + stack.push(key); + const appender = this.appenders.get(key); + if (appender?.appenderRefs) { + appender.appenderRefs.forEach((ref) => checkCircularRefs(ref, [...stack])); + } + return appender; + }; + + return checkCircularRefs(appenderRef, []); + } + private async applyBaseConfig(newBaseConfig: LoggingConfig) { const computedConfig = [...this.contextConfigs.values()].reduce( (baseConfig, contextConfig) => baseConfig.extend(contextConfig), @@ -167,6 +187,23 @@ export class LoggingSystem implements LoggerFactory { this.appenders.set(appenderKey, Appenders.create(appenderConfig)); } + // Once all appenders have been created, check for any that have explicitly + // declared `appenderRefs` dependencies, and look up those dependencies to + // attach to the appender. This enables appenders to act as a sort of + // middleware and call `append` on each other if needed. + for (const [key, appender] of this.appenders) { + if (!appender.addAppender || !appender.appenderRefs) { + continue; + } + for (const ref of appender.appenderRefs) { + const foundAppender = this.getAppenderByRef(ref); + if (!foundAppender) { + throw new Error(`Appender "${key}" config contains unknown appender key "${ref}".`); + } + appender.addAppender(ref, foundAppender); + } + } + for (const [loggerKey, loggerAdapter] of this.loggers) { loggerAdapter.updateLogger(this.createLogger(loggerKey, computedConfig)); } diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md index 2177da84b2b53..cc1fb05c0c7dd 100644 --- a/src/core/server/server.api.md +++ b/src/core/server/server.api.md @@ -180,10 +180,11 @@ export interface AppCategory { // Warning: (ae-forgotten-export) The symbol "ConsoleAppenderConfig" needs to be exported by the entry point index.d.ts // Warning: (ae-forgotten-export) The symbol "FileAppenderConfig" needs to be exported by the entry point index.d.ts // Warning: (ae-forgotten-export) The symbol "LegacyAppenderConfig" needs to be exported by the entry point index.d.ts +// Warning: (ae-forgotten-export) The symbol "RewriteAppenderConfig" needs to be exported by the entry point index.d.ts // Warning: (ae-forgotten-export) The symbol "RollingFileAppenderConfig" needs to be exported by the entry point index.d.ts // // @public (undocumented) -export type AppenderConfigType = ConsoleAppenderConfig | FileAppenderConfig | LegacyAppenderConfig | RollingFileAppenderConfig; +export type AppenderConfigType = ConsoleAppenderConfig | FileAppenderConfig | LegacyAppenderConfig | RewriteAppenderConfig | RollingFileAppenderConfig; // @public @deprecated (undocumented) export interface AssistanceAPIResponse {