From 1b6822cb381dcdc7ca18c0f2f62ab5efa0aaff55 Mon Sep 17 00:00:00 2001 From: Gerard Soldevila Date: Wed, 15 Nov 2023 13:09:40 +0100 Subject: [PATCH 1/5] Retry parsing logs if we do not find the predicate we were looking for (#171116) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Attempt at fixing https://github.com/elastic/kibana/issues/166007 The logs must always be present in order for the tests to pass. I assume the flakiness is due to a race condition and that we are sometimes reading the logs before the logger actually writes the statements we are looking for. This PR aims at re-processing the whole log file **only** if we fail to find a predicate we were looking for. NB we cannot wait for logs to be "complete" as Kibana is running in the background and keeps adding new entries non-stop. --- ➡️ Flaky Test Runner Pipeline - 50x 🟢 https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/3976 ➡️ Addressed PR feedback to reenable `lnsMetric` individual test. ➡️ Flaky Test Runner Pipeline - 300x 🟢 https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/3995 --- .../tests/browser.ts | 98 +++++++++---------- 1 file changed, 47 insertions(+), 51 deletions(-) diff --git a/x-pack/test/functional_execution_context/tests/browser.ts b/x-pack/test/functional_execution_context/tests/browser.ts index 1696b7ef3c312..f7d265316decb 100644 --- a/x-pack/test/functional_execution_context/tests/browser.ts +++ b/x-pack/test/functional_execution_context/tests/browser.ts @@ -12,8 +12,28 @@ import { assertLogContains, isExecutionContextLog, readLogFile } from '../test_u export default function ({ getService, getPageObjects }: FtrProviderContext) { const PageObjects = getPageObjects(['common', 'dashboard', 'header', 'home', 'timePicker']); - // Failing: See https://github.com/elastic/kibana/issues/166007 - describe.skip('Browser apps', () => { + describe('Browser apps', () => { + let logs: Ecs[]; + const retry = getService('retry'); + + const logContains = async ({ + predicate, + description, + }: { + predicate: (record: Ecs) => boolean; + description: string; + }) => { + return retry.try(async () => { + try { + await assertLogContains({ logs, predicate, description }); + } catch (err) { + // if we did not find our predicate in the logs, wait a bit and parse them again + await new Promise((resolve) => setTimeout(resolve, 10000)); + logs = await readLogFile(); + throw err; + } + }); + }; before(async () => { await PageObjects.common.navigateToUrl('home', '/tutorial_directory/sampleData', { useActualUrl: true, @@ -31,8 +51,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); describe('discover app', () => { - let logs: Ecs[]; - before(async () => { await PageObjects.common.navigateToApp('discover'); await PageObjects.timePicker.setCommonlyUsedTime('Last_7 days'); @@ -53,16 +71,15 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('propagates context for Discover', () => { it('propagates to Elasticsearch via "x-opaque-id" header', async () => { - await assertLogContains({ + await logContains({ description: 'execution context propagates to Elasticsearch via "x-opaque-id" header', predicate: (record) => Boolean(record.http?.request?.id?.includes('kibana:application:discover')), - logs, }); }); it('propagates to Kibana logs (fetch documents)', async () => { - await assertLogContains({ + await logContains({ description: 'execution context propagates to Kibana logs (fetch documents)', predicate: checkExecutionContextEntry({ type: 'application', @@ -77,12 +94,11 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { description: 'fetch documents', }, }), - logs, }); }); it('propagates to Kibana logs (fetch chart data and total hits)', async () => { - await assertLogContains({ + await logContains({ description: 'execution context propagates to Kibana logs (fetch chart data and total hits)', predicate: checkExecutionContextEntry({ @@ -105,15 +121,12 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }, }, }), - logs, }); }); }); }); describe('dashboard app', () => { - let logs: Ecs[]; - before(async () => { await PageObjects.common.navigateToApp('dashboard'); await PageObjects.dashboard.loadSavedDashboard('[Flights] Global Flight Dashboard'); @@ -170,17 +183,16 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('propagates context for Lens visualizations', () => { describe('lnsXY', () => { it('propagates to Elasticsearch via "x-opaque-id" header', async () => { - await assertLogContains({ + await logContains({ description: 'execution context propagates to Elasticsearch via "x-opaque-id" header', predicate: checkHttpRequestId( 'dashboard:dashboards:7adfa750-4c81-11e8-b3d7-01146121b73d;lens:lnsXY:086ac2e9-dd16-4b45-92b8-1e43ff7e3f65' ), - logs, }); }); it('propagates to Kibana logs', async () => { - await assertLogContains({ + await logContains({ description: 'execution context propagates to Kibana logs', predicate: checkExecutionContextEntry({ type: 'application', @@ -202,24 +214,22 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }, }, }), - logs, }); }); }); - describe.skip('lnsMetric', () => { + describe('lnsMetric', () => { it('propagates to Elasticsearch via "x-opaque-id" header', async () => { - await assertLogContains({ + await logContains({ description: 'execution context propagates to Elasticsearch via "x-opaque-id" header', predicate: checkHttpRequestId( - 'dashboard:dashboards:7adfa750-4c81-11e8-b3d7-01146121b73d;lens:lnsLegacyMetric:b766e3b8-4544-46ed-99e6-9ecc4847e2a2' + 'dashboard:dashboards:7adfa750-4c81-11e8-b3d7-01146121b73d;lens:lnsMetric:b766e3b8-4544-46ed-99e6-9ecc4847e2a2' ), - logs, }); }); it('propagates to Kibana logs', async () => { - await assertLogContains({ + await logContains({ description: 'execution context propagates to Kibana logs', predicate: checkExecutionContextEntry({ name: 'dashboards', @@ -234,31 +244,29 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { description: '[Flights] Global Flight Dashboard', child: { type: 'lens', - name: 'lnsLegacyMetric', + name: 'lnsMetric', id: 'b766e3b8-4544-46ed-99e6-9ecc4847e2a2', description: '', url: '/app/lens#/edit_by_value', }, }, }), - logs, }); }); }); describe('lnsDatatable', () => { it('propagates to Elasticsearch via "x-opaque-id" header', async () => { - await assertLogContains({ + await logContains({ description: 'execution context propagates to Elasticsearch via "x-opaque-id" header', predicate: checkHttpRequestId( 'dashboard:dashboards:7adfa750-4c81-11e8-b3d7-01146121b73d;lens:lnsDatatable:fb86b32f-fb7a-45cf-9511-f366fef51bbd' ), - logs, }); }); it('propagates to Kibana logs', async () => { - await assertLogContains({ + await logContains({ description: 'execution context propagates to Kibana logs', predicate: checkExecutionContextEntry({ name: 'dashboards', @@ -280,24 +288,22 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }, }, }), - logs, }); }); }); describe('lnsPie', () => { it('propagates to Elasticsearch via "x-opaque-id" header', async () => { - await assertLogContains({ + await logContains({ description: 'execution context propagates to Elasticsearch via "x-opaque-id" header', predicate: checkHttpRequestId( 'dashboard:dashboards:7adfa750-4c81-11e8-b3d7-01146121b73d;lens:lnsPie:5d53db36-2d5a-4adc-af7b-cec4c1a294e0' ), - logs, }); }); it('propagates to Kibana logs', async () => { - await assertLogContains({ + await logContains({ description: 'execution context propagates to Kibana logs', predicate: checkExecutionContextEntry({ name: 'dashboards', @@ -319,7 +325,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }, }, }), - logs, }); }); }); @@ -327,17 +332,16 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('propagates context for built-in Discover', () => { it('propagates to Elasticsearch via "x-opaque-id" header', async () => { - await assertLogContains({ + await logContains({ description: 'execution context propagates to Elasticsearch via "x-opaque-id" header', predicate: checkHttpRequestId( 'dashboard:dashboards:7adfa750-4c81-11e8-b3d7-01146121b73d;search:discover:571aaf70-4c88-11e8-b3d7-01146121b73d' ), - logs, }); }); it('propagates to Kibana logs', async () => { - await assertLogContains({ + await logContains({ description: 'execution context propagates to Kibana logs', predicate: checkExecutionContextEntry({ type: 'application', @@ -359,22 +363,20 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }, }, }), - logs, }); }); }); describe.skip('propagates context for TSVB visualizations', () => { it('propagates to Elasticsearch via "x-opaque-id" header', async () => { - await assertLogContains({ + await logContains({ description: 'execution context propagates to Elasticsearch via "x-opaque-id" header', predicate: checkHttpRequestId('agg_based:metrics:bcb63b50-4c89-11e8-b3d7-01146121b73d'), - logs, }); }); it('propagates to Kibana logs', async () => { - await assertLogContains({ + await logContains({ description: 'execution context propagates to Kibana logs', predicate: checkExecutionContextEntry({ name: 'dashboards', @@ -390,24 +392,23 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { url: '/app/visualize#/edit/bcb63b50-4c89-11e8-b3d7-01146121b73d', }, }), - logs, }); }); }); describe('propagates context for Vega visualizations', () => { + // CHECKPOINT this is the test that failed and caused the global .skip() it('propagates to Elasticsearch via "x-opaque-id" header', async () => { - await assertLogContains({ + await logContains({ description: 'execution context propagates to Elasticsearch via "x-opaque-id" header', predicate: checkHttpRequestId( 'dashboard:dashboards:7adfa750-4c81-11e8-b3d7-01146121b73d;agg_based:vega:ed78a660-53a0-11e8-acbd-0be0ad9d822b' ), - logs, }); }); it('propagates to Kibana logs', async () => { - await assertLogContains({ + await logContains({ description: 'execution context propagates to Kibana logs', predicate: checkExecutionContextEntry({ name: 'dashboards', @@ -429,24 +430,22 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }, }, }), - logs, }); }); }); describe.skip('propagates context for Tag Cloud visualization', () => { it('propagates to Elasticsearch via "x-opaque-id" header', async () => { - await assertLogContains({ + await logContains({ description: 'execution context propagates to Elasticsearch via "x-opaque-id" header', predicate: checkHttpRequestId( 'dashboard:dashboards:7adfa750-4c81-11e8-b3d7-01146121b73d;agg_based:tagcloud:293b5a30-4c8f-11e8-b3d7-01146121b73d' ), - logs, }); }); it('propagates to Kibana logs', async () => { - await assertLogContains({ + await logContains({ description: 'execution context propagates to Kibana logs', predicate: checkExecutionContextEntry({ name: 'dashboards', @@ -468,24 +467,22 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }, }, }), - logs, }); }); }); describe.skip('propagates context for Vertical bar visualization', () => { it('propagates to Elasticsearch via "x-opaque-id" header', async () => { - await assertLogContains({ + await logContains({ description: 'execution context propagates to Elasticsearch via "x-opaque-id" header', predicate: checkHttpRequestId( 'dashboard:dashboards:7adfa750-4c81-11e8-b3d7-01146121b73d;agg_based:histogram:9886b410-4c8b-11e8-b3d7-01146121b73d' ), - logs, }); }); it('propagates to Kibana logs', async () => { - await assertLogContains({ + await logContains({ description: 'execution context propagates to Kibana logs', predicate: checkExecutionContextEntry({ type: 'application', @@ -507,7 +504,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }, }, }), - logs, }); }); }); From e7cf51b63caa861618af7e5835f23f7e25998328 Mon Sep 17 00:00:00 2001 From: Konrad Szwarc Date: Wed, 15 Nov 2023 13:37:11 +0100 Subject: [PATCH 2/5] [EDR Workflows] Osquery Timeout (#169925) https://github.com/elastic/security-team/issues/7775 This PR adds `Timeout` form field to all the places where Osquery query can be configured. Behind the scenes all the scenarios lead to the value being passed down to the fleet. Live Query ![Screenshot 2023-11-06 at 16 20 05](https://github.com/elastic/kibana/assets/29123534/e87dd693-0f29-42a3-affb-20b0b42ff8ab) Live Pack Query ![Screenshot 2023-11-06 at 17 12 41](https://github.com/elastic/kibana/assets/29123534/35424508-b97c-43fa-befd-01f494b61ffd) Saved Query ![Screenshot 2023-11-06 at 17 11 58](https://github.com/elastic/kibana/assets/29123534/e5100fa6-7251-46a2-b83c-f843d157e889) Osquery Response Action Query ![Screenshot 2023-11-06 at 17 13 40](https://github.com/elastic/kibana/assets/29123534/b6ac4dd6-0d85-4ed5-9d4d-c1c485182a70) --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../current_mappings.json | 105 ++++++++-------- .../src/live_query/index.ts | 6 + .../kbn-osquery-io-ts-types/tsconfig.json | 3 + .../check_registered_types.test.ts | 6 +- .../api/live_query/create_live_query_route.ts | 2 + .../saved_query/create_saved_query_route.ts | 2 + .../saved_query/update_saved_query_route.ts | 2 + x-pack/plugins/osquery/common/constants.ts | 5 + .../osquery/cypress/e2e/all/live_query.cy.ts | 16 ++- .../cypress/e2e/all/live_query_packs.cy.ts | 3 + .../cypress/e2e/all/packs_create_edit.cy.ts | 3 + .../osquery/cypress/tasks/live_query.ts | 6 + x-pack/plugins/osquery/public/form/index.ts | 1 + .../osquery/public/form/timeout_field.tsx | 114 ++++++++++++++++++ .../public/live_queries/form/index.tsx | 7 +- .../form/live_query_query_field.tsx | 6 + .../public/packs/form/queries_field.tsx | 3 + .../osquery/public/packs/form/utils.ts | 1 + .../public/packs/queries/query_flyout.tsx | 24 +++- .../packs/queries/use_pack_query_form.tsx | 4 + .../routes/saved_queries/list/index.tsx | 1 + .../public/saved_queries/form/index.tsx | 9 ++ .../form/use_saved_query_form.tsx | 5 + .../saved_queries/saved_queries_dropdown.tsx | 2 +- .../osquery_response_action_type/index.tsx | 5 + x-pack/plugins/osquery/server/common/types.ts | 2 + .../handlers/action/create_action_handler.ts | 8 +- .../server/handlers/action/create_queries.ts | 7 +- .../lib/saved_query/saved_object_mappings.ts | 23 ++++ .../saved_object_model_versions.ts | 49 ++++++++ .../osquery/server/routes/pack/utils.ts | 12 +- .../saved_query/create_saved_query_route.ts | 9 +- .../saved_query/find_saved_query_route.ts | 2 + .../saved_query/read_saved_query_route.ts | 2 + .../server/routes/saved_query/types.ts | 2 + .../saved_query/update_saved_query_route.ts | 3 + 36 files changed, 389 insertions(+), 71 deletions(-) create mode 100644 x-pack/plugins/osquery/public/form/timeout_field.tsx create mode 100644 x-pack/plugins/osquery/server/lib/saved_query/saved_object_model_versions.ts diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index d2fde2f5a10a8..603f0efd54a87 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -42,10 +42,6 @@ } } }, - "metrics-data-source": { - "dynamic": false, - "properties": {} - }, "url": { "dynamic": false, "properties": { @@ -1473,6 +1469,10 @@ } } }, + "metrics-data-source": { + "dynamic": false, + "properties": {} + }, "canvas-element": { "dynamic": false, "properties": { @@ -2168,6 +2168,9 @@ "interval": { "type": "keyword" }, + "timeout": { + "type": "short" + }, "ecs_mapping": { "dynamic": false, "properties": {} @@ -2216,6 +2219,9 @@ "interval": { "type": "text" }, + "timeout": { + "type": "short" + }, "platform": { "type": "keyword" }, @@ -2258,6 +2264,9 @@ "interval": { "type": "text" }, + "timeout": { + "type": "short" + }, "platform": { "type": "keyword" }, @@ -2679,50 +2688,6 @@ } } }, - "apm-telemetry": { - "dynamic": false, - "properties": {} - }, - "apm-server-schema": { - "properties": { - "schemaJson": { - "type": "text", - "index": false - } - } - }, - "apm-service-group": { - "properties": { - "groupName": { - "type": "keyword" - }, - "kuery": { - "type": "text" - }, - "description": { - "type": "text" - }, - "color": { - "type": "text" - } - } - }, - "apm-custom-dashboards": { - "properties": { - "dashboardSavedObjectId": { - "type": "keyword" - }, - "kuery": { - "type": "text" - }, - "serviceEnvironmentFilterEnabled": { - "type": "boolean" - }, - "serviceNameFilterEnabled": { - "type": "boolean" - } - } - }, "enterprise_search_telemetry": { "dynamic": false, "properties": {} @@ -3196,5 +3161,49 @@ "index": false } } + }, + "apm-telemetry": { + "dynamic": false, + "properties": {} + }, + "apm-server-schema": { + "properties": { + "schemaJson": { + "type": "text", + "index": false + } + } + }, + "apm-service-group": { + "properties": { + "groupName": { + "type": "keyword" + }, + "kuery": { + "type": "text" + }, + "description": { + "type": "text" + }, + "color": { + "type": "text" + } + } + }, + "apm-custom-dashboards": { + "properties": { + "dashboardSavedObjectId": { + "type": "keyword" + }, + "kuery": { + "type": "text" + }, + "serviceEnvironmentFilterEnabled": { + "type": "boolean" + }, + "serviceNameFilterEnabled": { + "type": "boolean" + } + } } } diff --git a/packages/kbn-osquery-io-ts-types/src/live_query/index.ts b/packages/kbn-osquery-io-ts-types/src/live_query/index.ts index 378bb446cafab..6fc9af23a6a72 100644 --- a/packages/kbn-osquery-io-ts-types/src/live_query/index.ts +++ b/packages/kbn-osquery-io-ts-types/src/live_query/index.ts @@ -7,6 +7,7 @@ */ import * as t from 'io-ts'; +import { inRangeRt } from '@kbn/io-ts-utils'; export const id = t.string; export type Id = t.TypeOf; @@ -49,6 +50,11 @@ export type Interval = t.TypeOf; export const intervalOrUndefined = t.union([interval, t.undefined]); export type IntervalOrUndefined = t.TypeOf; +export const timeout = inRangeRt(60, 60 * 15); +export type Timeout = t.TypeOf; +export const timeoutOrUndefined = t.union([timeout, t.undefined]); +export type TimeoutOrUndefined = t.TypeOf; + export const snapshot = t.boolean; export type Snapshot = t.TypeOf; export const snapshotOrUndefined = t.union([snapshot, t.undefined]); diff --git a/packages/kbn-osquery-io-ts-types/tsconfig.json b/packages/kbn-osquery-io-ts-types/tsconfig.json index b72f7b0a15c5c..780216a67a247 100644 --- a/packages/kbn-osquery-io-ts-types/tsconfig.json +++ b/packages/kbn-osquery-io-ts-types/tsconfig.json @@ -12,5 +12,8 @@ ], "exclude": [ "target/**/*", + ], + "kbn_references": [ + "@kbn/io-ts-utils", ] } diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index dd736740a9377..2c7d540132139 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -125,9 +125,9 @@ describe('checking migration metadata changes on all registered SO types', () => "monitoring-telemetry": "5d91bf75787d9d4dd2fae954d0b3f76d33d2e559", "observability-onboarding-state": "b16064c516aac64ae699c737d7d10b6e199bfded", "osquery-manager-usage-metric": "983bcbc3b7dda0aad29b20907db233abba709bcc", - "osquery-pack": "6ab4358ca4304a12dcfc1777c8135b75cffb4397", - "osquery-pack-asset": "b14101d3172c4b60eb5404696881ce5275c84152", - "osquery-saved-query": "44f1161e165defe3f9b6ad643c68c542a765fcdb", + "osquery-pack": "702e86b1a936153b39f65b0781bdc136e186e123", + "osquery-pack-asset": "cd140bc2e4b092e93692b587bf6e38051ef94c75", + "osquery-saved-query": "6095e288750aa3164dfe186c74bc5195c2bf2bd4", "policy-settings-protection-updates-note": "33924bb246f9e5bcb876109cc83e3c7a28308352", "query": "21cbbaa09abb679078145ce90087b1e88b7eae95", "risk-engine-configuration": "b105d4a3c6adce40708d729d12e5ef3c8fbd9508", diff --git a/x-pack/plugins/osquery/common/api/live_query/create_live_query_route.ts b/x-pack/plugins/osquery/common/api/live_query/create_live_query_route.ts index 462490fdbae96..bd2187ec9494e 100644 --- a/x-pack/plugins/osquery/common/api/live_query/create_live_query_route.ts +++ b/x-pack/plugins/osquery/common/api/live_query/create_live_query_route.ts @@ -13,6 +13,7 @@ import { packIdOrUndefined, queryOrUndefined, arrayQueries, + timeoutOrUndefined, } from '@kbn/osquery-io-ts-types'; export const createLiveQueryRequestBodySchema = t.partial({ @@ -23,6 +24,7 @@ export const createLiveQueryRequestBodySchema = t.partial({ query: queryOrUndefined, queries: arrayQueries, saved_query_id: savedQueryIdOrUndefined, + timeout: timeoutOrUndefined, ecs_mapping: ecsMappingOrUndefined, pack_id: packIdOrUndefined, alert_ids: t.array(t.string), diff --git a/x-pack/plugins/osquery/common/api/saved_query/create_saved_query_route.ts b/x-pack/plugins/osquery/common/api/saved_query/create_saved_query_route.ts index b55f9b62c02ed..22d70b6d284b0 100644 --- a/x-pack/plugins/osquery/common/api/saved_query/create_saved_query_route.ts +++ b/x-pack/plugins/osquery/common/api/saved_query/create_saved_query_route.ts @@ -18,6 +18,7 @@ import { snapshotOrUndefined, removedOrUndefined, ecsMappingOrUndefined, + timeoutOrUndefined, } from '@kbn/osquery-io-ts-types'; export const createSavedQueryRequestSchema = t.type({ @@ -27,6 +28,7 @@ export const createSavedQueryRequestSchema = t.type({ query, version: versionOrUndefined, interval, + timeout: timeoutOrUndefined, snapshot: snapshotOrUndefined, removed: removedOrUndefined, ecs_mapping: ecsMappingOrUndefined, diff --git a/x-pack/plugins/osquery/common/api/saved_query/update_saved_query_route.ts b/x-pack/plugins/osquery/common/api/saved_query/update_saved_query_route.ts index 274b52094b1b4..e13014c28bb0d 100644 --- a/x-pack/plugins/osquery/common/api/saved_query/update_saved_query_route.ts +++ b/x-pack/plugins/osquery/common/api/saved_query/update_saved_query_route.ts @@ -7,12 +7,14 @@ import * as t from 'io-ts'; import { toNumberRt } from '@kbn/io-ts-utils'; +import { timeoutOrUndefined } from '@kbn/osquery-io-ts-types'; export const updateSavedQueryRequestBodySchema = t.type({ id: t.string, query: t.string, description: t.union([t.string, t.undefined]), interval: t.union([toNumberRt, t.undefined]), + timeout: timeoutOrUndefined, snapshot: t.union([t.boolean, t.undefined]), removed: t.union([t.boolean, t.undefined]), platform: t.union([t.string, t.undefined]), diff --git a/x-pack/plugins/osquery/common/constants.ts b/x-pack/plugins/osquery/common/constants.ts index fe3454090277a..5887d783d4ce4 100644 --- a/x-pack/plugins/osquery/common/constants.ts +++ b/x-pack/plugins/osquery/common/constants.ts @@ -29,3 +29,8 @@ export const API_VERSIONS = { v1: '1', }, }; + +export enum QUERY_TIMEOUT { + DEFAULT = 60, // 60 seconds + MAX = 60 * 15, +} diff --git a/x-pack/plugins/osquery/cypress/e2e/all/live_query.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/live_query.cy.ts index 3e84dc001eed5..e43dbf4b5d487 100644 --- a/x-pack/plugins/osquery/cypress/e2e/all/live_query.cy.ts +++ b/x-pack/plugins/osquery/cypress/e2e/all/live_query.cy.ts @@ -8,6 +8,7 @@ import { navigateTo } from '../../tasks/navigation'; import { checkResults, + fillInQueryTimeout, inputQuery, selectAllAgents, submitQuery, @@ -36,13 +37,26 @@ describe('ALL - Live Query', { tags: ['@ess', '@serverless'] }, () => { cy.contains('Query is a required field').should('not.exist'); checkResults(); getAdvancedButton().click(); + fillInQueryTimeout('91'); + submitQuery(); + cy.contains('Timeout value must be lower than 900 seconds.'); + fillInQueryTimeout('89'); + submitQuery(); + cy.contains('Timeout value must be lower than 900 seconds.').should('not.exist'); typeInOsqueryFieldInput('days{downArrow}{enter}'); submitQuery(); cy.contains('ECS field is required.'); typeInECSFieldInput('message{downArrow}{enter}'); + + cy.intercept('POST', '/api/osquery/live_queries').as('postQuery'); submitQuery(); cy.contains('ECS field is required.').should('not.exist'); - + cy.wait('@postQuery').then((interception) => { + expect(interception.request.body).to.have.property('query', 'select * from uptime;'); + expect(interception.request.body).to.have.property('timeout', 890); + expect(interception.response?.statusCode).to.eq(200); + expect(interception.response?.body.data.queries[0]).to.have.property('timeout', 890); + }); checkResults(); cy.get('[data-gridcell-column-index="0"][data-gridcell-row-index="0"]').should('exist').click(); cy.url().should('include', 'app/fleet/agents/'); diff --git a/x-pack/plugins/osquery/cypress/e2e/all/live_query_packs.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/live_query_packs.cy.ts index da598382bf0ee..85ae94dd3266d 100644 --- a/x-pack/plugins/osquery/cypress/e2e/all/live_query_packs.cy.ts +++ b/x-pack/plugins/osquery/cypress/e2e/all/live_query_packs.cy.ts @@ -29,18 +29,21 @@ describe('ALL - Live Query Packs', { tags: ['@ess', '@serverless'] }, () => { system_memory_linux_elastic: { ecs_mapping: {}, interval: 3600, + timeout: 700, platform: 'linux', query: 'SELECT * FROM memory_info;', }, system_info_elastic: { ecs_mapping: {}, interval: 3600, + timeout: 200, platform: 'linux,windows,darwin', query: 'SELECT * FROM system_info;', }, failingQuery: { ecs_mapping: {}, interval: 10, + timeout: 90, query: 'select opera_extensions.* from users join opera_extensions using (uid);', }, }, diff --git a/x-pack/plugins/osquery/cypress/e2e/all/packs_create_edit.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/packs_create_edit.cy.ts index 32e2496f8ea06..ca83482cea37b 100644 --- a/x-pack/plugins/osquery/cypress/e2e/all/packs_create_edit.cy.ts +++ b/x-pack/plugins/osquery/cypress/e2e/all/packs_create_edit.cy.ts @@ -197,18 +197,21 @@ describe('Packs - Create and Edit', { tags: ['@ess', '@serverless'] }, () => { const queries = { Query1: { interval: 3600, + timeout: 60, query: 'select * from uptime;', removed: true, snapshot: false, }, Query2: { interval: 3600, + timeout: 60, query: 'select * from uptime;', removed: false, snapshot: false, }, Query3: { interval: 3600, + timeout: 60, query: 'select * from uptime;', }, }; diff --git a/x-pack/plugins/osquery/cypress/tasks/live_query.ts b/x-pack/plugins/osquery/cypress/tasks/live_query.ts index d57f778e2c966..cc9319633befc 100644 --- a/x-pack/plugins/osquery/cypress/tasks/live_query.ts +++ b/x-pack/plugins/osquery/cypress/tasks/live_query.ts @@ -40,6 +40,12 @@ export const submitQuery = () => { cy.contains('Submit').click(); }; +export const fillInQueryTimeout = (timeout: string) => { + cy.getBySel('advanced-accordion-content').within(() => { + cy.getBySel('timeout-input').clear().type(timeout); + }); +}; + // sometimes the results get stuck in the tests, this is a workaround export const checkResults = () => { cy.getBySel('osqueryResultsTable').then(($table) => { diff --git a/x-pack/plugins/osquery/public/form/index.ts b/x-pack/plugins/osquery/public/form/index.ts index 623477aaa0d16..1da4865905166 100644 --- a/x-pack/plugins/osquery/public/form/index.ts +++ b/x-pack/plugins/osquery/public/form/index.ts @@ -10,3 +10,4 @@ export { QueryDescriptionField } from './query_description_field'; export { IntervalField } from './interval_field'; export { QueryIdField } from './query_id_field'; export { ResultsTypeField } from './results_type_field'; +export { TimeoutField } from './timeout_field'; diff --git a/x-pack/plugins/osquery/public/form/timeout_field.tsx b/x-pack/plugins/osquery/public/form/timeout_field.tsx new file mode 100644 index 0000000000000..a9ad26d11e173 --- /dev/null +++ b/x-pack/plugins/osquery/public/form/timeout_field.tsx @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React, { useCallback, useMemo } from 'react'; +import deepEqual from 'fast-deep-equal'; +import { useController } from 'react-hook-form'; +import type { EuiFieldNumberProps } from '@elastic/eui'; +import { + EuiFieldNumber, + EuiFlexGroup, + EuiFlexItem, + EuiFormRow, + EuiIconTip, + EuiText, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; + +import { QUERY_TIMEOUT } from '../../common/constants'; + +const timeoutFieldValidations = { + min: { + message: i18n.translate('xpack.osquery.pack.queryFlyoutForm.timeoutFieldMinNumberError', { + defaultMessage: 'Timeout value must be greater than {than} seconds.', + values: { than: QUERY_TIMEOUT.DEFAULT }, + }), + value: QUERY_TIMEOUT.DEFAULT, + }, + max: { + message: i18n.translate('xpack.osquery.pack.queryFlyoutForm.timeoutFieldMaxNumberError', { + defaultMessage: 'Timeout value must be lower than {than} seconds.', + values: { than: QUERY_TIMEOUT.MAX }, + }), + value: QUERY_TIMEOUT.MAX, + }, +}; + +interface TimeoutFieldProps { + euiFieldProps?: Record; +} + +const TimeoutFieldComponent = ({ euiFieldProps }: TimeoutFieldProps) => { + const { + field: { onChange, value }, + fieldState: { error }, + } = useController({ + name: 'timeout', + defaultValue: QUERY_TIMEOUT.DEFAULT, + rules: { + ...timeoutFieldValidations, + }, + }); + const handleChange = useCallback( + (e: React.ChangeEvent) => { + const numberValue = e.target.valueAsNumber ? e.target.valueAsNumber : 0; + onChange(numberValue); + }, + [onChange] + ); + const hasError = useMemo(() => !!error?.message, [error?.message]); + + return ( + + + + + + + + + } + fullWidth + error={error?.message} + isInvalid={hasError} + labelAppend={ + + + + + + } + > + + + ); +}; + +export const TimeoutField = React.memo(TimeoutFieldComponent, deepEqual); diff --git a/x-pack/plugins/osquery/public/live_queries/form/index.tsx b/x-pack/plugins/osquery/public/live_queries/form/index.tsx index 5f256b505043f..9f4f225868a03 100644 --- a/x-pack/plugins/osquery/public/live_queries/form/index.tsx +++ b/x-pack/plugins/osquery/public/live_queries/form/index.tsx @@ -10,7 +10,7 @@ import { FormattedMessage } from '@kbn/i18n-react'; import type { ECSMapping } from '@kbn/osquery-io-ts-types'; import React, { useCallback, useContext, useEffect, useMemo, useState } from 'react'; import { useForm as useHookForm, FormProvider } from 'react-hook-form'; -import { isEmpty, find, pickBy } from 'lodash'; +import { isEmpty, find, pickBy, isNumber } from 'lodash'; import { containsDynamicQuery, @@ -39,6 +39,7 @@ export interface LiveQueryFormFields { savedQueryId?: string | null; ecs_mapping: ECSMapping; packId: string[]; + timeout?: number; queryType: 'query' | 'pack'; } @@ -151,10 +152,10 @@ const LiveQueryFormComponent: React.FC = ({ alert_ids: values.alertIds, pack_id: queryType === 'pack' && values?.packId?.length ? values?.packId[0] : undefined, ecs_mapping: values.ecs_mapping, + timeout: values.timeout, }, - (value) => !isEmpty(value) + (value) => !isEmpty(value) || isNumber(value) ) as unknown as LiveQueryFormFields; - await mutateAsync(serializedData); }, [alertAttachmentContext, mutateAsync, queryType] diff --git a/x-pack/plugins/osquery/public/live_queries/form/live_query_query_field.tsx b/x-pack/plugins/osquery/public/live_queries/form/live_query_query_field.tsx index 100319f7c16a5..23ef0524dbb15 100644 --- a/x-pack/plugins/osquery/public/live_queries/form/live_query_query_field.tsx +++ b/x-pack/plugins/osquery/public/live_queries/form/live_query_query_field.tsx @@ -11,6 +11,8 @@ import { EuiCodeBlock, EuiFormRow, EuiAccordion, EuiSpacer } from '@elastic/eui' import React, { useCallback, useMemo, useState } from 'react'; import { useController, useFormContext } from 'react-hook-form'; import { i18n } from '@kbn/i18n'; +import { QUERY_TIMEOUT } from '../../../common/constants'; +import { TimeoutField } from '../../form/timeout_field'; import type { LiveQueryFormFields } from '.'; import { OsqueryEditor } from '../../editor'; import { useKibana } from '../../common/lib/kibana'; @@ -68,6 +70,7 @@ const LiveQueryQueryFieldComponent: React.FC = ({ resetField('query', { defaultValue: savedQuery.query }); resetField('savedQueryId', { defaultValue: savedQuery.savedQueryId }); resetField('ecs_mapping', { defaultValue: savedQuery.ecs_mapping ?? {} }); + resetField('timeout', { defaultValue: savedQuery.timeout ?? QUERY_TIMEOUT.DEFAULT }); if (!isEmpty(savedQuery.ecs_mapping)) { setAdvancedContentState('open'); @@ -122,6 +125,7 @@ const LiveQueryQueryFieldComponent: React.FC = ({ {!isSavedQueryDisabled && ( )} + = ({ data-test-subj="advanced-accordion-content" > + + )} diff --git a/x-pack/plugins/osquery/public/packs/form/queries_field.tsx b/x-pack/plugins/osquery/public/packs/form/queries_field.tsx index d40532bf80814..3e62a5639e1f7 100644 --- a/x-pack/plugins/osquery/public/packs/form/queries_field.tsx +++ b/x-pack/plugins/osquery/public/packs/form/queries_field.tsx @@ -14,6 +14,7 @@ import { FormattedMessage } from '@kbn/i18n-react'; import deepEqual from 'fast-deep-equal'; import { useController, useFormContext, useWatch, useFieldArray } from 'react-hook-form'; +import { QUERY_TIMEOUT } from '../../../common/constants'; import { PackQueriesTable } from '../pack_queries_table'; import { QueryFlyout } from '../queries/query_flyout'; import { OsqueryPackUploader } from './pack_uploader'; @@ -84,6 +85,7 @@ const QueriesFieldComponent: React.FC = ({ euiFieldProps }) = draft.id = updatedQuery.id; draft.interval = updatedQuery.interval; draft.query = updatedQuery.query; + draft.timeout = updatedQuery.timeout; if (updatedQuery.platform?.length) { draft.platform = updatedQuery.platform; @@ -137,6 +139,7 @@ const QueriesFieldComponent: React.FC = ({ euiFieldProps }) = { id: newQueryId, interval: newQuery.interval ?? parsedContent.interval ?? '3600', + timeout: newQuery.timeout ?? parsedContent.timeout ?? QUERY_TIMEOUT.DEFAULT, query: newQuery.query, version: newQuery.version ?? parsedContent.version, snapshot: newQuery.snapshot ?? parsedContent.snapshot, diff --git a/x-pack/plugins/osquery/public/packs/form/utils.ts b/x-pack/plugins/osquery/public/packs/form/utils.ts index e04db1ed5b237..c44fb468bb80d 100644 --- a/x-pack/plugins/osquery/public/packs/form/utils.ts +++ b/x-pack/plugins/osquery/public/packs/form/utils.ts @@ -17,6 +17,7 @@ export const convertPackQueriesToSO = (queries: Record = ({ }); resetField('version', { defaultValue: savedQuery.version ? [savedQuery.version] : [] }); resetField('interval', { defaultValue: savedQuery.interval ? savedQuery.interval : 3600 }); + resetField('timeout', { + defaultValue: savedQuery.timeout ? savedQuery.timeout : QUERY_TIMEOUT.DEFAULT, + }); resetField('snapshot', { defaultValue: savedQuery.snapshot ?? true }); resetField('removed', { defaultValue: savedQuery.removed }); resetField('ecs_mapping', { defaultValue: savedQuery.ecs_mapping ?? {} }); @@ -146,9 +155,14 @@ const QueryFlyoutComponent: React.FC = ({ - - - + + + + + + + + diff --git a/x-pack/plugins/osquery/public/packs/queries/use_pack_query_form.tsx b/x-pack/plugins/osquery/public/packs/queries/use_pack_query_form.tsx index 1a0ecdbe71c59..ea6f2a2105b41 100644 --- a/x-pack/plugins/osquery/public/packs/queries/use_pack_query_form.tsx +++ b/x-pack/plugins/osquery/public/packs/queries/use_pack_query_form.tsx @@ -11,6 +11,7 @@ import type { Draft } from 'immer'; import { produce } from 'immer'; import { useMemo } from 'react'; import type { ECSMapping } from '@kbn/osquery-io-ts-types'; +import { QUERY_TIMEOUT } from '../../../common/constants'; import type { Shard } from '../../../common/utils/converters'; export interface UsePackQueryFormProps { @@ -22,6 +23,7 @@ export interface PackSOQueryFormData { id: string; query: string; interval: string; + timeout?: number; snapshot?: boolean; removed?: boolean; platform?: string | undefined; @@ -37,6 +39,7 @@ export interface PackQueryFormData { description?: string; query: string; interval: number; + timeout?: number; snapshot?: boolean; removed?: boolean; platform?: string | undefined; @@ -48,6 +51,7 @@ const deserializer = (payload: PackSOQueryFormData): PackQueryFormData => ({ id: payload.id, query: payload.query, interval: payload.interval ? parseInt(payload.interval, 10) : 3600, + timeout: payload.timeout || QUERY_TIMEOUT.DEFAULT, snapshot: payload.snapshot, removed: payload.removed, platform: payload.platform, diff --git a/x-pack/plugins/osquery/public/routes/saved_queries/list/index.tsx b/x-pack/plugins/osquery/public/routes/saved_queries/list/index.tsx index 767e323fa5254..b09acd4ae754d 100644 --- a/x-pack/plugins/osquery/public/routes/saved_queries/list/index.tsx +++ b/x-pack/plugins/osquery/public/routes/saved_queries/list/index.tsx @@ -34,6 +34,7 @@ export interface SavedQuerySO { saved_object_id: string; description?: string; query: string; + timeout?: number; ecs_mapping: ECSMapping; updated_at: string; prebuilt?: boolean; diff --git a/x-pack/plugins/osquery/public/saved_queries/form/index.tsx b/x-pack/plugins/osquery/public/saved_queries/form/index.tsx index 0035fbd18f040..8d5eea2f4ec22 100644 --- a/x-pack/plugins/osquery/public/saved_queries/form/index.tsx +++ b/x-pack/plugins/osquery/public/saved_queries/form/index.tsx @@ -23,6 +23,7 @@ import { QueryDescriptionField, VersionField, ResultsTypeField, + TimeoutField, } from '../../form'; import { PlatformCheckBoxGroupField } from '../../packs/queries/platform_checkbox_group_field'; import { ALL_OSQUERY_VERSIONS_OPTIONS } from '../../packs/queries/constants'; @@ -89,12 +90,20 @@ const SavedQueryFormComponent: React.FC = ({ + + + + + + + + {!viewMode && hasPlayground && ( diff --git a/x-pack/plugins/osquery/public/saved_queries/form/use_saved_query_form.tsx b/x-pack/plugins/osquery/public/saved_queries/form/use_saved_query_form.tsx index 453fb9b87c6cd..340541eb5d598 100644 --- a/x-pack/plugins/osquery/public/saved_queries/form/use_saved_query_form.tsx +++ b/x-pack/plugins/osquery/public/saved_queries/form/use_saved_query_form.tsx @@ -11,6 +11,7 @@ import type { Draft } from 'immer'; import produce from 'immer'; import { useMemo } from 'react'; import type { ECSMapping } from '@kbn/osquery-io-ts-types'; +import { QUERY_TIMEOUT } from '../../../common/constants'; import { useSavedQueries } from '../use_saved_queries'; export interface SavedQuerySOFormData { @@ -18,6 +19,7 @@ export interface SavedQuerySOFormData { description?: string; query?: string; interval?: string; + timeout?: number; snapshot?: boolean; removed?: boolean; platform?: string; @@ -30,6 +32,7 @@ export interface SavedQueryFormData { description?: string; query?: string; interval?: number; + timeout?: number; snapshot?: boolean; removed?: boolean; platform?: string; @@ -46,6 +49,7 @@ const deserializer = (payload: SavedQuerySOFormData): SavedQueryFormData => ({ description: payload.description, query: payload.query, interval: payload.interval ? parseInt(payload.interval, 10) : 3600, + timeout: payload.timeout ?? QUERY_TIMEOUT.DEFAULT, snapshot: payload.snapshot ?? true, removed: payload.removed ?? false, platform: payload.platform, @@ -95,6 +99,7 @@ export const useSavedQueryForm = ({ defaultValue }: UseSavedQueryFormProps) => { id: '', query: '', interval: 3600, + timeout: QUERY_TIMEOUT.DEFAULT, ecs_mapping: {}, snapshot: true, }, diff --git a/x-pack/plugins/osquery/public/saved_queries/saved_queries_dropdown.tsx b/x-pack/plugins/osquery/public/saved_queries/saved_queries_dropdown.tsx index 411813c655e24..690341fd2221f 100644 --- a/x-pack/plugins/osquery/public/saved_queries/saved_queries_dropdown.tsx +++ b/x-pack/plugins/osquery/public/saved_queries/saved_queries_dropdown.tsx @@ -25,7 +25,7 @@ export interface SavedQueriesDropdownProps { disabled?: boolean; onChange: ( value: - | (Pick & { + | (Pick & { savedQueryId: string; }) | null diff --git a/x-pack/plugins/osquery/public/shared_components/osquery_response_action_type/index.tsx b/x-pack/plugins/osquery/public/shared_components/osquery_response_action_type/index.tsx index 064e55d9350b2..cba12d5ef3e6e 100644 --- a/x-pack/plugins/osquery/public/shared_components/osquery_response_action_type/index.tsx +++ b/x-pack/plugins/osquery/public/shared_components/osquery_response_action_type/index.tsx @@ -24,23 +24,27 @@ interface OsqueryResponseActionsValues { id?: string; ecsMapping?: ECSMapping; query?: string; + timeout: number; packId?: string; queries?: Array<{ id: string; ecs_mapping: ECSMapping; query: string; + timeout?: number; }>; } interface OsqueryResponseActionsParamsFormFields { savedQueryId: string | null; ecs_mapping: ECSMapping; + timeout: number; query: string; packId?: string[]; queries: Array<{ id: string; ecs_mapping: ECSMapping; query: string; + timeout?: number; }>; queryType: 'query' | 'pack'; } @@ -115,6 +119,7 @@ const OsqueryResponseActionParamsFormComponent = ({ : { savedQueryId: formData.savedQueryId, query: formData.query, + timeout: formData.timeout, ecsMapping: formData.ecs_mapping, } ); diff --git a/x-pack/plugins/osquery/server/common/types.ts b/x-pack/plugins/osquery/server/common/types.ts index 999041c0debcf..364d4d95328c7 100644 --- a/x-pack/plugins/osquery/server/common/types.ts +++ b/x-pack/plugins/osquery/server/common/types.ts @@ -21,6 +21,7 @@ export interface PackSavedObject { name: string; query: string; interval: number; + timeout?: number; snapshot?: boolean; removed?: boolean; ecs_mapping?: Record; @@ -41,6 +42,7 @@ export interface SavedQuerySavedObject { description: string | undefined; query: string; interval: number | string; + timeout?: number; snapshot?: boolean; removed?: boolean; platform: string; diff --git a/x-pack/plugins/osquery/server/handlers/action/create_action_handler.ts b/x-pack/plugins/osquery/server/handlers/action/create_action_handler.ts index d60c7bae8d9b3..79e0cdb2d2a48 100644 --- a/x-pack/plugins/osquery/server/handlers/action/create_action_handler.ts +++ b/x-pack/plugins/osquery/server/handlers/action/create_action_handler.ts @@ -7,7 +7,7 @@ import { v4 as uuidv4 } from 'uuid'; import moment from 'moment'; -import { filter, isEmpty, map, omit, pick, pickBy, some } from 'lodash'; +import { filter, isEmpty, isNumber, map, omit, pick, pickBy, some } from 'lodash'; import type { SavedObjectsClientContract } from '@kbn/core/server'; import type { ParsedTechnicalFields } from '@kbn/rule-registry-plugin/common'; import type { CreateLiveQueryRequestBodySchema } from '../../../common/api'; @@ -17,7 +17,7 @@ import { parseAgentSelection } from '../../lib/parse_agent_groups'; import { packSavedObjectType } from '../../../common/types'; import type { OsqueryAppContext } from '../../lib/osquery_app_context_services'; import { convertSOQueriesToPack } from '../../routes/pack/utils'; -import { ACTIONS_INDEX } from '../../../common/constants'; +import { ACTIONS_INDEX, QUERY_TIMEOUT } from '../../../common/constants'; import { TELEMETRY_EBT_LIVE_QUERY_EVENT } from '../../lib/telemetry/constants'; import type { PackSavedObject } from '../../common/types'; import { CustomHttpRequestError } from '../../common/error'; @@ -100,9 +100,10 @@ export const createActionHandler = async ( ecs_mapping: packQuery.ecs_mapping, version: packQuery.version, platform: packQuery.platform, + timeout: packQuery.timeout, agents: selectedAgents, }, - (value) => !isEmpty(value) + (value) => !isEmpty(value) || isNumber(value) ); }) : await createDynamicQueries({ @@ -125,6 +126,7 @@ export const createActionHandler = async ( input_type: 'osquery', agents: query.agents, user_id: metadata?.currentUser, + ...(query.timeout !== QUERY_TIMEOUT.DEFAULT ? { timeout: query.timeout } : {}), data: pick(query, ['id', 'query', 'ecs_mapping', 'version', 'platform']), }) ) diff --git a/x-pack/plugins/osquery/server/handlers/action/create_queries.ts b/x-pack/plugins/osquery/server/handlers/action/create_queries.ts index 78092859a3990..cf7f2563b3fbb 100644 --- a/x-pack/plugins/osquery/server/handlers/action/create_queries.ts +++ b/x-pack/plugins/osquery/server/handlers/action/create_queries.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { isEmpty, map, pickBy } from 'lodash'; +import { isEmpty, isNumber, map, pickBy } from 'lodash'; import { v4 as uuidv4 } from 'uuid'; import type { ParsedTechnicalFields } from '@kbn/rule-registry-plugin/common'; @@ -43,7 +43,7 @@ export const createDynamicQueries = async ({ alert_ids: params.alert_ids, agents, }, - (value) => !isEmpty(value) || value === true + (value) => !isEmpty(value) || value === true || isNumber(value) ); }) : [ @@ -61,10 +61,11 @@ export const createDynamicQueries = async ({ : undefined, ecs_mapping: params.ecs_mapping, alert_ids: params.alert_ids, + timeout: params.timeout, agents, ...(error ? { error } : {}), }, - (value) => !isEmpty(value) + (value) => !isEmpty(value) || isNumber(value) ), ]; diff --git a/x-pack/plugins/osquery/server/lib/saved_query/saved_object_mappings.ts b/x-pack/plugins/osquery/server/lib/saved_query/saved_object_mappings.ts index 70bee246ae886..a28dc7bafae7f 100644 --- a/x-pack/plugins/osquery/server/lib/saved_query/saved_object_mappings.ts +++ b/x-pack/plugins/osquery/server/lib/saved_query/saved_object_mappings.ts @@ -8,6 +8,11 @@ import { produce } from 'immer'; import type { SavedObjectsType } from '@kbn/core/server'; import { SECURITY_SOLUTION_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; +import { + packAssetSavedObjectModelVersion1, + packSavedObjectModelVersion1, + savedQueryModelVersion1, +} from './saved_object_model_versions'; import { savedQuerySavedObjectType, packSavedObjectType, @@ -67,6 +72,9 @@ export const savedQuerySavedObjectMappings: SavedObjectsType['mappings'] = { interval: { type: 'keyword', }, + timeout: { + type: 'short', + }, ecs_mapping: { dynamic: false, properties: {}, @@ -80,6 +88,9 @@ export const savedQueryType: SavedObjectsType = { hidden: false, namespaceType: 'multiple-isolated', mappings: savedQuerySavedObjectMappings, + modelVersions: { + 1: savedQueryModelVersion1, + }, management: { importableAndExportable: true, getTitle: (savedObject) => savedObject.attributes.id, @@ -145,6 +156,9 @@ export const packSavedObjectMappings: SavedObjectsType['mappings'] = { interval: { type: 'text', }, + timeout: { + type: 'short', + }, platform: { type: 'keyword', }, @@ -166,6 +180,9 @@ export const packType: SavedObjectsType = { hidden: false, namespaceType: 'multiple-isolated', mappings: packSavedObjectMappings, + modelVersions: { + 1: packSavedObjectModelVersion1, + }, management: { defaultSearchField: 'name', importableAndExportable: true, @@ -219,6 +236,9 @@ export const packAssetSavedObjectMappings: SavedObjectsType['mappings'] = { interval: { type: 'text', }, + timeout: { + type: 'short', + }, platform: { type: 'keyword', }, @@ -242,6 +262,9 @@ export const packAssetType: SavedObjectsType = { importableAndExportable: true, visibleInManagement: false, }, + modelVersions: { + 1: packAssetSavedObjectModelVersion1, + }, namespaceType: 'agnostic', mappings: packAssetSavedObjectMappings, }; diff --git a/x-pack/plugins/osquery/server/lib/saved_query/saved_object_model_versions.ts b/x-pack/plugins/osquery/server/lib/saved_query/saved_object_model_versions.ts new file mode 100644 index 0000000000000..9ab021c7f0cfc --- /dev/null +++ b/x-pack/plugins/osquery/server/lib/saved_query/saved_object_model_versions.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsModelVersion } from '@kbn/core-saved-objects-server'; + +export const savedQueryModelVersion1: SavedObjectsModelVersion = { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + timeout: { type: 'short' }, + }, + }, + ], +}; + +export const packSavedObjectModelVersion1: SavedObjectsModelVersion = { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + queries: { + properties: { + timeout: { type: 'short' }, + }, + }, + }, + }, + ], +}; + +export const packAssetSavedObjectModelVersion1: SavedObjectsModelVersion = { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + queries: { + properties: { + timeout: { type: 'short' }, + }, + }, + }, + }, + ], +}; diff --git a/x-pack/plugins/osquery/server/routes/pack/utils.ts b/x-pack/plugins/osquery/server/routes/pack/utils.ts index a458e6edadfa1..3f3c9fa90c746 100644 --- a/x-pack/plugins/osquery/server/routes/pack/utils.ts +++ b/x-pack/plugins/osquery/server/routes/pack/utils.ts @@ -21,7 +21,16 @@ export const convertPackQueriesToSO = (queries) => const ecsMapping = value.ecs_mapping && convertECSMappingToArray(value.ecs_mapping); acc.push({ id: key, - ...pick(value, ['name', 'query', 'interval', 'platform', 'version', 'snapshot', 'removed']), + ...pick(value, [ + 'name', + 'query', + 'interval', + 'platform', + 'version', + 'snapshot', + 'removed', + 'timeout', + ]), ...(ecsMapping ? { ecs_mapping: ecsMapping } : {}), }); @@ -32,6 +41,7 @@ export const convertPackQueriesToSO = (queries) => name: string; query: string; interval: number; + timeout?: number; snapshot?: boolean; removed?: boolean; ecs_mapping?: Record; diff --git a/x-pack/plugins/osquery/server/routes/saved_query/create_saved_query_route.ts b/x-pack/plugins/osquery/server/routes/saved_query/create_saved_query_route.ts index bccd521f6323d..d59f1c5ba0f8d 100644 --- a/x-pack/plugins/osquery/server/routes/saved_query/create_saved_query_route.ts +++ b/x-pack/plugins/osquery/server/routes/saved_query/create_saved_query_route.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { isEmpty, pickBy, some, isBoolean } from 'lodash'; +import { isEmpty, pickBy, some, isBoolean, isNumber } from 'lodash'; import type { IRouter } from '@kbn/core/server'; import type { CreateSavedQueryRequestSchemaDecoded } from '../../../common/api'; import { API_VERSIONS } from '../../../common/constants'; @@ -50,6 +50,7 @@ export const createSavedQueryRoute = (router: IRouter, osqueryContext: OsqueryAp interval, snapshot, removed, + timeout, // eslint-disable-next-line @typescript-eslint/naming-convention ecs_mapping, } = request.body; @@ -80,13 +81,14 @@ export const createSavedQueryRoute = (router: IRouter, osqueryContext: OsqueryAp interval, snapshot, removed, + timeout, ecs_mapping: convertECSMappingToArray(ecs_mapping), created_by: currentUser, created_at: new Date().toISOString(), updated_by: currentUser, updated_at: new Date().toISOString(), }, - (value) => !isEmpty(value) || isBoolean(value) + (value) => !isEmpty(value) || isBoolean(value) || isNumber(value) ) ); @@ -102,6 +104,7 @@ export const createSavedQueryRoute = (router: IRouter, osqueryContext: OsqueryAp snapshot: attributes.snapshot, version: attributes.version, interval: attributes.interval, + timeout: attributes.timeout, platform: attributes.platform, query: attributes.query, updated_at: attributes.updated_at, @@ -109,7 +112,7 @@ export const createSavedQueryRoute = (router: IRouter, osqueryContext: OsqueryAp saved_object_id: savedQuerySO.id, ecs_mapping, }, - (value) => !isEmpty(value) + (value) => !isEmpty(value) || isNumber(value) ); return response.ok({ diff --git a/x-pack/plugins/osquery/server/routes/saved_query/find_saved_query_route.ts b/x-pack/plugins/osquery/server/routes/saved_query/find_saved_query_route.ts index a176d581329b5..88ccc120d0fd6 100644 --- a/x-pack/plugins/osquery/server/routes/saved_query/find_saved_query_route.ts +++ b/x-pack/plugins/osquery/server/routes/saved_query/find_saved_query_route.ts @@ -73,6 +73,7 @@ export const findSavedQueryRoute = (router: IRouter, osqueryContext: OsqueryAppC description, id, interval, + timeout, platform, query, removed, @@ -94,6 +95,7 @@ export const findSavedQueryRoute = (router: IRouter, osqueryContext: OsqueryAppC version, ecs_mapping: ecsMapping, interval, + timeout, platform, query, updated_at: updatedAt, diff --git a/x-pack/plugins/osquery/server/routes/saved_query/read_saved_query_route.ts b/x-pack/plugins/osquery/server/routes/saved_query/read_saved_query_route.ts index fb355c8b732b0..706304300c40a 100644 --- a/x-pack/plugins/osquery/server/routes/saved_query/read_saved_query_route.ts +++ b/x-pack/plugins/osquery/server/routes/saved_query/read_saved_query_route.ts @@ -64,6 +64,7 @@ export const readSavedQueryRoute = (router: IRouter, osqueryContext: OsqueryAppC description, id, interval, + timeout, platform, query, removed, @@ -85,6 +86,7 @@ export const readSavedQueryRoute = (router: IRouter, osqueryContext: OsqueryAppC version, ecs_mapping: ecsMapping, interval, + timeout, platform, query, updated_at: updatedAt, diff --git a/x-pack/plugins/osquery/server/routes/saved_query/types.ts b/x-pack/plugins/osquery/server/routes/saved_query/types.ts index 3e3ecf9d18844..07f1a812cf9f6 100644 --- a/x-pack/plugins/osquery/server/routes/saved_query/types.ts +++ b/x-pack/plugins/osquery/server/routes/saved_query/types.ts @@ -11,6 +11,7 @@ export interface SavedQueryResponse { description: string | undefined; query: string; interval: number | string; + timeout?: number; snapshot?: boolean; removed?: boolean; platform?: string; @@ -29,6 +30,7 @@ export interface UpdateSavedQueryResponse { description: string | undefined; query: string; interval: number | string; + timeout?: number; snapshot?: boolean; removed?: boolean; platform?: string; diff --git a/x-pack/plugins/osquery/server/routes/saved_query/update_saved_query_route.ts b/x-pack/plugins/osquery/server/routes/saved_query/update_saved_query_route.ts index d6cd11805c57a..f6bcad9d4bed4 100644 --- a/x-pack/plugins/osquery/server/routes/saved_query/update_saved_query_route.ts +++ b/x-pack/plugins/osquery/server/routes/saved_query/update_saved_query_route.ts @@ -60,6 +60,7 @@ export const updateSavedQueryRoute = (router: IRouter, osqueryContext: OsqueryAp query, version, interval, + timeout, snapshot, removed, // eslint-disable-next-line @typescript-eslint/naming-convention @@ -102,6 +103,7 @@ export const updateSavedQueryRoute = (router: IRouter, osqueryContext: OsqueryAp query, version, interval, + timeout, snapshot, removed, ecs_mapping: convertECSMappingToArray(ecs_mapping), @@ -133,6 +135,7 @@ export const updateSavedQueryRoute = (router: IRouter, osqueryContext: OsqueryAp version: attributes.version, ecs_mapping: attributes.ecs_mapping, interval: attributes.interval, + timeout: attributes.timeout, platform: attributes.platform, query: attributes.query, updated_at: attributes.updated_at, From daaa4fd7ab0a592061b9fc82dc2afb8365272c0a Mon Sep 17 00:00:00 2001 From: Abdon Pijpelink Date: Wed, 15 Nov 2023 13:39:24 +0100 Subject: [PATCH 3/5] ES|QL in-product help: DATE_TRUNC arg order (#171270) A small fix: the arguments for ES|QL's DATE_TRUNC function were swapped a while ago, but that was never reflected in the in-product help. This PR fixes that. --- .../kbn-text-based-editor/src/esql_documentation_sections.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-text-based-editor/src/esql_documentation_sections.tsx b/packages/kbn-text-based-editor/src/esql_documentation_sections.tsx index 6f244647922ef..4114ab021bf71 100644 --- a/packages/kbn-text-based-editor/src/esql_documentation_sections.tsx +++ b/packages/kbn-text-based-editor/src/esql_documentation_sections.tsx @@ -1180,7 +1180,7 @@ Rounds down a date to the closest interval. \`\`\` FROM employees -| EVAL year_hired = DATE_TRUNC(hire_date, 1 year) +| EVAL year_hired = DATE_TRUNC(1 year, hire_date) | STATS count(emp_no) BY year_hired | SORT year_hired \`\`\` From 77bf0cf59700f300e7cc19568250ebe99c70f13b Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Wed, 15 Nov 2023 07:09:29 -0600 Subject: [PATCH 4/5] skip failing test suite (#163257, #163258, #163259) --- .../serve/integration_tests/serverless_config_flag.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/cli/serve/integration_tests/serverless_config_flag.test.ts b/src/cli/serve/integration_tests/serverless_config_flag.test.ts index 9d20722ecbfc5..77c15f832081a 100644 --- a/src/cli/serve/integration_tests/serverless_config_flag.test.ts +++ b/src/cli/serve/integration_tests/serverless_config_flag.test.ts @@ -15,7 +15,10 @@ import { filter, firstValueFrom, from, concatMap } from 'rxjs'; import { REPO_ROOT } from '@kbn/repo-info'; import { getConfigDirectory } from '@kbn/utils'; -describe('cli serverless project type', () => { +// Failing: See https://github.com/elastic/kibana/issues/163257 +// Failing: See https://github.com/elastic/kibana/issues/163258 +// Failing: See https://github.com/elastic/kibana/issues/163259 +describe.skip('cli serverless project type', () => { let child: ChildProcessWithoutNullStreams | undefined; afterEach(() => { From 214f209bf6325d1b9786e67da81e6056fb6cf5ca Mon Sep 17 00:00:00 2001 From: Marta Bondyra <4283304+mbondyra@users.noreply.github.com> Date: Wed, 15 Nov 2023 14:11:01 +0100 Subject: [PATCH 5/5] [Unified Fieldlist] field top stats values truncate to 3 lines (#171076) ## Summary Makes the values for field popover to truncate to 3 lines instead of to 1 line. before: Screenshot 2023-11-13 at 13 53 43 after: Screenshot 2023-11-13 at 13 53 25 Also makes the container stretching more to fit the content: no widening: Screenshot 2023-11-13 at 14 01 59 widening: Screenshot 2023-11-13 at 14 03 16 --- .../src/components/field_popover/field_popover.scss | 2 +- .../field_stats/field_top_values_bucket.tsx | 13 +++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/packages/kbn-unified-field-list/src/components/field_popover/field_popover.scss b/packages/kbn-unified-field-list/src/components/field_popover/field_popover.scss index 8e5a7d5963a8b..b8fe997d01c13 100644 --- a/packages/kbn-unified-field-list/src/components/field_popover/field_popover.scss +++ b/packages/kbn-unified-field-list/src/components/field_popover/field_popover.scss @@ -3,5 +3,5 @@ .unifiedFieldList__fieldPopover__fieldPopoverPanel { min-width: $euiSizeXXL * 6.5 !important; /* 1 */ - max-width: $euiSizeXXL * 7.5 !important; + max-width: $euiSizeXXL * 10 !important; } diff --git a/packages/kbn-unified-field-list/src/components/field_stats/field_top_values_bucket.tsx b/packages/kbn-unified-field-list/src/components/field_stats/field_top_values_bucket.tsx index ccae2a3dfffc1..f296fe74906eb 100755 --- a/packages/kbn-unified-field-list/src/components/field_stats/field_top_values_bucket.tsx +++ b/packages/kbn-unified-field-list/src/components/field_stats/field_top_values_bucket.tsx @@ -13,6 +13,7 @@ import { EuiFlexItem, EuiProgress, EuiText, + EuiTextBlockTruncate, EuiToolTip, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; @@ -84,14 +85,18 @@ const FieldTopValuesBucket: React.FC = ({ {(formattedFieldValue?.length ?? 0) > 0 ? ( - - {formattedFieldValue} - + + + {formattedFieldValue} + + ) : (