From 5262e34b7ccea3e698d9255bb886cd7bc0329166 Mon Sep 17 00:00:00 2001 From: Spencer Date: Tue, 31 Mar 2020 17:52:00 -0700 Subject: [PATCH 1/3] =?UTF-8?q?[7.7]=20[ci/es]=20always=20pull=20snapshots?= =?UTF-8?q?,=20break=20early=20when=20they're=E2=80=A6=20(#62067)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [ci/es] always pull snapshots, break early when they're not available * move to setup.sh after deps are installed Co-authored-by: spalger Co-authored-by: spalger Co-authored-by: Elastic Machine --- src/dev/ci_setup/setup.sh | 8 ++++++++ test/scripts/jenkins_build_kibana.sh | 3 --- test/scripts/jenkins_xpack_build_kibana.sh | 3 --- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/dev/ci_setup/setup.sh b/src/dev/ci_setup/setup.sh index 6f604f631cb0e..a9ba182836b73 100755 --- a/src/dev/ci_setup/setup.sh +++ b/src/dev/ci_setup/setup.sh @@ -16,6 +16,14 @@ echo " -- TEST_ES_SNAPSHOT_VERSION='$TEST_ES_SNAPSHOT_VERSION'" echo " -- installing node.js dependencies" yarn kbn bootstrap --prefer-offline +### +### Download es snapshots +### +echo " -- downloading es snapshot" +node scripts/es snapshot --download-only; +node scripts/es snapshot --license=oss --download-only; + + ### ### verify no git modifications ### diff --git a/test/scripts/jenkins_build_kibana.sh b/test/scripts/jenkins_build_kibana.sh index 2878af09bb1bb..efef91b8581a3 100755 --- a/test/scripts/jenkins_build_kibana.sh +++ b/test/scripts/jenkins_build_kibana.sh @@ -11,9 +11,6 @@ node scripts/build_kibana_platform_plugins \ # doesn't persist, also set in kibanaPipeline.groovy export KBN_NP_PLUGINS_BUILT=true -echo " -> downloading es snapshot" -node scripts/es snapshot --license=oss --download-only; - echo " -> Ensuring all functional tests are in a ciGroup" yarn run grunt functionalTests:ensureAllTestsInCiGroup; diff --git a/test/scripts/jenkins_xpack_build_kibana.sh b/test/scripts/jenkins_xpack_build_kibana.sh index c9796d6d6772f..808df0df73591 100755 --- a/test/scripts/jenkins_xpack_build_kibana.sh +++ b/test/scripts/jenkins_xpack_build_kibana.sh @@ -12,9 +12,6 @@ node scripts/build_kibana_platform_plugins \ # doesn't persist, also set in kibanaPipeline.groovy export KBN_NP_PLUGINS_BUILT=true -echo " -> downloading es snapshot" -node scripts/es snapshot --download-only; - echo " -> Ensuring all functional tests are in a ciGroup" cd "$XPACK_DIR" node scripts/functional_tests --assert-none-excluded \ From 2072b927f91746ccb606521afe05f53caf3a0764 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 31 Mar 2020 21:50:45 -0500 Subject: [PATCH 2/3] [7.7] Fix race condition in integration tests (#62064) (#62098) There's a race condition with our rule creation tests where if they're executed we'll get a failure message in the response, but if they haven't yet executed we won't. This ultimately seems like a bug with this removeServerGeneratedProperties helpers, which has been updated to remove those failure properties as well. --- .../security_and_spaces/tests/utils.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/utils.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/utils.ts index f1404b79a07af..7b725a7830c56 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/utils.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/utils.ts @@ -18,6 +18,8 @@ export const removeServerGeneratedProperties = ( created_at, updated_at, id, + last_failure_at, + last_failure_message, last_success_at, last_success_message, status, From 43c052f20900f90f919095efcbea2be58db47e8f Mon Sep 17 00:00:00 2001 From: Lukas Olson Date: Tue, 31 Mar 2020 20:04:14 -0700 Subject: [PATCH 3/3] [Search service] Shim total hits in async search response (#61565) (#62096) * Shim total hits in async search response * Resolve types * Fix types * Fix tests Co-authored-by: Elastic Machine Co-authored-by: Elastic Machine --- .../server/search/es_search_strategy.ts | 14 +++-- .../server/search/shim_hits_total.test.ts | 56 +++++++++++++++++++ .../server/search/shim_hits_total.ts | 18 ++++++ 3 files changed, 84 insertions(+), 4 deletions(-) create mode 100644 x-pack/plugins/data_enhanced/server/search/shim_hits_total.test.ts create mode 100644 x-pack/plugins/data_enhanced/server/search/shim_hits_total.ts diff --git a/x-pack/plugins/data_enhanced/server/search/es_search_strategy.ts b/x-pack/plugins/data_enhanced/server/search/es_search_strategy.ts index 95ffd7b06df21..77fdf2b3ea2f7 100644 --- a/x-pack/plugins/data_enhanced/server/search/es_search_strategy.ts +++ b/x-pack/plugins/data_enhanced/server/search/es_search_strategy.ts @@ -19,6 +19,7 @@ import { getTotalLoaded, } from '../../../../../src/plugins/data/server'; import { IEnhancedEsSearchRequest } from '../../common'; +import { shimHitsTotal } from './shim_hits_total'; export interface AsyncSearchResponse { id: string; @@ -56,22 +57,27 @@ async function asyncSearch( request: IEnhancedEsSearchRequest, options?: ISearchOptions ) { - const { body = undefined, index = undefined, ...params } = request.id ? {} : request.params; + const { timeout = undefined, restTotalHitsAsInt = undefined, ...params } = { + trackTotalHits: true, // Get the exact count of hits + ...request.params, + }; // If we have an ID, then just poll for that ID, otherwise send the entire request body + const { body = undefined, index = undefined, ...queryParams } = request.id ? {} : params; + const method = request.id ? 'GET' : 'POST'; const path = encodeURI(request.id ? `_async_search/${request.id}` : `${index}/_async_search`); // Wait up to 1s for the response to return - const query = toSnakeCase({ /* waitForCompletion: '1s', */ ...params }); + const query = toSnakeCase({ /* waitForCompletion: '1s', */ ...queryParams }); - const { response: rawResponse, id } = (await caller( + const { response, id } = (await caller( 'transport.request', { method, path, body, query }, options )) as AsyncSearchResponse; - return { id, rawResponse, ...getTotalLoaded(rawResponse._shards) }; + return { id, rawResponse: shimHitsTotal(response), ...getTotalLoaded(response._shards) }; } async function rollupSearch( diff --git a/x-pack/plugins/data_enhanced/server/search/shim_hits_total.test.ts b/x-pack/plugins/data_enhanced/server/search/shim_hits_total.test.ts new file mode 100644 index 0000000000000..61740b97299da --- /dev/null +++ b/x-pack/plugins/data_enhanced/server/search/shim_hits_total.test.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { shimHitsTotal } from './shim_hits_total'; + +describe('shimHitsTotal', () => { + test('returns the total if it is already numeric', () => { + const result = shimHitsTotal({ + hits: { + total: 5, + }, + } as any); + expect(result).toEqual({ + hits: { + total: 5, + }, + }); + }); + + test('returns the total if it is inside `value`', () => { + const result = shimHitsTotal({ + hits: { + total: { + value: 5, + }, + }, + } as any); + expect(result).toEqual({ + hits: { + total: 5, + }, + }); + }); + + test('returns other properties from the response', () => { + const result = shimHitsTotal({ + _shards: {}, + hits: { + hits: [], + total: { + value: 5, + }, + }, + } as any); + expect(result).toEqual({ + _shards: {}, + hits: { + hits: [], + total: 5, + }, + }); + }); +}); diff --git a/x-pack/plugins/data_enhanced/server/search/shim_hits_total.ts b/x-pack/plugins/data_enhanced/server/search/shim_hits_total.ts new file mode 100644 index 0000000000000..10d45be01563a --- /dev/null +++ b/x-pack/plugins/data_enhanced/server/search/shim_hits_total.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchResponse } from 'elasticsearch'; + +/** + * Temporary workaround until https://github.com/elastic/kibana/issues/26356 is addressed. + * Since we are setting `track_total_hits` in the request, `hits.total` will be an object + * containing the `value`. + */ +export function shimHitsTotal(response: SearchResponse) { + const total = (response.hits?.total as any)?.value ?? response.hits?.total; + const hits = { ...response.hits, total }; + return { ...response, hits }; +}