From c32d7eaecb5cda6407b3ba8a9596428b4cd6237c Mon Sep 17 00:00:00 2001 From: Stratoula Kalafateli Date: Fri, 2 Feb 2024 18:30:24 +0200 Subject: [PATCH] [ES|QL] Distinguish among empty and available fields (#174585) --- packages/kbn-es-types/src/search.ts | 4 ++ packages/kbn-field-utils/src/types.ts | 1 + .../src/components/source_document.tsx | 2 +- .../src/hooks/use_grouped_fields.test.tsx | 39 +++++++++++++++++++ .../src/hooks/use_grouped_fields.ts | 12 +++++- .../data/common/search/expressions/esql.ts | 29 +++++++++++++- .../esql_async_search_strategy.ts | 13 ++++++- .../esql_search/esql_search_strategy.ts | 3 +- .../common/fields/data_view_field.ts | 8 ++++ src/plugins/data_views/common/types.ts | 4 ++ .../components/sidebar/lib/get_field_list.ts | 1 + .../sidebar/lib/sidebar_reducer.test.ts | 3 ++ .../main/utils/fetch_text_based.ts | 15 ++++--- .../expression_types/specs/datatable.ts | 1 + test/api_integration/apis/search/bsearch.ts | 2 + .../apps/discover/group3/_sidebar.ts | 26 +++++++++++++ .../apps/discover/group4/_esql_view.ts | 18 +++++++++ 17 files changed, 165 insertions(+), 16 deletions(-) diff --git a/packages/kbn-es-types/src/search.ts b/packages/kbn-es-types/src/search.ts index 56e2cb56c8f71..461a32f149842 100644 --- a/packages/kbn-es-types/src/search.ts +++ b/packages/kbn-es-types/src/search.ts @@ -663,6 +663,10 @@ export type ESQLRow = unknown[]; export interface ESQLSearchReponse { columns: ESQLColumn[]; + // In case of ?drop_null_columns in the query, then + // all_columns will have available and empty fields + // while columns only the available ones (non nulls) + all_columns?: ESQLColumn[]; values: ESQLRow[]; } diff --git a/packages/kbn-field-utils/src/types.ts b/packages/kbn-field-utils/src/types.ts index a004497549981..43b790ec9326b 100644 --- a/packages/kbn-field-utils/src/types.ts +++ b/packages/kbn-field-utils/src/types.ts @@ -21,6 +21,7 @@ export interface FieldBase { timeSeriesMetric?: DataViewField['timeSeriesMetric']; esTypes?: DataViewField['esTypes']; scripted?: DataViewField['scripted']; + isNull?: DataViewField['isNull']; conflictDescriptions?: Record; } diff --git a/packages/kbn-unified-data-table/src/components/source_document.tsx b/packages/kbn-unified-data-table/src/components/source_document.tsx index fd221f63fcc64..15924fc02521e 100644 --- a/packages/kbn-unified-data-table/src/components/source_document.tsx +++ b/packages/kbn-unified-data-table/src/components/source_document.tsx @@ -66,7 +66,7 @@ export function SourceDocument({ {pairs.map(([fieldDisplayName, value, fieldName]) => { // temporary solution for text based mode. As there are a lot of unsupported fields we want to // hide the empty one from the Document view - if (isPlainRecord && fieldName && row.flattened[fieldName] === null) return null; + if (isPlainRecord && fieldName && !row.flattened[fieldName]) return null; return ( diff --git a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.test.tsx b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.test.tsx index 4a937f86bd5c0..c2a2379a676e1 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.test.tsx +++ b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.test.tsx @@ -472,6 +472,45 @@ describe('UnifiedFieldList useGroupedFields()', () => { expect(fieldListGroupedProps.fieldsExistInIndex).toBe(true); }); + it('should work correctly for text-based queries (no data view) with isNull fields', async () => { + const allFieldsEmpty = [...new Array(2)].flatMap((_, index) => + allFields.map((field) => { + return new DataViewField({ + ...field.toSpec(), + name: `${field.name}${index || ''}`, + isNull: true, + }); + }) + ); + const { result } = renderHook(useGroupedFields, { + initialProps: { + dataViewId: null, + allFields: allFieldsEmpty, + services: mockedServices, + }, + }); + + const fieldListGroupedProps = result.current.fieldListGroupedProps; + const fieldGroups = fieldListGroupedProps.fieldGroups; + + expect( + Object.keys(fieldGroups!).map( + (key) => `${key}-${fieldGroups![key as FieldsGroupNames]?.fields.length}` + ) + ).toStrictEqual([ + 'SpecialFields-0', + 'SelectedFields-0', + 'PopularFields-0', + 'AvailableFields-0', + 'UnmappedFields-0', + 'EmptyFields-56', + 'MetaFields-0', + ]); + + expect(fieldListGroupedProps.fieldsExistenceStatus).toBe(ExistenceFetchStatus.succeeded); + expect(fieldListGroupedProps.fieldsExistInIndex).toBe(true); + }); + it('should work correctly when details are overwritten', async () => { const onOverrideFieldGroupDetails: GroupedFieldsParams['onOverrideFieldGroupDetails'] = jest.fn((groupName) => { diff --git a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts index 7853c7e67800b..e368b55b78392 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts @@ -156,6 +156,10 @@ export function useGroupedFields({ if (field.type === 'nested') { return 'availableFields'; } + + if (field?.isNull) { + return 'emptyFields'; + } if (dataView?.getFieldByName && !dataView.getFieldByName(field.name)) { return 'unmappedFields'; } @@ -303,8 +307,12 @@ export function useGroupedFields({ }, }; - // do not show empty field accordion if there is no existence information - if (fieldsExistenceInfoUnavailable) { + // the fieldsExistenceInfoUnavailable check should happen only for dataview based + const dataViewFieldsExistenceUnavailable = dataViewId && fieldsExistenceInfoUnavailable; + // for textbased queries, rely on the empty fields length + const textBasedFieldsExistenceUnavailable = !dataViewId && !groupedFields.emptyFields.length; + + if (dataViewFieldsExistenceUnavailable || textBasedFieldsExistenceUnavailable) { delete fieldGroupDefinitions.EmptyFields; } diff --git a/src/plugins/data/common/search/expressions/esql.ts b/src/plugins/data/common/search/expressions/esql.ts index c997da17e5cda..3275a296568fd 100644 --- a/src/plugins/data/common/search/expressions/esql.ts +++ b/src/plugins/data/common/search/expressions/esql.ts @@ -11,6 +11,7 @@ import { castEsToKbnFieldTypeName, ES_FIELD_TYPES, KBN_FIELD_TYPES } from '@kbn/ import { i18n } from '@kbn/i18n'; import type { Datatable, + DatatableColumn, DatatableColumnType, ExpressionFunctionDefinition, } from '@kbn/expressions-plugin/common'; @@ -243,7 +244,31 @@ export const getEsqlFn = ({ getStartDependencies }: EsqlFnArguments) => { name, meta: { type: normalizeType(type) }, })) ?? []; - const columnNames = columns.map(({ name }) => name); + // all_columns in the response means that there is a separation between + // columns with data and empty columns + // columns contain only columns with data while all_columns everything + const hasEmptyColumns = + body.all_columns && body.all_columns?.length > body.columns.length; + + let emptyColumns: DatatableColumn[] = []; + + if (hasEmptyColumns) { + const difference = + body.all_columns?.filter((col1) => { + return !body.columns.some((col2) => { + return col1.name === col2.name; + }); + }) ?? []; + emptyColumns = + difference?.map(({ name, type }) => ({ + id: name, + name, + meta: { type: normalizeType(type) }, + isNull: true, + })) ?? []; + } + const allColumns = [...columns, ...emptyColumns]; + const columnNames = allColumns.map(({ name }) => name); const rows = body.values.map((row) => zipObject(columnNames, row)); return { @@ -251,7 +276,7 @@ export const getEsqlFn = ({ getStartDependencies }: EsqlFnArguments) => { meta: { type: 'es_ql', }, - columns, + columns: allColumns, rows, warning, } as Datatable; diff --git a/src/plugins/data/server/search/strategies/esql_async_search/esql_async_search_strategy.ts b/src/plugins/data/server/search/strategies/esql_async_search/esql_async_search_strategy.ts index f41b4ece598b7..f052e6749e726 100644 --- a/src/plugins/data/server/search/strategies/esql_async_search/esql_async_search_strategy.ts +++ b/src/plugins/data/server/search/strategies/esql_async_search/esql_async_search_strategy.ts @@ -67,11 +67,20 @@ export const esqlAsyncSearchStrategyProvider = ( }; const { body, headers, meta } = id ? await client.transport.request( - { method: 'GET', path: `/_query/async/${id}`, querystring: { ...params } }, + { + method: 'GET', + path: `/_query/async/${id}`, + querystring: { ...params }, + }, { ...options.transport, signal: options.abortSignal, meta: true } ) : await client.transport.request( - { method: 'POST', path: `/_query/async`, body: params }, + { + method: 'POST', + path: `/_query/async`, + body: params, + querystring: 'drop_null_columns', + }, { ...options.transport, signal: options.abortSignal, meta: true } ); diff --git a/src/plugins/data/server/search/strategies/esql_search/esql_search_strategy.ts b/src/plugins/data/server/search/strategies/esql_search/esql_search_strategy.ts index 1ad5b8fe0a5dd..e9a6499b4aa1b 100644 --- a/src/plugins/data/server/search/strategies/esql_search/esql_search_strategy.ts +++ b/src/plugins/data/server/search/strategies/esql_search/esql_search_strategy.ts @@ -36,7 +36,8 @@ export const esqlSearchStrategyProvider = ( const { headers, body, meta } = await esClient.asCurrentUser.transport.request( { method: 'POST', - path: '/_query', + path: `/_query`, + querystring: 'drop_null_columns', body: { ...requestParams, }, diff --git a/src/plugins/data_views/common/fields/data_view_field.ts b/src/plugins/data_views/common/fields/data_view_field.ts index 02474b6a41c2e..36cd78682aa97 100644 --- a/src/plugins/data_views/common/fields/data_view_field.ts +++ b/src/plugins/data_views/common/fields/data_view_field.ts @@ -305,6 +305,14 @@ export class DataViewField implements DataViewFieldBase { return this.aggregatable && !notVisualizableFieldTypes.includes(this.spec.type); } + /** + * Returns true if field is Empty + */ + + public get isNull() { + return Boolean(this.spec.isNull); + } + /** * Returns true if field is subtype nested */ diff --git a/src/plugins/data_views/common/types.ts b/src/plugins/data_views/common/types.ts index 6ba4b6819a045..2177f51621fec 100644 --- a/src/plugins/data_views/common/types.ts +++ b/src/plugins/data_views/common/types.ts @@ -407,6 +407,10 @@ export type FieldSpec = DataViewFieldBase & { * True if field is aggregatable */ aggregatable: boolean; + /** + * True if field is empty + */ + isNull?: boolean; /** * True if can be read from doc values */ diff --git a/src/plugins/discover/public/application/main/components/sidebar/lib/get_field_list.ts b/src/plugins/discover/public/application/main/components/sidebar/lib/get_field_list.ts index 66dfab4e53c58..487f0faf7a046 100644 --- a/src/plugins/discover/public/application/main/components/sidebar/lib/get_field_list.ts +++ b/src/plugins/discover/public/application/main/components/sidebar/lib/get_field_list.ts @@ -74,6 +74,7 @@ export function getTextBasedQueryFieldList( type: column.meta?.type ?? 'unknown', searchable: false, aggregatable: false, + isNull: Boolean(column?.isNull), }) ); } diff --git a/src/plugins/discover/public/application/main/components/sidebar/lib/sidebar_reducer.test.ts b/src/plugins/discover/public/application/main/components/sidebar/lib/sidebar_reducer.test.ts index c5ea2878f800e..26e183481787b 100644 --- a/src/plugins/discover/public/application/main/components/sidebar/lib/sidebar_reducer.test.ts +++ b/src/plugins/discover/public/application/main/components/sidebar/lib/sidebar_reducer.test.ts @@ -109,6 +109,7 @@ describe('sidebar reducer', function () { meta: { type: 'number', }, + isNull: true, }, { id: '2', @@ -127,12 +128,14 @@ describe('sidebar reducer', function () { name: 'text1', type: 'number', aggregatable: false, + isNull: true, searchable: false, }), new DataViewField({ name: 'text2', type: 'keyword', aggregatable: false, + isNull: false, searchable: false, }), ], diff --git a/src/plugins/discover/public/application/main/utils/fetch_text_based.ts b/src/plugins/discover/public/application/main/utils/fetch_text_based.ts index a1aa14e47d79b..98e8bc5846186 100644 --- a/src/plugins/discover/public/application/main/utils/fetch_text_based.ts +++ b/src/plugins/discover/public/application/main/utils/fetch_text_based.ts @@ -62,14 +62,13 @@ export function fetchTextBased( const rows = table?.rows ?? []; textBasedQueryColumns = table?.columns ?? undefined; textBasedHeaderWarning = table.warning ?? undefined; - finalData = rows.map( - (row: Record, idx: number) => - ({ - id: String(idx), - raw: row, - flattened: row, - } as unknown as DataTableRecord) - ); + finalData = rows.map((row: Record, idx: number) => { + return { + id: String(idx), + raw: row, + flattened: row, + } as unknown as DataTableRecord; + }); } }); return lastValueFrom(execution).then(() => { diff --git a/src/plugins/expressions/common/expression_types/specs/datatable.ts b/src/plugins/expressions/common/expression_types/specs/datatable.ts index fbba26f3d4dc8..c2ee9aa7f22da 100644 --- a/src/plugins/expressions/common/expression_types/specs/datatable.ts +++ b/src/plugins/expressions/common/expression_types/specs/datatable.ts @@ -89,6 +89,7 @@ export interface DatatableColumn { id: string; name: string; meta: DatatableColumnMeta; + isNull?: boolean; } /** diff --git a/test/api_integration/apis/search/bsearch.ts b/test/api_integration/apis/search/bsearch.ts index 96b4bbbf622cf..867ae83864a74 100644 --- a/test/api_integration/apis/search/bsearch.ts +++ b/test/api_integration/apis/search/bsearch.ts @@ -428,6 +428,7 @@ export default function ({ getService }: FtrProviderContext) { expect(jsonBody[0].result.requestParams).to.eql({ method: 'POST', path: '/_query', + querystring: 'drop_null_columns', }); }); @@ -456,6 +457,7 @@ export default function ({ getService }: FtrProviderContext) { expect(jsonBody[0].error.attributes.requestParams).to.eql({ method: 'POST', path: '/_query', + querystring: 'drop_null_columns', }); }); }); diff --git a/test/functional/apps/discover/group3/_sidebar.ts b/test/functional/apps/discover/group3/_sidebar.ts index cae06dd375b46..f2ada0f1927ef 100644 --- a/test/functional/apps/discover/group3/_sidebar.ts +++ b/test/functional/apps/discover/group3/_sidebar.ts @@ -107,6 +107,10 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.discover.selectTextBaseLang(); + const testQuery = `from logstash-* | limit 10000`; + await monacoEditor.setCodeEditorValue(testQuery); + await testSubjects.click('querySubmitButton'); + await PageObjects.header.waitUntilLoadingHasFinished(); await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded(); await PageObjects.unifiedFieldList.openSidebarFieldFilter(); options = await find.allByCssSelector('[data-test-subj*="typeFilter"]'); @@ -125,6 +129,25 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { ); }); }); + + it('should show empty fields in text-based view', async function () { + await kibanaServer.uiSettings.update({ 'discover:enableESQL': true }); + await browser.refresh(); + + await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded(); + await PageObjects.discover.selectTextBaseLang(); + + const testQuery = `from logstash-* | limit 10 | keep machine.ram_range, bytes `; + await monacoEditor.setCodeEditorValue(testQuery); + await testSubjects.click('querySubmitButton'); + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded(); + await PageObjects.unifiedFieldList.openSidebarFieldFilter(); + + expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be( + '2 selected fields. 1 available field. 1 empty field.' + ); + }); }); describe('search', function () { @@ -422,6 +445,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { ); await PageObjects.discover.selectTextBaseLang(); + await monacoEditor.setCodeEditorValue('from logstash-* | limit 10000'); + await testSubjects.click('querySubmitButton'); + await PageObjects.header.waitUntilLoadingHasFinished(); await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded(); expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be( diff --git a/test/functional/apps/discover/group4/_esql_view.ts b/test/functional/apps/discover/group4/_esql_view.ts index fd9060f9b9ec8..fb388d4277fe1 100644 --- a/test/functional/apps/discover/group4/_esql_view.ts +++ b/test/functional/apps/discover/group4/_esql_view.ts @@ -133,6 +133,24 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const cell = await dataGrid.getCellElement(0, 2); expect(await cell.getVisibleText()).to.be('1'); }); + + it('should render correctly if there are empty fields', async function () { + await PageObjects.discover.selectTextBaseLang(); + const testQuery = `from logstash-* | limit 10 | keep machine.ram_range, bytes`; + + await monacoEditor.setCodeEditorValue(testQuery); + await testSubjects.click('querySubmitButton'); + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + const cell = await dataGrid.getCellElement(0, 3); + expect(await cell.getVisibleText()).to.be(' - '); + expect(await dataGrid.getHeaders()).to.eql([ + 'Control column', + 'Select column', + 'Numberbytes', + 'machine.ram_range', + ]); + }); }); describe('errors', () => { it('should show error messages for syntax errors in query', async function () {