[ES|QL] Distinguish among empty and available fields

packages/kbn-es-types/src/search.ts

@@ -663,6 +663,10 @@ export type ESQLRow = unknown[];
 
 export interface ESQLSearchReponse {
   columns: ESQLColumn[];
+  // In case of ?drop_null_columns in the query, then
+  // all_columns will have available and empty fields
+  // while columns only the available ones (non nulls)
+  all_columns?: ESQLColumn[];
   values: ESQLRow[];
 }
 
@@ -674,4 +678,5 @@ export interface ESQLSearchParams {
   query: string;
   filter?: unknown;
   locale?: string;
+  dropNullColumns?: boolean;
 }

packages/kbn-field-utils/src/types.ts

@@ -21,6 +21,7 @@ export interface FieldBase {
   timeSeriesMetric?: DataViewField['timeSeriesMetric'];
   esTypes?: DataViewField['esTypes'];
   scripted?: DataViewField['scripted'];
+  isNull?: DataViewField['isNull'];
   conflictDescriptions?: Record<string, string[]>;
 }
 

packages/kbn-unified-data-table/src/components/source_document.tsx

@@ -66,7 +66,7 @@ export function SourceDocument({
       {pairs.map(([fieldDisplayName, value, fieldName]) => {
         // temporary solution for text based mode. As there are a lot of unsupported fields we want to
         // hide the empty one from the Document view
-        if (isPlainRecord && fieldName && row.flattened[fieldName] === null) return null;
+        if (isPlainRecord && fieldName && !row.flattened[fieldName]) return null;
         return (
           <Fragment key={fieldDisplayName}>
             <EuiDescriptionListTitle className="unifiedDataTable__descriptionListTitle">

packages/kbn-unified-field-list/src/hooks/use_grouped_fields.test.tsx

@@ -472,6 +472,45 @@ describe('UnifiedFieldList useGroupedFields()', () => {
     expect(fieldListGroupedProps.fieldsExistInIndex).toBe(true);
   });
 
+  it('should work correctly for text-based queries (no data view) with isNull fields', async () => {
+    const allFieldsEmpty = [...new Array(2)].flatMap((_, index) =>
+      allFields.map((field) => {
+        return new DataViewField({
+          ...field.toSpec(),
+          name: `${field.name}${index || ''}`,
+          isNull: true,
+        });
+      })
+    );
+    const { result } = renderHook(useGroupedFields, {
+      initialProps: {
+        dataViewId: null,
+        allFields: allFieldsEmpty,
+        services: mockedServices,
+      },
+    });
+
+    const fieldListGroupedProps = result.current.fieldListGroupedProps;
+    const fieldGroups = fieldListGroupedProps.fieldGroups;
+
+    expect(
+      Object.keys(fieldGroups!).map(
+        (key) => `${key}-${fieldGroups![key as FieldsGroupNames]?.fields.length}`
+      )
+    ).toStrictEqual([
+      'SpecialFields-0',
+      'SelectedFields-0',
+      'PopularFields-0',
+      'AvailableFields-0',
+      'UnmappedFields-0',
+      'EmptyFields-56',
+      'MetaFields-0',
+    ]);
+
+    expect(fieldListGroupedProps.fieldsExistenceStatus).toBe(ExistenceFetchStatus.succeeded);
+    expect(fieldListGroupedProps.fieldsExistInIndex).toBe(true);
+  });
+
   it('should work correctly when details are overwritten', async () => {
     const onOverrideFieldGroupDetails: GroupedFieldsParams<DataViewField>['onOverrideFieldGroupDetails'] =
       jest.fn((groupName) => {

packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts

@@ -156,6 +156,10 @@ export function useGroupedFields<T extends FieldListItem = DataViewField>({
         if (field.type === 'nested') {
           return 'availableFields';
         }
+
+        if (field?.isNull) {
+          return 'emptyFields';
+        }
         if (dataView?.getFieldByName && !dataView.getFieldByName(field.name)) {
           return 'unmappedFields';
         }
@@ -303,8 +307,13 @@ export function useGroupedFields<T extends FieldListItem = DataViewField>({
       },
     };
 
-    // do not show empty field accordion if there is no existence information
-    if (fieldsExistenceInfoUnavailable) {
+    // dataViewId is null for text based queries
+    // the fieldsExistenceInfoUnavailable check should happen only for dataview based
+    // for textbased queries, rely on the empty fields length
+    if (
+      (fieldsExistenceInfoUnavailable && dataViewId) ||
+      (!dataViewId && !groupedFields.emptyFields.length)
+    ) {
       delete fieldGroupDefinitions.EmptyFields;
     }
 

src/plugins/data/common/query/text_based_query_state_to_ast.ts

@@ -20,6 +20,7 @@ import {
 interface Args extends QueryState {
   timeFieldName?: string;
   inputQuery?: Query;
+  dropNullColumns?: boolean;
 }
 
 /**
@@ -34,6 +35,7 @@ export function textBasedQueryStateToExpressionAst({
   inputQuery,
   time,
   timeFieldName,
+  dropNullColumns,
 }: Args) {
   const kibana = buildExpressionFunction<ExpressionFunctionKibana>('kibana', {});
   let q;
@@ -51,7 +53,7 @@ export function textBasedQueryStateToExpressionAst({
     const mode = getAggregateQueryMode(query);
     for (const esMode of ['sql', 'esql']) {
       if (mode === esMode && esMode in query) {
-        const essql = aggregateQueryToAst(query, timeFieldName);
+        const essql = aggregateQueryToAst(query, timeFieldName, dropNullColumns);
 
         if (essql) {
           ast.chain.push(essql);

src/plugins/data/common/query/text_based_query_state_to_ast_with_validation.ts

@@ -14,6 +14,7 @@ interface Args extends QueryState {
   dataView?: DataView;
   inputQuery?: Query;
   timeFieldName?: string;
+  dropNullColumns?: boolean;
 }
 
 /**
@@ -28,6 +29,7 @@ export async function textBasedQueryStateToAstWithValidation({
   inputQuery,
   time,
   dataView,
+  dropNullColumns,
 }: Args) {
   let ast;
   if (query && isOfAggregateQueryType(query)) {
@@ -37,6 +39,7 @@ export async function textBasedQueryStateToAstWithValidation({
       inputQuery,
       time,
       timeFieldName: dataView?.timeFieldName,
+      dropNullColumns,
     });
   }
   return ast;

src/plugins/data/common/search/expressions/aggregate_query_to_ast.ts

@@ -13,7 +13,8 @@ import { EsqlExpressionFunctionDefinition } from './esql';
 
 export const aggregateQueryToAst = (
   query: AggregateQuery,
-  timeField?: string
+  timeField?: string,
+  dropNullColumns?: boolean
 ): undefined | ExpressionAstFunction => {
   if ('sql' in query) {
     return buildExpressionFunction<EssqlExpressionFunctionDefinition>('essql', {
@@ -26,6 +27,7 @@ export const aggregateQueryToAst = (
       query: query.esql,
       timeField,
       locale: i18n.getLocale(),
+      dropNullColumns,
     }).toAst();
   }
 };

src/plugins/data/common/search/expressions/esql.ts

@@ -43,6 +43,7 @@ interface Arguments {
   // timezone?: string;
   timeField?: string;
   locale?: string;
+  dropNullColumns?: boolean;
 }
 
 export type EsqlExpressionFunctionDefinition = ExpressionFunctionDefinition<
@@ -125,10 +126,17 @@ export const getEsqlFn = ({ getStartDependencies }: EsqlFnArguments) => {
           defaultMessage: 'The locale to use.',
         }),
       },
+      dropNullColumns: {
+        aliases: ['dropNullColumns'],
+        types: ['boolean'],
+        help: i18n.translate('data.search.essql.dropNull.help', {
+          defaultMessage: 'If true adds all_columns in a separate property',
+        }),
+      },
     },
     fn(
       input,
-      { query, /* timezone, */ timeField, locale },
+      { query, /* timezone, */ timeField, locale, dropNullColumns },
       { abortSignal, inspectorAdapters, getKibanaRequest }
     ) {
       return defer(() =>
@@ -149,6 +157,7 @@ export const getEsqlFn = ({ getStartDependencies }: EsqlFnArguments) => {
             query,
             // time_zone: timezone,
             locale,
+            dropNullColumns,
           };
           if (input) {
             const esQueryConfigs = getEsQueryConfig(
@@ -243,15 +252,34 @@ export const getEsqlFn = ({ getStartDependencies }: EsqlFnArguments) => {
               name,
               meta: { type: normalizeType(type) },
             })) ?? [];
-          const columnNames = columns.map(({ name }) => name);
+
+          // all_columns in the response means that there is a separation between
+          // columns with data and empty columns
+          // columns contain only columns with data while all_columns everything
+          const difference =
+            body.all_columns?.filter((col1) => {
+              return !body.columns.some((col2) => {
+                return col1.name === col2.name;
+              });
+            }) ?? [];
+          const emptyColumns =
+            difference?.map(({ name, type }) => ({
+              id: name,
+              name,
+              meta: { type: normalizeType(type) },
+              isNull: true,
+            })) ?? [];
+
+          const allColumns = [...columns, ...emptyColumns];
+          const columnNames = allColumns.map(({ name }) => name);
           const rows = body.values.map((row) => zipObject(columnNames, row));
 
           return {
             type: 'datatable',
             meta: {
               type: 'es_ql',
             },
-            columns,
+            columns: allColumns,
             rows,
             warning,
           } as Datatable;

src/plugins/data/server/search/strategies/esql_async_search/esql_async_search_strategy.ts

@@ -22,12 +22,16 @@ import { IKibanaSearchRequest, IKibanaSearchResponse, pollSearch } from '../../.
 import { toAsyncKibanaSearchResponse } from './response_utils';
 import { SearchConfigSchema } from '../../../../config';
 
+interface ESQLQueryRequest extends SqlQueryRequest {
+  body?: SqlQueryRequest['body'] & { dropNullColumns?: boolean };
+}
+
 export const esqlAsyncSearchStrategyProvider = (
   searchConfig: SearchConfigSchema,
   logger: Logger,
   useInternalUser: boolean = false
 ): ISearchStrategy<
-  IKibanaSearchRequest<SqlQueryRequest['body']>,
+  IKibanaSearchRequest<ESQLQueryRequest['body']>,
   IKibanaSearchResponse<SqlGetAsyncResponse>
 > => {
   function cancelAsyncSearch(id: string, esClient: IScopedClusterClient) {
@@ -46,13 +50,15 @@ export const esqlAsyncSearchStrategyProvider = (
   }
 
   function asyncSearch(
-    { id, ...request }: IKibanaSearchRequest<SqlQueryRequest['body']>,
+    { id, ...request }: IKibanaSearchRequest<ESQLQueryRequest['body']>,
     options: IAsyncSearchOptions,
     { esClient, uiSettingsClient }: SearchStrategyDependencies
   ) {
     const client = useInternalUser ? esClient.asInternalUser : esClient.asCurrentUser;
 
     const search = async () => {
+      const { dropNullColumns, ...requestParams } = request.params ?? {};
+      const queryParams = dropNullColumns ? '?drop_null_columns' : '';
       const params = id
         ? {
             ...getCommonDefaultAsyncGetParams(searchConfig, options),
@@ -63,15 +69,19 @@ export const esqlAsyncSearchStrategyProvider = (
           }
         : {
             ...(await getCommonDefaultAsyncSubmitParams(searchConfig, options)),
-            ...request.params,
+            ...requestParams,
           };
       const { body, headers, meta } = id
         ? await client.transport.request<SqlGetAsyncResponse>(
-            { method: 'GET', path: `/_query/async/${id}`, querystring: { ...params } },
+            {
+              method: 'GET',
+              path: `/_query/async/${id}`,
+              querystring: { ...params },
+            },
             { ...options.transport, signal: options.abortSignal, meta: true }
           )
         : await client.transport.request<SqlGetAsyncResponse>(
-            { method: 'POST', path: `/_query/async`, body: params },
+            { method: 'POST', path: `/_query/async${queryParams}`, body: params },
             { ...options.transport, signal: options.abortSignal, meta: true }
           );
 

src/plugins/data/server/search/strategies/esql_search/esql_search_strategy.ts

@@ -32,11 +32,12 @@ export const esqlSearchStrategyProvider = (
 
     const search = async () => {
       try {
-        const { terminateAfter, ...requestParams } = request.params ?? {};
+        const { terminateAfter, dropNullColumns, ...requestParams } = request.params ?? {};
+        const queryParams = dropNullColumns ? '?drop_null_columns' : '';
         const { headers, body, meta } = await esClient.asCurrentUser.transport.request(
           {
             method: 'POST',
-            path: '/_query',
+            path: `/_query${queryParams}`,
             body: {
               ...requestParams,
             },

src/plugins/data_views/common/fields/data_view_field.ts

@@ -305,6 +305,14 @@ export class DataViewField implements DataViewFieldBase {
     return this.aggregatable && !notVisualizableFieldTypes.includes(this.spec.type);
   }
 
+  /**
+   * Returns true if field is Empty
+   */
+
+  public get isNull() {
+    return Boolean(this.spec.isNull);
+  }
+
   /**
    * Returns true if field is subtype nested
    */

src/plugins/data_views/common/types.ts

@@ -407,6 +407,10 @@ export type FieldSpec = DataViewFieldBase & {
    * True if field is aggregatable
    */
   aggregatable: boolean;
+  /**
+   * True if field is empty
+   */
+  isNull?: boolean;
   /**
    * True if can be read from doc values
    */

src/plugins/discover/public/application/main/components/sidebar/lib/get_field_list.ts

@@ -74,6 +74,7 @@ export function getTextBasedQueryFieldList(
         type: column.meta?.type ?? 'unknown',
         searchable: false,
         aggregatable: false,
+        isNull: Boolean(column?.isNull),
       })
   );
 }

src/plugins/discover/public/application/main/components/sidebar/lib/sidebar_reducer.test.ts

@@ -109,6 +109,7 @@ describe('sidebar reducer', function () {
             meta: {
               type: 'number',
             },
+            isNull: true,
           },
           {
             id: '2',
@@ -127,12 +128,14 @@ describe('sidebar reducer', function () {
           name: 'text1',
           type: 'number',
           aggregatable: false,
+          empty: true,
           searchable: false,
         }),
         new DataViewField({
           name: 'text2',
           type: 'keyword',
           aggregatable: false,
+          empty: false,
           searchable: false,
         }),
       ],