[Discover] Show ignored field values

src/plugins/data/common/search/tabify/tabify_docs.ts

@@ -55,8 +55,17 @@ export interface TabifyDocsOptions {
    * merged into the flattened document.
    */
   source?: boolean;
+  /**
+   * If set to `true` values that have been ignored in ES (ignored_field_values)
+   * will be merged into the flattened document.
+   */
+  includeIgnoredValues?: boolean;
 }
 
+// This is an overwrite of the SearchHit type to add the ignored_field_values.
+// Can be removed once the estypes.SearchHit knows about ignored_field_values
+type Hit<T = unknown> = estypes.SearchHit<T> & { ignored_field_values?: Record<string, unknown[]> };
+
 /**
  * Flattens an individual hit (from an ES response) into an object. This will
  * create flattened field names, like `user.name`.
@@ -65,11 +74,7 @@ export interface TabifyDocsOptions {
  * @param indexPattern The index pattern for the requested index if available.
  * @param params Parameters how to flatten the hit
  */
-export function flattenHit(
-  hit: estypes.SearchHit,
-  indexPattern?: IndexPattern,
-  params?: TabifyDocsOptions
-) {
+export function flattenHit(hit: Hit, indexPattern?: IndexPattern, params?: TabifyDocsOptions) {
   const flat = {} as Record<string, any>;
 
   function flatten(obj: Record<string, any>, keyPrefix: string = '') {
@@ -111,6 +116,20 @@ export function flattenHit(
     flatten(hit._source as Record<string, any>);
   }
 
+  if (params?.includeIgnoredValues && hit.ignored_field_values) {
+    Object.entries(hit.ignored_field_values).forEach(([fieldName, fieldValue]) => {
+      if (flat[fieldName]) {
+        if (Array.isArray(flat[fieldName])) {
+          flat[fieldName] = [...flat[fieldName], ...fieldValue];
+        } else {
+          flat[fieldName] = [flat[fieldName], ...fieldValue];
+        }
+      } else {
+        flat[fieldName] = fieldValue;
+      }
+    });
+  }
+
   // Merge all valid meta fields into the flattened object
   // expect for _source (in case that was specified as a meta field)
   indexPattern?.metaFields?.forEach((metaFieldName) => {

src/plugins/discover/kibana.json

@@ -8,6 +8,7 @@
     "data",
     "embeddable",
     "inspector",
+    "fieldFormats",
     "kibanaLegacy",
     "urlForwarding",
     "navigation",
@@ -16,7 +17,7 @@
     "indexPatternFieldEditor"
   ],
   "optionalPlugins": ["home", "share", "usageCollection", "spaces"],
-  "requiredBundles": ["kibanaUtils", "home", "kibanaReact", "fieldFormats", "dataViews"],
+  "requiredBundles": ["kibanaUtils", "home", "kibanaReact", "dataViews"],
   "extraPublicDirs": ["common"],
   "owner": {
     "name": "Data Discovery",

src/plugins/discover/public/__mocks__/index_pattern.ts

@@ -6,8 +6,7 @@
  * Side Public License, v 1.
  */
 
-import type { estypes } from '@elastic/elasticsearch';
-import { flattenHit, IIndexPatternFieldList } from '../../../data/common';
+import { IIndexPatternFieldList } from '../../../data/common';
 import { IndexPattern } from '../../../data/common';
 
 const fields = [
@@ -86,10 +85,5 @@ const indexPattern = {
 } as unknown as IndexPattern;
 
 indexPattern.isTimeBased = () => !!indexPattern.timeFieldName;
-indexPattern.formatField = (hit: Record<string, unknown>, fieldName: string) => {
-  return fieldName === '_source'
-    ? hit._source
-    : flattenHit(hit as unknown as estypes.SearchHit, indexPattern)[fieldName];
-};
 
 export const indexPatternMock = indexPattern;

src/plugins/discover/public/__mocks__/index_pattern_with_timefield.ts

@@ -6,9 +6,8 @@
  * Side Public License, v 1.
  */
 
-import { flattenHit, IIndexPatternFieldList } from '../../../data/common';
+import { IIndexPatternFieldList } from '../../../data/common';
 import { IndexPattern } from '../../../data/common';
-import type { estypes } from '@elastic/elasticsearch';
 
 const fields = [
   {
@@ -77,10 +76,5 @@ const indexPattern = {
 } as unknown as IndexPattern;
 
 indexPattern.isTimeBased = () => !!indexPattern.timeFieldName;
-indexPattern.formatField = (hit: Record<string, unknown>, fieldName: string) => {
-  return fieldName === '_source'
-    ? hit._source
-    : flattenHit(hit as unknown as estypes.SearchHit, indexPattern)[fieldName];
-};
 
 export const indexPatternWithTimefieldMock = indexPattern;

src/plugins/discover/public/application/apps/main/components/doc_table/components/table_row.tsx

@@ -10,6 +10,7 @@ import React, { Fragment, useCallback, useMemo, useState } from 'react';
 import classNames from 'classnames';
 import { i18n } from '@kbn/i18n';
 import { EuiButtonEmpty, EuiIcon } from '@elastic/eui';
+import { formatFieldValue } from '../../../../../helpers/format_value';
 import { flattenHit } from '../../../../../../../../data/common';
 import { DocViewer } from '../../../../../components/doc_viewer/doc_viewer';
 import { FilterManager, IndexPattern } from '../../../../../../../../data/public';
@@ -58,7 +59,10 @@ export const TableRow = ({
   });
   const anchorDocTableRowSubj = row.isAnchor ? ' docTableAnchorRow' : '';
 
-  const flattenedRow = useMemo(() => flattenHit(row, indexPattern), [indexPattern, row]);
+  const flattenedRow = useMemo(
+    () => flattenHit(row, indexPattern, { includeIgnoredValues: true }),
+    [indexPattern, row]
+  );
   const mapping = useMemo(() => indexPattern.fields.getByName, [indexPattern]);
 
   // toggle display of the rows details, a full list of the fields from each row
@@ -68,7 +72,12 @@ export const TableRow = ({
    * Fill an element with the value of a field
    */
   const displayField = (fieldName: string) => {
-    const formattedField = indexPattern.formatField(row, fieldName);
+    const formattedField = formatFieldValue(
+      flattenedRow[fieldName],
+      row,
+      indexPattern,
+      mapping(fieldName)
+    );
 
     // field formatters take care of escaping
     // eslint-disable-next-line react/no-danger
@@ -142,9 +151,9 @@ export const TableRow = ({
   } else {
     columns.forEach(function (column: string) {
       // when useNewFieldsApi is true, addressing to the fields property is safe
-      if (useNewFieldsApi && !mapping(column) && !row.fields![column]) {
+      if (useNewFieldsApi && !mapping(column) && row.fields && !row.fields[column]) {
         const innerColumns = Object.fromEntries(
-          Object.entries(row.fields!).filter(([key]) => {
+          Object.entries(row.fields).filter(([key]) => {
             return key.indexOf(`${column}.`) === 0;
           })
         );
@@ -161,7 +170,13 @@ export const TableRow = ({
           />
         );
       } else {
-        const isFilterable = Boolean(mapping(column)?.filterable && filter);
+        // Check whether the field is defined as filterable in the mapping and does
+        // NOT have ignored values in it to determine whether we want to allow filtering.
+        // We should improve this and show a helpful tooltip why the filter buttons are not
+        // there/disabled when there are ignored values.
+        const isFilterable = Boolean(
+          mapping(column)?.filterable && filter && !row._ignored?.includes(column)
+        );
         rowCells.push(
           <TableCell
             key={column}

src/plugins/discover/public/application/apps/main/components/sidebar/lib/field_calculator.js

@@ -13,7 +13,7 @@ import { flattenHit } from '../../../../../../../../data/common';
 function getFieldValues(hits, field, indexPattern) {
   const name = field.name;
   return map(hits, function (hit) {
-    return flattenHit(hit, indexPattern)[name];
+    return flattenHit(hit, indexPattern, { includeIgnoredValues: true })[name];
   });
 }
 

src/plugins/discover/public/application/apps/main/utils/calc_field_counts.ts

@@ -22,7 +22,7 @@ export function calcFieldCounts(
     return {};
   }
   for (const hit of rows) {
-    const fields = Object.keys(flattenHit(hit, indexPattern));
+    const fields = Object.keys(flattenHit(hit, indexPattern, { includeIgnoredValues: true }));
     for (const fieldName of fields) {
       counts[fieldName] = (counts[fieldName] || 0) + 1;
     }

src/plugins/discover/public/application/components/discover_grid/discover_grid.tsx

@@ -271,7 +271,11 @@ export const DiscoverGrid = ({
       getRenderCellValueFn(
         indexPattern,
         displayedRows,
-        displayedRows ? displayedRows.map((hit) => flattenHit(hit, indexPattern)) : [],
+        displayedRows
+          ? displayedRows.map((hit) =>
+              flattenHit(hit, indexPattern, { includeIgnoredValues: true })
+            )
+          : [],
         useNewFieldsApi,
         fieldsToShow,
         services.uiSettings.get(MAX_DOC_FIELDS_DISPLAYED)

src/plugins/discover/public/application/components/discover_grid/get_render_cell_value.tsx

@@ -22,6 +22,7 @@ import { DiscoverGridContext } from './discover_grid_context';
 import { JsonCodeEditor } from '../json_code_editor/json_code_editor';
 import { defaultMonacoEditorWidth } from './constants';
 import { EsHitRecord } from '../../types';
+import { formatFieldValue } from '../../helpers/format_value';
 
 export const getRenderCellValueFn =
   (
@@ -191,12 +192,12 @@ export const getRenderCellValueFn =
       return <span>{JSON.stringify(rowFlattened[columnId])}</span>;
     }
 
-    const valueFormatted = indexPattern.formatField(row, columnId);
+    const valueFormatted = formatFieldValue(rowFlattened[columnId], row, indexPattern, field);
     if (typeof valueFormatted === 'undefined') {
       return <span>-</span>;
     }
     return (
       // eslint-disable-next-line react/no-danger
-      <span dangerouslySetInnerHTML={{ __html: indexPattern.formatField(row, columnId) }} />
+      <span dangerouslySetInnerHTML={{ __html: valueFormatted }} />
     );
   };

src/plugins/discover/public/application/components/table/table.tsx

@@ -20,6 +20,8 @@ import {
 } from '../../doc_views/doc_views_types';
 import { ACTIONS_COLUMN, MAIN_COLUMNS } from './table_columns';
 import { getFieldsToShow } from '../../helpers/get_fields_to_show';
+import { getIgnoredReason, IgnoredReason } from '../../helpers/get_ignored_reason';
+import { formatFieldValue } from '../../helpers/format_value';
 
 export interface DocViewerTableProps {
   columns?: string[];
@@ -46,6 +48,7 @@ export interface FieldRecord {
   };
   value: {
     formattedValue: string;
+    ignored?: IgnoredReason;
   };
 }
 
@@ -64,8 +67,6 @@ export const DocViewerTable = ({
     [indexPattern?.fields]
   );
 
-  const formattedHit = useMemo(() => indexPattern?.formatHit(hit, 'html'), [hit, indexPattern]);
-
   const tableColumns = useMemo(() => {
     return filter ? [ACTIONS_COLUMN, ...MAIN_COLUMNS] : MAIN_COLUMNS;
   }, [filter]);
@@ -96,7 +97,7 @@ export const DocViewerTable = ({
     return null;
   }
 
-  const flattened = flattenHit(hit, indexPattern, { source: true });
+  const flattened = flattenHit(hit, indexPattern, { source: true, includeIgnoredValues: true });
   const fieldsToShow = getFieldsToShow(Object.keys(flattened), indexPattern, showMultiFields);
 
   const items: FieldRecord[] = Object.keys(flattened)
@@ -115,6 +116,8 @@ export const DocViewerTable = ({
       const displayName = fieldMapping?.displayName ?? field;
       const fieldType = isNestedFieldParent(field, indexPattern) ? 'nested' : fieldMapping?.type;
 
+      const ignored = getIgnoredReason(fieldMapping ?? field, hit._ignored);
+
       return {
         action: {
           onToggleColumn,
@@ -130,7 +133,8 @@ export const DocViewerTable = ({
           scripted: Boolean(fieldMapping?.scripted),
         },
         value: {
-          formattedValue: formattedHit[field],
+          formattedValue: formatFieldValue(flattened[field], hit, indexPattern, fieldMapping),
+          ignored,
         },
       };
     });

src/plugins/discover/public/application/components/table/table_cell_value.tsx

@@ -8,14 +8,24 @@
 
 import classNames from 'classnames';
 import React, { Fragment, useState } from 'react';
+import { IgnoredReason } from '../../helpers/get_ignored_reason';
 import { FieldRecord } from './table';
 import { DocViewTableRowBtnCollapse } from './table_row_btn_collapse';
 
 const COLLAPSE_LINE_LENGTH = 350;
 
-type TableFieldValueProps = FieldRecord['value'] & Pick<FieldRecord['field'], 'field'>;
+type TableFieldValueProps = Pick<FieldRecord['field'], 'field'> & {
+  formattedValue: FieldRecord['value']['formattedValue'];
+  rawValue: unknown;
+  ignoreReason?: IgnoredReason;
+};
 
-export const TableFieldValue = ({ formattedValue, field }: TableFieldValueProps) => {
+export const TableFieldValue = ({
+  formattedValue,
+  field,
+  rawValue,
+  ignoreReason,
+}: TableFieldValueProps) => {
   const [fieldOpen, setFieldOpen] = useState(false);
 
   const value = String(formattedValue);
@@ -30,11 +40,18 @@ export const TableFieldValue = ({ formattedValue, field }: TableFieldValueProps)
 
   const onToggleCollapse = () => setFieldOpen((fieldOpenPrev) => !fieldOpenPrev);
 
+  const multiValue = Array.isArray(rawValue) && rawValue.length > 1;
+
   return (
     <Fragment>
       {isCollapsible && (
         <DocViewTableRowBtnCollapse onClick={onToggleCollapse} isCollapsed={isCollapsed} />
       )}
+      {ignoreReason && (
+        <em>
+          {ignoreReason.toString()} (multiValue: {String(multiValue)})
+        </em>
+      )}
       <div
         className={valueClassName}
         data-test-subj={`tableDocViewRow-${field}-value`}

src/plugins/discover/public/application/components/table/table_columns.tsx

@@ -82,8 +82,18 @@ export const MAIN_COLUMNS: Array<EuiBasicTableColumn<FieldRecord>> = [
         </strong>
       </EuiText>
     ),
-    render: ({ formattedValue }: FieldRecord['value'], { field: { field } }: FieldRecord) => {
-      return <TableFieldValue field={field} formattedValue={formattedValue} />;
+    render: (
+      { formattedValue, ignored }: FieldRecord['value'],
+      { field: { field }, action: { flattenedField } }: FieldRecord
+    ) => {
+      return (
+        <TableFieldValue
+          field={field}
+          formattedValue={formattedValue}
+          rawValue={flattenedField}
+          ignoreReason={ignored}
+        />
+      );
     },
   },
 ];

src/plugins/discover/public/application/helpers/format_value.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { estypes } from '@elastic/elasticsearch';
+import { DataView, DataViewField, KBN_FIELD_TYPES } from '../../../../data/common';
+import { getServices } from '../../kibana_services';
+
+// TODO: need more test coverage
+
+/**
+ * Formats the value of a specific field using the appropriate field formatter if available
+ * or the default string field formatter otherwise.
+ *
+ * @param value The value to format
+ * @param hit The actual search hit (required to get highlight information from)
+ * @param dataView The data view if available
+ * @param field The field that value was from if available
+ */
+export function formatFieldValue(
+  value: unknown,
+  hit: estypes.SearchHit,
+  dataView?: DataView,
+  field?: DataViewField
+): string {
+  if (!dataView || !field) {
+    // If either no field is available or no data view, we'll use the default
+    // string formatter to format that field.
+    return getServices()
+      .fieldFormats.getDefaultInstance(KBN_FIELD_TYPES.STRING)
+      .convert(value, 'html', { hit, field, indexPattern: dataView });
+  }
+
+  // If we have a data view and field we use that fields field formatter
+  return dataView
+    .getFormatterForField(field)
+    .convert(value, 'html', { hit, field, indexPattern: dataView });
+}