[Data Plugin] Allow server-side date formatters to accept custom timezone

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.fieldformats.md

@@ -13,7 +13,6 @@ fieldFormats: {
     BoolFormat: typeof BoolFormat;
     BytesFormat: typeof BytesFormat;
     ColorFormat: typeof ColorFormat;
-    DateNanosFormat: typeof DateNanosFormat;
     DurationFormat: typeof DurationFormat;
     IpFormat: typeof IpFormat;
     NumberFormat: typeof NumberFormat;

src/plugins/data/common/field_formats/constants/base_formatters.ts

@@ -23,7 +23,6 @@ import {
   BoolFormat,
   BytesFormat,
   ColorFormat,
-  DateNanosFormat,
   DurationFormat,
   IpFormat,
   NumberFormat,
@@ -40,7 +39,6 @@ export const baseFormatters: FieldFormatInstanceType[] = [
   BoolFormat,
   BytesFormat,
   ColorFormat,
-  DateNanosFormat,
   DurationFormat,
   IpFormat,
   NumberFormat,

src/plugins/data/common/field_formats/converters/index.ts

@@ -19,7 +19,6 @@
 
 export { UrlFormat } from './url';
 export { BytesFormat } from './bytes';
-export { DateNanosFormat } from './date_nanos';
 export { RelativeDateFormat } from './relative_date';
 export { DurationFormat } from './duration';
 export { IpFormat } from './ip';

src/plugins/data/common/field_formats/field_formats_registry.ts

@@ -180,10 +180,18 @@ export class FieldFormatsRegistry {
    * @param  {ES_FIELD_TYPES[]} esTypes
    * @return {FieldFormat}
    */
-  getDefaultInstancePlain(fieldType: KBN_FIELD_TYPES, esTypes?: ES_FIELD_TYPES[]): FieldFormat {
+  getDefaultInstancePlain(
+    fieldType: KBN_FIELD_TYPES,
+    esTypes?: ES_FIELD_TYPES[],
+    params: Record<string, any> = {}
+  ): FieldFormat {
     const conf = this.getDefaultConfig(fieldType, esTypes);
+    const instanceParams = {
+      ...conf.params,
+      ...params,
+    };
 
-    return this.getInstance(conf.id, conf.params);
+    return this.getInstance(conf.id, instanceParams);
   }
   /**
    * Returns a cache key built by the given variables for caching in memoized

src/plugins/data/common/field_formats/index.ts

@@ -27,7 +27,6 @@ export {
   BoolFormat,
   BytesFormat,
   ColorFormat,
-  DateNanosFormat,
   DurationFormat,
   IpFormat,
   NumberFormat,

src/plugins/data/common/field_formats/converters/date_nanos.ts → src/plugins/data/public/field_formats/converters/date_nanos.ts

@@ -18,11 +18,14 @@
  */
 
 import { i18n } from '@kbn/i18n';
-import moment, { Moment } from 'moment';
 import { memoize, noop } from 'lodash';
-import { KBN_FIELD_TYPES } from '../../kbn_field_types/types';
-import { FieldFormat } from '../field_format';
-import { TextContextTypeConvert, FIELD_FORMAT_IDS } from '../types';
+import moment, { Moment } from 'moment';
+import {
+  FieldFormat,
+  FIELD_FORMAT_IDS,
+  KBN_FIELD_TYPES,
+  TextContextTypeConvert,
+} from '../../../common';
 
 /**
  * Analyse the given moment.js format pattern for the fractional sec part (S,SS,SSS...)

src/plugins/data/public/field_formats/converters/index.ts

@@ -18,3 +18,4 @@
  */
 
 export { DateFormat } from './date';
+export { DateNanosFormat } from './date_nanos';

src/plugins/data/public/field_formats/index.ts

@@ -18,5 +18,5 @@
  */
 
 export { FieldFormatsService, FieldFormatsSetup, FieldFormatsStart } from './field_formats_service';
-export { DateFormat } from './converters';
+export { DateFormat, DateNanosFormat } from './converters';
 export { baseFormattersPublic } from './constants';

src/plugins/data/public/index.ts

@@ -157,7 +157,6 @@ import {
   BoolFormat,
   BytesFormat,
   ColorFormat,
-  DateNanosFormat,
   DurationFormat,
   IpFormat,
   NumberFormat,
@@ -170,7 +169,7 @@ import {
   TruncateFormat,
 } from '../common/field_formats';
 
-import { DateFormat } from './field_formats';
+import { DateNanosFormat, DateFormat } from './field_formats';
 export { baseFormattersPublic } from './field_formats';
 
 // Field formats helpers namespace:

src/plugins/data/public/public.api.md

@@ -1954,42 +1954,42 @@ export const UI_SETTINGS: {
 // src/plugins/data/public/index.ts:136:21 - (ae-forgotten-export) The symbol "getEsQueryConfig" needs to be exported by the entry point index.d.ts
 // src/plugins/data/public/index.ts:136:21 - (ae-forgotten-export) The symbol "luceneStringToDsl" needs to be exported by the entry point index.d.ts
 // src/plugins/data/public/index.ts:136:21 - (ae-forgotten-export) The symbol "decorateQuery" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:177:26 - (ae-forgotten-export) The symbol "FieldFormatsRegistry" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:177:26 - (ae-forgotten-export) The symbol "BoolFormat" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:177:26 - (ae-forgotten-export) The symbol "BytesFormat" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:177:26 - (ae-forgotten-export) The symbol "ColorFormat" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:177:26 - (ae-forgotten-export) The symbol "DateNanosFormat" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:177:26 - (ae-forgotten-export) The symbol "DurationFormat" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:177:26 - (ae-forgotten-export) The symbol "IpFormat" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:177:26 - (ae-forgotten-export) The symbol "NumberFormat" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:177:26 - (ae-forgotten-export) The symbol "PercentFormat" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:177:26 - (ae-forgotten-export) The symbol "RelativeDateFormat" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:177:26 - (ae-forgotten-export) The symbol "SourceFormat" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:177:26 - (ae-forgotten-export) The symbol "StaticLookupFormat" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:177:26 - (ae-forgotten-export) The symbol "UrlFormat" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:177:26 - (ae-forgotten-export) The symbol "StringFormat" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:177:26 - (ae-forgotten-export) The symbol "TruncateFormat" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:233:27 - (ae-forgotten-export) The symbol "isFilterable" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:233:27 - (ae-forgotten-export) The symbol "isNestedField" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:233:27 - (ae-forgotten-export) The symbol "validateIndexPattern" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:233:27 - (ae-forgotten-export) The symbol "getFromSavedObject" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:233:27 - (ae-forgotten-export) The symbol "flattenHitWrapper" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:233:27 - (ae-forgotten-export) The symbol "formatHitProvider" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:370:20 - (ae-forgotten-export) The symbol "getRequestInspectorStats" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:370:20 - (ae-forgotten-export) The symbol "getResponseInspectorStats" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:370:20 - (ae-forgotten-export) The symbol "tabifyAggResponse" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:370:20 - (ae-forgotten-export) The symbol "tabifyGetColumns" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:372:1 - (ae-forgotten-export) The symbol "CidrMask" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:373:1 - (ae-forgotten-export) The symbol "dateHistogramInterval" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:382:1 - (ae-forgotten-export) The symbol "InvalidEsCalendarIntervalError" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:383:1 - (ae-forgotten-export) The symbol "InvalidEsIntervalFormatError" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:384:1 - (ae-forgotten-export) The symbol "Ipv4Address" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:385:1 - (ae-forgotten-export) The symbol "isDateHistogramBucketAggConfig" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:389:1 - (ae-forgotten-export) The symbol "isValidEsInterval" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:390:1 - (ae-forgotten-export) The symbol "isValidInterval" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:393:1 - (ae-forgotten-export) The symbol "parseInterval" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:394:1 - (ae-forgotten-export) The symbol "propFilter" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:397:1 - (ae-forgotten-export) The symbol "toAbsoluteDates" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:176:26 - (ae-forgotten-export) The symbol "FieldFormatsRegistry" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:176:26 - (ae-forgotten-export) The symbol "BoolFormat" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:176:26 - (ae-forgotten-export) The symbol "BytesFormat" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:176:26 - (ae-forgotten-export) The symbol "ColorFormat" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:176:26 - (ae-forgotten-export) The symbol "DateNanosFormat" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:176:26 - (ae-forgotten-export) The symbol "DurationFormat" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:176:26 - (ae-forgotten-export) The symbol "IpFormat" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:176:26 - (ae-forgotten-export) The symbol "NumberFormat" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:176:26 - (ae-forgotten-export) The symbol "PercentFormat" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:176:26 - (ae-forgotten-export) The symbol "RelativeDateFormat" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:176:26 - (ae-forgotten-export) The symbol "SourceFormat" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:176:26 - (ae-forgotten-export) The symbol "StaticLookupFormat" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:176:26 - (ae-forgotten-export) The symbol "UrlFormat" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:176:26 - (ae-forgotten-export) The symbol "StringFormat" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:176:26 - (ae-forgotten-export) The symbol "TruncateFormat" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:232:27 - (ae-forgotten-export) The symbol "isFilterable" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:232:27 - (ae-forgotten-export) The symbol "isNestedField" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:232:27 - (ae-forgotten-export) The symbol "validateIndexPattern" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:232:27 - (ae-forgotten-export) The symbol "getFromSavedObject" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:232:27 - (ae-forgotten-export) The symbol "flattenHitWrapper" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:232:27 - (ae-forgotten-export) The symbol "formatHitProvider" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:369:20 - (ae-forgotten-export) The symbol "getRequestInspectorStats" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:369:20 - (ae-forgotten-export) The symbol "getResponseInspectorStats" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:369:20 - (ae-forgotten-export) The symbol "tabifyAggResponse" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:369:20 - (ae-forgotten-export) The symbol "tabifyGetColumns" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:371:1 - (ae-forgotten-export) The symbol "CidrMask" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:372:1 - (ae-forgotten-export) The symbol "dateHistogramInterval" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:381:1 - (ae-forgotten-export) The symbol "InvalidEsCalendarIntervalError" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:382:1 - (ae-forgotten-export) The symbol "InvalidEsIntervalFormatError" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:383:1 - (ae-forgotten-export) The symbol "Ipv4Address" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:384:1 - (ae-forgotten-export) The symbol "isDateHistogramBucketAggConfig" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:388:1 - (ae-forgotten-export) The symbol "isValidEsInterval" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:389:1 - (ae-forgotten-export) The symbol "isValidInterval" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:392:1 - (ae-forgotten-export) The symbol "parseInterval" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:393:1 - (ae-forgotten-export) The symbol "propFilter" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:396:1 - (ae-forgotten-export) The symbol "toAbsoluteDates" needs to be exported by the entry point index.d.ts
 // src/plugins/data/public/query/state_sync/connect_to_query_state.ts:40:60 - (ae-forgotten-export) The symbol "FilterStateStore" needs to be exported by the entry point index.d.ts
 // src/plugins/data/public/types.ts:52:5 - (ae-forgotten-export) The symbol "createFiltersFromValueClickAction" needs to be exported by the entry point index.d.ts
 // src/plugins/data/public/types.ts:53:5 - (ae-forgotten-export) The symbol "createFiltersFromRangeSelectAction" needs to be exported by the entry point index.d.ts

src/plugins/data/server/field_formats/converters/date_nanos_server.ts

@@ -0,0 +1,141 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { i18n } from '@kbn/i18n';
+import { memoize, noop } from 'lodash';
+import moment, { Moment } from 'moment-timezone';
+import {
+  FieldFormat,
+  FIELD_FORMAT_IDS,
+  KBN_FIELD_TYPES,
+  TextContextTypeConvert,
+} from '../../../common';
+
+/**
+ * Analyse the given moment.js format pattern for the fractional sec part (S,SS,SSS...)
+ * returning length, match, pattern and an escaped pattern, that excludes the fractional
+ * part when formatting with moment.js -> e.g. [SSS]
+ */
+export function analysePatternForFract(pattern: string) {
+  const fracSecMatch = pattern.match('S+') as any; // extract fractional seconds sub-pattern
+  const fracSecMatchStr = fracSecMatch ? fracSecMatch[0] : '';
+
+  return {
+    length: fracSecMatchStr.length,
+    patternNanos: fracSecMatchStr,
+    pattern,
+    patternEscaped: fracSecMatchStr ? pattern.replace(fracSecMatch, `[${fracSecMatch}]`) : '',
+  };
+}
+
+/**
+ * Format a given moment.js date object
+ * Since momentjs would loose the exact value for fractional seconds with a higher resolution than
+ * milliseconds, the fractional pattern is replaced by the fractional value of the raw timestamp
+ */
+export function formatWithNanos(
+  dateMomentObj: Moment,
+  valRaw: string,
+  fracPatternObj: Record<string, any>
+) {
+  if (fracPatternObj.length <= 3) {
+    // S,SS,SSS is formatted correctly by moment.js
+    return dateMomentObj.format(fracPatternObj.pattern);
+  } else {
+    // Beyond SSS the precise value of the raw datetime string is used
+    const valFormatted = dateMomentObj.format(fracPatternObj.patternEscaped);
+    // Extract fractional value of ES formatted timestamp, zero pad if necessary:
+    // 2020-05-18T20:45:05.957Z -> 957000000
+    // 2020-05-18T20:45:05.957000123Z -> 957000123
+    // we do not need to take care of the year 10000 bug since max year of date_nanos is 2262
+    const valNanos = valRaw
+      .substr(20, valRaw.length - 21) // remove timezone(Z)
+      .padEnd(9, '0') // pad shorter fractionals
+      .substr(0, fracPatternObj.patternNanos.length);
+    return valFormatted.replace(fracPatternObj.patternNanos, valNanos);
+  }
+}
+
+export class DateNanosFormat extends FieldFormat {
+  static id = FIELD_FORMAT_IDS.DATE_NANOS;
+  static title = i18n.translate('data.fieldFormats.date_nanos.title', {
+    defaultMessage: 'Date nanos',
+  });
+  static fieldType = KBN_FIELD_TYPES.DATE;
+
+  private memoizedConverter: Function = noop;
+  private memoizedPattern: string = '';
+  private timeZone: string = '';
+
+  getParamDefaults() {
+    return {
+      pattern: this.getConfig!('dateNanosFormat'),
+      fallbackPattern: this.getConfig!('dateFormat'),
+      timezone: this.getConfig!('dateFormat:tz'),
+    };
+  }
+
+  textConvert: TextContextTypeConvert = (val) => {
+    // don't give away our ref to converter so
+    // we can hot-swap when config changes
+    const pattern = this.param('pattern');
+    const timezone = this.param('timezone');
+    const fractPattern = analysePatternForFract(pattern);
+    const fallbackPattern = this.param('patternFallback');
+
+    const timezoneChanged = this.timeZone !== timezone;
+    const datePatternChanged = this.memoizedPattern !== pattern;
+    if (timezoneChanged || datePatternChanged) {
+      this.timeZone = timezone;
+      this.memoizedPattern = pattern;
+
+      this.memoizedConverter = memoize((value: any) => {
+        if (value === null || value === undefined) {
+          return '-';
+        }
+
+        /* On the server, importing moment returns a new instance. Unlike on
+         * the client side, it doesn't have the dateFormat:tz configuration
+         * baked in.
+         * We need to set the timezone manually here. The date is taken in as
+         * UTC and converted into the desired timezone. */
+        let date;
+        if (this.timeZone === 'Browser') {
+          // Assume a warning has been logged that this can be unpredictable. It
+          // would be too verbose to log anything here.
+          date = moment.utc(val);
+        } else {
+          date = moment.utc(val).tz(this.timeZone);
+        }
+
+        if (typeof value !== 'string' && date.isValid()) {
+          // fallback for max/min aggregation, where unixtime in ms is returned as a number
+          // aggregations in Elasticsearch generally just return ms
+          return date.format(fallbackPattern);
+        } else if (date.isValid()) {
+          return formatWithNanos(date, value, fractPattern);
+        } else {
+          return value;
+        }
+      });
+    }
+
+    return this.memoizedConverter(val);
+  };
+}