diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.gettime.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.gettime.md index 04a0d871cab2d..3969a97fa7789 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.gettime.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.gettime.md @@ -7,7 +7,10 @@ Signature: ```typescript -export declare function getTime(indexPattern: IIndexPattern | undefined, timeRange: TimeRange, forceNow?: Date): import("../..").RangeFilter | undefined; +export declare function getTime(indexPattern: IIndexPattern | undefined, timeRange: TimeRange, options?: { + forceNow?: Date; + fieldName?: string; +}): import("../..").RangeFilter | undefined; ``` ## Parameters @@ -16,7 +19,7 @@ export declare function getTime(indexPattern: IIndexPattern | undefined, timeRan | --- | --- | --- | | indexPattern | IIndexPattern | undefined | | | timeRange | TimeRange | | -| forceNow | Date | | +| options | {
forceNow?: Date;
fieldName?: string;
} | | Returns: diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.iindexpattern.gettimefield.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.iindexpattern.gettimefield.md new file mode 100644 index 0000000000000..c3998876c9712 --- /dev/null +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.iindexpattern.gettimefield.md @@ -0,0 +1,15 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [IIndexPattern](./kibana-plugin-plugins-data-public.iindexpattern.md) > [getTimeField](./kibana-plugin-plugins-data-public.iindexpattern.gettimefield.md) + +## IIndexPattern.getTimeField() method + +Signature: + +```typescript +getTimeField?(): IFieldType | undefined; +``` +Returns: + +`IFieldType | undefined` + diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.iindexpattern.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.iindexpattern.md index 1bbd6cf67f0ce..1cb89822eb605 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.iindexpattern.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.iindexpattern.md @@ -21,3 +21,9 @@ export interface IIndexPattern | [title](./kibana-plugin-plugins-data-public.iindexpattern.title.md) | string | | | [type](./kibana-plugin-plugins-data-public.iindexpattern.type.md) | string | | +## Methods + +| Method | Description | +| --- | --- | +| [getTimeField()](./kibana-plugin-plugins-data-public.iindexpattern.gettimefield.md) | | + diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.md index 0fd82ffb2240c..e1df493143b73 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.md @@ -43,7 +43,7 @@ | [getEsPreference(uiSettings, sessionId)](./kibana-plugin-plugins-data-public.getespreference.md) | | | [getQueryLog(uiSettings, storage, appName, language)](./kibana-plugin-plugins-data-public.getquerylog.md) | | | [getSearchErrorType({ message })](./kibana-plugin-plugins-data-public.getsearcherrortype.md) | | -| [getTime(indexPattern, timeRange, forceNow)](./kibana-plugin-plugins-data-public.gettime.md) | | +| [getTime(indexPattern, timeRange, options)](./kibana-plugin-plugins-data-public.gettime.md) | | | [plugin(initializerContext)](./kibana-plugin-plugins-data-public.plugin.md) | | ## Interfaces diff --git a/src/plugins/data/common/index_patterns/types.ts b/src/plugins/data/common/index_patterns/types.ts index 698edbf9cd6a8..e21d27a70e02a 100644 --- a/src/plugins/data/common/index_patterns/types.ts +++ b/src/plugins/data/common/index_patterns/types.ts @@ -26,6 +26,7 @@ export interface IIndexPattern { id?: string; type?: string; timeFieldName?: string; + getTimeField?(): IFieldType | undefined; fieldFormatMap?: Record< string, { diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index 86560b3ccf7b1..91dea66f06a94 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -699,7 +699,10 @@ export function getSearchErrorType({ message }: Pick): " // Warning: (ae-missing-release-tag) "getTime" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export function getTime(indexPattern: IIndexPattern | undefined, timeRange: TimeRange, forceNow?: Date): import("../..").RangeFilter | undefined; +export function getTime(indexPattern: IIndexPattern | undefined, timeRange: TimeRange, options?: { + forceNow?: Date; + fieldName?: string; +}): import("../..").RangeFilter | undefined; // Warning: (ae-missing-release-tag) "IAggConfig" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // @@ -842,6 +845,8 @@ export interface IIndexPattern { // (undocumented) fields: IFieldType[]; // (undocumented) + getTimeField?(): IFieldType | undefined; + // (undocumented) id?: string; // (undocumented) timeFieldName?: string; diff --git a/src/plugins/data/public/query/timefilter/get_time.test.ts b/src/plugins/data/public/query/timefilter/get_time.test.ts index a8eb3a3fe8102..4dba157a6f554 100644 --- a/src/plugins/data/public/query/timefilter/get_time.test.ts +++ b/src/plugins/data/public/query/timefilter/get_time.test.ts @@ -51,5 +51,43 @@ describe('get_time', () => { }); clock.restore(); }); + + test('build range filter for non-primary field', () => { + const clock = sinon.useFakeTimers(moment.utc([2000, 1, 1, 0, 0, 0, 0]).valueOf()); + + const filter = getTime( + { + id: 'test', + title: 'test', + timeFieldName: 'date', + fields: [ + { + name: 'date', + type: 'date', + esTypes: ['date'], + aggregatable: true, + searchable: true, + filterable: true, + }, + { + name: 'myCustomDate', + type: 'date', + esTypes: ['date'], + aggregatable: true, + searchable: true, + filterable: true, + }, + ], + } as any, + { from: 'now-60y', to: 'now' }, + { fieldName: 'myCustomDate' } + ); + expect(filter!.range.myCustomDate).toEqual({ + gte: '1940-02-01T00:00:00.000Z', + lte: '2000-02-01T00:00:00.000Z', + format: 'strict_date_optional_time', + }); + clock.restore(); + }); }); }); diff --git a/src/plugins/data/public/query/timefilter/get_time.ts b/src/plugins/data/public/query/timefilter/get_time.ts index fa15406189041..9cdd25d3213ce 100644 --- a/src/plugins/data/public/query/timefilter/get_time.ts +++ b/src/plugins/data/public/query/timefilter/get_time.ts @@ -19,7 +19,7 @@ import dateMath from '@elastic/datemath'; import { IIndexPattern } from '../..'; -import { TimeRange, IFieldType, buildRangeFilter } from '../../../common'; +import { TimeRange, buildRangeFilter } from '../../../common'; interface CalculateBoundsOptions { forceNow?: Date; @@ -35,18 +35,27 @@ export function calculateBounds(timeRange: TimeRange, options: CalculateBoundsOp export function getTime( indexPattern: IIndexPattern | undefined, timeRange: TimeRange, + options?: { forceNow?: Date; fieldName?: string } +) { + return createTimeRangeFilter( + indexPattern, + timeRange, + options?.fieldName || indexPattern?.timeFieldName, + options?.forceNow + ); +} + +function createTimeRangeFilter( + indexPattern: IIndexPattern | undefined, + timeRange: TimeRange, + fieldName?: string, forceNow?: Date ) { if (!indexPattern) { - // in CI, we sometimes seem to fail here. return; } - - const timefield: IFieldType | undefined = indexPattern.fields.find( - field => field.name === indexPattern.timeFieldName - ); - - if (!timefield) { + const field = indexPattern.fields.find(f => f.name === (fieldName || indexPattern.timeFieldName)); + if (!field) { return; } @@ -55,7 +64,7 @@ export function getTime( return; } return buildRangeFilter( - timefield, + field, { ...(bounds.min && { gte: bounds.min.toISOString() }), ...(bounds.max && { lte: bounds.max.toISOString() }), diff --git a/src/plugins/data/public/query/timefilter/index.ts b/src/plugins/data/public/query/timefilter/index.ts index a6260e782c12f..034af03842ab8 100644 --- a/src/plugins/data/public/query/timefilter/index.ts +++ b/src/plugins/data/public/query/timefilter/index.ts @@ -22,6 +22,6 @@ export { TimefilterService, TimefilterSetup } from './timefilter_service'; export * from './types'; export { Timefilter, TimefilterContract } from './timefilter'; export { TimeHistory, TimeHistoryContract } from './time_history'; -export { getTime } from './get_time'; +export { getTime, calculateBounds } from './get_time'; export { changeTimeFilter } from './lib/change_time_filter'; export { extractTimeFilter } from './lib/extract_time_filter'; diff --git a/src/plugins/data/public/query/timefilter/timefilter.ts b/src/plugins/data/public/query/timefilter/timefilter.ts index 4fbdac47fb3b0..86ef69be572a9 100644 --- a/src/plugins/data/public/query/timefilter/timefilter.ts +++ b/src/plugins/data/public/query/timefilter/timefilter.ts @@ -164,7 +164,9 @@ export class Timefilter { }; public createFilter = (indexPattern: IndexPattern, timeRange?: TimeRange) => { - return getTime(indexPattern, timeRange ? timeRange : this._time, this.getForceNow()); + return getTime(indexPattern, timeRange ? timeRange : this._time, { + forceNow: this.getForceNow(), + }); }; public getBounds(): TimeRangeBounds { diff --git a/src/plugins/data/public/search/aggs/buckets/date_histogram.ts b/src/plugins/data/public/search/aggs/buckets/date_histogram.ts index 57f3aa85ad944..3ecdc17cb57f3 100644 --- a/src/plugins/data/public/search/aggs/buckets/date_histogram.ts +++ b/src/plugins/data/public/search/aggs/buckets/date_histogram.ts @@ -45,7 +45,7 @@ const updateTimeBuckets = ( customBuckets?: IBucketDateHistogramAggConfig['buckets'] ) => { const bounds = - agg.params.timeRange && agg.fieldIsTimeField() + agg.params.timeRange && (agg.fieldIsTimeField() || agg.params.interval === 'auto') ? timefilter.calculateBounds(agg.params.timeRange) : undefined; const buckets = customBuckets || agg.buckets; diff --git a/src/plugins/data/public/search/expressions/esaggs.ts b/src/plugins/data/public/search/expressions/esaggs.ts index 087b83127079f..eec75b0841133 100644 --- a/src/plugins/data/public/search/expressions/esaggs.ts +++ b/src/plugins/data/public/search/expressions/esaggs.ts @@ -32,8 +32,15 @@ import { Adapters } from '../../../../../plugins/inspector/public'; import { IAggConfigs } from '../aggs'; import { ISearchSource } from '../search_source'; import { tabifyAggResponse } from '../tabify'; -import { Filter, Query, serializeFieldFormat, TimeRange } from '../../../common'; -import { FilterManager, getTime } from '../../query'; +import { + Filter, + Query, + serializeFieldFormat, + TimeRange, + IIndexPattern, + isRangeFilter, +} from '../../../common'; +import { FilterManager, calculateBounds, getTime } from '../../query'; import { getSearchService, getQueryService, getIndexPatterns } from '../../services'; import { buildTabularInspectorData } from './build_tabular_inspector_data'; import { getRequestInspectorStats, getResponseInspectorStats, serializeAggConfig } from './utils'; @@ -42,6 +49,8 @@ export interface RequestHandlerParams { searchSource: ISearchSource; aggs: IAggConfigs; timeRange?: TimeRange; + timeFields?: string[]; + indexPattern?: IIndexPattern; query?: Query; filters?: Filter[]; forceFetch: boolean; @@ -65,12 +74,15 @@ interface Arguments { partialRows: boolean; includeFormatHints: boolean; aggConfigs: string; + timeFields?: string[]; } const handleCourierRequest = async ({ searchSource, aggs, timeRange, + timeFields, + indexPattern, query, filters, forceFetch, @@ -111,9 +123,19 @@ const handleCourierRequest = async ({ return aggs.onSearchRequestStart(paramSearchSource, options); }); - if (timeRange) { + // If timeFields have been specified, use the specified ones, otherwise use primary time field of index + // pattern if it's available. + const defaultTimeField = indexPattern?.getTimeField?.(); + const defaultTimeFields = defaultTimeField ? [defaultTimeField.name] : []; + const allTimeFields = timeFields && timeFields.length > 0 ? timeFields : defaultTimeFields; + + // If a timeRange has been specified and we had at least one timeField available, create range + // filters for that those time fields + if (timeRange && allTimeFields.length > 0) { timeFilterSearchSource.setField('filter', () => { - return getTime(searchSource.getField('index'), timeRange); + return allTimeFields + .map(fieldName => getTime(indexPattern, timeRange, { fieldName })) + .filter(isRangeFilter); }); } @@ -181,11 +203,13 @@ const handleCourierRequest = async ({ (searchSource as any).finalResponse = resp; - const parsedTimeRange = timeRange ? getTime(aggs.indexPattern, timeRange) : null; + const parsedTimeRange = timeRange ? calculateBounds(timeRange) : null; const tabifyParams = { metricsAtAllLevels, partialRows, - timeRange: parsedTimeRange ? parsedTimeRange.range : undefined, + timeRange: parsedTimeRange + ? { from: parsedTimeRange.min, to: parsedTimeRange.max, timeFields: allTimeFields } + : undefined, }; const tabifyCacheHash = calculateObjectHash({ tabifyAggs: aggs, ...tabifyParams }); @@ -242,6 +266,11 @@ export const esaggs = (): ExpressionFunctionDefinition { const check = (aggResp: any, count: number, keys: string[]) => { @@ -187,9 +192,9 @@ describe('Buckets wrapper', () => { }, }; const timeRange = { - gte: 150, - lte: 350, - name: 'date', + from: moment(150), + to: moment(350), + timeFields: ['date'], }; const buckets = new TabifyBuckets(aggResp, aggParams, timeRange); @@ -204,9 +209,9 @@ describe('Buckets wrapper', () => { }, }; const timeRange = { - gte: 150, - lte: 350, - name: 'date', + from: moment(150), + to: moment(350), + timeFields: ['date'], }; const buckets = new TabifyBuckets(aggResp, aggParams, timeRange); @@ -221,9 +226,9 @@ describe('Buckets wrapper', () => { }, }; const timeRange = { - gte: 100, - lte: 400, - name: 'date', + from: moment(100), + to: moment(400), + timeFields: ['date'], }; const buckets = new TabifyBuckets(aggResp, aggParams, timeRange); @@ -238,13 +243,47 @@ describe('Buckets wrapper', () => { }, }; const timeRange = { - gte: 150, - lte: 350, - name: 'date', + from: moment(150), + to: moment(350), + timeFields: ['date'], }; const buckets = new TabifyBuckets(aggResp, aggParams, timeRange); expect(buckets).toHaveLength(4); }); + + test('does drop bucket when multiple time fields specified', () => { + const aggParams = { + drop_partials: true, + field: { + name: 'date', + }, + }; + const timeRange = { + from: moment(100), + to: moment(350), + timeFields: ['date', 'other_datefield'], + }; + const buckets = new TabifyBuckets(aggResp, aggParams, timeRange); + + expect(buckets.buckets.map((b: Bucket) => b.key)).toEqual([100, 200]); + }); + + test('does not drop bucket when no timeFields have been specified', () => { + const aggParams = { + drop_partials: true, + field: { + name: 'date', + }, + }; + const timeRange = { + from: moment(100), + to: moment(350), + timeFields: [], + }; + const buckets = new TabifyBuckets(aggResp, aggParams, timeRange); + + expect(buckets.buckets.map((b: Bucket) => b.key)).toEqual([0, 100, 200, 300]); + }); }); }); diff --git a/src/plugins/data/public/search/tabify/buckets.ts b/src/plugins/data/public/search/tabify/buckets.ts index 971e820ac6ddf..cd52a09caeaad 100644 --- a/src/plugins/data/public/search/tabify/buckets.ts +++ b/src/plugins/data/public/search/tabify/buckets.ts @@ -20,7 +20,7 @@ import { get, isPlainObject, keys, findKey } from 'lodash'; import moment from 'moment'; import { IAggConfig } from '../aggs'; -import { AggResponseBucket, TabbedRangeFilterParams } from './types'; +import { AggResponseBucket, TabbedRangeFilterParams, TimeRangeInformation } from './types'; type AggParams = IAggConfig['params'] & { drop_partials: boolean; @@ -36,7 +36,7 @@ export class TabifyBuckets { buckets: any; _keys: any[] = []; - constructor(aggResp: any, aggParams?: AggParams, timeRange?: TabbedRangeFilterParams) { + constructor(aggResp: any, aggParams?: AggParams, timeRange?: TimeRangeInformation) { if (aggResp && aggResp.buckets) { this.buckets = aggResp.buckets; } else if (aggResp) { @@ -107,12 +107,12 @@ export class TabifyBuckets { // dropPartials should only be called if the aggParam setting is enabled, // and the agg field is the same as the Time Range. - private dropPartials(params: AggParams, timeRange?: TabbedRangeFilterParams) { + private dropPartials(params: AggParams, timeRange?: TimeRangeInformation) { if ( !timeRange || this.buckets.length <= 1 || this.objectMode || - params.field.name !== timeRange.name + !timeRange.timeFields.includes(params.field.name) ) { return; } @@ -120,10 +120,10 @@ export class TabifyBuckets { const interval = this.buckets[1].key - this.buckets[0].key; this.buckets = this.buckets.filter((bucket: AggResponseBucket) => { - if (moment(bucket.key).isBefore(timeRange.gte)) { + if (moment(bucket.key).isBefore(timeRange.from)) { return false; } - if (moment(bucket.key + interval).isAfter(timeRange.lte)) { + if (moment(bucket.key + interval).isAfter(timeRange.to)) { return false; } return true; diff --git a/src/plugins/data/public/search/tabify/tabify.ts b/src/plugins/data/public/search/tabify/tabify.ts index e93e989034252..9cb55f94537c5 100644 --- a/src/plugins/data/public/search/tabify/tabify.ts +++ b/src/plugins/data/public/search/tabify/tabify.ts @@ -20,7 +20,7 @@ import { get } from 'lodash'; import { TabbedAggResponseWriter } from './response_writer'; import { TabifyBuckets } from './buckets'; -import { TabbedResponseWriterOptions, TabbedRangeFilterParams } from './types'; +import { TabbedResponseWriterOptions } from './types'; import { AggResponseBucket } from './types'; import { AggGroupNames, IAggConfigs } from '../aggs'; @@ -54,7 +54,7 @@ export function tabifyAggResponse( switch (agg.type.type) { case AggGroupNames.Buckets: const aggBucket = get(bucket, agg.id); - const tabifyBuckets = new TabifyBuckets(aggBucket, agg.params, timeRange); + const tabifyBuckets = new TabifyBuckets(aggBucket, agg.params, respOpts?.timeRange); if (tabifyBuckets.length) { tabifyBuckets.forEach((subBucket, tabifyBucketKey) => { @@ -153,20 +153,6 @@ export function tabifyAggResponse( doc_count: esResponse.hits.total, }; - let timeRange: TabbedRangeFilterParams | undefined; - - // Extract the time range object if provided - if (respOpts && respOpts.timeRange) { - const [timeRangeKey] = Object.keys(respOpts.timeRange); - - if (timeRangeKey) { - timeRange = { - name: timeRangeKey, - ...respOpts.timeRange[timeRangeKey], - }; - } - } - collectBucket(aggConfigs, write, topLevelBucket, '', 1); return write.response(); diff --git a/src/plugins/data/public/search/tabify/types.ts b/src/plugins/data/public/search/tabify/types.ts index 1e051880d3f19..72e91eb58c8a9 100644 --- a/src/plugins/data/public/search/tabify/types.ts +++ b/src/plugins/data/public/search/tabify/types.ts @@ -17,6 +17,7 @@ * under the License. */ +import { Moment } from 'moment'; import { RangeFilterParams } from '../../../common'; import { IAggConfig } from '../aggs'; @@ -25,11 +26,18 @@ export interface TabbedRangeFilterParams extends RangeFilterParams { name: string; } +/** @internal */ +export interface TimeRangeInformation { + from?: Moment; + to?: Moment; + timeFields: string[]; +} + /** @internal **/ export interface TabbedResponseWriterOptions { metricsAtAllLevels: boolean; partialRows: boolean; - timeRange?: { [key: string]: RangeFilterParams }; + timeRange?: TimeRangeInformation; } /** @internal */ diff --git a/src/plugins/data/server/server.api.md b/src/plugins/data/server/server.api.md index 5d94b6516c2ba..df4ba23244b4d 100644 --- a/src/plugins/data/server/server.api.md +++ b/src/plugins/data/server/server.api.md @@ -408,6 +408,8 @@ export interface IIndexPattern { // (undocumented) fields: IFieldType[]; // (undocumented) + getTimeField?(): IFieldType | undefined; + // (undocumented) id?: string; // (undocumented) timeFieldName?: string; diff --git a/test/interpreter_functional/test_suites/run_pipeline/esaggs.ts b/test/interpreter_functional/test_suites/run_pipeline/esaggs.ts new file mode 100644 index 0000000000000..5ea151dffdc8e --- /dev/null +++ b/test/interpreter_functional/test_suites/run_pipeline/esaggs.ts @@ -0,0 +1,93 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import expect from '@kbn/expect'; +import { ExpectExpression, expectExpressionProvider } from './helpers'; +import { FtrProviderContext } from '../../../functional/ftr_provider_context'; + +function getCell(esaggsResult: any, column: number, row: number): unknown | undefined { + const columnId = esaggsResult?.columns[column]?.id; + if (!columnId) { + return; + } + return esaggsResult?.rows[row]?.[columnId]; +} + +export default function({ + getService, + updateBaselines, +}: FtrProviderContext & { updateBaselines: boolean }) { + let expectExpression: ExpectExpression; + describe('esaggs pipeline expression tests', () => { + before(() => { + expectExpression = expectExpressionProvider({ getService, updateBaselines }); + }); + + describe('correctly renders tagcloud', () => { + it('filters on index pattern primary date field by default', async () => { + const aggConfigs = [{ id: 1, enabled: true, type: 'count', schema: 'metric', params: {} }]; + const timeRange = { + from: '2006-09-21T00:00:00Z', + to: '2015-09-22T00:00:00Z', + }; + const expression = ` + kibana_context timeRange='${JSON.stringify(timeRange)}' + | esaggs index='logstash-*' aggConfigs='${JSON.stringify(aggConfigs)}' + `; + const result = await expectExpression('esaggs_primary_timefield', expression).getResponse(); + expect(getCell(result, 0, 0)).to.be(9375); + }); + + it('filters on the specified date field', async () => { + const aggConfigs = [{ id: 1, enabled: true, type: 'count', schema: 'metric', params: {} }]; + const timeRange = { + from: '2006-09-21T00:00:00Z', + to: '2015-09-22T00:00:00Z', + }; + const expression = ` + kibana_context timeRange='${JSON.stringify(timeRange)}' + | esaggs index='logstash-*' timeFields='relatedContent.article:published_time' aggConfigs='${JSON.stringify( + aggConfigs + )}' + `; + const result = await expectExpression('esaggs_other_timefield', expression).getResponse(); + expect(getCell(result, 0, 0)).to.be(11134); + }); + + it('filters on multiple specified date field', async () => { + const aggConfigs = [{ id: 1, enabled: true, type: 'count', schema: 'metric', params: {} }]; + const timeRange = { + from: '2006-09-21T00:00:00Z', + to: '2015-09-22T00:00:00Z', + }; + const expression = ` + kibana_context timeRange='${JSON.stringify(timeRange)}' + | esaggs index='logstash-*' timeFields='relatedContent.article:published_time' timeFields='@timestamp' aggConfigs='${JSON.stringify( + aggConfigs + )}' + `; + const result = await expectExpression( + 'esaggs_multiple_timefields', + expression + ).getResponse(); + expect(getCell(result, 0, 0)).to.be(7452); + }); + }); + }); +} diff --git a/test/interpreter_functional/test_suites/run_pipeline/index.ts b/test/interpreter_functional/test_suites/run_pipeline/index.ts index 031a0e3576ccc..9590f9f8c1794 100644 --- a/test/interpreter_functional/test_suites/run_pipeline/index.ts +++ b/test/interpreter_functional/test_suites/run_pipeline/index.ts @@ -46,5 +46,6 @@ export default function({ getService, getPageObjects, loadTestFile }: FtrProvide loadTestFile(require.resolve('./basic')); loadTestFile(require.resolve('./tag_cloud')); loadTestFile(require.resolve('./metric')); + loadTestFile(require.resolve('./esaggs')); }); } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/auto_date.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/auto_date.test.ts deleted file mode 100644 index 5f35ef650a08c..0000000000000 --- a/x-pack/plugins/lens/public/indexpattern_datasource/auto_date.test.ts +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { dataPluginMock } from '../../../../../src/plugins/data/public/mocks'; -import { getAutoDate } from './auto_date'; - -describe('auto_date', () => { - let autoDate: ReturnType; - - beforeEach(() => { - autoDate = getAutoDate({ data: dataPluginMock.createSetupContract() }); - }); - - it('should do nothing if no time range is provided', () => { - const result = autoDate.fn( - { - type: 'kibana_context', - }, - { - aggConfigs: 'canttouchthis', - }, - // eslint-disable-next-line - {} as any - ); - - expect(result).toEqual('canttouchthis'); - }); - - it('should not change anything if there are no auto date histograms', () => { - const aggConfigs = JSON.stringify([ - { type: 'date_histogram', params: { interval: '35h' } }, - { type: 'count' }, - ]); - const result = autoDate.fn( - { - timeRange: { - from: 'now-10d', - to: 'now', - }, - type: 'kibana_context', - }, - { - aggConfigs, - }, - // eslint-disable-next-line - {} as any - ); - - expect(result).toEqual(aggConfigs); - }); - - it('should change auto date histograms', () => { - const aggConfigs = JSON.stringify([ - { type: 'date_histogram', params: { interval: 'auto' } }, - { type: 'count' }, - ]); - const result = autoDate.fn( - { - timeRange: { - from: 'now-10d', - to: 'now', - }, - type: 'kibana_context', - }, - { - aggConfigs, - }, - // eslint-disable-next-line - {} as any - ); - - const interval = JSON.parse(result).find( - (agg: { type: string }) => agg.type === 'date_histogram' - ).params.interval; - - expect(interval).toBeTruthy(); - expect(typeof interval).toEqual('string'); - expect(interval).not.toEqual('auto'); - }); -}); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/auto_date.ts b/x-pack/plugins/lens/public/indexpattern_datasource/auto_date.ts deleted file mode 100644 index 97a46f4a3e176..0000000000000 --- a/x-pack/plugins/lens/public/indexpattern_datasource/auto_date.ts +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { DataPublicPluginSetup } from '../../../../../src/plugins/data/public'; -import { - ExpressionFunctionDefinition, - KibanaContext, -} from '../../../../../src/plugins/expressions/public'; - -interface LensAutoDateProps { - aggConfigs: string; -} - -export function getAutoDate(deps: { - data: DataPublicPluginSetup; -}): ExpressionFunctionDefinition< - 'lens_auto_date', - KibanaContext | null, - LensAutoDateProps, - string -> { - function autoIntervalFromContext(ctx?: KibanaContext | null) { - if (!ctx || !ctx.timeRange) { - return; - } - - return deps.data.search.aggs.calculateAutoTimeExpression(ctx.timeRange); - } - - /** - * Convert all 'auto' date histograms into a concrete value (e.g. 2h). - * This allows us to support 'auto' on all date fields, and opens the - * door to future customizations (e.g. adjusting the level of detail, etc). - */ - return { - name: 'lens_auto_date', - aliases: [], - help: '', - inputTypes: ['kibana_context', 'null'], - args: { - aggConfigs: { - types: ['string'], - default: '""', - help: '', - }, - }, - fn(input, args) { - const interval = autoIntervalFromContext(input); - - if (!interval) { - return args.aggConfigs; - } - - const configs = JSON.parse(args.aggConfigs) as Array<{ - type: string; - params: { interval: string }; - }>; - - const updatedConfigs = configs.map(c => { - if (c.type !== 'date_histogram' || !c.params || c.params.interval !== 'auto') { - return c; - } - - return { - ...c, - params: { - ...c.params, - interval, - }, - }; - }); - - return JSON.stringify(updatedConfigs); - }, - }; -} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/index.ts index fe14f472341af..73fd144b9c7f8 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/index.ts @@ -8,7 +8,6 @@ import { CoreSetup } from 'kibana/public'; import { Storage } from '../../../../../src/plugins/kibana_utils/public'; import { getIndexPatternDatasource } from './indexpattern'; import { renameColumns } from './rename_columns'; -import { getAutoDate } from './auto_date'; import { ExpressionsSetup } from '../../../../../src/plugins/expressions/public'; import { DataPublicPluginSetup, @@ -31,10 +30,9 @@ export class IndexPatternDatasource { setup( core: CoreSetup, - { data: dataSetup, expressions, editorFrame }: IndexPatternDatasourceSetupPlugins + { expressions, editorFrame }: IndexPatternDatasourceSetupPlugins ) { expressions.registerFunction(renameColumns); - expressions.registerFunction(getAutoDate({ data: dataSetup })); editorFrame.registerDatasource( core.getStartServices().then(([coreStart, { data }]) => diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts index e4f3677d0fe88..06635e663361d 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts @@ -10,6 +10,7 @@ import { DatasourcePublicAPI, Operation, Datasource } from '../types'; import { coreMock } from 'src/core/public/mocks'; import { IndexPatternPersistedState, IndexPatternPrivateState } from './types'; import { dataPluginMock } from '../../../../../src/plugins/data/public/mocks'; +import { Ast } from '@kbn/interpreter/common'; jest.mock('./loader'); jest.mock('../id_generator'); @@ -262,20 +263,7 @@ describe('IndexPattern Data Source', () => { Object { "arguments": Object { "aggConfigs": Array [ - Object { - "chain": Array [ - Object { - "arguments": Object { - "aggConfigs": Array [ - "[{\\"id\\":\\"col1\\",\\"enabled\\":true,\\"type\\":\\"count\\",\\"schema\\":\\"metric\\",\\"params\\":{}},{\\"id\\":\\"col2\\",\\"enabled\\":true,\\"type\\":\\"date_histogram\\",\\"schema\\":\\"segment\\",\\"params\\":{\\"field\\":\\"timestamp\\",\\"useNormalizedEsInterval\\":true,\\"interval\\":\\"1d\\",\\"drop_partials\\":false,\\"min_doc_count\\":0,\\"extended_bounds\\":{}}}]", - ], - }, - "function": "lens_auto_date", - "type": "function", - }, - ], - "type": "expression", - }, + "[{\\"id\\":\\"col1\\",\\"enabled\\":true,\\"type\\":\\"count\\",\\"schema\\":\\"metric\\",\\"params\\":{}},{\\"id\\":\\"col2\\",\\"enabled\\":true,\\"type\\":\\"date_histogram\\",\\"schema\\":\\"segment\\",\\"params\\":{\\"field\\":\\"timestamp\\",\\"useNormalizedEsInterval\\":true,\\"interval\\":\\"1d\\",\\"drop_partials\\":false,\\"min_doc_count\\":0,\\"extended_bounds\\":{}}}]", ], "includeFormatHints": Array [ true, @@ -289,6 +277,9 @@ describe('IndexPattern Data Source', () => { "partialRows": Array [ false, ], + "timeFields": Array [ + "timestamp", + ], }, "function": "esaggs", "type": "function", @@ -307,6 +298,89 @@ describe('IndexPattern Data Source', () => { } `); }); + + it('should put all time fields used in date_histograms to the esaggs timeFields parameter', async () => { + const queryPersistedState: IndexPatternPersistedState = { + currentIndexPatternId: '1', + layers: { + first: { + indexPatternId: '1', + columnOrder: ['col1', 'col2', 'col3'], + columns: { + col1: { + label: 'Count of records', + dataType: 'number', + isBucketed: false, + sourceField: 'Records', + operationType: 'count', + }, + col2: { + label: 'Date', + dataType: 'date', + isBucketed: true, + operationType: 'date_histogram', + sourceField: 'timestamp', + params: { + interval: 'auto', + }, + }, + col3: { + label: 'Date 2', + dataType: 'date', + isBucketed: true, + operationType: 'date_histogram', + sourceField: 'another_datefield', + params: { + interval: 'auto', + }, + }, + }, + }, + }, + }; + + const state = stateFromPersistedState(queryPersistedState); + + const ast = indexPatternDatasource.toExpression(state, 'first') as Ast; + expect(ast.chain[0].arguments.timeFields).toEqual(['timestamp', 'another_datefield']); + }); + + it('should not put date fields used outside date_histograms to the esaggs timeFields parameter', async () => { + const queryPersistedState: IndexPatternPersistedState = { + currentIndexPatternId: '1', + layers: { + first: { + indexPatternId: '1', + columnOrder: ['col1', 'col2'], + columns: { + col1: { + label: 'Count of records', + dataType: 'date', + isBucketed: false, + sourceField: 'timefield', + operationType: 'cardinality', + }, + col2: { + label: 'Date', + dataType: 'date', + isBucketed: true, + operationType: 'date_histogram', + sourceField: 'timestamp', + params: { + interval: 'auto', + }, + }, + }, + }, + }, + }; + + const state = stateFromPersistedState(queryPersistedState); + + const ast = indexPatternDatasource.toExpression(state, 'first') as Ast; + expect(ast.chain[0].arguments.timeFields).toEqual(['timestamp']); + expect(ast.chain[0].arguments.timeFields).not.toContain('timefield'); + }); }); describe('#insertLayer', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts index 3ab51b5fa3f2b..1308fa3b7ca60 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts @@ -10,6 +10,7 @@ import { IndexPatternColumn } from './indexpattern'; import { operationDefinitionMap } from './operations'; import { IndexPattern, IndexPatternPrivateState } from './types'; import { OriginalColumn } from './rename_columns'; +import { dateHistogramOperation } from './operations/definitions'; function getExpressionForLayer( indexPattern: IndexPattern, @@ -68,6 +69,12 @@ function getExpressionForLayer( return base; }); + const allDateHistogramFields = Object.values(columns) + .map(column => + column.operationType === dateHistogramOperation.type ? column.sourceField : null + ) + .filter((field): field is string => Boolean(field)); + return { type: 'expression', chain: [ @@ -79,20 +86,8 @@ function getExpressionForLayer( metricsAtAllLevels: [false], partialRows: [false], includeFormatHints: [true], - aggConfigs: [ - { - type: 'expression', - chain: [ - { - type: 'function', - function: 'lens_auto_date', - arguments: { - aggConfigs: [JSON.stringify(aggs)], - }, - }, - ], - }, - ], + timeFields: allDateHistogramFields, + aggConfigs: [JSON.stringify(aggs)], }, }, { diff --git a/x-pack/plugins/lens/server/migrations.test.ts b/x-pack/plugins/lens/server/migrations.test.ts index e80308cc9acdb..4cc330d40efd7 100644 --- a/x-pack/plugins/lens/server/migrations.test.ts +++ b/x-pack/plugins/lens/server/migrations.test.ts @@ -158,4 +158,124 @@ describe('Lens migrations', () => { ]); }); }); + + describe('7.8.0 auto timestamp', () => { + const context = {} as SavedObjectMigrationContext; + + const example = { + type: 'lens', + attributes: { + expression: `kibana + | kibana_context query="{\\"query\\":\\"\\",\\"language\\":\\"kuery\\"}" filters="[]" + | lens_merge_tables layerIds="bd09dc71-a7e2-42d0-83bd-85df8291f03c" + tables={esaggs + index="ff959d40-b880-11e8-a6d9-e546fe2bba5f" + metricsAtAllLevels=false + partialRows=false + includeFormatHints=true + aggConfigs={ + lens_auto_date + aggConfigs="[{\\"id\\":\\"1d9cc16c-1460-41de-88f8-471932ecbc97\\",\\"enabled\\":true,\\"type\\":\\"date_histogram\\",\\"schema\\":\\"segment\\",\\"params\\":{\\"field\\":\\"products.created_on\\",\\"useNormalizedEsInterval\\":true,\\"interval\\":\\"auto\\",\\"drop_partials\\":false,\\"min_doc_count\\":0,\\"extended_bounds\\":{}}},{\\"id\\":\\"66115819-8481-4917-a6dc-8ffb10dd02df\\",\\"enabled\\":true,\\"type\\":\\"count\\",\\"schema\\":\\"metric\\",\\"params\\":{}}]" + } + | lens_rename_columns idMap="{\\"col-0-1d9cc16c-1460-41de-88f8-471932ecbc97\\":{\\"label\\":\\"products.created_on\\",\\"dataType\\":\\"date\\",\\"operationType\\":\\"date_histogram\\",\\"sourceField\\":\\"products.created_on\\",\\"isBucketed\\":true,\\"scale\\":\\"interval\\",\\"params\\":{\\"interval\\":\\"auto\\"},\\"id\\":\\"1d9cc16c-1460-41de-88f8-471932ecbc97\\"},\\"col-1-66115819-8481-4917-a6dc-8ffb10dd02df\\":{\\"label\\":\\"Count of records\\",\\"dataType\\":\\"number\\",\\"operationType\\":\\"count\\",\\"suggestedPriority\\":0,\\"isBucketed\\":false,\\"scale\\":\\"ratio\\",\\"sourceField\\":\\"Records\\",\\"id\\":\\"66115819-8481-4917-a6dc-8ffb10dd02df\\"}}" + } + | lens_xy_chart + xTitle="products.created_on" + yTitle="Count of records" + legend={lens_xy_legendConfig isVisible=true position="right"} + layers={lens_xy_layer + layerId="bd09dc71-a7e2-42d0-83bd-85df8291f03c" + hide=false + xAccessor="1d9cc16c-1460-41de-88f8-471932ecbc97" + yScaleType="linear" + xScaleType="time" + isHistogram=true + seriesType="bar_stacked" + accessors="66115819-8481-4917-a6dc-8ffb10dd02df" + columnToLabel="{\\"66115819-8481-4917-a6dc-8ffb10dd02df\\":\\"Count of records\\"}" + } + `, + state: { + datasourceStates: { + indexpattern: { + currentIndexPatternId: 'ff959d40-b880-11e8-a6d9-e546fe2bba5f', + layers: { + 'bd09dc71-a7e2-42d0-83bd-85df8291f03c': { + indexPatternId: 'ff959d40-b880-11e8-a6d9-e546fe2bba5f', + columns: { + '1d9cc16c-1460-41de-88f8-471932ecbc97': { + label: 'products.created_on', + dataType: 'date', + operationType: 'date_histogram', + sourceField: 'products.created_on', + isBucketed: true, + scale: 'interval', + params: { interval: 'auto' }, + }, + '66115819-8481-4917-a6dc-8ffb10dd02df': { + label: 'Count of records', + dataType: 'number', + operationType: 'count', + suggestedPriority: 0, + isBucketed: false, + scale: 'ratio', + sourceField: 'Records', + }, + }, + columnOrder: [ + '1d9cc16c-1460-41de-88f8-471932ecbc97', + '66115819-8481-4917-a6dc-8ffb10dd02df', + ], + }, + }, + }, + }, + datasourceMetaData: { + filterableIndexPatterns: [ + { id: 'ff959d40-b880-11e8-a6d9-e546fe2bba5f', title: 'kibana_sample_data_ecommerce' }, + ], + }, + visualization: { + legend: { isVisible: true, position: 'right' }, + preferredSeriesType: 'bar_stacked', + layers: [ + { + layerId: 'bd09dc71-a7e2-42d0-83bd-85df8291f03c', + accessors: ['66115819-8481-4917-a6dc-8ffb10dd02df'], + position: 'top', + seriesType: 'bar_stacked', + showGridlines: false, + xAccessor: '1d9cc16c-1460-41de-88f8-471932ecbc97', + }, + ], + }, + query: { query: '', language: 'kuery' }, + filters: [], + }, + title: 'Bar chart', + visualizationType: 'lnsXY', + }, + }; + + it('should remove the lens_auto_date expression', () => { + const result = migrations['7.8.0'](example, context); + expect(result.attributes.expression).toContain(`timeFields=\"products.created_on\"`); + }); + + it('should handle pre-migrated expression', () => { + const input = { + type: 'lens', + attributes: { + ...example.attributes, + expression: `kibana +| kibana_context query="{\\"query\\":\\"\\",\\"language\\":\\"kuery\\"}" filters="[]" +| lens_merge_tables layerIds="bd09dc71-a7e2-42d0-83bd-85df8291f03c" + tables={esaggs index="ff959d40-b880-11e8-a6d9-e546fe2bba5f" metricsAtAllLevels=false partialRows=false includeFormatHints=true aggConfigs="[{\\"id\\":\\"1d9cc16c-1460-41de-88f8-471932ecbc97\\",\\"enabled\\":true,\\"type\\":\\"date_histogram\\",\\"schema\\":\\"segment\\",\\"params\\":{\\"field\\":\\"products.created_on\\",\\"useNormalizedEsInterval\\":true,\\"interval\\":\\"auto\\",\\"drop_partials\\":false,\\"min_doc_count\\":0,\\"extended_bounds\\":{}}},{\\"id\\":\\"66115819-8481-4917-a6dc-8ffb10dd02df\\",\\"enabled\\":true,\\"type\\":\\"count\\",\\"schema\\":\\"metric\\",\\"params\\":{}}]" timeFields=\"products.created_on\"} +| lens_xy_chart xTitle="products.created_on" yTitle="Count of records" legend={lens_xy_legendConfig isVisible=true position="right"} layers={}`, + }, + }; + const result = migrations['7.8.0'](input, context); + expect(result).toEqual(input); + }); + }); }); diff --git a/x-pack/plugins/lens/server/migrations.ts b/x-pack/plugins/lens/server/migrations.ts index 3d238723b7438..51fcd3b6198c3 100644 --- a/x-pack/plugins/lens/server/migrations.ts +++ b/x-pack/plugins/lens/server/migrations.ts @@ -4,7 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ -import { cloneDeep } from 'lodash'; +import { cloneDeep, flow } from 'lodash'; +import { fromExpression, toExpression, Ast, ExpressionFunctionAST } from '@kbn/interpreter/common'; import { SavedObjectMigrationFn } from 'src/core/server'; interface XYLayerPre77 { @@ -14,6 +15,122 @@ interface XYLayerPre77 { accessors: string[]; } +/** + * Removes the `lens_auto_date` subexpression from a stored expression + * string. For example: aggConfigs={lens_auto_date aggConfigs="JSON string"} + */ +const removeLensAutoDate: SavedObjectMigrationFn = (doc, context) => { + const expression: string = doc.attributes?.expression; + try { + const ast = fromExpression(expression); + const newChain: ExpressionFunctionAST[] = ast.chain.map(topNode => { + if (topNode.function !== 'lens_merge_tables') { + return topNode; + } + return { + ...topNode, + arguments: { + ...topNode.arguments, + tables: (topNode.arguments.tables as Ast[]).map(middleNode => { + return { + type: 'expression', + chain: middleNode.chain.map(node => { + // Check for sub-expression in aggConfigs + if ( + node.function === 'esaggs' && + typeof node.arguments.aggConfigs[0] !== 'string' + ) { + return { + ...node, + arguments: { + ...node.arguments, + aggConfigs: (node.arguments.aggConfigs[0] as Ast).chain[0].arguments + .aggConfigs, + }, + }; + } + return node; + }), + }; + }), + }, + }; + }); + + return { + ...doc, + attributes: { + ...doc.attributes, + expression: toExpression({ ...ast, chain: newChain }), + }, + }; + } catch (e) { + context.log.warning(e.message); + return { ...doc }; + } +}; + +/** + * Adds missing timeField arguments to esaggs in the Lens expression + */ +const addTimeFieldToEsaggs: SavedObjectMigrationFn = (doc, context) => { + const expression: string = doc.attributes?.expression; + + try { + const ast = fromExpression(expression); + const newChain: ExpressionFunctionAST[] = ast.chain.map(topNode => { + if (topNode.function !== 'lens_merge_tables') { + return topNode; + } + return { + ...topNode, + arguments: { + ...topNode.arguments, + tables: (topNode.arguments.tables as Ast[]).map(middleNode => { + return { + type: 'expression', + chain: middleNode.chain.map(node => { + // Skip if there are any timeField arguments already, because that indicates + // the fix is already applied + if (node.function !== 'esaggs' || node.arguments.timeFields) { + return node; + } + const timeFields: string[] = []; + JSON.parse(node.arguments.aggConfigs[0] as string).forEach( + (agg: { type: string; params: { field: string } }) => { + if (agg.type !== 'date_histogram') { + return; + } + timeFields.push(agg.params.field); + } + ); + return { + ...node, + arguments: { + ...node.arguments, + timeFields, + }, + }; + }), + }; + }), + }, + }; + }); + + return { + ...doc, + attributes: { + ...doc.attributes, + expression: toExpression({ ...ast, chain: newChain }), + }, + }; + } catch (e) { + context.log.warning(e.message); + return { ...doc }; + } +}; + export const migrations: Record = { '7.7.0': doc => { const newDoc = cloneDeep(doc); @@ -34,4 +151,7 @@ export const migrations: Record = { } return newDoc; }, + // The order of these migrations matter, since the timefield migration relies on the aggConfigs + // sitting directly on the esaggs as an argument and not a nested function (which lens_auto_date was). + '7.8.0': flow(removeLensAutoDate, addTimeFieldToEsaggs), };