diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.cumulative_sum.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.cumulative_sum.md new file mode 100644 index 0000000000000..ad1de0cc5f45b --- /dev/null +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.cumulative_sum.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-expressions-public](./kibana-plugin-plugins-expressions-public.md) > [ExpressionFunctionDefinitions](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.md) > [cumulative\_sum](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.cumulative_sum.md) + +## ExpressionFunctionDefinitions.cumulative\_sum property + +Signature: + +```typescript +cumulative_sum: ExpressionFunctionCumulativeSum; +``` diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.md index 914c5d6ebe2f6..d1703a1e019e6 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.md @@ -17,6 +17,7 @@ export interface ExpressionFunctionDefinitions | Property | Type | Description | | --- | --- | --- | | [clog](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.clog.md) | ExpressionFunctionClog | | +| [cumulative\_sum](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.cumulative_sum.md) | ExpressionFunctionCumulativeSum | | | [font](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.font.md) | ExpressionFunctionFont | | | [kibana\_context](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.kibana_context.md) | ExpressionFunctionKibanaContext | | | [kibana](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.kibana.md) | ExpressionFunctionKibana | | diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.cumulative_sum.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.cumulative_sum.md new file mode 100644 index 0000000000000..2fb8cde92e877 --- /dev/null +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.cumulative_sum.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-expressions-server](./kibana-plugin-plugins-expressions-server.md) > [ExpressionFunctionDefinitions](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.md) > [cumulative\_sum](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.cumulative_sum.md) + +## ExpressionFunctionDefinitions.cumulative\_sum property + +Signature: + +```typescript +cumulative_sum: ExpressionFunctionCumulativeSum; +``` diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.md index 71cd0b98a68c2..05b4ddce4ccde 100644 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.md +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.md @@ -17,6 +17,7 @@ export interface ExpressionFunctionDefinitions | Property | Type | Description | | --- | --- | --- | | [clog](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.clog.md) | ExpressionFunctionClog | | +| [cumulative\_sum](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.cumulative_sum.md) | ExpressionFunctionCumulativeSum | | | [font](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.font.md) | ExpressionFunctionFont | | | [kibana\_context](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.kibana_context.md) | ExpressionFunctionKibanaContext | | | [kibana](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.kibana.md) | ExpressionFunctionKibana | | diff --git a/src/plugins/expressions/common/expression_functions/specs/cumulative_sum.ts b/src/plugins/expressions/common/expression_functions/specs/cumulative_sum.ts new file mode 100644 index 0000000000000..970015638794f --- /dev/null +++ b/src/plugins/expressions/common/expression_functions/specs/cumulative_sum.ts @@ -0,0 +1,170 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { i18n } from '@kbn/i18n'; +import { ExpressionFunctionDefinition } from '../types'; +import { Datatable, DatatableRow } from '../../expression_types'; + +export interface CumulativeSumArgs { + by?: string[]; + inputColumnId: string; + outputColumnId: string; + outputColumnName?: string; +} + +export type ExpressionFunctionCumulativeSum = ExpressionFunctionDefinition< + 'cumulative_sum', + Datatable, + CumulativeSumArgs, + Datatable +>; + +/** + * Returns a string identifying the group of a row by a list of columns to group by + */ +function getBucketIdentifier(row: DatatableRow, groupColumns?: string[]) { + return (groupColumns || []) + .map((groupColumnId) => (row[groupColumnId] == null ? '' : String(row[groupColumnId]))) + .join('|'); +} + +/** + * Calculates the cumulative sum of a specified column in the data table. + * + * Also supports multiple series in a single data table - use the `by` argument + * to specify the columns to split the calculation by. + * For each unique combination of all `by` columns a separate cumulative sum will be calculated. + * The order of rows won't be changed - this function is not modifying any existing columns, it's only + * adding the specified `outputColumnId` column to every row of the table without adding or removing rows. + * + * Behavior: + * * Will write the cumulative sum of `inputColumnId` into `outputColumnId` + * * If provided will use `outputColumnName` as name for the newly created column. Otherwise falls back to `outputColumnId` + * * Cumulative sums always start with 0, a cell will contain its own value plus the values of + * all cells of the same series further up in the table. + * + * Edge cases: + * * Will return the input table if `inputColumnId` does not exist + * * Will throw an error if `outputColumnId` exists already in provided data table + * * If the row value contains `null` or `undefined`, it will be ignored and overwritten with the cumulative sum of + * all cells of the same series further up in the table. + * * For all values besides `null` and `undefined`, the value will be cast to a number before it's added to the + * cumulative sum of the current series - if this results in `NaN` (like in case of objects), all cells of the + * current series will be set to `NaN`. + * * To determine separate series defined by the `by` columns, the values of these columns will be cast to strings + * before comparison. If the values are objects, the return value of their `toString` method will be used for comparison. + * Missing values (`null` and `undefined`) will be treated as empty strings. + */ +export const cumulativeSum: ExpressionFunctionCumulativeSum = { + name: 'cumulative_sum', + type: 'datatable', + + inputTypes: ['datatable'], + + help: i18n.translate('expressions.functions.cumulativeSum.help', { + defaultMessage: 'Calculates the cumulative sum of a column in a data table', + }), + + args: { + by: { + help: i18n.translate('expressions.functions.cumulativeSum.args.byHelpText', { + defaultMessage: 'Column to split the cumulative sum calculation by', + }), + multi: true, + types: ['string'], + required: false, + }, + inputColumnId: { + help: i18n.translate('expressions.functions.cumulativeSum.args.inputColumnIdHelpText', { + defaultMessage: 'Column to calculate the cumulative sum of', + }), + types: ['string'], + required: true, + }, + outputColumnId: { + help: i18n.translate('expressions.functions.cumulativeSum.args.outputColumnIdHelpText', { + defaultMessage: 'Column to store the resulting cumulative sum in', + }), + types: ['string'], + required: true, + }, + outputColumnName: { + help: i18n.translate('expressions.functions.cumulativeSum.args.outputColumnNameHelpText', { + defaultMessage: 'Name of the column to store the resulting cumulative sum in', + }), + types: ['string'], + required: false, + }, + }, + + fn(input, { by, inputColumnId, outputColumnId, outputColumnName }) { + if (input.columns.some((column) => column.id === outputColumnId)) { + throw new Error( + i18n.translate('expressions.functions.cumulativeSum.columnConflictMessage', { + defaultMessage: + 'Specified outputColumnId {columnId} already exists. Please pick another column id.', + values: { + columnId: outputColumnId, + }, + }) + ); + } + + const inputColumnDefinition = input.columns.find((column) => column.id === inputColumnId); + + if (!inputColumnDefinition) { + return input; + } + + const outputColumnDefinition = { + ...inputColumnDefinition, + id: outputColumnId, + name: outputColumnName || outputColumnId, + }; + + const resultColumns = [...input.columns]; + // add output column after input column in the table + resultColumns.splice( + resultColumns.indexOf(inputColumnDefinition) + 1, + 0, + outputColumnDefinition + ); + + const accumulators: Partial> = {}; + return { + ...input, + columns: resultColumns, + rows: input.rows.map((row) => { + const newRow = { ...row }; + + const bucketIdentifier = getBucketIdentifier(row, by); + const accumulatorValue = accumulators[bucketIdentifier] ?? 0; + const currentValue = newRow[inputColumnId]; + if (currentValue != null) { + newRow[outputColumnId] = Number(currentValue) + accumulatorValue; + accumulators[bucketIdentifier] = newRow[outputColumnId]; + } else { + newRow[outputColumnId] = accumulatorValue; + } + + return newRow; + }), + }; + }, +}; diff --git a/src/plugins/expressions/common/expression_functions/specs/index.ts b/src/plugins/expressions/common/expression_functions/specs/index.ts index 5b9562dae5f2e..aadea5882b9c0 100644 --- a/src/plugins/expressions/common/expression_functions/specs/index.ts +++ b/src/plugins/expressions/common/expression_functions/specs/index.ts @@ -25,6 +25,7 @@ import { variableSet } from './var_set'; import { variable } from './var'; import { AnyExpressionFunctionDefinition } from '../types'; import { theme } from './theme'; +import { cumulativeSum } from './cumulative_sum'; export const functionSpecs: AnyExpressionFunctionDefinition[] = [ clog, @@ -34,6 +35,7 @@ export const functionSpecs: AnyExpressionFunctionDefinition[] = [ variableSet, variable, theme, + cumulativeSum, ]; export * from './clog'; @@ -43,3 +45,4 @@ export * from './kibana_context'; export * from './var_set'; export * from './var'; export * from './theme'; +export * from './cumulative_sum'; diff --git a/src/plugins/expressions/common/expression_functions/specs/tests/cumulative_sum.test.ts b/src/plugins/expressions/common/expression_functions/specs/tests/cumulative_sum.test.ts new file mode 100644 index 0000000000000..037b3ddc25f89 --- /dev/null +++ b/src/plugins/expressions/common/expression_functions/specs/tests/cumulative_sum.test.ts @@ -0,0 +1,361 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { functionWrapper } from './utils'; +import { cumulativeSum, CumulativeSumArgs } from '../cumulative_sum'; +import { ExecutionContext } from '../../../execution/types'; +import { Datatable } from '../../../expression_types/specs/datatable'; + +describe('interpreter/functions#cumulative_sum', () => { + const fn = functionWrapper(cumulativeSum); + const runFn = (input: Datatable, args: CumulativeSumArgs) => + fn(input, args, {} as ExecutionContext) as Datatable; + + it('calculates cumulative sum', () => { + const result = runFn( + { + type: 'datatable', + columns: [{ id: 'val', name: 'val', meta: { type: 'number' } }], + rows: [{ val: 5 }, { val: 7 }, { val: 3 }, { val: 2 }], + }, + { inputColumnId: 'val', outputColumnId: 'output' } + ); + expect(result.columns).toContainEqual({ + id: 'output', + name: 'output', + meta: { type: 'number' }, + }); + expect(result.rows.map((row) => row.output)).toEqual([5, 12, 15, 17]); + }); + + it('replaces null or undefined data with zeroes until there is real data', () => { + const result = runFn( + { + type: 'datatable', + columns: [{ id: 'val', name: 'val', meta: { type: 'number' } }], + rows: [{}, { val: null }, { val: undefined }, { val: 1 }], + }, + { inputColumnId: 'val', outputColumnId: 'output' } + ); + expect(result.columns).toContainEqual({ + id: 'output', + name: 'output', + meta: { type: 'number' }, + }); + expect(result.rows.map((row) => row.output)).toEqual([0, 0, 0, 1]); + }); + + it('calculates cumulative sum for multiple series', () => { + const result = runFn( + { + type: 'datatable', + columns: [ + { id: 'val', name: 'val', meta: { type: 'number' } }, + { id: 'split', name: 'split', meta: { type: 'string' } }, + ], + rows: [ + { val: 1, split: 'A' }, + { val: 2, split: 'B' }, + { val: 3, split: 'B' }, + { val: 4, split: 'A' }, + { val: 5, split: 'A' }, + { val: 6, split: 'A' }, + { val: 7, split: 'B' }, + { val: 8, split: 'B' }, + ], + }, + { inputColumnId: 'val', outputColumnId: 'output', by: ['split'] } + ); + + expect(result.rows.map((row) => row.output)).toEqual([ + 1, + 2, + 2 + 3, + 1 + 4, + 1 + 4 + 5, + 1 + 4 + 5 + 6, + 2 + 3 + 7, + 2 + 3 + 7 + 8, + ]); + }); + + it('treats missing split column as separate series', () => { + const result = runFn( + { + type: 'datatable', + columns: [ + { id: 'val', name: 'val', meta: { type: 'number' } }, + { id: 'split', name: 'split', meta: { type: 'string' } }, + ], + rows: [ + { val: 1, split: 'A' }, + { val: 2, split: 'B' }, + { val: 3 }, + { val: 4, split: 'A' }, + { val: 5 }, + { val: 6, split: 'A' }, + { val: 7, split: 'B' }, + { val: 8, split: 'B' }, + ], + }, + { inputColumnId: 'val', outputColumnId: 'output', by: ['split'] } + ); + expect(result.rows.map((row) => row.output)).toEqual([ + 1, + 2, + 3, + 1 + 4, + 3 + 5, + 1 + 4 + 6, + 2 + 7, + 2 + 7 + 8, + ]); + }); + + it('treats null like undefined and empty string for split columns', () => { + const result = runFn( + { + type: 'datatable', + columns: [ + { id: 'val', name: 'val', meta: { type: 'number' } }, + { id: 'split', name: 'split', meta: { type: 'string' } }, + ], + rows: [ + { val: 1, split: 'A' }, + { val: 2, split: 'B' }, + { val: 3 }, + { val: 4, split: 'A' }, + { val: 5 }, + { val: 6, split: 'A' }, + { val: 7, split: null }, + { val: 8, split: 'B' }, + { val: 9, split: '' }, + ], + }, + { inputColumnId: 'val', outputColumnId: 'output', by: ['split'] } + ); + expect(result.rows.map((row) => row.output)).toEqual([ + 1, + 2, + 3, + 1 + 4, + 3 + 5, + 1 + 4 + 6, + 3 + 5 + 7, + 2 + 8, + 3 + 5 + 7 + 9, + ]); + }); + + it('calculates cumulative sum for multiple series by multiple split columns', () => { + const result = runFn( + { + type: 'datatable', + columns: [ + { id: 'val', name: 'val', meta: { type: 'number' } }, + { id: 'split', name: 'split', meta: { type: 'string' } }, + { id: 'split2', name: 'split2', meta: { type: 'string' } }, + ], + rows: [ + { val: 1, split: 'A', split2: 'C' }, + { val: 2, split: 'B', split2: 'C' }, + { val: 3, split2: 'C' }, + { val: 4, split: 'A', split2: 'C' }, + { val: 5 }, + { val: 6, split: 'A', split2: 'D' }, + { val: 7, split: 'B', split2: 'D' }, + { val: 8, split: 'B', split2: 'D' }, + ], + }, + { inputColumnId: 'val', outputColumnId: 'output', by: ['split', 'split2'] } + ); + expect(result.rows.map((row) => row.output)).toEqual([1, 2, 3, 1 + 4, 5, 6, 7, 7 + 8]); + }); + + it('splits separate series by the string representation of the cell values', () => { + const result = runFn( + { + type: 'datatable', + columns: [ + { id: 'val', name: 'val', meta: { type: 'number' } }, + { id: 'split', name: 'split', meta: { type: 'string' } }, + ], + rows: [ + { val: 1, split: { anObj: 3 } }, + { val: 2, split: { anotherObj: 5 } }, + { val: 10, split: 5 }, + { val: 11, split: '5' }, + ], + }, + { inputColumnId: 'val', outputColumnId: 'output', by: ['split'] } + ); + + expect(result.rows.map((row) => row.output)).toEqual([1, 1 + 2, 10, 21]); + }); + + it('casts values to number before calculating cumulative sum', () => { + const result = runFn( + { + type: 'datatable', + columns: [{ id: 'val', name: 'val', meta: { type: 'number' } }], + rows: [{ val: 5 }, { val: '7' }, { val: '3' }, { val: 2 }], + }, + { inputColumnId: 'val', outputColumnId: 'output' } + ); + expect(result.rows.map((row) => row.output)).toEqual([5, 12, 15, 17]); + }); + + it('casts values to number before calculating cumulative sum for NaN like values', () => { + const result = runFn( + { + type: 'datatable', + columns: [{ id: 'val', name: 'val', meta: { type: 'number' } }], + rows: [{ val: 5 }, { val: '7' }, { val: {} }, { val: 2 }], + }, + { inputColumnId: 'val', outputColumnId: 'output' } + ); + expect(result.rows.map((row) => row.output)).toEqual([5, 12, NaN, NaN]); + }); + + it('skips undefined and null values', () => { + const result = runFn( + { + type: 'datatable', + columns: [{ id: 'val', name: 'val', meta: { type: 'number' } }], + rows: [ + { val: null }, + { val: 7 }, + { val: undefined }, + { val: undefined }, + { val: undefined }, + { val: undefined }, + { val: '3' }, + { val: 2 }, + { val: null }, + ], + }, + { inputColumnId: 'val', outputColumnId: 'output' } + ); + expect(result.rows.map((row) => row.output)).toEqual([0, 7, 7, 7, 7, 7, 10, 12, 12]); + }); + + it('copies over meta information from the source column', () => { + const result = runFn( + { + type: 'datatable', + columns: [ + { + id: 'val', + name: 'val', + meta: { + type: 'number', + + field: 'afield', + index: 'anindex', + params: { id: 'number', params: { pattern: '000' } }, + source: 'synthetic', + sourceParams: { + some: 'params', + }, + }, + }, + ], + rows: [{ val: 5 }], + }, + { inputColumnId: 'val', outputColumnId: 'output' } + ); + expect(result.columns).toContainEqual({ + id: 'output', + name: 'output', + meta: { + type: 'number', + + field: 'afield', + index: 'anindex', + params: { id: 'number', params: { pattern: '000' } }, + source: 'synthetic', + sourceParams: { + some: 'params', + }, + }, + }); + }); + + it('sets output name on output column if specified', () => { + const result = runFn( + { + type: 'datatable', + columns: [ + { + id: 'val', + name: 'val', + meta: { + type: 'number', + }, + }, + ], + rows: [{ val: 5 }], + }, + { inputColumnId: 'val', outputColumnId: 'output', outputColumnName: 'Output name' } + ); + expect(result.columns).toContainEqual({ + id: 'output', + name: 'Output name', + meta: { type: 'number' }, + }); + }); + + it('returns source table if input column does not exist', () => { + const input: Datatable = { + type: 'datatable', + columns: [ + { + id: 'val', + name: 'val', + meta: { + type: 'number', + }, + }, + ], + rows: [{ val: 5 }], + }; + expect(runFn(input, { inputColumnId: 'nonexisting', outputColumnId: 'output' })).toBe(input); + }); + + it('throws an error if output column exists already', () => { + expect(() => + runFn( + { + type: 'datatable', + columns: [ + { + id: 'val', + name: 'val', + meta: { + type: 'number', + }, + }, + ], + rows: [{ val: 5 }], + }, + { inputColumnId: 'val', outputColumnId: 'val' } + ) + ).toThrow(); + }); +}); diff --git a/src/plugins/expressions/common/expression_functions/types.ts b/src/plugins/expressions/common/expression_functions/types.ts index caaef541aefd5..fb1823e85b391 100644 --- a/src/plugins/expressions/common/expression_functions/types.ts +++ b/src/plugins/expressions/common/expression_functions/types.ts @@ -29,6 +29,7 @@ import { ExpressionFunctionVarSet, ExpressionFunctionVar, ExpressionFunctionTheme, + ExpressionFunctionCumulativeSum, } from './specs'; import { ExpressionAstFunction } from '../ast'; import { PersistableStateDefinition } from '../../../kibana_utils/common'; @@ -131,4 +132,5 @@ export interface ExpressionFunctionDefinitions { var_set: ExpressionFunctionVarSet; var: ExpressionFunctionVar; theme: ExpressionFunctionTheme; + cumulative_sum: ExpressionFunctionCumulativeSum; } diff --git a/src/plugins/expressions/public/public.api.md b/src/plugins/expressions/public/public.api.md index 95ee651d433ac..4739b9434bdaa 100644 --- a/src/plugins/expressions/public/public.api.md +++ b/src/plugins/expressions/public/public.api.md @@ -377,6 +377,10 @@ export interface ExpressionFunctionDefinitions { // // (undocumented) clog: ExpressionFunctionClog; + // Warning: (ae-forgotten-export) The symbol "ExpressionFunctionCumulativeSum" needs to be exported by the entry point index.d.ts + // + // (undocumented) + cumulative_sum: ExpressionFunctionCumulativeSum; // Warning: (ae-forgotten-export) The symbol "ExpressionFunctionFont" needs to be exported by the entry point index.d.ts // // (undocumented) diff --git a/src/plugins/expressions/server/server.api.md b/src/plugins/expressions/server/server.api.md index d5da60af8f8e5..fcdfd5ef3246c 100644 --- a/src/plugins/expressions/server/server.api.md +++ b/src/plugins/expressions/server/server.api.md @@ -349,6 +349,10 @@ export interface ExpressionFunctionDefinitions { // // (undocumented) clog: ExpressionFunctionClog; + // Warning: (ae-forgotten-export) The symbol "ExpressionFunctionCumulativeSum" needs to be exported by the entry point index.d.ts + // + // (undocumented) + cumulative_sum: ExpressionFunctionCumulativeSum; // Warning: (ae-forgotten-export) The symbol "ExpressionFunctionFont" needs to be exported by the entry point index.d.ts // // (undocumented)