From bd87d98fbdeabe93ac5a2a020a20cf68b36b938f Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 10 May 2021 10:29:44 +0200 Subject: [PATCH 1/3] refactoring --- .../config_panel/config_panel.tsx | 3 +- .../dimension_panel/dimension_editor.tsx | 2 +- .../formula/{ => editor}/formula.scss | 0 .../formula/editor/formula_editor.tsx | 547 +++++++++++ .../formula/editor/formula_help.tsx | 175 ++++ .../definitions/formula/editor/index.ts | 8 + .../{ => editor}/math_completion.test.ts | 0 .../formula/{ => editor}/math_completion.ts | 8 +- .../{ => editor}/math_tokenization.tsx | 0 .../definitions/formula/formula.tsx | 907 +----------------- .../definitions/formula/generate.ts | 90 ++ .../operations/definitions/formula/parse.ts | 150 +++ .../operations/definitions/formula/util.ts | 69 +- .../definitions/formula/validation.ts | 23 +- .../operations/layer_helpers.ts | 30 +- 15 files changed, 1031 insertions(+), 981 deletions(-) rename x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/{ => editor}/formula.scss (100%) create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/index.ts rename x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/{ => editor}/math_completion.test.ts (100%) rename x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/{ => editor}/math_completion.ts (98%) rename x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/{ => editor}/math_tokenization.tsx (100%) create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/generate.ts create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx index 0c26530b6172c..79c7882a8d56e 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx @@ -63,7 +63,8 @@ export function LayerPanels( () => (datasourceId: string, newState: unknown) => { dispatch({ type: 'UPDATE_DATASOURCE_STATE', - updater: () => newState, + updater: (prevState: unknown) => + typeof newState === 'function' ? newState(prevState) : newState, datasourceId, clearStagedPreview: false, }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index 28a5438b7af08..c85bc9188f083 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -144,7 +144,7 @@ export function DimensionEditor(props: DimensionEditorProps) { }; const setIsCloseable = (isCloseable: boolean) => { - setState({ ...state, isDimensionClosePrevented: !isCloseable }); + setState((prevState) => ({ ...prevState, isDimensionClosePrevented: !isCloseable })); }; const selectedOperationDefinition = diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.scss b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss similarity index 100% rename from x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.scss rename to x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx new file mode 100644 index 0000000000000..7b96aec4194a4 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -0,0 +1,547 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useCallback, useEffect, useState } from 'react'; +import { i18n } from '@kbn/i18n'; +import { EuiButtonEmpty, EuiFlexGroup, EuiFlexItem, EuiSpacer, EuiPopover } from '@elastic/eui'; +import { monaco } from '@kbn/monaco'; +import classNames from 'classnames'; +import { CodeEditor } from '../../../../../../../../../src/plugins/kibana_react/public'; +import type { CodeEditorProps } from '../../../../../../../../../src/plugins/kibana_react/public'; +import { ParamEditorProps } from '../../index'; +import { getManagedColumnsFrom } from '../../../layer_helpers'; +import { ErrorWrapper, runASTValidation, tryToParse } from '../validation'; +import { useDebounceWithOptions } from '../../helpers'; +import { + LensMathSuggestion, + SUGGESTION_TYPE, + suggest, + getSuggestion, + getSignatureHelp, + getHover, + getTokenInfo, + offsetToRowColumn, + monacoPositionToOffset, +} from './math_completion'; +import { LANGUAGE_ID } from './math_tokenization'; +import { MemoizedFormulaHelp } from './formula_help'; + +import './formula.scss'; +import { FormulaIndexPatternColumn, regenerateLayerFromAst } from '../formula'; + +export function FormulaEditor({ + layer, + updateLayer, + currentColumn, + columnId, + indexPattern, + operationDefinitionMap, + data, + toggleFullscreen, + isFullscreen, + setIsCloseable, +}: ParamEditorProps) { + const [text, setText] = useState(currentColumn.params.formula); + const [isHelpOpen, setIsHelpOpen] = useState(false); + const editorModel = React.useRef( + monaco.editor.createModel(text ?? '', LANGUAGE_ID) + ); + const overflowDiv1 = React.useRef(); + const disposables = React.useRef([]); + const editor1 = React.useRef(); + + // The Monaco editor needs to have the overflowDiv in the first render. Using an effect + // requires a second render to work, so we are using an if statement to guarantee it happens + // on first render + if (!overflowDiv1?.current) { + const node1 = (overflowDiv1.current = document.createElement('div')); + node1.setAttribute('data-test-subj', 'lnsFormulaWidget'); + // Monaco CSS is targeted on the monaco-editor class + node1.classList.add('lnsFormulaOverflow', 'monaco-editor'); + document.body.appendChild(node1); + } + + // Clean up the monaco editor and DOM on unmount + useEffect(() => { + const model = editorModel.current; + const allDisposables = disposables.current; + const editor1ref = editor1.current; + return () => { + model.dispose(); + overflowDiv1.current?.parentNode?.removeChild(overflowDiv1.current); + editor1ref?.dispose(); + allDisposables?.forEach((d) => d.dispose()); + }; + }, []); + + useDebounceWithOptions( + () => { + if (!editorModel.current) return; + + if (!text) { + monaco.editor.setModelMarkers(editorModel.current, 'LENS', []); + if (currentColumn.params.formula) { + // Only submit if valid + const { newLayer } = regenerateLayerFromAst( + text || '', + layer, + columnId, + currentColumn, + indexPattern, + operationDefinitionMap + ); + updateLayer(newLayer); + } + + return; + } + + let errors: ErrorWrapper[] = []; + + const { root, error } = tryToParse(text); + if (error) { + errors = [error]; + } else if (root) { + const validationErrors = runASTValidation( + root, + layer, + indexPattern, + operationDefinitionMap + ); + if (validationErrors.length) { + errors = validationErrors; + } + } + + if (errors.length) { + monaco.editor.setModelMarkers( + editorModel.current, + 'LENS', + errors.flatMap((innerError) => { + if (innerError.locations.length) { + return innerError.locations.map((location) => { + const startPosition = offsetToRowColumn(text, location.min); + const endPosition = offsetToRowColumn(text, location.max); + return { + message: innerError.message, + startColumn: startPosition.column + 1, + startLineNumber: startPosition.lineNumber, + endColumn: endPosition.column + 1, + endLineNumber: endPosition.lineNumber, + severity: + innerError.severity === 'warning' + ? monaco.MarkerSeverity.Warning + : monaco.MarkerSeverity.Error, + }; + }); + } else { + // Parse errors return no location info + const startPosition = offsetToRowColumn(text, 0); + const endPosition = offsetToRowColumn(text, text.length - 1); + return [ + { + message: innerError.message, + startColumn: startPosition.column + 1, + startLineNumber: startPosition.lineNumber, + endColumn: endPosition.column + 1, + endLineNumber: endPosition.lineNumber, + severity: + innerError.severity === 'warning' + ? monaco.MarkerSeverity.Warning + : monaco.MarkerSeverity.Error, + }, + ]; + } + }) + ); + } else { + monaco.editor.setModelMarkers(editorModel.current, 'LENS', []); + + // Only submit if valid + const { newLayer, locations } = regenerateLayerFromAst( + text || '', + layer, + columnId, + currentColumn, + indexPattern, + operationDefinitionMap + ); + updateLayer(newLayer); + + const managedColumns = getManagedColumnsFrom(columnId, newLayer.columns); + const markers: monaco.editor.IMarkerData[] = managedColumns + .flatMap(([id, column]) => { + if (locations[id]) { + const def = operationDefinitionMap[column.operationType]; + if (def.getErrorMessage) { + const messages = def.getErrorMessage( + newLayer, + id, + indexPattern, + operationDefinitionMap + ); + if (messages) { + const startPosition = offsetToRowColumn(text, locations[id].min); + const endPosition = offsetToRowColumn(text, locations[id].max); + return [ + { + message: messages.join(', '), + startColumn: startPosition.column + 1, + startLineNumber: startPosition.lineNumber, + endColumn: endPosition.column + 1, + endLineNumber: endPosition.lineNumber, + severity: monaco.MarkerSeverity.Warning, + }, + ]; + } + } + } + return []; + }) + .filter((marker) => marker); + monaco.editor.setModelMarkers(editorModel.current, 'LENS', markers); + } + }, + // Make it validate on flyout open in case of a broken formula left over + // from a previous edit + { skipFirstRender: text == null }, + 256, + [text] + ); + + /** + * The way that Monaco requests autocompletion is not intuitive, but the way we use it + * we fetch new suggestions in these scenarios: + * + * - If the user types one of the trigger characters, suggestions are always fetched + * - When the user selects the kql= suggestion, we tell Monaco to trigger new suggestions after + * - When the user types the first character into an empty text box, Monaco requests suggestions + * + * Monaco also triggers suggestions automatically when there are no suggestions being displayed + * and the user types a non-whitespace character. + * + * While suggestions are being displayed, Monaco uses an in-memory cache of the last known suggestions. + */ + const provideCompletionItems = useCallback( + async ( + model: monaco.editor.ITextModel, + position: monaco.Position, + context: monaco.languages.CompletionContext + ) => { + const innerText = model.getValue(); + const textRange = model.getFullModelRange(); + let wordRange: monaco.Range; + let aSuggestions: { list: LensMathSuggestion[]; type: SUGGESTION_TYPE } = { + list: [], + type: SUGGESTION_TYPE.FIELD, + }; + + const lengthAfterPosition = model.getValueLengthInRange({ + startLineNumber: position.lineNumber, + startColumn: position.column, + endLineNumber: textRange.endLineNumber, + endColumn: textRange.endColumn, + }); + + if (context.triggerCharacter === '(') { + const wordUntil = model.getWordAtPosition(position.delta(0, -3)); + if (wordUntil) { + wordRange = new monaco.Range( + position.lineNumber, + position.column, + position.lineNumber, + position.column + ); + + // Retrieve suggestions for subexpressions + // TODO: make this work for expressions nested more than one level deep + aSuggestions = await suggest({ + expression: innerText.substring(0, innerText.length - lengthAfterPosition) + ')', + position: innerText.length - lengthAfterPosition, + context, + indexPattern, + operationDefinitionMap, + data, + }); + } + } else { + aSuggestions = await suggest({ + expression: innerText, + position: innerText.length - lengthAfterPosition, + context, + indexPattern, + operationDefinitionMap, + data, + }); + } + + return { + suggestions: aSuggestions.list.map((s) => + getSuggestion(s, aSuggestions.type, wordRange, operationDefinitionMap) + ), + }; + }, + [indexPattern, operationDefinitionMap, data] + ); + + const provideSignatureHelp = useCallback( + async ( + model: monaco.editor.ITextModel, + position: monaco.Position, + token: monaco.CancellationToken, + context: monaco.languages.SignatureHelpContext + ) => { + const innerText = model.getValue(); + const textRange = model.getFullModelRange(); + + const lengthAfterPosition = model.getValueLengthInRange({ + startLineNumber: position.lineNumber, + startColumn: position.column, + endLineNumber: textRange.endLineNumber, + endColumn: textRange.endColumn, + }); + return getSignatureHelp( + model.getValue(), + innerText.length - lengthAfterPosition, + operationDefinitionMap + ); + }, + [operationDefinitionMap] + ); + + const provideHover = useCallback( + async ( + model: monaco.editor.ITextModel, + position: monaco.Position, + token: monaco.CancellationToken + ) => { + const innerText = model.getValue(); + const textRange = model.getFullModelRange(); + + const lengthAfterPosition = model.getValueLengthInRange({ + startLineNumber: position.lineNumber, + startColumn: position.column, + endLineNumber: textRange.endLineNumber, + endColumn: textRange.endColumn, + }); + return getHover( + model.getValue(), + innerText.length - lengthAfterPosition, + operationDefinitionMap + ); + }, + [operationDefinitionMap] + ); + + const onTypeHandler = useCallback( + (e: monaco.editor.IModelContentChangedEvent, editor: monaco.editor.IStandaloneCodeEditor) => { + if (e.isFlush || e.isRedoing || e.isUndoing) { + return; + } + if (e.changes.length === 1 && e.changes[0].text === '=') { + const currentPosition = e.changes[0].range; + if (currentPosition) { + const tokenInfo = getTokenInfo( + editor.getValue(), + monacoPositionToOffset( + editor.getValue(), + new monaco.Position(currentPosition.startLineNumber, currentPosition.startColumn) + ) + ); + // Make sure that we are only adding kql='' or lucene='', and also + // check that the = sign isn't inside the KQL expression like kql='=' + if ( + !tokenInfo || + typeof tokenInfo.ast === 'number' || + tokenInfo.ast.type !== 'namedArgument' || + (tokenInfo.ast.name !== 'kql' && tokenInfo.ast.name !== 'lucene') || + tokenInfo.ast.value !== 'LENS_MATH_MARKER' + ) { + return; + } + + // Timeout is required because otherwise the cursor position is not updated. + setTimeout(() => { + editor.executeEdits( + 'LENS', + [ + { + range: { + ...currentPosition, + // Insert after the current char + startColumn: currentPosition.startColumn + 1, + endColumn: currentPosition.startColumn + 1, + }, + text: `''`, + }, + ], + [ + // After inserting, move the cursor in between the single quotes + new monaco.Selection( + currentPosition.startLineNumber, + currentPosition.startColumn + 2, + currentPosition.startLineNumber, + currentPosition.startColumn + 2 + ), + ] + ); + editor.trigger('lens', 'editor.action.triggerSuggest', {}); + }, 0); + } + } + }, + [] + ); + + const codeEditorOptions: CodeEditorProps = { + languageId: LANGUAGE_ID, + value: text ?? '', + onChange: setText, + options: { + automaticLayout: false, + fontSize: 14, + folding: false, + lineNumbers: 'off', + scrollBeyondLastLine: false, + minimap: { enabled: false }, + wordWrap: 'on', + // Disable suggestions that appear when we don't provide a default suggestion + wordBasedSuggestions: false, + autoIndent: 'brackets', + wrappingIndent: 'none', + dimension: { width: 290, height: 200 }, + fixedOverflowWidgets: true, + }, + }; + + useEffect(() => { + // Because the monaco model is owned by Lens, we need to manually attach and remove handlers + const { dispose: dispose1 } = monaco.languages.registerCompletionItemProvider(LANGUAGE_ID, { + triggerCharacters: ['.', '(', '=', ' ', ':', `'`], + provideCompletionItems, + }); + const { dispose: dispose2 } = monaco.languages.registerSignatureHelpProvider(LANGUAGE_ID, { + signatureHelpTriggerCharacters: ['(', '='], + provideSignatureHelp, + }); + const { dispose: dispose3 } = monaco.languages.registerHoverProvider(LANGUAGE_ID, { + provideHover, + }); + return () => { + dispose1(); + dispose2(); + dispose3(); + }; + }, [provideCompletionItems, provideSignatureHelp, provideHover]); + + // The Monaco editor will lazily load Monaco, which takes a render cycle to trigger. This can cause differences + // in the behavior of Monaco when it's first loaded and then reloaded. + return ( +
+
+ + + + + { + toggleFullscreen(); + }} + iconType="fullScreen" + size="s" + color="text" + flush="right" + > + {isFullscreen + ? i18n.translate('xpack.lens.formula.fullScreenCloseLabel', { + defaultMessage: 'Collapse formula', + }) + : i18n.translate('xpack.lens.formula.fullScreenEditorLabel', { + defaultMessage: 'Expand formula', + })} + + + +
+
+ { + editor1.current = editor; + disposables.current.push( + editor.onDidFocusEditorWidget(() => { + setIsCloseable(false); + }) + ); + disposables.current.push( + editor.onDidBlurEditorWidget(() => { + setIsCloseable(true); + }) + ); + // If we ever introduce a second Monaco editor, we need to toggle + // the typing handler to the active editor to maintain the cursor + disposables.current.push( + editor.onDidChangeModelContent((e) => { + onTypeHandler(e, editor); + }) + ); + }} + /> + +
+
+ + + {isFullscreen ? ( + + ) : ( + setIsHelpOpen(false)} + button={ + setIsHelpOpen(!isHelpOpen)} + iconType="help" + size="s" + color="text" + > + {i18n.translate('xpack.lens.formula.functionReferenceEditorLabel', { + defaultMessage: 'Function reference', + })} + + } + anchorPosition="leftDown" + > + + + )} + + + {/* Errors go here */} + +
+
+ ); +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx new file mode 100644 index 0000000000000..1335cfe7e3efa --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx @@ -0,0 +1,175 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useState } from 'react'; +import { i18n } from '@kbn/i18n'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiText, + EuiSelectable, + EuiSelectableOption, +} from '@elastic/eui'; +import { Markdown } from '../../../../../../../../../src/plugins/kibana_react/public'; +import { GenericOperationDefinition, ParamEditorProps } from '../../index'; +import { IndexPattern } from '../../../../types'; +import { tinymathFunctions } from '../util'; +import { getPossibleFunctions } from './math_completion'; + +import { FormulaIndexPatternColumn } from '../formula'; + +function FormulaHelp({ + indexPattern, + operationDefinitionMap, +}: { + indexPattern: IndexPattern; + operationDefinitionMap: Record; +}) { + const [selectedFunction, setSelectedFunction] = useState(); + + const helpItems: Array = []; + + helpItems.push({ label: 'Math', isGroupLabel: true }); + + helpItems.push( + ...getPossibleFunctions(indexPattern) + .filter((key) => key in tinymathFunctions) + .map((key) => ({ + label: `${key}`, + description: , + checked: selectedFunction === key ? ('on' as const) : undefined, + })) + ); + + helpItems.push({ label: 'Elasticsearch', isGroupLabel: true }); + + // Es aggs + helpItems.push( + ...getPossibleFunctions(indexPattern) + .filter((key) => key in operationDefinitionMap) + .map((key) => ({ + label: `${key}: ${operationDefinitionMap[key].displayName}`, + description: getHelpText(key, operationDefinitionMap), + checked: + selectedFunction === `${key}: ${operationDefinitionMap[key].displayName}` + ? ('on' as const) + : undefined, + })) + ); + + return ( + + + { + const chosenType = newOptions.find(({ checked }) => checked === 'on')!; + if (!chosenType) { + setSelectedFunction(undefined); + } else { + setSelectedFunction(chosenType.label); + } + }} + > + {(list, search) => ( + <> + {search} + {list} + + )} + + + + + {selectedFunction ? ( + helpItems.find(({ label }) => label === selectedFunction)?.description + ) : ( + + )} + + + + ); +} + +export const MemoizedFormulaHelp = React.memo(FormulaHelp); + +// TODO: i18n this whole thing, or move examples into the operation definitions with i18n +function getHelpText( + type: string, + operationDefinitionMap: ParamEditorProps['operationDefinitionMap'] +) { + const definition = operationDefinitionMap[type]; + + if (type === 'count') { + return ( + +

Example: count()

+ + ); + } + + return ( + + {definition.input === 'field' ?

Example: {type}(bytes)

: null} + {definition.input === 'fullReference' && !('operationParams' in definition) ? ( +

Example: {type}(sum(bytes))

+ ) : null} + + {'operationParams' in definition && definition.operationParams ? ( +

+

+ Example: {type}(sum(bytes),{' '} + {definition.operationParams.map((p) => `${p.name}=5`).join(', ')}) +

+

+ ) : null} + + ); +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/index.ts new file mode 100644 index 0000000000000..4b6acefa6b30a --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './formula_editor'; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts similarity index 100% rename from x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.test.ts rename to x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts similarity index 98% rename from x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts rename to x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts index 1ae5da9d6db1d..e8c16fe64651a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts @@ -15,10 +15,10 @@ import { TinymathNamedArgument, } from '@kbn/tinymath'; import { DataPublicPluginStart, QuerySuggestion } from 'src/plugins/data/public'; -import { IndexPattern } from '../../../types'; -import { memoizedGetAvailableOperationsByMetadata } from '../../operations'; -import { tinymathFunctions, groupArgsByType } from './util'; -import type { GenericOperationDefinition } from '..'; +import { IndexPattern } from '../../../../types'; +import { memoizedGetAvailableOperationsByMetadata } from '../../../operations'; +import { tinymathFunctions, groupArgsByType } from '../util'; +import type { GenericOperationDefinition } from '../..'; export enum SUGGESTION_TYPE { FIELD = 'field', diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_tokenization.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_tokenization.tsx similarity index 100% rename from x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_tokenization.tsx rename to x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_tokenization.tsx diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 584ea5da38957..6f0abe8f55568 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -5,61 +5,16 @@ * 2.0. */ -import React, { useCallback, useEffect, useState } from 'react'; -import { isObject } from 'lodash'; import { i18n } from '@kbn/i18n'; -import type { TinymathAST, TinymathVariable, TinymathLocation } from '@kbn/tinymath'; -import { - EuiButtonEmpty, - EuiFlexGroup, - EuiFlexItem, - EuiText, - EuiSpacer, - EuiPopover, - EuiSelectable, - EuiSelectableOption, -} from '@elastic/eui'; -import { monaco } from '@kbn/monaco'; -import classNames from 'classnames'; -import { CodeEditor, Markdown } from '../../../../../../../../src/plugins/kibana_react/public'; -import type { CodeEditorProps } from '../../../../../../../../src/plugins/kibana_react/public'; -import { - OperationDefinition, - GenericOperationDefinition, - IndexPatternColumn, - ParamEditorProps, -} from '../index'; +import type { TinymathLocation } from '@kbn/tinymath'; +import { OperationDefinition, GenericOperationDefinition } from '../index'; import { ReferenceBasedIndexPatternColumn } from '../column_types'; import { IndexPattern, IndexPatternLayer } from '../../../types'; -import { getColumnOrder, getManagedColumnsFrom } from '../../layer_helpers'; -import { mathOperation } from './math'; -import { documentField } from '../../../document_field'; -import { ErrorWrapper, runASTValidation, shouldHaveFieldArgument, tryToParse } from './validation'; -import { - extractParamsForFormula, - findVariables, - getOperationParams, - getSafeFieldName, - groupArgsByType, - hasMathNode, - tinymathFunctions, -} from './util'; -import { useDebounceWithOptions } from '../helpers'; -import { - LensMathSuggestion, - SUGGESTION_TYPE, - suggest, - getSuggestion, - getPossibleFunctions, - getSignatureHelp, - getHover, - getTokenInfo, - offsetToRowColumn, - monacoPositionToOffset, -} from './math_completion'; -import { LANGUAGE_ID } from './math_tokenization'; - -import './formula.scss'; +import { getColumnOrder } from '../../layer_helpers'; +import { runASTValidation, tryToParse } from './validation'; +import { FormulaEditor } from './editor'; +import { parseAndExtract } from './parse'; +import { generateFormula } from './generate'; const defaultLabel = i18n.translate('xpack.lens.indexPattern.formulaLabel', { defaultMessage: 'Formula', @@ -153,41 +108,12 @@ export const formulaOperation: OperationDefinition< buildColumn({ previousColumn, layer, indexPattern }, _, operationDefinitionMap) { let previousFormula = ''; if (previousColumn) { - if ('references' in previousColumn) { - const metric = layer.columns[previousColumn.references[0]]; - if (metric && 'sourceField' in metric && metric.dataType === 'number') { - const fieldName = getSafeFieldName(metric.sourceField); - // TODO need to check the input type from the definition - previousFormula += `${previousColumn.operationType}(${metric.operationType}(${fieldName})`; - } - } else { - if ( - previousColumn && - 'sourceField' in previousColumn && - previousColumn.dataType === 'number' - ) { - previousFormula += `${previousColumn.operationType}(${getSafeFieldName( - previousColumn?.sourceField - )}`; - } - } - const formulaNamedArgs = extractParamsForFormula(previousColumn, operationDefinitionMap); - if (formulaNamedArgs.length) { - previousFormula += - ', ' + formulaNamedArgs.map(({ name, value }) => `${name}=${value}`).join(', '); - } - if (previousColumn.filter) { - if (previousColumn.operationType !== 'count') { - previousFormula += ', '; - } - previousFormula += - (previousColumn.filter.language === 'kuery' ? 'kql=' : 'lucene=') + - `'${previousColumn.filter.query.replace(/'/g, `\\'`)}'`; // replace all - } - if (previousFormula) { - // close the formula at the end - previousFormula += ')'; - } + previousFormula = generateFormula( + previousColumn, + layer, + previousFormula, + operationDefinitionMap + ); } // carry over the format settings from previous operation for seamless transfer // NOTE: this works only for non-default formatters set in Lens @@ -207,11 +133,8 @@ export const formulaOperation: OperationDefinition< references: [], }; }, - isTransferable: (column, newIndexPattern, operationDefinitionMap) => { - // Basic idea: if it has any math operation in it, probably it cannot be transferable - const { root, error } = tryToParse(column.params.formula || ''); - if (!root) return true; - return Boolean(!error && !hasMathNode(root)); + isTransferable: () => { + return true; }, createCopy(layer, sourceId, targetId, indexPattern, operationDefinitionMap) { const currentColumn = layer.columns[sourceId] as FormulaIndexPatternColumn; @@ -236,660 +159,6 @@ export const formulaOperation: OperationDefinition< paramEditor: FormulaEditor, }; -function FormulaEditor({ - layer, - updateLayer, - currentColumn, - columnId, - indexPattern, - operationDefinitionMap, - data, - toggleFullscreen, - isFullscreen, - setIsCloseable, -}: ParamEditorProps) { - const [text, setText] = useState(currentColumn.params.formula); - const [isHelpOpen, setIsHelpOpen] = useState(false); - const editorModel = React.useRef( - monaco.editor.createModel(text ?? '', LANGUAGE_ID) - ); - const overflowDiv1 = React.useRef(); - const disposables = React.useRef([]); - const editor1 = React.useRef(); - - // The Monaco editor needs to have the overflowDiv in the first render. Using an effect - // requires a second render to work, so we are using an if statement to guarantee it happens - // on first render - if (!overflowDiv1?.current) { - const node1 = (overflowDiv1.current = document.createElement('div')); - node1.setAttribute('data-test-subj', 'lnsFormulaWidget'); - // Monaco CSS is targeted on the monaco-editor class - node1.classList.add('lnsFormulaOverflow', 'monaco-editor'); - document.body.appendChild(node1); - } - - // Clean up the monaco editor and DOM on unmount - useEffect(() => { - const model = editorModel.current; - const allDisposables = disposables.current; - const editor1ref = editor1.current; - return () => { - model.dispose(); - overflowDiv1.current?.parentNode?.removeChild(overflowDiv1.current); - editor1ref?.dispose(); - allDisposables?.forEach((d) => d.dispose()); - }; - }, []); - - useDebounceWithOptions( - () => { - if (!editorModel.current) return; - - if (!text) { - monaco.editor.setModelMarkers(editorModel.current, 'LENS', []); - if (currentColumn.params.formula) { - // Only submit if valid - const { newLayer } = regenerateLayerFromAst( - text || '', - layer, - columnId, - currentColumn, - indexPattern, - operationDefinitionMap - ); - updateLayer(newLayer); - } - - return; - } - - let errors: ErrorWrapper[] = []; - - const { root, error } = tryToParse(text); - if (error) { - errors = [error]; - } else if (root) { - const validationErrors = runASTValidation( - root, - layer, - indexPattern, - operationDefinitionMap - ); - if (validationErrors.length) { - errors = validationErrors; - } - } - - if (errors.length) { - monaco.editor.setModelMarkers( - editorModel.current, - 'LENS', - errors.flatMap((innerError) => { - if (innerError.locations.length) { - return innerError.locations.map((location) => { - const startPosition = offsetToRowColumn(text, location.min); - const endPosition = offsetToRowColumn(text, location.max); - return { - message: innerError.message, - startColumn: startPosition.column + 1, - startLineNumber: startPosition.lineNumber, - endColumn: endPosition.column + 1, - endLineNumber: endPosition.lineNumber, - severity: - innerError.severity === 'warning' - ? monaco.MarkerSeverity.Warning - : monaco.MarkerSeverity.Error, - }; - }); - } else { - // Parse errors return no location info - const startPosition = offsetToRowColumn(text, 0); - const endPosition = offsetToRowColumn(text, text.length - 1); - return [ - { - message: innerError.message, - startColumn: startPosition.column + 1, - startLineNumber: startPosition.lineNumber, - endColumn: endPosition.column + 1, - endLineNumber: endPosition.lineNumber, - severity: - innerError.severity === 'warning' - ? monaco.MarkerSeverity.Warning - : monaco.MarkerSeverity.Error, - }, - ]; - } - }) - ); - } else { - monaco.editor.setModelMarkers(editorModel.current, 'LENS', []); - - // Only submit if valid - const { newLayer, locations } = regenerateLayerFromAst( - text || '', - layer, - columnId, - currentColumn, - indexPattern, - operationDefinitionMap - ); - updateLayer(newLayer); - - const managedColumns = getManagedColumnsFrom(columnId, newLayer.columns); - const markers: monaco.editor.IMarkerData[] = managedColumns - .flatMap(([id, column]) => { - if (locations[id]) { - const def = operationDefinitionMap[column.operationType]; - if (def.getErrorMessage) { - const messages = def.getErrorMessage( - newLayer, - id, - indexPattern, - operationDefinitionMap - ); - if (messages) { - const startPosition = offsetToRowColumn(text, locations[id].min); - const endPosition = offsetToRowColumn(text, locations[id].max); - return [ - { - message: messages.join(', '), - startColumn: startPosition.column + 1, - startLineNumber: startPosition.lineNumber, - endColumn: endPosition.column + 1, - endLineNumber: endPosition.lineNumber, - severity: monaco.MarkerSeverity.Warning, - }, - ]; - } - } - } - return []; - }) - .filter((marker) => marker); - monaco.editor.setModelMarkers(editorModel.current, 'LENS', markers); - } - }, - // Make it validate on flyout open in case of a broken formula left over - // from a previous edit - { skipFirstRender: text == null }, - 256, - [text] - ); - - /** - * The way that Monaco requests autocompletion is not intuitive, but the way we use it - * we fetch new suggestions in these scenarios: - * - * - If the user types one of the trigger characters, suggestions are always fetched - * - When the user selects the kql= suggestion, we tell Monaco to trigger new suggestions after - * - When the user types the first character into an empty text box, Monaco requests suggestions - * - * Monaco also triggers suggestions automatically when there are no suggestions being displayed - * and the user types a non-whitespace character. - * - * While suggestions are being displayed, Monaco uses an in-memory cache of the last known suggestions. - */ - const provideCompletionItems = useCallback( - async ( - model: monaco.editor.ITextModel, - position: monaco.Position, - context: monaco.languages.CompletionContext - ) => { - const innerText = model.getValue(); - const textRange = model.getFullModelRange(); - let wordRange: monaco.Range; - let aSuggestions: { list: LensMathSuggestion[]; type: SUGGESTION_TYPE } = { - list: [], - type: SUGGESTION_TYPE.FIELD, - }; - - const lengthAfterPosition = model.getValueLengthInRange({ - startLineNumber: position.lineNumber, - startColumn: position.column, - endLineNumber: textRange.endLineNumber, - endColumn: textRange.endColumn, - }); - - if (context.triggerCharacter === '(') { - const wordUntil = model.getWordAtPosition(position.delta(0, -3)); - if (wordUntil) { - wordRange = new monaco.Range( - position.lineNumber, - position.column, - position.lineNumber, - position.column - ); - - // Retrieve suggestions for subexpressions - // TODO: make this work for expressions nested more than one level deep - aSuggestions = await suggest({ - expression: innerText.substring(0, innerText.length - lengthAfterPosition) + ')', - position: innerText.length - lengthAfterPosition, - context, - indexPattern, - operationDefinitionMap, - data, - }); - } - } else { - aSuggestions = await suggest({ - expression: innerText, - position: innerText.length - lengthAfterPosition, - context, - indexPattern, - operationDefinitionMap, - data, - }); - } - - return { - suggestions: aSuggestions.list.map((s) => - getSuggestion(s, aSuggestions.type, wordRange, operationDefinitionMap) - ), - }; - }, - [indexPattern, operationDefinitionMap, data] - ); - - const provideSignatureHelp = useCallback( - async ( - model: monaco.editor.ITextModel, - position: monaco.Position, - token: monaco.CancellationToken, - context: monaco.languages.SignatureHelpContext - ) => { - const innerText = model.getValue(); - const textRange = model.getFullModelRange(); - - const lengthAfterPosition = model.getValueLengthInRange({ - startLineNumber: position.lineNumber, - startColumn: position.column, - endLineNumber: textRange.endLineNumber, - endColumn: textRange.endColumn, - }); - return getSignatureHelp( - model.getValue(), - innerText.length - lengthAfterPosition, - operationDefinitionMap - ); - }, - [operationDefinitionMap] - ); - - const provideHover = useCallback( - async ( - model: monaco.editor.ITextModel, - position: monaco.Position, - token: monaco.CancellationToken - ) => { - const innerText = model.getValue(); - const textRange = model.getFullModelRange(); - - const lengthAfterPosition = model.getValueLengthInRange({ - startLineNumber: position.lineNumber, - startColumn: position.column, - endLineNumber: textRange.endLineNumber, - endColumn: textRange.endColumn, - }); - return getHover( - model.getValue(), - innerText.length - lengthAfterPosition, - operationDefinitionMap - ); - }, - [operationDefinitionMap] - ); - - const onTypeHandler = useCallback( - (e: monaco.editor.IModelContentChangedEvent, editor: monaco.editor.IStandaloneCodeEditor) => { - if (e.isFlush || e.isRedoing || e.isUndoing) { - return; - } - if (e.changes.length === 1 && e.changes[0].text === '=') { - const currentPosition = e.changes[0].range; - if (currentPosition) { - const tokenInfo = getTokenInfo( - editor.getValue(), - monacoPositionToOffset( - editor.getValue(), - new monaco.Position(currentPosition.startLineNumber, currentPosition.startColumn) - ) - ); - // Make sure that we are only adding kql='' or lucene='', and also - // check that the = sign isn't inside the KQL expression like kql='=' - if ( - !tokenInfo || - typeof tokenInfo.ast === 'number' || - tokenInfo.ast.type !== 'namedArgument' || - (tokenInfo.ast.name !== 'kql' && tokenInfo.ast.name !== 'lucene') || - tokenInfo.ast.value !== 'LENS_MATH_MARKER' - ) { - return; - } - - // Timeout is required because otherwise the cursor position is not updated. - setTimeout(() => { - editor.executeEdits( - 'LENS', - [ - { - range: { - ...currentPosition, - // Insert after the current char - startColumn: currentPosition.startColumn + 1, - endColumn: currentPosition.startColumn + 1, - }, - text: `''`, - }, - ], - [ - // After inserting, move the cursor in between the single quotes - new monaco.Selection( - currentPosition.startLineNumber, - currentPosition.startColumn + 2, - currentPosition.startLineNumber, - currentPosition.startColumn + 2 - ), - ] - ); - editor.trigger('lens', 'editor.action.triggerSuggest', {}); - }, 0); - } - } - }, - [] - ); - - const codeEditorOptions: CodeEditorProps = { - languageId: LANGUAGE_ID, - value: text ?? '', - onChange: setText, - options: { - automaticLayout: false, - fontSize: 14, - folding: false, - lineNumbers: 'off', - scrollBeyondLastLine: false, - minimap: { enabled: false }, - wordWrap: 'on', - // Disable suggestions that appear when we don't provide a default suggestion - wordBasedSuggestions: false, - autoIndent: 'brackets', - wrappingIndent: 'none', - dimension: { width: 290, height: 200 }, - fixedOverflowWidgets: true, - }, - }; - - useEffect(() => { - // Because the monaco model is owned by Lens, we need to manually attach and remove handlers - const { dispose: dispose1 } = monaco.languages.registerCompletionItemProvider(LANGUAGE_ID, { - triggerCharacters: ['.', '(', '=', ' ', ':', `'`], - provideCompletionItems, - }); - const { dispose: dispose2 } = monaco.languages.registerSignatureHelpProvider(LANGUAGE_ID, { - signatureHelpTriggerCharacters: ['(', '='], - provideSignatureHelp, - }); - const { dispose: dispose3 } = monaco.languages.registerHoverProvider(LANGUAGE_ID, { - provideHover, - }); - return () => { - dispose1(); - dispose2(); - dispose3(); - }; - }, [provideCompletionItems, provideSignatureHelp, provideHover]); - - // The Monaco editor will lazily load Monaco, which takes a render cycle to trigger. This can cause differences - // in the behavior of Monaco when it's first loaded and then reloaded. - return ( -
-
- - - - - { - toggleFullscreen(); - }} - iconType="fullScreen" - size="s" - color="text" - flush="right" - > - {isFullscreen - ? i18n.translate('xpack.lens.formula.fullScreenCloseLabel', { - defaultMessage: 'Collapse formula', - }) - : i18n.translate('xpack.lens.formula.fullScreenEditorLabel', { - defaultMessage: 'Expand formula', - })} - - - -
-
- { - editor1.current = editor; - disposables.current.push( - editor.onDidFocusEditorWidget(() => { - setIsCloseable(false); - }) - ); - disposables.current.push( - editor.onDidBlurEditorWidget(() => { - setIsCloseable(true); - }) - ); - // If we ever introduce a second Monaco editor, we need to toggle - // the typing handler to the active editor to maintain the cursor - disposables.current.push( - editor.onDidChangeModelContent((e) => { - onTypeHandler(e, editor); - }) - ); - }} - /> - -
-
- - - {isFullscreen ? ( - - ) : ( - setIsHelpOpen(false)} - button={ - setIsHelpOpen(!isHelpOpen)} - iconType="help" - size="s" - color="text" - > - {i18n.translate('xpack.lens.formula.functionReferenceEditorLabel', { - defaultMessage: 'Function reference', - })} - - } - anchorPosition="leftDown" - > - - - )} - - - {/* Errors go here */} - -
-
- ); -} - -function FormulaHelp({ - indexPattern, - operationDefinitionMap, -}: { - indexPattern: IndexPattern; - operationDefinitionMap: Record; -}) { - const [selectedFunction, setSelectedFunction] = useState(); - - const helpItems: Array = []; - - helpItems.push({ label: 'Math', isGroupLabel: true }); - - helpItems.push( - ...getPossibleFunctions(indexPattern) - .filter((key) => key in tinymathFunctions) - .map((key) => ({ - label: `${key}`, - description: , - checked: selectedFunction === key ? ('on' as const) : undefined, - })) - ); - - helpItems.push({ label: 'Elasticsearch', isGroupLabel: true }); - - // Es aggs - helpItems.push( - ...getPossibleFunctions(indexPattern) - .filter((key) => key in operationDefinitionMap) - .map((key) => ({ - label: `${key}: ${operationDefinitionMap[key].displayName}`, - description: getHelpText(key, operationDefinitionMap), - checked: - selectedFunction === `${key}: ${operationDefinitionMap[key].displayName}` - ? ('on' as const) - : undefined, - })) - ); - - return ( - - - { - const chosenType = newOptions.find(({ checked }) => checked === 'on')!; - if (!chosenType) { - setSelectedFunction(undefined); - } else { - setSelectedFunction(chosenType.label); - } - }} - > - {(list, search) => ( - <> - {search} - {list} - - )} - - - - - {selectedFunction ? ( - helpItems.find(({ label }) => label === selectedFunction)?.description - ) : ( - - )} - - - - ); -} - -const MemoizedFormulaHelp = React.memo(FormulaHelp); - -function parseAndExtract( - text: string, - layer: IndexPatternLayer, - columnId: string, - indexPattern: IndexPattern, - operationDefinitionMap: Record -) { - const { root, error } = tryToParse(text); - if (error || !root) { - return { extracted: [], isValid: false }; - } - // before extracting the data run the validation task and throw if invalid - const errors = runASTValidation(root, layer, indexPattern, operationDefinitionMap); - if (errors.length) { - return { extracted: [], isValid: false }; - } - /* - { name: 'add', args: [ { name: 'abc', args: [5] }, 5 ] } - */ - const extracted = extractColumns(columnId, operationDefinitionMap, root, layer, indexPattern); - return { extracted, isValid: true }; -} - export function regenerateLayerFromAst( text: string, layer: IndexPatternLayer, @@ -947,149 +216,3 @@ export function regenerateLayerFromAst( // turn ast into referenced columns // set state } - -function extractColumns( - idPrefix: string, - operations: Record, - ast: TinymathAST, - layer: IndexPatternLayer, - indexPattern: IndexPattern -): Array<{ column: IndexPatternColumn; location?: TinymathLocation }> { - const columns: Array<{ column: IndexPatternColumn; location?: TinymathLocation }> = []; - - function parseNode(node: TinymathAST) { - if (typeof node === 'number' || node.type !== 'function') { - // leaf node - return node; - } - - const nodeOperation = operations[node.name]; - if (!nodeOperation) { - // it's a regular math node - const consumedArgs = node.args.map(parseNode).filter(Boolean) as Array< - number | TinymathVariable - >; - return { - ...node, - args: consumedArgs, - }; - } - - // split the args into types for better TS experience - const { namedArguments, variables, functions } = groupArgsByType(node.args); - - // operation node - if (nodeOperation.input === 'field') { - const [fieldName] = variables.filter((v): v is TinymathVariable => isObject(v)); - // a validation task passed before executing this and checked already there's a field - const field = shouldHaveFieldArgument(node) - ? indexPattern.getFieldByName(fieldName.value)! - : documentField; - - const mappedParams = getOperationParams(nodeOperation, namedArguments || []); - - const newCol = (nodeOperation as OperationDefinition< - IndexPatternColumn, - 'field' - >).buildColumn( - { - layer, - indexPattern, - field, - }, - mappedParams - ); - const newColId = `${idPrefix}X${columns.length}`; - newCol.customLabel = true; - newCol.label = newColId; - columns.push({ column: newCol, location: node.location }); - // replace by new column id - return newColId; - } - - if (nodeOperation.input === 'fullReference') { - const [referencedOp] = functions; - const consumedParam = parseNode(referencedOp); - - const subNodeVariables = consumedParam ? findVariables(consumedParam) : []; - const mathColumn = mathOperation.buildColumn({ - layer, - indexPattern, - }); - mathColumn.references = subNodeVariables.map(({ value }) => value); - mathColumn.params.tinymathAst = consumedParam!; - columns.push({ column: mathColumn }); - mathColumn.customLabel = true; - mathColumn.label = `${idPrefix}X${columns.length - 1}`; - - const mappedParams = getOperationParams(nodeOperation, namedArguments || []); - const newCol = (nodeOperation as OperationDefinition< - IndexPatternColumn, - 'fullReference' - >).buildColumn( - { - layer, - indexPattern, - referenceIds: [`${idPrefix}X${columns.length - 1}`], - }, - mappedParams - ); - const newColId = `${idPrefix}X${columns.length}`; - newCol.customLabel = true; - newCol.label = newColId; - columns.push({ column: newCol, location: node.location }); - // replace by new column id - return newColId; - } - } - const root = parseNode(ast); - if (root === undefined) { - return []; - } - const variables = findVariables(root); - const mathColumn = mathOperation.buildColumn({ - layer, - indexPattern, - }); - mathColumn.references = variables.map(({ value }) => value); - mathColumn.params.tinymathAst = root!; - const newColId = `${idPrefix}X${columns.length}`; - mathColumn.customLabel = true; - mathColumn.label = newColId; - columns.push({ column: mathColumn }); - return columns; -} - -// TODO: i18n this whole thing, or move examples into the operation definitions with i18n -function getHelpText( - type: string, - operationDefinitionMap: ParamEditorProps['operationDefinitionMap'] -) { - const definition = operationDefinitionMap[type]; - - if (type === 'count') { - return ( - -

Example: count()

- - ); - } - - return ( - - {definition.input === 'field' ?

Example: {type}(bytes)

: null} - {definition.input === 'fullReference' && !('operationParams' in definition) ? ( -

Example: {type}(sum(bytes))

- ) : null} - - {'operationParams' in definition && definition.operationParams ? ( -

-

- Example: {type}(sum(bytes),{' '} - {definition.operationParams.map((p) => `${p.name}=5`).join(', ')}) -

-

- ) : null} - - ); -} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/generate.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/generate.ts new file mode 100644 index 0000000000000..e44cd50ae9c41 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/generate.ts @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isObject } from 'lodash'; +import { GenericOperationDefinition, IndexPatternColumn } from '../index'; +import { ReferenceBasedIndexPatternColumn } from '../column_types'; +import { IndexPatternLayer } from '../../../types'; + +// Just handle two levels for now +type OperationParams = Record>; + +export function getSafeFieldName(fieldName: string | undefined) { + // clean up the "Records" field for now + if (!fieldName || fieldName === 'Records') { + return ''; + } + return fieldName; +} + +export function generateFormula( + previousColumn: ReferenceBasedIndexPatternColumn | IndexPatternColumn, + layer: IndexPatternLayer, + previousFormula: string, + operationDefinitionMap: Record | undefined +) { + if ('references' in previousColumn) { + const metric = layer.columns[previousColumn.references[0]]; + if (metric && 'sourceField' in metric && metric.dataType === 'number') { + const fieldName = getSafeFieldName(metric.sourceField); + // TODO need to check the input type from the definition + previousFormula += `${previousColumn.operationType}(${metric.operationType}(${fieldName})`; + } + } else { + if (previousColumn && 'sourceField' in previousColumn && previousColumn.dataType === 'number') { + previousFormula += `${previousColumn.operationType}(${getSafeFieldName( + previousColumn?.sourceField + )}`; + } + } + const formulaNamedArgs = extractParamsForFormula(previousColumn, operationDefinitionMap); + if (formulaNamedArgs.length) { + previousFormula += + ', ' + formulaNamedArgs.map(({ name, value }) => `${name}=${value}`).join(', '); + } + if (previousColumn.filter) { + if (previousColumn.operationType !== 'count') { + previousFormula += ', '; + } + previousFormula += + (previousColumn.filter.language === 'kuery' ? 'kql=' : 'lucene=') + + `'${previousColumn.filter.query.replace(/'/g, `\\'`)}'`; // replace all + } + if (previousFormula) { + // close the formula at the end + previousFormula += ')'; + } + return previousFormula; +} + +function extractParamsForFormula( + column: IndexPatternColumn | ReferenceBasedIndexPatternColumn, + operationDefinitionMap: Record | undefined +) { + if (!operationDefinitionMap) { + return []; + } + const def = operationDefinitionMap[column.operationType]; + if ('operationParams' in def && column.params) { + return (def.operationParams || []).flatMap(({ name, required }) => { + const value = (column.params as OperationParams)![name]; + if (isObject(value)) { + return Object.keys(value).map((subName) => ({ + name: `${name}-${subName}`, + value: value[subName] as string | number, + required, + })); + } + return { + name, + value, + required, + }; + }); + } + return []; +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts new file mode 100644 index 0000000000000..9ddc1973044f8 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts @@ -0,0 +1,150 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isObject } from 'lodash'; +import type { TinymathAST, TinymathVariable, TinymathLocation } from '@kbn/tinymath'; +import { OperationDefinition, GenericOperationDefinition, IndexPatternColumn } from '../index'; +import { IndexPattern, IndexPatternLayer } from '../../../types'; +import { mathOperation } from './math'; +import { documentField } from '../../../document_field'; +import { runASTValidation, shouldHaveFieldArgument, tryToParse } from './validation'; +import { findVariables, getOperationParams, groupArgsByType } from './util'; + +export function parseAndExtract( + text: string, + layer: IndexPatternLayer, + columnId: string, + indexPattern: IndexPattern, + operationDefinitionMap: Record +) { + const { root, error } = tryToParse(text); + if (error || !root) { + return { extracted: [], isValid: false }; + } + // before extracting the data run the validation task and throw if invalid + const errors = runASTValidation(root, layer, indexPattern, operationDefinitionMap); + if (errors.length) { + return { extracted: [], isValid: false }; + } + /* + { name: 'add', args: [ { name: 'abc', args: [5] }, 5 ] } + */ + const extracted = extractColumns(columnId, operationDefinitionMap, root, layer, indexPattern); + return { extracted, isValid: true }; +} + +function extractColumns( + idPrefix: string, + operations: Record, + ast: TinymathAST, + layer: IndexPatternLayer, + indexPattern: IndexPattern +): Array<{ column: IndexPatternColumn; location?: TinymathLocation }> { + const columns: Array<{ column: IndexPatternColumn; location?: TinymathLocation }> = []; + + function parseNode(node: TinymathAST) { + if (typeof node === 'number' || node.type !== 'function') { + // leaf node + return node; + } + + const nodeOperation = operations[node.name]; + if (!nodeOperation) { + // it's a regular math node + const consumedArgs = node.args.map(parseNode).filter(Boolean) as Array< + number | TinymathVariable + >; + return { + ...node, + args: consumedArgs, + }; + } + + // split the args into types for better TS experience + const { namedArguments, variables, functions } = groupArgsByType(node.args); + + // operation node + if (nodeOperation.input === 'field') { + const [fieldName] = variables.filter((v): v is TinymathVariable => isObject(v)); + // a validation task passed before executing this and checked already there's a field + const field = shouldHaveFieldArgument(node) + ? indexPattern.getFieldByName(fieldName.value)! + : documentField; + + const mappedParams = getOperationParams(nodeOperation, namedArguments || []); + + const newCol = (nodeOperation as OperationDefinition< + IndexPatternColumn, + 'field' + >).buildColumn( + { + layer, + indexPattern, + field, + }, + mappedParams + ); + const newColId = `${idPrefix}X${columns.length}`; + newCol.customLabel = true; + newCol.label = newColId; + columns.push({ column: newCol, location: node.location }); + // replace by new column id + return newColId; + } + + if (nodeOperation.input === 'fullReference') { + const [referencedOp] = functions; + const consumedParam = parseNode(referencedOp); + + const subNodeVariables = consumedParam ? findVariables(consumedParam) : []; + const mathColumn = mathOperation.buildColumn({ + layer, + indexPattern, + }); + mathColumn.references = subNodeVariables.map(({ value }) => value); + mathColumn.params.tinymathAst = consumedParam!; + columns.push({ column: mathColumn }); + mathColumn.customLabel = true; + mathColumn.label = `${idPrefix}X${columns.length - 1}`; + + const mappedParams = getOperationParams(nodeOperation, namedArguments || []); + const newCol = (nodeOperation as OperationDefinition< + IndexPatternColumn, + 'fullReference' + >).buildColumn( + { + layer, + indexPattern, + referenceIds: [`${idPrefix}X${columns.length - 1}`], + }, + mappedParams + ); + const newColId = `${idPrefix}X${columns.length}`; + newCol.customLabel = true; + newCol.label = newColId; + columns.push({ column: newCol, location: node.location }); + // replace by new column id + return newColId; + } + } + const root = parseNode(ast); + if (root === undefined) { + return []; + } + const variables = findVariables(root); + const mathColumn = mathOperation.buildColumn({ + layer, + indexPattern, + }); + mathColumn.references = variables.map(({ value }) => value); + mathColumn.params.tinymathAst = root!; + const newColId = `${idPrefix}X${columns.length}`; + mathColumn.customLabel = true; + mathColumn.label = newColId; + columns.push({ column: mathColumn }); + return columns; +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts index 17ca19839a216..5d9a8647eb7ab 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts @@ -10,12 +10,10 @@ import { i18n } from '@kbn/i18n'; import type { TinymathAST, TinymathFunction, - TinymathLocation, TinymathNamedArgument, TinymathVariable, } from 'packages/kbn-tinymath'; -import { ReferenceBasedIndexPatternColumn } from '../column_types'; -import type { OperationDefinition, IndexPatternColumn, GenericOperationDefinition } from '../index'; +import type { OperationDefinition, IndexPatternColumn } from '../index'; import type { GroupedNodes } from './types'; export function groupArgsByType(args: TinymathAST[]) { @@ -43,45 +41,6 @@ export function getValueOrName(node: TinymathAST) { return node.name; } -export function getSafeFieldName(fieldName: string | undefined) { - // clean up the "Records" field for now - if (!fieldName || fieldName === 'Records') { - return ''; - } - return fieldName; -} - -// Just handle two levels for now -type OeprationParams = Record>; - -export function extractParamsForFormula( - column: IndexPatternColumn | ReferenceBasedIndexPatternColumn, - operationDefinitionMap: Record | undefined -) { - if (!operationDefinitionMap) { - return []; - } - const def = operationDefinitionMap[column.operationType]; - if ('operationParams' in def && column.params) { - return (def.operationParams || []).flatMap(({ name, required }) => { - const value = (column.params as OeprationParams)![name]; - if (isObject(value)) { - return Object.keys(value).map((subName) => ({ - name: `${name}-${subName}`, - value: value[subName] as string | number, - required, - })); - } - return { - name, - value, - required, - }; - }); - } - return []; -} - export function getOperationParams( operation: | OperationDefinition @@ -332,32 +291,6 @@ export function findMathNodes(root: TinymathAST | string): TinymathFunction[] { return flattenMathNodes(root); } -export function hasMathNode(root: TinymathAST): boolean { - return Boolean(findMathNodes(root).length); -} - -function findFunctionNodes(root: TinymathAST | string): TinymathFunction[] { - function flattenFunctionNodes(node: TinymathAST | string): TinymathFunction[] { - if (!isObject(node) || node.type !== 'function') { - return []; - } - return [node, ...node.args.flatMap(flattenFunctionNodes)].filter(Boolean); - } - return flattenFunctionNodes(root); -} - -export function hasInvalidOperations( - node: TinymathAST | string, - operations: Record -): { names: string[]; locations: TinymathLocation[] } { - const nodes = findFunctionNodes(node).filter((v) => !isMathNode(v) && !operations[v.name]); - return { - // avoid duplicates - names: Array.from(new Set(nodes.map(({ name }) => name))), - locations: nodes.map(({ location }) => location), - }; -} - // traverse a tree and find all string leaves export function findVariables(node: TinymathAST | string): TinymathVariable[] { if (typeof node === 'string') { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts index cb52e22302cbe..4e5ae21e576e4 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -16,7 +16,6 @@ import { getOperationParams, getValueOrName, groupArgsByType, - hasInvalidOperations, isMathNode, tinymathFunctions, } from './util'; @@ -74,6 +73,28 @@ export function isParsingError(message: string) { return message.includes(validationErrors.failedParsing.message); } +function findFunctionNodes(root: TinymathAST | string): TinymathFunction[] { + function flattenFunctionNodes(node: TinymathAST | string): TinymathFunction[] { + if (!isObject(node) || node.type !== 'function') { + return []; + } + return [node, ...node.args.flatMap(flattenFunctionNodes)].filter(Boolean); + } + return flattenFunctionNodes(root); +} + +export function hasInvalidOperations( + node: TinymathAST | string, + operations: Record +): { names: string[]; locations: TinymathLocation[] } { + const nodes = findFunctionNodes(node).filter((v) => !isMathNode(v) && !operations[v.name]); + return { + // avoid duplicates + names: Array.from(new Set(nodes.map(({ name }) => name))), + locations: nodes.map(({ location }) => location), + }; +} + export const getQueryValidationError = ( query: string, language: 'kql' | 'lucene', diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index 49366f2421b7b..4fd429820379f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -1079,11 +1079,21 @@ export function getExistingColumnGroups(layer: IndexPatternLayer): [string[], st /** * Returns true if the given column can be applied to the given index pattern */ -export function isColumnTransferable(column: IndexPatternColumn, newIndexPattern: IndexPattern) { - return operationDefinitionMap[column.operationType].isTransferable( - column, - newIndexPattern, - operationDefinitionMap +export function isColumnTransferable( + column: IndexPatternColumn, + newIndexPattern: IndexPattern, + layer: IndexPatternLayer +): boolean { + return ( + operationDefinitionMap[column.operationType].isTransferable( + column, + newIndexPattern, + operationDefinitionMap + ) && + (!('references' in column) || + column.references.every((columnId) => + isColumnTransferable(layer.columns[columnId], newIndexPattern, layer) + )) ); } @@ -1092,15 +1102,7 @@ export function updateLayerIndexPattern( newIndexPattern: IndexPattern ): IndexPatternLayer { const keptColumns: IndexPatternLayer['columns'] = _.pickBy(layer.columns, (column) => { - if ('references' in column) { - return ( - isColumnTransferable(column, newIndexPattern) && - column.references.every((columnId) => - isColumnTransferable(layer.columns[columnId], newIndexPattern) - ) - ); - } - return isColumnTransferable(column, newIndexPattern); + return isColumnTransferable(column, newIndexPattern, layer); }); const newColumns: IndexPatternLayer['columns'] = _.mapValues(keptColumns, (column) => { const operationDefinition = operationDefinitionMap[column.operationType]; From 571501a26592ff34aab397e4190c4f66a0502d65 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 10 May 2021 10:35:12 +0200 Subject: [PATCH 2/3] move main column generation into parse module --- .../formula/editor/formula_editor.tsx | 3 +- .../definitions/formula/formula.tsx | 66 +------------------ .../operations/definitions/formula/parse.ts | 62 ++++++++++++++++- 3 files changed, 66 insertions(+), 65 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx index 7b96aec4194a4..42f4d9cf6ca33 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -31,7 +31,8 @@ import { LANGUAGE_ID } from './math_tokenization'; import { MemoizedFormulaHelp } from './formula_help'; import './formula.scss'; -import { FormulaIndexPatternColumn, regenerateLayerFromAst } from '../formula'; +import { FormulaIndexPatternColumn } from '../formula'; +import { regenerateLayerFromAst } from '../parse'; export function FormulaEditor({ layer, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 6f0abe8f55568..6494c47548f2f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -6,14 +6,12 @@ */ import { i18n } from '@kbn/i18n'; -import type { TinymathLocation } from '@kbn/tinymath'; -import { OperationDefinition, GenericOperationDefinition } from '../index'; +import { OperationDefinition } from '../index'; import { ReferenceBasedIndexPatternColumn } from '../column_types'; -import { IndexPattern, IndexPatternLayer } from '../../../types'; -import { getColumnOrder } from '../../layer_helpers'; +import { IndexPattern } from '../../../types'; import { runASTValidation, tryToParse } from './validation'; import { FormulaEditor } from './editor'; -import { parseAndExtract } from './parse'; +import { regenerateLayerFromAst } from './parse'; import { generateFormula } from './generate'; const defaultLabel = i18n.translate('xpack.lens.indexPattern.formulaLabel', { @@ -158,61 +156,3 @@ export const formulaOperation: OperationDefinition< paramEditor: FormulaEditor, }; - -export function regenerateLayerFromAst( - text: string, - layer: IndexPatternLayer, - columnId: string, - currentColumn: FormulaIndexPatternColumn, - indexPattern: IndexPattern, - operationDefinitionMap: Record -) { - const { extracted, isValid } = parseAndExtract( - text, - layer, - columnId, - indexPattern, - operationDefinitionMap - ); - - const columns = { ...layer.columns }; - - const locations: Record = {}; - - Object.keys(columns).forEach((k) => { - if (k.startsWith(columnId)) { - delete columns[k]; - } - }); - - extracted.forEach(({ column, location }, index) => { - columns[`${columnId}X${index}`] = column; - if (location) locations[`${columnId}X${index}`] = location; - }); - - columns[columnId] = { - ...currentColumn, - params: { - ...currentColumn.params, - formula: text, - isFormulaBroken: !isValid, - }, - references: !isValid ? [] : [`${columnId}X${extracted.length - 1}`], - }; - - return { - newLayer: { - ...layer, - columns, - columnOrder: getColumnOrder({ - ...layer, - columns, - }), - }, - locations, - }; - - // TODO - // turn ast into referenced columns - // set state -} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts index 9ddc1973044f8..70ed2f36dfd1c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts @@ -13,8 +13,10 @@ import { mathOperation } from './math'; import { documentField } from '../../../document_field'; import { runASTValidation, shouldHaveFieldArgument, tryToParse } from './validation'; import { findVariables, getOperationParams, groupArgsByType } from './util'; +import { FormulaIndexPatternColumn } from './formula'; +import { getColumnOrder } from '../../layer_helpers'; -export function parseAndExtract( +function parseAndExtract( text: string, layer: IndexPatternLayer, columnId: string, @@ -148,3 +150,61 @@ function extractColumns( columns.push({ column: mathColumn }); return columns; } + +export function regenerateLayerFromAst( + text: string, + layer: IndexPatternLayer, + columnId: string, + currentColumn: FormulaIndexPatternColumn, + indexPattern: IndexPattern, + operationDefinitionMap: Record +) { + const { extracted, isValid } = parseAndExtract( + text, + layer, + columnId, + indexPattern, + operationDefinitionMap + ); + + const columns = { ...layer.columns }; + + const locations: Record = {}; + + Object.keys(columns).forEach((k) => { + if (k.startsWith(columnId)) { + delete columns[k]; + } + }); + + extracted.forEach(({ column, location }, index) => { + columns[`${columnId}X${index}`] = column; + if (location) locations[`${columnId}X${index}`] = location; + }); + + columns[columnId] = { + ...currentColumn, + params: { + ...currentColumn.params, + formula: text, + isFormulaBroken: !isValid, + }, + references: !isValid ? [] : [`${columnId}X${extracted.length - 1}`], + }; + + return { + newLayer: { + ...layer, + columns, + columnOrder: getColumnOrder({ + ...layer, + columns, + }), + }, + locations, + }; + + // TODO + // turn ast into referenced columns + // set state +} From 70a6b86c86e82f255959d51be70f60803a9678c2 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 10 May 2021 10:49:08 +0200 Subject: [PATCH 3/3] fix tests --- .../formula/editor/math_completion.test.ts | 12 ++++++------ .../operations/definitions/formula/formula.test.tsx | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts index 9b5e77b7b90db..9e29160b6747b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts @@ -6,12 +6,12 @@ */ import { monaco } from '@kbn/monaco'; -import { createMockedIndexPattern } from '../../../mocks'; -import { GenericOperationDefinition } from '../index'; -import type { IndexPatternField } from '../../../types'; -import type { OperationMetadata } from '../../../../types'; -import { dataPluginMock } from '../../../../../../../../src/plugins/data/public/mocks'; -import { tinymathFunctions } from './util'; +import { createMockedIndexPattern } from '../../../../mocks'; +import { GenericOperationDefinition } from '../../index'; +import type { IndexPatternField } from '../../../../types'; +import type { OperationMetadata } from '../../../../../types'; +import { dataPluginMock } from '../../../../../../../../../src/plugins/data/public/mocks'; +import { tinymathFunctions } from '../util'; import { getSignatureHelp, getHover, suggest } from './math_completion'; const buildGenericColumn = (type: string) => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx index 433e21eb13345..ce7b48aa1875e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx @@ -7,7 +7,8 @@ import { createMockedIndexPattern } from '../../../mocks'; import { formulaOperation, GenericOperationDefinition, IndexPatternColumn } from '../index'; -import { FormulaIndexPatternColumn, regenerateLayerFromAst } from './formula'; +import { FormulaIndexPatternColumn } from './formula'; +import { regenerateLayerFromAst } from './parse'; import type { IndexPattern, IndexPatternField, IndexPatternLayer } from '../../../types'; import { tinymathFunctions } from './util';