forked from opensearch-project/OpenSearch-Dashboards
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[MQL] support different query languages (opensearch-project#6595)
* [MQL] support different query languages Only adds the quick startup for OpenSearch cluster with a SQL plugin and observability with: ``` yarn opensearch snapshot --sql ``` Also, adds SQL to the language selector stolen shamelessly from opensearch-project#5623 Next steps to intercept and send to SQL API or just transform basic syntax to DSL Signed-off-by: Kawika Avilla <kavilla414@gmail.com> Working query enhance Signed-off-by: Kawika Avilla <kavilla414@gmail.com> Reget search enhance Signed-off-by: Kawika Avilla <kavilla414@gmail.com> data frames Signed-off-by: Kawika Avilla <kavilla414@gmail.com> temp state need to pass df Signed-off-by: Kawika Avilla <kavilla414@gmail.com> sending data frame Signed-off-by: Kawika Avilla <kavilla414@gmail.com> gets df type Signed-off-by: Kawika Avilla <kavilla414@gmail.com> add some small ui things Signed-off-by: Kawika Avilla <kavilla414@gmail.com> move back query language switcher Signed-off-by: Kawika Avilla <kavilla414@gmail.com> updating side panel Signed-off-by: Kawika Avilla <kavilla414@gmail.com> add ui config on query enhancement Signed-off-by: Kawika Avilla <kavilla414@gmail.com> support query string input Signed-off-by: Kawika Avilla <kavilla414@gmail.com> update to say search bar Signed-off-by: Kawika Avilla <kavilla414@gmail.com> update ppl to call once Signed-off-by: Kawika Avilla <kavilla414@gmail.com> update when fields update Signed-off-by: Kawika Avilla <kavilla414@gmail.com> add unknown Signed-off-by: Kawika Avilla <kavilla414@gmail.com> * clean up Signed-off-by: Kawika Avilla <kavilla414@gmail.com> * more clean up Signed-off-by: Kawika Avilla <kavilla414@gmail.com> * clean up licenses Signed-off-by: Kawika Avilla <kavilla414@gmail.com> --------- Signed-off-by: Kawika Avilla <kavilla414@gmail.com> Cleanup unused helper function Signed-off-by: Kawika Avilla <kavilla414@gmail.com> create index pattern somewhat working Signed-off-by: Kawika Avilla <kavilla414@gmail.com>
- Loading branch information
Showing
56 changed files
with
1,059 additions
and
168 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| import { IDataFrame } from '..'; | ||
|
|
||
| export interface DfCache { | ||
| get: () => IDataFrame | undefined; | ||
| set: (value: IDataFrame) => IDataFrame; | ||
| clear: () => void; | ||
| } | ||
|
|
||
| export function createDataFrameCache(): DfCache { | ||
| let df: IDataFrame | undefined; | ||
| const cache: DfCache = { | ||
| get: () => { | ||
| return df; | ||
| }, | ||
| set: (prom: IDataFrame) => { | ||
| df = prom; | ||
| return prom; | ||
| }, | ||
| clear: () => { | ||
| df = undefined; | ||
| }, | ||
| }; | ||
| return cache; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| export * from './types'; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| export interface IFieldType { | ||
| name: string; | ||
| type: string; | ||
| values: any[]; | ||
| count?: number; | ||
| aggregatable?: boolean; | ||
| filterable?: boolean; | ||
| searchable?: boolean; | ||
| sortable?: boolean; | ||
| visualizable?: boolean; | ||
| displayName?: string; | ||
| format?: any; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| export * from './types'; | ||
| export * from './utils'; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| import { IFieldType } from './fields'; | ||
|
|
||
| export * from './_df_cache'; | ||
|
|
||
| /** @public **/ | ||
| export enum DATA_FRAME_TYPES { | ||
| DEFAULT = 'data_frame', | ||
| POLLING = 'data_frame_polling', | ||
| } | ||
|
|
||
| export interface IDataFrame { | ||
| name?: string; | ||
| schema?: Array<Partial<IFieldType>>; | ||
| fields: IFieldType[]; | ||
| size: number; | ||
| } | ||
|
|
||
| export interface DataFrameAgg { | ||
| key: string; | ||
| value: number; | ||
| } | ||
|
|
||
| export interface PartialDataFrame extends Omit<IDataFrame, 'fields' | 'size'> { | ||
| fields: Array<Partial<IFieldType>>; | ||
| } | ||
|
|
||
| /** | ||
| * To be utilize with aggregations and will map to buckets | ||
| * Plugins can get the aggreted value by their own logic | ||
| * Setting to null will disable the aggregation if plugin wishes | ||
| * In future, if the plugin doesn't intentionally set the value to null, | ||
| * we can calculate the value based on the fields. | ||
| */ | ||
| export interface IDataFrameWithAggs extends IDataFrame { | ||
| aggs: DataFrameAgg[] | null; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,147 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| import { SearchResponse } from 'elasticsearch'; | ||
| import datemath from '@elastic/datemath'; | ||
| import { DATA_FRAME_TYPES, IDataFrame, IDataFrameWithAggs, PartialDataFrame } from './types'; | ||
| import { IFieldType } from './fields'; | ||
| import { IndexPatternFieldMap, IndexPatternSpec } from '../index_patterns'; | ||
| import { IOpenSearchDashboardsSearchRequest } from '../search'; | ||
|
|
||
| export interface IDataFrameResponse extends SearchResponse<any> { | ||
| type: DATA_FRAME_TYPES; | ||
| body: IDataFrame | IDataFrameWithAggs; | ||
| took: number; | ||
| } | ||
|
|
||
| export const getRawQueryString = ( | ||
| searchRequest: IOpenSearchDashboardsSearchRequest | ||
| ): string | undefined => { | ||
| return searchRequest.params?.body?.query?.queries[0]?.query; | ||
| }; | ||
|
|
||
| export const convertResult = (response: IDataFrameResponse): SearchResponse<any> => { | ||
| const data = response.body; | ||
| const hits: any[] = []; | ||
| for (let index = 0; index < data.size; index++) { | ||
| const hit: { [key: string]: any } = {}; | ||
| data.fields.forEach((field) => { | ||
| hit[field.name] = field.values[index]; | ||
| }); | ||
| hits.push({ | ||
| _index: data.name ?? '', | ||
| _type: '', | ||
| _id: '', | ||
| _score: 0, | ||
| _source: hit, | ||
| }); | ||
| } | ||
| const searchResponse: SearchResponse<any> = { | ||
| took: response.took, | ||
| timed_out: false, | ||
| _shards: { | ||
| total: 1, | ||
| successful: 1, | ||
| skipped: 0, | ||
| failed: 0, | ||
| }, | ||
| hits: { | ||
| total: 0, | ||
| max_score: 0, | ||
| hits, | ||
| }, | ||
| }; | ||
|
|
||
| if (data.hasOwnProperty('aggs')) { | ||
| const dataWithAggs = data as IDataFrameWithAggs; | ||
| if (!dataWithAggs.aggs) { | ||
| // TODO: SQL best guess, get timestamp field and caculate it here | ||
| return searchResponse; | ||
| } | ||
| searchResponse.aggregations = { | ||
| 2: { | ||
| buckets: dataWithAggs.aggs.map((agg) => { | ||
| searchResponse.hits.total += agg.value; | ||
| return { | ||
| key: new Date(agg.key).getTime(), | ||
| key_as_string: agg.key, | ||
| doc_count: agg.value, | ||
| }; | ||
| }), | ||
| }, | ||
| }; | ||
| } | ||
|
|
||
| return searchResponse; | ||
| }; | ||
|
|
||
| export const formatFieldValue = (field: IFieldType | Partial<IFieldType>, value: any): any => { | ||
| return field.format && field.format.convert ? field.format.convert(value) : value; | ||
| }; | ||
|
|
||
| export const getFieldType = (field: IFieldType | Partial<IFieldType>): string | undefined => { | ||
| if (field.name) { | ||
| const fieldName = field.name.toLowerCase(); | ||
| // TODO: feels little biased to check if timestamp. | ||
| // Has to be a better way to know so to be fair to all data sources | ||
| if (fieldName.includes('date') || fieldName.includes('timestamp')) { | ||
| return 'date'; | ||
| } | ||
| } | ||
| if (!field.values) return field.type; | ||
| const firstValue = field.values.filter((value) => value !== null && value !== undefined)[0]; | ||
| if (firstValue instanceof Date || datemath.isDateTime(firstValue)) { | ||
| return 'date'; | ||
| } | ||
| return field.type; | ||
| }; | ||
|
|
||
| export const getTimeField = (data: IDataFrame): IFieldType | undefined => { | ||
| return data.fields.find((field) => field.type === 'date'); | ||
| }; | ||
|
|
||
| export const createDataFrame = (partial: PartialDataFrame): IDataFrame | IDataFrameWithAggs => { | ||
| let size = 0; | ||
| const fields = partial.fields.map((field) => { | ||
| if (!field.values) { | ||
| field.values = new Array(size); | ||
| } else if (field.values.length > size) { | ||
| size = field.values.length; | ||
| } | ||
| field.type = getFieldType(field); | ||
| // if (!field.type) { | ||
| // need to think if this needs to be mapped to OSD field type for example | ||
| // PPL type for date is TIMESTAMP | ||
| // OSD is expecting date | ||
| // field.type = get type | ||
| // } | ||
| // get timeseries field | ||
| return field as IFieldType; | ||
| }); | ||
|
|
||
| return { | ||
| ...partial, | ||
| fields, | ||
| size, | ||
| }; | ||
| }; | ||
|
|
||
| export const dataFrameToSpec = (dataFrame: IDataFrame): IndexPatternSpec => { | ||
| return { | ||
| id: DATA_FRAME_TYPES.DEFAULT, | ||
| title: dataFrame.name, | ||
| timeFieldName: getTimeField(dataFrame)?.name, | ||
| fields: dataFrame.fields.reduce((acc, field) => { | ||
| acc[field.name] = { | ||
| name: field.name, | ||
| type: field.type, | ||
| aggregatable: true, | ||
| searchable: true, | ||
| }; | ||
| return acc; | ||
| }, {} as IndexPatternFieldMap), | ||
| // TODO: SQL dataSourceRef | ||
| }; | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.