diff --git a/changelogs/fragments/7368.yml b/changelogs/fragments/7368.yml deleted file mode 100644 index c8316dc939f0..000000000000 --- a/changelogs/fragments/7368.yml +++ /dev/null @@ -1,2 +0,0 @@ -feat: -- [Discover] Adds a dataset selector for Discover ([#7368](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/7368)) \ No newline at end of file diff --git a/package.json b/package.json index e86a19ae5d77..30cc500cdb66 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,7 @@ "start": "scripts/use_node scripts/opensearch_dashboards --dev", "start:docker": "scripts/use_node scripts/opensearch_dashboards --dev --opensearch.hosts=$OPENSEARCH_HOSTS --opensearch.ignoreVersionMismatch=true --server.host=$SERVER_HOST", "start:security": "scripts/use_node scripts/opensearch_dashboards --dev --security", - "start:enhancements": "scripts/use_node scripts/opensearch_dashboards --dev --uiSettings.overrides['query:enhancements:enabled']=true --uiSettings.overrides['home:useNewHomePage']=true", + "start:enhancements": "scripts/use_node scripts/opensearch_dashboards --dev --uiSettings.overrides['query:enhancements:enabled']=true", "debug": "scripts/use_node --nolazy --inspect scripts/opensearch_dashboards --dev", "debug-break": "scripts/use_node --nolazy --inspect-brk scripts/opensearch_dashboards --dev", "lint": "yarn run lint:es && yarn run lint:style", diff --git a/src/plugins/data/common/data_frames/utils.ts b/src/plugins/data/common/data_frames/utils.ts index 0da02cda295f..31df2626a98a 100644 --- a/src/plugins/data/common/data_frames/utils.ts +++ b/src/plugins/data/common/data_frames/utils.ts @@ -45,6 +45,29 @@ export const getRawQueryString = ( ); }; +/** + * Parses a raw query string and extracts the query string and data source. + * @param rawQueryString - The raw query string to parse. + * @returns An object containing the parsed query string and data source (if found). + */ +export const parseRawQueryString = (rawQueryString: string) => { + const rawDataSource = rawQueryString.match(/::(.*?)::/); + return { + qs: rawQueryString.replace(/::.*?::/, ''), + formattedQs(key: string = '.'): string { + const parts = rawQueryString.split('::'); + if (parts.length > 1) { + return (parts.slice(0, 1).join('') + parts.slice(1).join(key)).replace( + new RegExp(key + '$'), + '' + ); + } + return rawQueryString; + }, + ...(rawDataSource && { dataSource: rawDataSource[1] }), + }; +}; + /** * Returns the raw aggregations from the search request. * diff --git a/src/plugins/data/common/data_sets/types.ts b/src/plugins/data/common/data_sets/types.ts deleted file mode 100644 index 23ab74bed030..000000000000 --- a/src/plugins/data/common/data_sets/types.ts +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -/** @public **/ -export enum SIMPLE_DATA_SOURCE_TYPES { - DEFAULT = 'data-source', - EXTERNAL = 'external-source', -} - -/** @public **/ -export enum SIMPLE_DATA_SET_TYPES { - INDEX_PATTERN = 'index-pattern', - TEMPORARY = 'temporary', - TEMPORARY_ASYNC = 'temporary-async', -} - -export interface SimpleObject { - id: string; - title?: string; - dataSourceRef?: SimpleDataSource; -} - -export interface SimpleDataSource { - id: string; - name: string; - indices?: SimpleObject[]; - tables?: SimpleObject[]; - type: SIMPLE_DATA_SOURCE_TYPES; -} - -export interface SimpleDataSet extends SimpleObject { - fields?: any[]; - timeFieldName?: string; - timeFields?: any[]; - type?: SIMPLE_DATA_SET_TYPES; -} diff --git a/src/plugins/data/common/index.ts b/src/plugins/data/common/index.ts index 0250a6ec2e01..d7b7e56e2280 100644 --- a/src/plugins/data/common/index.ts +++ b/src/plugins/data/common/index.ts @@ -31,7 +31,6 @@ export * from './constants'; export * from './opensearch_query'; export * from './data_frames'; -export * from './data_sets'; export * from './field_formats'; export * from './field_mapping'; export * from './index_patterns'; diff --git a/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts b/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts index 3d0dbe15dab7..3d7bd8fbb4a2 100644 --- a/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts +++ b/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts @@ -433,13 +433,11 @@ export class IndexPatternsService { /** * Get an index pattern by id. Cache optimized * @param id - * @param onlyCheckCache - Only check cache for index pattern if it doesn't exist it will not error out */ - get = async (id: string, onlyCheckCache: boolean = false): Promise => { + get = async (id: string): Promise => { const cache = indexPatternCache.get(id); - - if (cache || onlyCheckCache) { + if (cache) { return cache; } diff --git a/src/plugins/data/common/search/opensearch_search/types.ts b/src/plugins/data/common/search/opensearch_search/types.ts index 6d24e8c36dd3..f90a3f1de245 100644 --- a/src/plugins/data/common/search/opensearch_search/types.ts +++ b/src/plugins/data/common/search/opensearch_search/types.ts @@ -48,10 +48,6 @@ export interface ISearchOptions { * Use this option to enable support for long numerals. */ withLongNumeralsSupport?: boolean; - /** - * Use this option to enable support for async. - */ - isAsync?: boolean; } export type ISearchRequestParams> = { diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index da00e73f54f0..d9518e6a6cab 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -94,6 +94,7 @@ import { convertResult, createDataFrame, getRawQueryString, + parseRawQueryString, } from '../../data_frames'; import { IOpenSearchSearchRequest, IOpenSearchSearchResponse, ISearchOptions } from '../..'; import { IOpenSearchDashboardsSearchRequest, IOpenSearchDashboardsSearchResponse } from '../types'; @@ -323,12 +324,7 @@ export class SearchSource { const dataFrame = createDataFrame({ name: searchRequest.index.title || searchRequest.index, fields: [], - ...(rawQueryString && { - meta: { - queryConfig: { qs: rawQueryString }, - ...(searchRequest.dataSourceId && { dataSource: searchRequest.dataSourceId }), - }, - }), + ...(rawQueryString && { meta: { queryConfig: parseRawQueryString(rawQueryString) } }), }); await this.setDataFrame(dataFrame); return this.getDataFrame(); @@ -430,8 +426,7 @@ export class SearchSource { private async fetchExternalSearch(searchRequest: SearchRequest, options: ISearchOptions) { const { search, getConfig, onResponse } = this.dependencies; - const currentDataframe = this.getDataFrame(); - if (!currentDataframe || currentDataframe.name !== searchRequest.index?.id) { + if (!this.getDataFrame()) { await this.createDataFrame(searchRequest); } diff --git a/src/plugins/data/common/types.ts b/src/plugins/data/common/types.ts index 1670fbf72d5d..6a1f6e5a99d3 100644 --- a/src/plugins/data/common/types.ts +++ b/src/plugins/data/common/types.ts @@ -35,7 +35,6 @@ export * from './query/types'; export * from './osd_field_types/types'; export * from './index_patterns/types'; export * from './data_frames/types'; -export * from './data_sets/types'; /** * If a service is being shared on both the client and the server, and diff --git a/src/plugins/data/public/antlr/shared/utils.ts b/src/plugins/data/public/antlr/shared/utils.ts index 8be6e6524fc5..b2658b304e0f 100644 --- a/src/plugins/data/public/antlr/shared/utils.ts +++ b/src/plugins/data/public/antlr/shared/utils.ts @@ -12,7 +12,7 @@ export interface IDataSourceRequestHandlerParams { } export const getRawSuggestionData$ = ( - connectionsService: any, + connectionsService, dataSourceReuqstHandler: ({ dataSourceId, title, @@ -21,11 +21,11 @@ export const getRawSuggestionData$ = ( ) => connectionsService.getSelectedConnection$().pipe( distinctUntilChanged(), - switchMap((connection: any) => { + switchMap((connection) => { if (connection === undefined) { return from(defaultReuqstHandler()); } - const dataSourceId = connection?.dataSource?.id; + const dataSourceId = connection?.id; const title = connection?.attributes?.title; return from(dataSourceReuqstHandler({ dataSourceId, title })); }) @@ -34,8 +34,8 @@ export const getRawSuggestionData$ = ( export const fetchData = ( tables: string[], queryFormatter: (table: string, dataSourceId?: string, title?: string) => any, - api: any, - connectionService: any + api, + connectionService ): Promise => { return new Promise((resolve, reject) => { getRawSuggestionData$( @@ -65,8 +65,8 @@ export const fetchData = ( ); } ).subscribe({ - next: (dataFrames: any) => resolve(dataFrames), - error: (err: any) => { + next: (dataFrames) => resolve(dataFrames), + error: (err) => { // TODO: pipe error to UI reject(err); }, @@ -74,11 +74,7 @@ export const fetchData = ( }); }; -export const fetchTableSchemas = ( - tables: string[], - api: any, - connectionService: any -): Promise => { +export const fetchTableSchemas = (tables: string[], api, connectionService): Promise => { return fetchData( tables, (table, dataSourceId, title) => ({ @@ -100,8 +96,8 @@ export const fetchTableSchemas = ( export const fetchColumnValues = ( tables: string[], column: string, - api: any, - connectionService: any + api, + connectionService ): Promise => { return fetchData( tables, diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts index 208359352e4b..f1ac419e9ec1 100644 --- a/src/plugins/data/public/index.ts +++ b/src/plugins/data/public/index.ts @@ -445,7 +445,6 @@ export { QueryEditorTopRow, // for BWC, keeping the old name IUiStart as DataPublicPluginStartUi, - DataSetNavigator, } from './ui'; /** diff --git a/src/plugins/data/public/query/dataset_manager/dataset_manager.mock.ts b/src/plugins/data/public/query/dataset_manager/dataset_manager.mock.ts deleted file mode 100644 index 2f1f5144274c..000000000000 --- a/src/plugins/data/public/query/dataset_manager/dataset_manager.mock.ts +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { DataSetContract } from '.'; - -const createSetupContractMock = () => { - const dataSetManagerMock: jest.Mocked = { - init: jest.fn(), - getDataSet: jest.fn(), - setDataSet: jest.fn(), - getUpdates$: jest.fn(), - getDefaultDataSet: jest.fn(), - }; - return dataSetManagerMock; -}; - -export const dataSetManagerMock = { - createSetupContract: createSetupContractMock, - createStartContract: createSetupContractMock, -}; diff --git a/src/plugins/data/public/query/dataset_manager/dataset_manager.test.ts b/src/plugins/data/public/query/dataset_manager/dataset_manager.test.ts deleted file mode 100644 index fcf91e6b8f89..000000000000 --- a/src/plugins/data/public/query/dataset_manager/dataset_manager.test.ts +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { DataSetManager } from './dataset_manager'; -import { coreMock } from '../../../../../core/public/mocks'; -import { SimpleDataSet } from '../../../common/data_sets'; - -describe('DataSetManager', () => { - let service: DataSetManager; - - beforeEach(() => { - service = new DataSetManager(coreMock.createSetup().uiSettings); - }); - - test('getUpdates$ is a cold emits only after query changes', () => { - const obs$ = service.getUpdates$(); - const emittedValues: SimpleDataSet[] = []; - obs$.subscribe((v) => { - emittedValues.push(v!); - }); - expect(emittedValues).toHaveLength(0); - - const newDataSet: SimpleDataSet = { id: 'test_dataset', title: 'Test Dataset' }; - service.setDataSet(newDataSet); - expect(emittedValues).toHaveLength(1); - expect(emittedValues[0]).toEqual(newDataSet); - - service.setDataSet({ ...newDataSet }); - expect(emittedValues).toHaveLength(2); - }); -}); diff --git a/src/plugins/data/public/query/dataset_manager/dataset_manager.ts b/src/plugins/data/public/query/dataset_manager/dataset_manager.ts deleted file mode 100644 index 018eba50ad73..000000000000 --- a/src/plugins/data/public/query/dataset_manager/dataset_manager.ts +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { BehaviorSubject } from 'rxjs'; -import { skip } from 'rxjs/operators'; -import { CoreStart } from 'opensearch-dashboards/public'; -import { - IndexPatternsService, - SIMPLE_DATA_SET_TYPES, - SimpleDataSet, - SimpleDataSource, -} from '../../../common'; - -export class DataSetManager { - private dataSet$: BehaviorSubject; - private indexPatterns?: IndexPatternsService; - - constructor(private readonly uiSettings: CoreStart['uiSettings']) { - this.dataSet$ = new BehaviorSubject(undefined); - } - - public init = (indexPatterns: IndexPatternsService) => { - this.indexPatterns = indexPatterns; - }; - - public getUpdates$ = () => { - return this.dataSet$.asObservable().pipe(skip(1)); - }; - - public getDataSet = () => { - return this.dataSet$.getValue(); - }; - - /** - * Updates the query. - * @param {Query} query - */ - public setDataSet = (dataSet: SimpleDataSet | undefined) => { - this.dataSet$.next(dataSet); - }; - - public getDefaultDataSet = async (): Promise => { - const defaultIndexPatternId = await this.uiSettings.get('defaultIndex'); - if (!defaultIndexPatternId) { - return undefined; - } - - const indexPattern = await this.indexPatterns?.get(defaultIndexPatternId); - if (!indexPattern) { - return undefined; - } - - if (!indexPattern.id) { - return undefined; - } - - return { - id: indexPattern.id, - title: indexPattern.title, - type: SIMPLE_DATA_SET_TYPES.INDEX_PATTERN, - timeFieldName: indexPattern.timeFieldName, - ...(indexPattern.dataSourceRef - ? { - dataSourceRef: { - id: indexPattern.dataSourceRef?.id, - name: indexPattern.dataSourceRef?.name, - type: indexPattern.dataSourceRef?.type, - } as SimpleDataSource, - } - : {}), - }; - }; -} - -export type DataSetContract = PublicMethodsOf; diff --git a/src/plugins/data/public/query/dataset_manager/index.ts b/src/plugins/data/public/query/dataset_manager/index.ts deleted file mode 100644 index 8a9a39b81127..000000000000 --- a/src/plugins/data/public/query/dataset_manager/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -export { DataSetContract, DataSetManager } from './dataset_manager'; diff --git a/src/plugins/data/public/query/index.tsx b/src/plugins/data/public/query/index.tsx index 42c6349bcc89..505f095aeda7 100644 --- a/src/plugins/data/public/query/index.tsx +++ b/src/plugins/data/public/query/index.tsx @@ -32,7 +32,6 @@ export * from './lib'; export * from './query_service'; export * from './filter_manager'; -export * from './dataset_manager'; export * from './timefilter'; export * from './saved_query'; export * from './persisted_log'; diff --git a/src/plugins/data/public/query/query_service.ts b/src/plugins/data/public/query/query_service.ts index 7d3bdca2f9b8..1b758d18bda3 100644 --- a/src/plugins/data/public/query/query_service.ts +++ b/src/plugins/data/public/query/query_service.ts @@ -37,8 +37,7 @@ import { TimefilterService, TimefilterSetup } from './timefilter'; import { createSavedQueryService } from './saved_query/saved_query_service'; import { createQueryStateObservable } from './state_sync/create_global_query_observable'; import { QueryStringManager, QueryStringContract } from './query_string'; -import { DataSetManager, DataSetContract } from './dataset_manager'; -import { buildOpenSearchQuery, getOpenSearchQueryConfig, IndexPatternsService } from '../../common'; +import { buildOpenSearchQuery, getOpenSearchQueryConfig } from '../../common'; import { getUiSettings } from '../services'; import { IndexPattern } from '..'; @@ -56,14 +55,12 @@ interface QueryServiceStartDependencies { savedObjectsClient: SavedObjectsClientContract; storage: IStorageWrapper; uiSettings: IUiSettingsClient; - indexPatterns: IndexPatternsService; } export class QueryService { filterManager!: FilterManager; timefilter!: TimefilterSetup; queryStringManager!: QueryStringContract; - dataSetManager!: DataSetContract; state$!: ReturnType; @@ -77,31 +74,22 @@ export class QueryService { }); this.queryStringManager = new QueryStringManager(storage, uiSettings); - this.dataSetManager = new DataSetManager(uiSettings); this.state$ = createQueryStateObservable({ filterManager: this.filterManager, timefilter: this.timefilter, queryString: this.queryStringManager, - dataSet: this.dataSetManager, }).pipe(share()); return { filterManager: this.filterManager, timefilter: this.timefilter, queryString: this.queryStringManager, - dataSet: this.dataSetManager, state$: this.state$, }; } - public start({ - savedObjectsClient, - storage, - uiSettings, - indexPatterns, - }: QueryServiceStartDependencies) { - this.dataSetManager.init(indexPatterns); + public start({ savedObjectsClient, storage, uiSettings }: QueryServiceStartDependencies) { return { addToQueryLog: createAddToQueryLog({ storage, @@ -109,7 +97,6 @@ export class QueryService { }), filterManager: this.filterManager, queryString: this.queryStringManager, - dataSet: this.dataSetManager, savedQueries: createSavedQueryService(savedObjectsClient), state$: this.state$, timefilter: this.timefilter, diff --git a/src/plugins/data/public/query/state_sync/connect_to_query_state.ts b/src/plugins/data/public/query/state_sync/connect_to_query_state.ts index 5db81e597b80..8b850b36eabc 100644 --- a/src/plugins/data/public/query/state_sync/connect_to_query_state.ts +++ b/src/plugins/data/public/query/state_sync/connect_to_query_state.ts @@ -48,21 +48,16 @@ import { validateTimeRange } from '../timefilter'; * @param OsdUrlStateStorage to use for syncing and store data * @param syncConfig app filter and query */ -export const connectStorageToQueryState = async ( +export const connectStorageToQueryState = ( { - dataSet, filterManager, queryString, state$, - }: Pick< - QueryStart | QuerySetup, - 'timefilter' | 'filterManager' | 'queryString' | 'dataSet' | 'state$' - >, + }: Pick, OsdUrlStateStorage: IOsdUrlStateStorage, syncConfig: { filters: FilterStateStore; query: boolean; - dataSet?: boolean; } ) => { try { @@ -73,14 +68,10 @@ export const connectStorageToQueryState = async ( if (syncConfig.filters === FilterStateStore.APP_STATE) { syncKeys.push('appFilters'); } - if (syncConfig.dataSet) { - syncKeys.push('dataSet'); - } const initialStateFromURL: QueryState = OsdUrlStateStorage.get('_q') ?? { query: queryString.getDefaultQuery(), filters: filterManager.getAppFilters(), - dataSet: await dataSet.getDefaultDataSet(), }; // set up initial '_q' flag in the URL to sync query and filter changes @@ -96,17 +87,6 @@ export const connectStorageToQueryState = async ( } } - if (syncConfig.dataSet && !_.isEqual(initialStateFromURL.dataSet, dataSet.getDataSet())) { - if (initialStateFromURL.dataSet) { - dataSet.setDataSet(_.cloneDeep(initialStateFromURL.dataSet)); - } else { - const defaultDataSet = await dataSet.getDefaultDataSet(); - if (defaultDataSet) { - dataSet.setDataSet(defaultDataSet); - } - } - } - if (syncConfig.filters === FilterStateStore.APP_STATE) { if ( !initialStateFromURL.filters || @@ -139,10 +119,6 @@ export const connectStorageToQueryState = async ( newState.filters = filterManager.getAppFilters(); } - if (syncConfig.dataSet && changes.dataSet) { - newState.dataSet = dataSet.getDataSet(); - } - return newState; }) ) @@ -167,24 +143,19 @@ export const connectStorageToQueryState = async ( * @param QueryService: either setup or start * @param stateContainer to use for syncing */ -export const connectToQueryState = async ( +export const connectToQueryState = ( { timefilter: { timefilter }, filterManager, queryString, - dataSet, state$, - }: Pick< - QueryStart | QuerySetup, - 'timefilter' | 'filterManager' | 'dataSet' | 'queryString' | 'state$' - >, + }: Pick, stateContainer: BaseStateContainer, syncConfig: { time?: boolean; refreshInterval?: boolean; filters?: FilterStateStore | boolean; query?: boolean; - dataSet?: boolean; } ) => { const syncKeys: Array = []; @@ -210,9 +181,6 @@ export const connectToQueryState = async ( break; } } - if (syncConfig.dataSet) { - syncKeys.push('dataSet'); - } // initial syncing // TODO: @@ -267,11 +235,6 @@ export const connectToQueryState = async ( } } - if (syncConfig.dataSet && !initialState.dataSet) { - initialState.dataSet = await dataSet.getDefaultDataSet(); - initialDirty = true; - } - if (initialDirty) { stateContainer.set({ ...stateContainer.get(), ...initialState }); } @@ -309,16 +272,13 @@ export const connectToQueryState = async ( newState.filters = filterManager.getAppFilters(); } } - if (syncConfig.dataSet && changes.dataSet) { - newState.dataSet = dataSet.getDataSet(); - } return newState; }) ) .subscribe((newState) => { stateContainer.set({ ...stateContainer.get(), ...newState }); }), - stateContainer.state$.subscribe(async (state) => { + stateContainer.state$.subscribe((state) => { updateInProgress = true; // cloneDeep is required because services are mutating passed objects @@ -371,21 +331,6 @@ export const connectToQueryState = async ( } } - if (syncConfig.dataSet) { - const currentDataSet = dataSet.getDataSet(); - if (!_.isEqual(state.dataSet, currentDataSet)) { - if (state.dataSet) { - dataSet.setDataSet(state.dataSet); - } else { - const defaultDataSet = await dataSet.getDefaultDataSet(); - if (defaultDataSet) { - dataSet.setDataSet(defaultDataSet); - stateContainer.set({ ...stateContainer.get(), dataSet: defaultDataSet }); - } - } - } - } - updateInProgress = false; }), ]; diff --git a/src/plugins/data/public/query/state_sync/create_global_query_observable.ts b/src/plugins/data/public/query/state_sync/create_global_query_observable.ts index 440ea836383e..8abcb3ece18d 100644 --- a/src/plugins/data/public/query/state_sync/create_global_query_observable.ts +++ b/src/plugins/data/public/query/state_sync/create_global_query_observable.ts @@ -36,18 +36,15 @@ import { QueryState, QueryStateChange } from './index'; import { createStateContainer } from '../../../../opensearch_dashboards_utils/public'; import { isFilterPinned, compareFilters, COMPARE_ALL_OPTIONS } from '../../../common'; import { QueryStringContract } from '../query_string'; -import { DataSetContract } from '../dataset_manager'; export function createQueryStateObservable({ timefilter: { timefilter }, filterManager, queryString, - dataSet, }: { timefilter: TimefilterSetup; filterManager: FilterManager; queryString: QueryStringContract; - dataSet: DataSetContract; }): Observable<{ changes: QueryStateChange; state: QueryState }> { return new Observable((subscriber) => { const state = createStateContainer({ @@ -55,7 +52,6 @@ export function createQueryStateObservable({ refreshInterval: timefilter.getRefreshInterval(), filters: filterManager.getFilters(), query: queryString.getQuery(), - dataSet: dataSet.getDataSet(), }); let currentChange: QueryStateChange = {}; @@ -64,10 +60,6 @@ export function createQueryStateObservable({ currentChange.query = true; state.set({ ...state.get(), query: queryString.getQuery() }); }), - dataSet.getUpdates$().subscribe(() => { - currentChange.dataSet = true; - state.set({ ...state.get(), dataSet: dataSet.getDataSet() }); - }), timefilter.getTimeUpdate$().subscribe(() => { currentChange.time = true; state.set({ ...state.get(), time: timefilter.getTime() }); diff --git a/src/plugins/data/public/query/state_sync/sync_state_with_url.ts b/src/plugins/data/public/query/state_sync/sync_state_with_url.ts index 3b92cff04d26..67245fd693ab 100644 --- a/src/plugins/data/public/query/state_sync/sync_state_with_url.ts +++ b/src/plugins/data/public/query/state_sync/sync_state_with_url.ts @@ -46,22 +46,17 @@ const GLOBAL_STATE_STORAGE_KEY = '_g'; * @param osdUrlStateStorage to use for syncing */ export const syncQueryStateWithUrl = ( - query: Pick< - QueryStart | QuerySetup, - 'filterManager' | 'timefilter' | 'queryString' | 'dataSet' | 'state$' - >, + query: Pick, osdUrlStateStorage: IOsdUrlStateStorage ) => { const { timefilter: { timefilter }, filterManager, - dataSet, } = query; const defaultState: QueryState = { time: timefilter.getTime(), refreshInterval: timefilter.getRefreshInterval(), filters: filterManager.getGlobalFilters(), - dataSet: dataSet.getDataSet(), }; // retrieve current state from `_g` url @@ -83,7 +78,6 @@ export const syncQueryStateWithUrl = ( refreshInterval: true, time: true, filters: FilterStateStore.GLOBAL_STATE, - dataSet: true, }); // if there weren't any initial state in url, @@ -113,8 +107,8 @@ export const syncQueryStateWithUrl = ( start(); return { - stop: async () => { - (await stopSyncingWithStateContainer)(); + stop: () => { + stopSyncingWithStateContainer(); stopSyncingWithUrl(); }, hasInheritedQueryFromUrl, diff --git a/src/plugins/data/public/query/state_sync/types.ts b/src/plugins/data/public/query/state_sync/types.ts index 8134a7208f13..0ee0ad1c463e 100644 --- a/src/plugins/data/public/query/state_sync/types.ts +++ b/src/plugins/data/public/query/state_sync/types.ts @@ -28,7 +28,7 @@ * under the License. */ -import { Filter, RefreshInterval, TimeRange, Query, SimpleDataSet } from '../../../common'; +import { Filter, RefreshInterval, TimeRange, Query } from '../../../common'; /** * All query state service state @@ -38,7 +38,6 @@ export interface QueryState { refreshInterval?: RefreshInterval; filters?: Filter[]; query?: Query; - dataSet?: SimpleDataSet; } type QueryStateChangePartial = { diff --git a/src/plugins/data/public/search/search_service.ts b/src/plugins/data/public/search/search_service.ts index d2a2dd67a6bd..712f437d2e21 100644 --- a/src/plugins/data/public/search/search_service.ts +++ b/src/plugins/data/public/search/search_service.ts @@ -64,8 +64,6 @@ import { createDataFrameCache, dataFrameToSpec, } from '../../common/data_frames'; -import { getQueryService, getUiService } from '../services'; -import { UI_SETTINGS } from '../../common'; /** @internal */ export interface SearchServiceSetupDependencies { @@ -135,21 +133,7 @@ export class SearchService implements Plugin { { fieldFormats, indexPatterns }: SearchServiceStartDependencies ): ISearchStart { const search = ((request, options) => { - const selectedLanguage = getQueryService().queryString.getQuery().language; - const uiService = getUiService(); - const enhancement = uiService.Settings.getQueryEnhancements(selectedLanguage); - uiService.Settings.setUiOverridesByUserQueryLanguage(selectedLanguage); - const isEnhancedEnabled = uiSettings.get(UI_SETTINGS.QUERY_ENHANCEMENTS_ENABLED); - - if (enhancement) { - if (!isEnhancedEnabled) { - notifications.toasts.addWarning( - `Query enhancements are disabled. Please enable to use: ${selectedLanguage}.` - ); - } - return enhancement.search.search(request, options); - } - return this.defaultSearchInterceptor.search(request, options); + return this.searchInterceptor.search(request, options); }) as ISearchGeneric; const loadingCount$ = new BehaviorSubject(0); @@ -172,8 +156,7 @@ export class SearchService implements Plugin { dataFrame.meta.queryConfig.dataSourceId = dataSource?.id; } this.dfCache.set(dataFrame); - const dataSetName = `${dataFrame.meta?.queryConfig?.dataSourceId ?? ''}.${dataFrame.name}`; - const existingIndexPattern = await indexPatterns.get(dataSetName, true); + const existingIndexPattern = indexPatterns.getByTitle(dataFrame.name!, true); const dataSet = await indexPatterns.create( dataFrameToSpec(dataFrame, existingIndexPattern?.id), !existingIndexPattern?.id @@ -183,6 +166,8 @@ export class SearchService implements Plugin { }, clear: () => { if (this.dfCache.get() === undefined) return; + // name because the id is not unique for temporary index pattern created + indexPatterns.clearCache(this.dfCache.get()!.name, false); this.dfCache.clear(); }, }; diff --git a/src/plugins/data/public/ui/_index.scss b/src/plugins/data/public/ui/_index.scss index 4aa425041f58..f7c738b8d09f 100644 --- a/src/plugins/data/public/ui/_index.scss +++ b/src/plugins/data/public/ui/_index.scss @@ -2,6 +2,5 @@ @import "./typeahead/index"; @import "./saved_query_management/index"; @import "./query_string_input/index"; -@import "./dataset_navigator/index"; @import "./query_editor/index"; @import "./shard_failure_modal/shard_failure_modal"; diff --git a/src/plugins/data/public/ui/dataset_navigator/_dataset_navigator.scss b/src/plugins/data/public/ui/dataset_navigator/_dataset_navigator.scss deleted file mode 100644 index 73a8c8719500..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/_dataset_navigator.scss +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -.datasetNavigator { - min-width: 350px; - border-bottom: $euiBorderThin !important; -} - -.dataSetNavigatorFormWrapper { - padding: $euiSizeS; -} - -.dataSetNavigator__loading { - padding: $euiSizeS; -} diff --git a/src/plugins/data/public/ui/dataset_navigator/_index.scss b/src/plugins/data/public/ui/dataset_navigator/_index.scss deleted file mode 100644 index 53acdffad43d..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/_index.scss +++ /dev/null @@ -1 +0,0 @@ -@import "./dataset_navigator"; diff --git a/src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx deleted file mode 100644 index c1ab4b3f846b..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import React from 'react'; -import { HttpStart, SavedObjectsClientContract } from 'opensearch-dashboards/public'; -import { DataSetNavigator, DataSetNavigatorProps } from './'; -import { DataSetContract } from '../../query'; - -// Updated function signature to include additional dependencies -export function createDataSetNavigator( - savedObjectsClient: SavedObjectsClientContract, - http: HttpStart, - dataSet: DataSetContract -) { - // Return a function that takes props, omitting the dependencies from the props type - return (props: Omit) => ( - - ); -} diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx deleted file mode 100644 index ad4490ff134d..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ /dev/null @@ -1,736 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import React, { useCallback, useEffect, useState } from 'react'; -import { - EuiButton, - EuiButtonEmpty, - EuiContextMenu, - EuiForm, - EuiFormRow, - EuiLoadingSpinner, - EuiPanel, - EuiPopover, - EuiSelect, -} from '@elastic/eui'; -import { HttpStart, SavedObjectsClientContract } from 'opensearch-dashboards/public'; -import _ from 'lodash'; -import { i18n } from '@osd/i18n'; -import { - SIMPLE_DATA_SET_TYPES, - SIMPLE_DATA_SOURCE_TYPES, - SimpleDataSet, - SimpleDataSource, - SimpleObject, -} from '../../../common'; -import { - useLoadDatabasesToCache, - useLoadExternalDataSourcesToCache, - useLoadTablesToCache, -} from './lib/catalog_cache/cache_loader'; -import { CatalogCacheManager } from './lib/catalog_cache/cache_manager'; -import { CachedDataSourceStatus, DirectQueryLoadingStatus } from './lib/types'; -import { - getIndexPatterns, - getNotifications, - getQueryService, - getSearchService, - getUiService, -} from '../../services'; -import { - fetchDataSources, - fetchIndexPatterns, - fetchIndices, - isCatalogCacheFetching, - fetchIfExternalDataSourcesEnabled, -} from './lib'; -import { useDataSetManager } from '../search_bar/lib/use_dataset_manager'; -import { DataSetContract } from '../../query'; - -export interface DataSetNavigatorProps { - savedObjectsClient?: SavedObjectsClientContract; - http?: HttpStart; - dataSet?: DataSetContract; -} - -interface DataSetNavigatorState { - isMounted: boolean; - isOpen: boolean; - isLoading: boolean; - isExternalDataSourcesEnabled: boolean; - indexPatterns: any[]; - dataSources: SimpleDataSource[]; - externalDataSources: SimpleDataSource[]; - cachedDatabases: any[]; - cachedTables: SimpleObject[]; -} - -interface SelectedDataSetState extends SimpleDataSet { - isExternal: boolean; - database?: any | undefined; -} - -export const DataSetNavigator = (props: DataSetNavigatorProps) => { - const { savedObjectsClient, http, dataSet: dataSetManager } = props; - const searchService = getSearchService(); - const queryService = getQueryService(); - const uiService = getUiService(); - const indexPatternsService = getIndexPatterns(); - const notifications = getNotifications(); - - const { dataSet } = useDataSetManager({ dataSetManager: dataSetManager! }); - - const [navigatorState, setNavigatorState] = useState({ - isOpen: false, - isLoading: false, - isMounted: false, - isExternalDataSourcesEnabled: false, - dataSources: [], - externalDataSources: [], - indexPatterns: [], - cachedDatabases: [], - cachedTables: [], - }); - - const [selectedDataSetState, setSelectedDataSetState] = useState({ - id: dataSet?.id ?? '', - title: dataSet?.title, - type: dataSet?.type, - isExternal: false, - dataSourceRef: dataSet?.dataSourceRef, - database: undefined, - timeFieldName: dataSet?.timeFieldName, - fields: dataSet?.fields, - }); - - const { - loadStatus: dataSourcesLoadStatus, - loadExternalDataSources: startLoadingDataSources, - } = useLoadExternalDataSourcesToCache(http!, notifications); - const { - loadStatus: databasesLoadStatus, - startLoading: startLoadingDatabases, - } = useLoadDatabasesToCache(http!, notifications); - const { loadStatus: tablesLoadStatus, startLoading: startLoadingTables } = useLoadTablesToCache( - http!, - notifications - ); - - const onClick = () => { - setNavigatorState((prevState) => ({ - ...prevState, - isOpen: !prevState.isOpen, - })); - }; - - const isLoading = (loading: boolean) => { - setNavigatorState((prevState) => ({ - ...prevState, - isLoading: loading, - })); - }; - - const closePopover = () => { - setNavigatorState((prevState) => ({ - ...prevState, - isOpen: false, - externalDataSources: [], - cachedDatabases: [], - cachedTables: [], - })); - }; - - const onRefresh = () => { - if (!isCatalogCacheFetching(dataSourcesLoadStatus) && navigatorState.dataSources.length > 0) { - startLoadingDataSources(navigatorState.dataSources.map((dataSource) => dataSource.id)); - } - }; - - useEffect(() => { - setNavigatorState((prevState) => ({ ...prevState, isMounted: true, isLoading: true })); - Promise.all([ - fetchIndexPatterns(savedObjectsClient!, ''), - fetchDataSources(savedObjectsClient!), - fetchIfExternalDataSourcesEnabled(http!), - ]) - .then(([indexPatterns, dataSources, isExternalDataSourcesEnabled]) => { - if (!navigatorState.isMounted) return; - setNavigatorState((prevState) => ({ - ...prevState, - isExternalDataSourcesEnabled, - indexPatterns, - dataSources, - })); - const selectedPattern = indexPatterns.find( - (pattern) => pattern.id === props.dataSet?.getDataSet()?.id - ); - if (selectedPattern) { - setSelectedDataSetState({ - id: selectedPattern.id, - title: selectedPattern.title, - type: SIMPLE_DATA_SET_TYPES.INDEX_PATTERN, - timeFieldName: selectedPattern.timeFieldName, - fields: selectedPattern.fields, - ...(selectedPattern.dataSourceRef - ? { - dataSourceRef: { - id: selectedPattern.dataSourceRef.id, - name: selectedPattern.dataSourceRef.name, - type: selectedPattern.dataSourceRef.type, - }, - } - : { dataSource: undefined }), - database: undefined, - isExternal: false, - }); - } - }) - .finally(() => { - isLoading(false); - }); - return () => { - setNavigatorState((prevState) => ({ ...prevState, isMounted: false })); - }; - }, [savedObjectsClient, http, navigatorState.isMounted, props.dataSet]); - - useEffect(() => { - const status = dataSourcesLoadStatus.toLowerCase(); - const externalDataSourcesCache = CatalogCacheManager.getExternalDataSourcesCache(); - if (status === DirectQueryLoadingStatus.SUCCESS) { - setNavigatorState((prevState) => ({ - ...prevState, - externalDataSources: externalDataSourcesCache.externalDataSources.map((ds) => ({ - id: ds.dataSourceRef, - name: ds.name, - type: SIMPLE_DATA_SOURCE_TYPES.EXTERNAL, - })), - })); - } else if ( - status === DirectQueryLoadingStatus.CANCELED || - status === DirectQueryLoadingStatus.FAILED - ) { - setNavigatorState((prevState) => ({ ...prevState, failed: true })); - } - }, [dataSourcesLoadStatus]); - - useEffect(() => { - const status = databasesLoadStatus.toLowerCase(); - if (selectedDataSetState.isExternal && selectedDataSetState.dataSourceRef) { - const dataSourceCache = CatalogCacheManager.getOrCreateDataSource( - selectedDataSetState.dataSourceRef.name, - selectedDataSetState.dataSourceRef.id - ); - if (status === DirectQueryLoadingStatus.SUCCESS) { - setNavigatorState((prevState) => ({ - ...prevState, - cachedDatabases: dataSourceCache.databases, - })); - } else if ( - status === DirectQueryLoadingStatus.CANCELED || - status === DirectQueryLoadingStatus.FAILED - ) { - setNavigatorState((prevState) => ({ ...prevState, failed: true })); - } - } - }, [databasesLoadStatus, selectedDataSetState.isExternal, selectedDataSetState.dataSourceRef]); - - const handleSelectExternalDataSource = useCallback( - async (dataSource) => { - if (selectedDataSetState.isExternal && dataSource) { - const dataSourceCache = CatalogCacheManager.getOrCreateDataSource( - dataSource.name, - dataSource.id - ); - if ( - (dataSourceCache.status === CachedDataSourceStatus.Empty || - dataSourceCache.status === CachedDataSourceStatus.Failed) && - !isCatalogCacheFetching(databasesLoadStatus) - ) { - await startLoadingDatabases({ - dataSourceName: dataSource.name, - dataSourceMDSId: dataSource.id, - }); - } else if (dataSourceCache.status === CachedDataSourceStatus.Updated) { - setNavigatorState((prevState) => ({ - ...prevState, - cachedDatabases: dataSourceCache.databases, - })); - setSelectedDataSetState((prevState) => ({ - ...prevState, - dataSource, - isExternal: true, - })); - } - } - }, - [databasesLoadStatus, selectedDataSetState.isExternal, startLoadingDatabases] - ); - - // Start loading tables for selected database - const handleSelectExternalDatabase = useCallback( - (externalDatabase: SimpleDataSource) => { - if (selectedDataSetState.dataSourceRef && externalDatabase) { - let databaseCache; - try { - databaseCache = CatalogCacheManager.getDatabase( - selectedDataSetState.dataSourceRef.name, - externalDatabase.name, - selectedDataSetState.dataSourceRef.id - ); - } catch (error) { - return; - } - if ( - databaseCache.status === CachedDataSourceStatus.Empty || - (databaseCache.status === CachedDataSourceStatus.Failed && - !isCatalogCacheFetching(tablesLoadStatus)) - ) { - startLoadingTables({ - dataSourceName: selectedDataSetState.dataSourceRef.name, - databaseName: externalDatabase.name, - dataSourceMDSId: selectedDataSetState.dataSourceRef.id, - }); - setSelectedDataSetState((prevState) => ({ - ...prevState, - database: externalDatabase, - })); - } else if (databaseCache.status === CachedDataSourceStatus.Updated) { - setNavigatorState((prevState) => ({ - ...prevState, - cachedTables: databaseCache.tables, - })); - } - } - }, - [selectedDataSetState.dataSourceRef, tablesLoadStatus, startLoadingTables] - ); - - // Retrieve tables from cache upon success - useEffect(() => { - if ( - selectedDataSetState.dataSourceRef && - selectedDataSetState.isExternal && - selectedDataSetState.database - ) { - const tablesStatus = tablesLoadStatus.toLowerCase(); - let databaseCache; - try { - databaseCache = CatalogCacheManager.getDatabase( - selectedDataSetState.dataSourceRef.name, - selectedDataSetState.database, - selectedDataSetState.dataSourceRef.id - ); - } catch (error) { - return; - } - if (tablesStatus === DirectQueryLoadingStatus.SUCCESS) { - setNavigatorState((prevState) => ({ - ...prevState, - cachedTables: databaseCache.tables, - })); - } else if ( - tablesStatus === DirectQueryLoadingStatus.CANCELED || - tablesStatus === DirectQueryLoadingStatus.FAILED - ) { - notifications.toasts.addWarning('Error loading tables'); - } - } - }, [ - tablesLoadStatus, - selectedDataSetState.dataSourceRef, - selectedDataSetState.isExternal, - selectedDataSetState.database, - notifications.toasts, - ]); - - const handleSelectedDataSource = useCallback( - async (source: SimpleDataSource) => { - if (source) { - isLoading(true); - const indices = await fetchIndices(searchService, source.id); - setSelectedDataSetState((prevState) => ({ - ...prevState, - isExternal: false, - dataSourceRef: { - ...source, - indices: indices.map((indexName: string) => ({ - id: indexName, - title: indexName, - dataSourceRef: { - id: source.id, - name: source.name, - type: source.type, - }, - })), - }, - })); - isLoading(false); - } - }, - [searchService] - ); - - const handleSelectedObject = useCallback( - async (object) => { - isLoading(true); - if (object) { - const fields = await indexPatternsService.getFieldsForWildcard({ - pattern: object.title, - dataSourceId: object.dataSourceRef?.id, - }); - - const timeFields = fields.filter((field: any) => field.type === 'date'); - const timeFieldName = timeFields?.length > 0 ? timeFields[0].name : undefined; - setSelectedDataSetState((prevState) => ({ - ...prevState, - id: object.id, - title: object.title, - fields, - timeFields, - timeFieldName, - type: SIMPLE_DATA_SET_TYPES.TEMPORARY, - })); - isLoading(false); - } - }, - [indexPatternsService] - ); - - const handleSelectedDataSet = useCallback( - async (ds?: SimpleDataSet) => { - const selectedDataSet = ds ?? selectedDataSetState; - if (!selectedDataSet || !selectedDataSet.id) return; - - const language = uiService.Settings.getUserQueryLanguage(); - const queryEnhancements = uiService.Settings.getQueryEnhancements(language); - const initialInput = queryEnhancements?.searchBar?.queryStringInput?.initialValue; - - // Update query - const query = initialInput - ? initialInput.replace('', selectedDataSet.title!) - : ''; - uiService.Settings.setUserQueryString(query); - queryService.queryString.setQuery({ query, language }); - - // Update dataset - queryService.dataSet.setDataSet(selectedDataSet); - - // Add to recent datasets - CatalogCacheManager.addRecentDataSet({ - id: selectedDataSet.id, - title: selectedDataSet.title ?? selectedDataSet.id!, - dataSourceRef: selectedDataSet.dataSourceRef, - timeFieldName: selectedDataSet.timeFieldName, - type: selectedDataSet.type, - }); - - // Update data set manager - dataSetManager!.setDataSet({ - id: selectedDataSet.id, - title: selectedDataSet.title, - dataSourceRef: selectedDataSet.dataSourceRef, - timeFieldName: selectedDataSet.timeFieldName, - type: selectedDataSet.type, - }); - - closePopover(); - }, - [ - dataSetManager, - queryService.dataSet, - queryService.queryString, - selectedDataSetState, - uiService.Settings, - ] - ); - - const indexPatternsLabel = i18n.translate('data.query.dataSetNavigator.indexPatternsName', { - defaultMessage: 'Index patterns', - }); - const indicesLabel = i18n.translate('data.query.dataSetNavigator.indicesName', { - defaultMessage: 'Indexes', - }); - const S3DataSourcesLabel = i18n.translate('data.query.dataSetNavigator.S3DataSourcesLabel', { - defaultMessage: 'S3', - }); - - const createRefreshButton = () => ( - - ); - - const createLoadingSpinner = () => ( - - - - ); - - const createIndexPatternsPanel = () => ({ - id: 1, - title: indexPatternsLabel, - items: navigatorState.indexPatterns.map((indexPattern) => ({ - name: indexPattern.title, - onClick: async () => await handleSelectedDataSet(indexPattern), - })), - content:
{navigatorState.indexPatterns.length === 0 && createLoadingSpinner()}
, - }); - - const createIndexesPanel = () => ({ - id: 2, - title: indicesLabel, - items: [ - ...navigatorState.dataSources.map((dataSource) => ({ - name: dataSource.name, - panel: 3, - onClick: async () => await handleSelectedDataSource(dataSource), - })), - ], - content:
{navigatorState.isLoading && createLoadingSpinner()}
, - }); - - const createDataSourcesPanel = () => ({ - id: 3, - title: selectedDataSetState.dataSourceRef?.name ?? indicesLabel, - items: selectedDataSetState.dataSourceRef?.indices?.map((object) => ({ - name: object.title, - panel: 7, - onClick: async () => - await handleSelectedObject({ ...object, type: SIMPLE_DATA_SET_TYPES.TEMPORARY }), - })), - content:
{navigatorState.isLoading && createLoadingSpinner()}
, - }); - - const createS3DataSourcesPanel = () => ({ - id: 4, - title: ( -
- {S3DataSourcesLabel} - {CatalogCacheManager.getExternalDataSourcesCache().status === - CachedDataSourceStatus.Updated && createRefreshButton()} -
- ), - items: [ - ...navigatorState.externalDataSources.map((dataSource) => ({ - name: dataSource.name, - onClick: async () => await handleSelectExternalDataSource(dataSource), - panel: 5, - })), - ], - content:
{dataSourcesLoadStatus && createLoadingSpinner()}
, - }); - - const createDatabasesPanel = () => ({ - id: 5, - title: selectedDataSetState.dataSourceRef?.name - ? selectedDataSetState.dataSourceRef?.name - : 'Databases', - items: [ - ...navigatorState.externalDataSources.map((db) => ({ - name: db.name, - onClick: async () => { - await handleSelectExternalDatabase(db); - }, - panel: 6, - })), - ], - content:
{isCatalogCacheFetching(databasesLoadStatus) && createLoadingSpinner()}
, - }); - - return ( - - {dataSet?.dataSourceRef?.name - ? `${dataSet.dataSourceRef?.name}::${dataSet?.title}` - : dataSet?.title} - - } - isOpen={navigatorState.isOpen} - closePopover={closePopover} - anchorPosition="downLeft" - panelPaddingSize="none" - > - 0 - ? [ - { - name: 'Recently Used', - panel: 8, - }, - ] - : []), - { - name: indexPatternsLabel, - panel: 1, - }, - { - name: indicesLabel, - panel: 2, - }, - ...(navigatorState.isExternalDataSourcesEnabled - ? [ - { - name: S3DataSourcesLabel, - panel: 4, - onClick: async () => { - const externalDataSourcesCache = CatalogCacheManager.getExternalDataSourcesCache(); - if ( - (externalDataSourcesCache.status === CachedDataSourceStatus.Empty || - externalDataSourcesCache.status === CachedDataSourceStatus.Failed) && - !isCatalogCacheFetching(dataSourcesLoadStatus) && - navigatorState.dataSources.length > 0 - ) { - startLoadingDataSources( - navigatorState.dataSources.map((dataSource) => dataSource.id) - ); - } else if ( - externalDataSourcesCache.status === CachedDataSourceStatus.Updated - ) { - setNavigatorState((prevState) => ({ - ...prevState, - externalDataSources: externalDataSourcesCache.externalDataSources.map( - (ds) => ({ - id: ds.name, - name: ds.name, - type: SIMPLE_DATA_SOURCE_TYPES.EXTERNAL, - }) - ), - })); - } - }, - }, - ] - : []), - ], - }, - createIndexPatternsPanel(), - createIndexesPanel(), - createDataSourcesPanel(), - createS3DataSourcesPanel(), - createDatabasesPanel(), - { - id: 6, - title: selectedDataSetState.database ? selectedDataSetState.database : 'Tables', - items: [ - ...navigatorState.cachedTables.map((table) => ({ - name: table.title, - onClick: async () => { - setSelectedDataSetState((prevState) => ({ - ...prevState, - object: { - id: `${selectedDataSetState.dataSourceRef!.name}.${ - selectedDataSetState.database - }.${table.title}`, - title: `${selectedDataSetState.dataSourceRef!.name}.${ - selectedDataSetState.database - }.${table.title}`, - dataSource: { - id: selectedDataSetState.dataSourceRef!.id, - name: selectedDataSetState.dataSourceRef!.name, - type: selectedDataSetState.dataSourceRef!.type, - }, - type: SIMPLE_DATA_SET_TYPES.TEMPORARY_ASYNC, - }, - })); - }, - })), - ], - content: ( -
{isCatalogCacheFetching(tablesLoadStatus) && createLoadingSpinner()}
- ), - }, - { - id: 7, - title: selectedDataSetState.title, - content: - navigatorState.isLoading || !selectedDataSetState.title ? ( -
{createLoadingSpinner()}
- ) : ( - - - 0 - ? [ - ...selectedDataSetState.timeFields!.map((field: any) => ({ - value: field.name, - text: field.name, - })), - ] - : []), - { value: 'no-time-filter', text: "I don't want to use a time filter" }, - ]} - onChange={(event) => { - setSelectedDataSetState((prevState) => ({ - ...prevState, - timeFieldName: - event.target.value !== 'no-time-filter' - ? event.target.value - : undefined, - })); - }} - aria-label="Select a date field" - /> - - { - await handleSelectedDataSet(); - }} - > - Select - - - ), - }, - { - id: 8, - title: 'Recently Used', - items: CatalogCacheManager.getRecentDataSets().map((ds) => ({ - name: ds.title, - onClick: async () => { - setSelectedDataSetState({ - id: ds.id ?? ds.title, - title: ds.title, - dataSourceRef: ds.dataSourceRef, - database: undefined, - isExternal: !ds.dataSourceRef?.type?.startsWith('data-source'), - timeFieldName: ds.timeFieldName, - }); - await handleSelectedDataSet(); - }, - })), - }, - ]} - /> - - ); -}; - -// eslint-disable-next-line import/no-default-export -export default DataSetNavigator; diff --git a/src/plugins/data/public/ui/dataset_navigator/index.tsx b/src/plugins/data/public/ui/dataset_navigator/index.tsx deleted file mode 100644 index e98e52c8421f..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/index.tsx +++ /dev/null @@ -1,7 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -export { DataSetNavigator, DataSetNavigatorProps } from './dataset_navigator'; -export { createDataSetNavigator } from './create_dataset_navigator'; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_intercept.ts b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_intercept.ts deleted file mode 100644 index 0526cfd51212..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_intercept.ts +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { HttpFetchOptionsWithPath, IHttpInterceptController } from 'opensearch-dashboards/public'; -import { SECURITY_DASHBOARDS_LOGOUT_URL } from '../constants'; -import { CatalogCacheManager } from './cache_manager'; - -export function catalogRequestIntercept(): any { - return ( - fetchOptions: Readonly, - _controller: IHttpInterceptController - ) => { - if (fetchOptions.path.includes(SECURITY_DASHBOARDS_LOGOUT_URL)) { - // Clears all user catalog cache details - CatalogCacheManager.clearDataSourceCache(); - CatalogCacheManager.clearAccelerationsCache(); - CatalogCacheManager.clearExternalDataSourcesCache(); - CatalogCacheManager.clearRecentDataSetsCache(); - } - }; -} diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_loader.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_loader.tsx deleted file mode 100644 index 9a42bfc74aa8..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_loader.tsx +++ /dev/null @@ -1,470 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { useEffect, useRef, useState } from 'react'; -import { HttpStart, NotificationsStart } from 'opensearch-dashboards/public'; -import { ASYNC_POLLING_INTERVAL, SPARK_HIVE_TABLE_REGEX, SPARK_PARTITION_INFO } from '../constants'; -import { - AsyncPollingResult, - CachedColumn, - CachedDataSourceStatus, - CachedTable, - LoadCacheType, - StartLoadingParams, - DirectQueryLoadingStatus, - DirectQueryRequest, -} from '../types'; -import { getAsyncSessionId, setAsyncSessionId } from '../utils/query_session_utils'; -import { - addBackticksIfNeeded, - combineSchemaAndDatarows, - get as getObjValue, - formatError, -} from '../utils/shared'; -import { usePolling } from '../utils/use_polling'; -import { SQLService } from '../requests/sql'; -import { CatalogCacheManager } from './cache_manager'; -import { fetchExternalDataSources } from '../utils'; - -export const updateDatabasesToCache = ( - dataSourceName: string, - pollingResult: AsyncPollingResult, - dataSourceMDSId?: string -) => { - const cachedDataSource = CatalogCacheManager.getOrCreateDataSource( - dataSourceName, - dataSourceMDSId - ); - - const currentTime = new Date().toUTCString(); - - if (!pollingResult) { - CatalogCacheManager.addOrUpdateDataSource( - { - ...cachedDataSource, - databases: [], - lastUpdated: currentTime, - status: CachedDataSourceStatus.Failed, - ...(dataSourceMDSId && { dataSourceMDSId }), - }, - dataSourceMDSId - ); - return; - } - - const combinedData = combineSchemaAndDatarows(pollingResult.schema, pollingResult.datarows); - const newDatabases = combinedData.map((row: any) => ({ - name: row.namespace, - tables: [], - lastUpdated: '', - status: CachedDataSourceStatus.Empty, - })); - - CatalogCacheManager.addOrUpdateDataSource( - { - ...cachedDataSource, - databases: newDatabases, - lastUpdated: currentTime, - status: CachedDataSourceStatus.Updated, - ...(dataSourceMDSId && { dataSourceMDSId }), - }, - dataSourceMDSId - ); -}; - -export const updateTablesToCache = ( - dataSourceName: string, - databaseName: string, - pollingResult: AsyncPollingResult, - dataSourceMDSId?: string -) => { - try { - const cachedDatabase = CatalogCacheManager.getDatabase( - dataSourceName, - databaseName, - dataSourceMDSId - ); - const currentTime = new Date().toUTCString(); - - if (!pollingResult) { - CatalogCacheManager.updateDatabase( - dataSourceName, - { - ...cachedDatabase, - tables: [], - lastUpdated: currentTime, - status: CachedDataSourceStatus.Failed, - }, - dataSourceMDSId - ); - return; - } - - const combinedData = combineSchemaAndDatarows(pollingResult.schema, pollingResult.datarows); - const newTables = combinedData - .filter((row: any) => !SPARK_HIVE_TABLE_REGEX.test(row.information)) - .map((row: any) => ({ - name: row.tableName, - })); - - CatalogCacheManager.updateDatabase( - dataSourceName, - { - ...cachedDatabase, - tables: newTables, - lastUpdated: currentTime, - status: CachedDataSourceStatus.Updated, - }, - dataSourceMDSId - ); - } catch (error) { - // eslint-disable-next-line no-console - console.error(error); - } -}; - -export const updateAccelerationsToCache = ( - dataSourceName: string, - pollingResult: AsyncPollingResult, - dataSourceMDSId?: string -) => { - const currentTime = new Date().toUTCString(); - - if (!pollingResult) { - CatalogCacheManager.addOrUpdateAccelerationsByDataSource({ - name: dataSourceName, - accelerations: [], - lastUpdated: currentTime, - status: CachedDataSourceStatus.Failed, - ...(dataSourceMDSId && { dataSourceMDSId }), - }); - return; - } - - const combinedData = combineSchemaAndDatarows(pollingResult.schema, pollingResult.datarows); - - const newAccelerations: any[] = combinedData.map((row: any) => ({ - flintIndexName: row.flint_index_name, - type: row.kind === 'mv' ? 'materialized' : row.kind, - database: row.database, - table: row.table, - indexName: row.index_name, - autoRefresh: row.auto_refresh, - status: row.status, - })); - - CatalogCacheManager.addOrUpdateAccelerationsByDataSource({ - name: dataSourceName, - accelerations: newAccelerations, - lastUpdated: currentTime, - status: CachedDataSourceStatus.Updated, - ...(dataSourceMDSId && { dataSourceMDSId }), - }); -}; - -export const updateTableColumnsToCache = ( - dataSourceName: string, - databaseName: string, - tableName: string, - pollingResult: AsyncPollingResult, - dataSourceMDSId?: string -) => { - try { - if (!pollingResult) { - return; - } - const cachedDatabase = CatalogCacheManager.getDatabase( - dataSourceName, - databaseName, - dataSourceMDSId - ); - const currentTime = new Date().toUTCString(); - - const combinedData: Array<{ col_name: string; data_type: string }> = combineSchemaAndDatarows( - pollingResult.schema, - pollingResult.datarows - ); - - const tableColumns: CachedColumn[] = []; - for (const row of combinedData) { - if (row.col_name === SPARK_PARTITION_INFO) { - break; - } - tableColumns.push({ - fieldName: row.col_name, - dataType: row.data_type, - }); - } - - const newTables: CachedTable[] = cachedDatabase.tables.map((ts) => - ts.name === tableName ? { ...ts, columns: tableColumns } : { ...ts } - ); - - if (cachedDatabase.status === CachedDataSourceStatus.Updated) { - CatalogCacheManager.updateDatabase( - dataSourceName, - { - ...cachedDatabase, - tables: newTables, - lastUpdated: currentTime, - status: CachedDataSourceStatus.Updated, - }, - dataSourceMDSId - ); - } - } catch (error) { - // eslint-disable-next-line no-console - console.error(error); - } -}; - -export const updateToCache = ( - pollResults: any, - loadCacheType: LoadCacheType, - dataSourceName: string, - databaseName?: string, - tableName?: string, - dataSourceMDSId?: string -) => { - switch (loadCacheType) { - case 'databases': - updateDatabasesToCache(dataSourceName, pollResults, dataSourceMDSId); - break; - case 'tables': - updateTablesToCache(dataSourceName, databaseName!, pollResults, dataSourceMDSId); - break; - case 'accelerations': - updateAccelerationsToCache(dataSourceName, pollResults, dataSourceMDSId); - break; - case 'tableColumns': - updateTableColumnsToCache( - dataSourceName, - databaseName!, - tableName!, - pollResults, - dataSourceMDSId - ); - default: - break; - } -}; - -export const createLoadQuery = ( - loadCacheType: LoadCacheType, - dataSourceName: string, - databaseName?: string, - tableName?: string -) => { - let query; - switch (loadCacheType) { - case 'databases': - query = `SHOW SCHEMAS IN ${addBackticksIfNeeded(dataSourceName)}`; - break; - case 'tables': - query = `SHOW TABLE EXTENDED IN ${addBackticksIfNeeded( - dataSourceName - )}.${addBackticksIfNeeded(databaseName!)} LIKE '*'`; - break; - case 'accelerations': - query = `SHOW FLINT INDEX in ${addBackticksIfNeeded(dataSourceName)}`; - break; - case 'tableColumns': - query = `DESC ${addBackticksIfNeeded(dataSourceName)}.${addBackticksIfNeeded( - databaseName! - )}.${addBackticksIfNeeded(tableName!)}`; - break; - default: - query = ''; - break; - } - return query; -}; - -export const useLoadToCache = ( - loadCacheType: LoadCacheType, - http: HttpStart, - notifications: NotificationsStart -) => { - const sqlService = new SQLService(http); - const [currentDataSourceName, setCurrentDataSourceName] = useState(''); - const [currentDatabaseName, setCurrentDatabaseName] = useState(''); - const [currentTableName, setCurrentTableName] = useState(''); - const [loadStatus, setLoadStatus] = useState( - DirectQueryLoadingStatus.INITIAL - ); - const dataSourceMDSClientId = useRef(''); - - const { - data: pollingResult, - loading: _pollingLoading, - error: pollingError, - startPolling, - stopPolling: stopLoading, - } = usePolling((params) => { - return sqlService.fetchWithJobId(params, dataSourceMDSClientId.current); - }, ASYNC_POLLING_INTERVAL); - - const onLoadingFailed = () => { - setLoadStatus(DirectQueryLoadingStatus.FAILED); - updateToCache( - null, - loadCacheType, - currentDataSourceName, - currentDatabaseName, - currentTableName, - dataSourceMDSClientId.current - ); - }; - - const startLoading = async ({ - dataSourceName, - dataSourceMDSId, - databaseName, - tableName, - }: StartLoadingParams) => { - setLoadStatus(DirectQueryLoadingStatus.SCHEDULED); - setCurrentDataSourceName(dataSourceName); - setCurrentDatabaseName(databaseName); - setCurrentTableName(tableName); - dataSourceMDSClientId.current = dataSourceMDSId || ''; - - let requestPayload: DirectQueryRequest = { - lang: 'sql', - query: createLoadQuery(loadCacheType, dataSourceName, databaseName, tableName), - datasource: dataSourceName, - }; - - const sessionId = getAsyncSessionId(dataSourceName); - if (sessionId) { - requestPayload = { ...requestPayload, sessionId }; - } - await sqlService - .fetch(requestPayload, dataSourceMDSId) - .then((result) => { - setAsyncSessionId(dataSourceName, getObjValue(result, 'sessionId', null)); - if (result.queryId) { - startPolling({ - queryId: result.queryId, - }); - } else { - // eslint-disable-next-line no-console - console.error('No query id found in response'); - onLoadingFailed(); - } - }) - .catch((e) => { - onLoadingFailed(); - const formattedError = formatError( - '', - 'The query failed to execute and the operation could not be complete.', - e.body?.message - ); - notifications.toasts.addError(formattedError, { - title: 'Query Failed', - }); - // eslint-disable-next-line no-console - console.error(e); - }); - }; - - useEffect(() => { - // cancel direct query - if (!pollingResult) return; - const { status: anyCaseStatus, datarows, error } = pollingResult; - const status = anyCaseStatus?.toLowerCase(); - - if (status === DirectQueryLoadingStatus.SUCCESS || datarows) { - setLoadStatus(status); - stopLoading(); - updateToCache( - pollingResult, - loadCacheType, - currentDataSourceName, - currentDatabaseName, - currentTableName, - dataSourceMDSClientId.current - ); - } else if (status === DirectQueryLoadingStatus.FAILED) { - onLoadingFailed(); - stopLoading(); - - const formattedError = formatError( - '', - 'The query failed to execute and the operation could not be complete.', - error - ); - notifications.toasts.addError(formattedError, { - title: 'Query Failed', - }); - } else { - setLoadStatus(status); - } - - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [pollingResult, pollingError]); - - return { loadStatus, startLoading, stopLoading }; -}; - -export const useLoadDatabasesToCache = (http: HttpStart, notifications: NotificationsStart) => { - const { loadStatus, startLoading, stopLoading } = useLoadToCache( - 'databases', - http, - notifications - ); - return { loadStatus, startLoading, stopLoading }; -}; - -export const useLoadTablesToCache = (http: HttpStart, notifications: NotificationsStart) => { - const { loadStatus, startLoading, stopLoading } = useLoadToCache('tables', http, notifications); - return { loadStatus, startLoading, stopLoading }; -}; - -export const useLoadTableColumnsToCache = (http: HttpStart, notifications: NotificationsStart) => { - const { loadStatus, startLoading, stopLoading } = useLoadToCache( - 'tableColumns', - http, - notifications - ); - return { loadStatus, startLoading, stopLoading }; -}; - -export const useLoadAccelerationsToCache = (http: HttpStart, notifications: NotificationsStart) => { - const { loadStatus, startLoading, stopLoading } = useLoadToCache( - 'accelerations', - http, - notifications - ); - return { loadStatus, startLoading, stopLoading }; -}; - -export const useLoadExternalDataSourcesToCache = ( - http: HttpStart, - notifications: NotificationsStart -) => { - const [loadStatus, setLoadStatus] = useState( - DirectQueryLoadingStatus.INITIAL - ); - - const loadExternalDataSources = async (connectedClusters: string[]) => { - setLoadStatus(DirectQueryLoadingStatus.SCHEDULED); - CatalogCacheManager.setExternalDataSourcesLoadingStatus(CachedDataSourceStatus.Empty); - - try { - const externalDataSources = await fetchExternalDataSources(http, connectedClusters); - CatalogCacheManager.updateExternalDataSources(externalDataSources); - setLoadStatus(DirectQueryLoadingStatus.SUCCESS); - CatalogCacheManager.setExternalDataSourcesLoadingStatus(CachedDataSourceStatus.Updated); - } catch (error) { - setLoadStatus(DirectQueryLoadingStatus.FAILED); - CatalogCacheManager.setExternalDataSourcesLoadingStatus(CachedDataSourceStatus.Failed); - notifications.toasts.addError(error, { - title: 'Failed to load external datasources', - }); - } - }; - - return { loadStatus, loadExternalDataSources }; -}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_manager.ts b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_manager.ts deleted file mode 100644 index 3d0a8e0c982d..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_manager.ts +++ /dev/null @@ -1,416 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { - ASYNC_QUERY_EXTERNAL_DATASOURCES_CACHE, - CATALOG_CACHE_VERSION, - RECENT_DATASET_OPTIONS_CACHE, -} from '../constants'; -import { ASYNC_QUERY_ACCELERATIONS_CACHE, ASYNC_QUERY_DATASOURCE_CACHE } from '../utils/shared'; -import { - AccelerationsCacheData, - CachedAccelerationByDataSource, - CachedDataSource, - CachedDataSourceStatus, - CachedDatabase, - DataSetOption, - DataSourceCacheData, - ExternalDataSource, - ExternalDataSourcesCacheData, - RecentDataSetOptionsCacheData, -} from '../types'; -import { SimpleDataSet, SimpleObject } from '../../../../../common'; - -/** - * Manages caching for catalog data including data sources and accelerations. - */ -export class CatalogCacheManager { - /** - * Key for the data source cache in local storage. - */ - private static readonly datasourceCacheKey = ASYNC_QUERY_DATASOURCE_CACHE; - - /** - * Key for the accelerations cache in local storage. - */ - private static readonly accelerationsCacheKey = ASYNC_QUERY_ACCELERATIONS_CACHE; - - /** - * Key for external datasources cache in local storage - */ - private static readonly externalDataSourcesCacheKey = ASYNC_QUERY_EXTERNAL_DATASOURCES_CACHE; - - /** - * Key for recently selected datasets in local storage - */ - private static readonly recentDataSetCacheKey = RECENT_DATASET_OPTIONS_CACHE; - - // TODO: make this an advanced setting - private static readonly maxRecentDataSet = 4; - - /** - * Saves data source cache to local storage. - * @param {DataSourceCacheData} cacheData - The data source cache data to save. - */ - static saveDataSourceCache(cacheData: DataSourceCacheData): void { - sessionStorage.setItem(this.datasourceCacheKey, JSON.stringify(cacheData)); - } - - /** - * Retrieves data source cache from local storage. - * @returns {DataSourceCacheData} The retrieved data source cache. - */ - static getDataSourceCache(): DataSourceCacheData { - const catalogData = sessionStorage.getItem(this.datasourceCacheKey); - - if (catalogData) { - return JSON.parse(catalogData); - } else { - const defaultCacheObject = { version: CATALOG_CACHE_VERSION, dataSources: [] }; - this.saveDataSourceCache(defaultCacheObject); - return defaultCacheObject; - } - } - - /** - * Saves accelerations cache to local storage. - * @param {AccelerationsCacheData} cacheData - The accelerations cache data to save. - */ - static saveAccelerationsCache(cacheData: AccelerationsCacheData): void { - sessionStorage.setItem(this.accelerationsCacheKey, JSON.stringify(cacheData)); - } - - /** - * Retrieves accelerations cache from local storage. - * @returns {AccelerationsCacheData} The retrieved accelerations cache. - */ - static getAccelerationsCache(): AccelerationsCacheData { - const accelerationCacheData = sessionStorage.getItem(this.accelerationsCacheKey); - - if (accelerationCacheData) { - return JSON.parse(accelerationCacheData); - } else { - const defaultCacheObject = { - version: CATALOG_CACHE_VERSION, - dataSources: [], - }; - this.saveAccelerationsCache(defaultCacheObject); - return defaultCacheObject; - } - } - - /** - * Adds or updates a data source in the accelerations cache. - * @param {CachedAccelerationByDataSource} dataSource - The data source to add or update. - */ - static addOrUpdateAccelerationsByDataSource( - dataSource: CachedAccelerationByDataSource, - dataSourceMDSId?: string - ): void { - let index = -1; - const accCacheData = this.getAccelerationsCache(); - if (dataSourceMDSId) { - index = accCacheData.dataSources.findIndex( - (ds: CachedAccelerationByDataSource) => - ds.name === dataSource.name && ds.dataSourceMDSId === dataSourceMDSId - ); - } else { - index = accCacheData.dataSources.findIndex( - (ds: CachedAccelerationByDataSource) => ds.name === dataSource.name - ); - } - if (index !== -1) { - accCacheData.dataSources[index] = dataSource; - } else { - accCacheData.dataSources.push(dataSource); - } - this.saveAccelerationsCache(accCacheData); - } - - /** - * Retrieves accelerations cache from local storage by the datasource name. - * @param {string} dataSourceName - The name of the data source. - * @returns {CachedAccelerationByDataSource} The retrieved accelerations by datasource in cache. - * @throws {Error} If the data source is not found. - */ - static getOrCreateAccelerationsByDataSource( - dataSourceName: string, - dataSourceMDSId?: string - ): CachedAccelerationByDataSource { - const accCacheData = this.getAccelerationsCache(); - let cachedDataSource; - if (dataSourceMDSId) { - cachedDataSource = accCacheData.dataSources.find( - (ds) => ds.name === dataSourceName && ds.dataSourceMDSId === dataSourceMDSId - ); - } else { - cachedDataSource = accCacheData.dataSources.find((ds) => ds.name === dataSourceName); - } - if (cachedDataSource) return cachedDataSource; - else { - let defaultDataSourceObject: CachedAccelerationByDataSource = { - name: dataSourceName, - lastUpdated: '', - status: CachedDataSourceStatus.Empty, - accelerations: [], - }; - - if (dataSourceMDSId !== '' && dataSourceMDSId !== undefined) { - defaultDataSourceObject = { ...defaultDataSourceObject, dataSourceMDSId }; - } - this.addOrUpdateAccelerationsByDataSource(defaultDataSourceObject, dataSourceMDSId); - return defaultDataSourceObject; - } - } - - /** - * Adds or updates a data source in the cache. - * @param {CachedDataSource} dataSource - The data source to add or update. - */ - static addOrUpdateDataSource(dataSource: CachedDataSource, dataSourceMDSId?: string): void { - const cacheData = this.getDataSourceCache(); - let index; - if (dataSourceMDSId) { - index = cacheData.dataSources.findIndex( - (ds: CachedDataSource) => - ds.name === dataSource.name && ds.dataSourceMDSId === dataSourceMDSId - ); - } - index = cacheData.dataSources.findIndex((ds: CachedDataSource) => ds.name === dataSource.name); - if (index !== -1) { - cacheData.dataSources[index] = dataSource; - } else { - cacheData.dataSources.push(dataSource); - } - this.saveDataSourceCache(cacheData); - } - - /** - * Retrieves or creates a data source with the specified name. - * @param {string} dataSourceName - The name of the data source. - * @returns {CachedDataSource} The retrieved or created data source. - */ - static getOrCreateDataSource(dataSourceName: string, dataSourceMDSId?: string): CachedDataSource { - let cachedDataSource; - if (dataSourceMDSId) { - cachedDataSource = this.getDataSourceCache().dataSources.find( - (ds) => ds.dataSourceMDSId === dataSourceMDSId && ds.name === dataSourceName - ); - } else { - cachedDataSource = this.getDataSourceCache().dataSources.find( - (ds) => ds.name === dataSourceName - ); - } - if (cachedDataSource) { - return cachedDataSource; - } else { - let defaultDataSourceObject: CachedDataSource = { - name: dataSourceName, - lastUpdated: '', - status: CachedDataSourceStatus.Empty, - databases: [], - }; - if (dataSourceMDSId !== '' && dataSourceMDSId !== undefined) { - defaultDataSourceObject = { ...defaultDataSourceObject, dataSourceMDSId }; - } - this.addOrUpdateDataSource(defaultDataSourceObject, dataSourceMDSId); - return defaultDataSourceObject; - } - } - - /** - * Retrieves a database from the cache. - * @param {string} dataSourceName - The name of the data source containing the database. - * @param {string} databaseName - The name of the database. - * @returns {CachedDatabase} The retrieved database. - * @throws {Error} If the data source or database is not found. - */ - static getDatabase( - dataSourceName: string, - databaseName: string, - dataSourceMDSId?: string - ): CachedDatabase { - let cachedDataSource; - if (dataSourceMDSId) { - cachedDataSource = this.getDataSourceCache().dataSources.find( - (ds) => ds.dataSourceMDSId === dataSourceMDSId && ds.name === dataSourceName - ); - } else { - cachedDataSource = this.getDataSourceCache().dataSources.find( - (ds) => ds.name === dataSourceName - ); - } - if (!cachedDataSource) { - throw new Error('DataSource not found exception: ' + dataSourceName); - } - - const cachedDatabase = cachedDataSource.databases.find((db) => db.name === databaseName); - if (!cachedDatabase) { - throw new Error('Database not found exception: ' + databaseName); - } - - return cachedDatabase; - } - - /** - * Retrieves a table from the cache. - * @param {string} dataSourceName - The name of the data source containing the database. - * @param {string} databaseName - The name of the database. - * @param {string} tableName - The name of the database. - * @returns {Cachedtable} The retrieved database. - * @throws {Error} If the data source, database or table is not found. - */ - static getTable( - dataSourceName: string, - databaseName: string, - tableName: string, - dataSourceMDSId?: string - ): SimpleObject { - const cachedDatabase = this.getDatabase(dataSourceName, databaseName, dataSourceMDSId); - - const cachedTable = cachedDatabase.tables!.find((table) => table.title === tableName); - if (!cachedTable) { - throw new Error('Table not found exception: ' + tableName); - } - return cachedTable; - } - - /** - * Updates a database in the cache. - * @param {string} dataSourceName - The name of the data source containing the database. - * @param {CachedDatabase} database - The database to be updated. - * @throws {Error} If the data source or database is not found. - */ - static updateDatabase( - dataSourceName: string, - database: CachedDatabase, - dataSourceMDSId?: string - ): void { - let cachedDataSource; - if (dataSourceMDSId) { - cachedDataSource = this.getDataSourceCache().dataSources.find( - (ds) => ds.dataSourceMDSId === dataSourceMDSId && ds.name === dataSourceName - ); - } else { - cachedDataSource = this.getDataSourceCache().dataSources.find( - (ds) => ds.name === dataSourceName - ); - } - - if (!cachedDataSource) { - throw new Error('DataSource not found exception: ' + dataSourceName); - } - - const index = cachedDataSource.databases.findIndex((db) => db.name === database.name); - if (index !== -1) { - cachedDataSource.databases[index] = database; - this.addOrUpdateDataSource(cachedDataSource, dataSourceMDSId); - } else { - throw new Error('Database not found exception: ' + database.name); - } - } - - /** - * Clears the data source cache from local storage. - */ - static clearDataSourceCache(): void { - sessionStorage.removeItem(this.datasourceCacheKey); - this.clearExternalDataSourcesCache(); - } - - /** - * Clears the accelerations cache from local storage. - */ - static clearAccelerationsCache(): void { - sessionStorage.removeItem(this.accelerationsCacheKey); - } - - static saveExternalDataSourcesCache(cacheData: ExternalDataSourcesCacheData): void { - sessionStorage.setItem(this.externalDataSourcesCacheKey, JSON.stringify(cacheData)); - } - - static getExternalDataSourcesCache(): ExternalDataSourcesCacheData { - const externalDataSourcesData = sessionStorage.getItem(this.externalDataSourcesCacheKey); - - if (externalDataSourcesData) { - return JSON.parse(externalDataSourcesData); - } else { - const defaultCacheObject: ExternalDataSourcesCacheData = { - version: CATALOG_CACHE_VERSION, - externalDataSources: [], - lastUpdated: '', - status: CachedDataSourceStatus.Empty, - }; - this.saveExternalDataSourcesCache(defaultCacheObject); - return defaultCacheObject; - } - } - - static updateExternalDataSources(externalDataSources: ExternalDataSource[]): void { - const currentTime = new Date().toUTCString(); - const cacheData = this.getExternalDataSourcesCache(); - cacheData.externalDataSources = externalDataSources; - cacheData.lastUpdated = currentTime; - cacheData.status = CachedDataSourceStatus.Updated; - this.saveExternalDataSourcesCache(cacheData); - } - - static getExternalDataSources(): ExternalDataSourcesCacheData { - return this.getExternalDataSourcesCache(); - } - - static clearExternalDataSourcesCache(): void { - sessionStorage.removeItem(this.externalDataSourcesCacheKey); - } - - static setExternalDataSourcesLoadingStatus(status: CachedDataSourceStatus): void { - const cacheData = this.getExternalDataSourcesCache(); - cacheData.status = status; - this.saveExternalDataSourcesCache(cacheData); - } - - static saveRecentDataSetsCache(cacheData: RecentDataSetOptionsCacheData): void { - sessionStorage.setItem(this.recentDataSetCacheKey, JSON.stringify(cacheData)); - } - - static getRecentDataSetsCache(): RecentDataSetOptionsCacheData { - const recentDataSetOptionsData = sessionStorage.getItem(this.recentDataSetCacheKey); - - if (recentDataSetOptionsData) { - return JSON.parse(recentDataSetOptionsData); - } else { - const defaultCacheObject: RecentDataSetOptionsCacheData = { - version: CATALOG_CACHE_VERSION, - recentDataSets: [], - }; - this.saveRecentDataSetsCache(defaultCacheObject); - return defaultCacheObject; - } - } - - static addRecentDataSet(dataSet: SimpleDataSet): void { - const cacheData = this.getRecentDataSetsCache(); - - cacheData.recentDataSets = cacheData.recentDataSets.filter( - (option) => option.id !== dataSet.id - ); - - cacheData.recentDataSets.push(dataSet); - - if (cacheData.recentDataSets.length > this.maxRecentDataSet) { - cacheData.recentDataSets.shift(); - } - - this.saveRecentDataSetsCache(cacheData); - } - - static getRecentDataSets(): SimpleDataSet[] { - return this.getRecentDataSetsCache().recentDataSets; - } - - static clearRecentDataSetsCache(): void { - sessionStorage.removeItem(this.recentDataSetCacheKey); - } -} diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/index.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/index.tsx deleted file mode 100644 index 5449277b2bd8..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/index.tsx +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -export * from './cache_intercept'; -export * from './cache_loader'; -export * from './cache_manager'; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/constants.ts b/src/plugins/data/public/ui/dataset_navigator/lib/constants.ts deleted file mode 100644 index e22da95ff4c6..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/constants.ts +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -export const ASYNC_QUERY_SESSION_ID = 'async-query-session-id'; -export const ASYNC_QUERY_EXTERNAL_DATASOURCES_CACHE = 'async_query_external_datasources_cache'; -export const RECENT_DATASET_OPTIONS_CACHE = 'recent_dataset_options_cache'; - -export const DATA_SOURCE_NAME_URL_PARAM_KEY = 'datasourceName'; -export const DATA_SOURCE_TYPE_URL_PARAM_KEY = 'datasourceType'; -export const OLLY_QUESTION_URL_PARAM_KEY = 'olly_q'; -export const INDEX_URL_PARAM_KEY = 'indexPattern'; -export const DEFAULT_DATA_SOURCE_TYPE = 'DEFAULT_INDEX_PATTERNS'; -export const DEFAULT_DATA_SOURCE_NAME = 'Default cluster'; -export const DEFAULT_DATA_SOURCE_OBSERVABILITY_DISPLAY_NAME = 'OpenSearch'; -export const DEFAULT_DATA_SOURCE_TYPE_NAME = 'Default Group'; -export const enum QUERY_LANGUAGE { - PPL = 'PPL', - SQL = 'SQL', - DQL = 'DQL', -} -export enum DATA_SOURCE_TYPES { - DEFAULT_CLUSTER_TYPE = DEFAULT_DATA_SOURCE_TYPE, - SPARK = 'spark', - S3Glue = 's3glue', -} -export const ASYNC_POLLING_INTERVAL = 2000; - -export const CATALOG_CACHE_VERSION = '1.0'; -export const ACCELERATION_DEFUALT_SKIPPING_INDEX_NAME = 'skipping'; -export const ACCELERATION_TIME_INTERVAL = [ - { text: 'millisecond(s)', value: 'millisecond' }, - { text: 'second(s)', value: 'second' }, - { text: 'minutes(s)', value: 'minute' }, - { text: 'hour(s)', value: 'hour' }, - { text: 'day(s)', value: 'day' }, - { text: 'week(s)', value: 'week' }, -]; -export const ACCELERATION_REFRESH_TIME_INTERVAL = [ - { text: 'minutes(s)', value: 'minute' }, - { text: 'hour(s)', value: 'hour' }, - { text: 'day(s)', value: 'day' }, - { text: 'week(s)', value: 'week' }, -]; - -export const ACCELERATION_ADD_FIELDS_TEXT = '(add fields here)'; -export const ACCELERATION_INDEX_NAME_REGEX = /^[a-z0-9_]+$/; -export const ACCELERATION_S3_URL_REGEX = /^(s3|s3a):\/\/[a-zA-Z0-9.\-]+/; -export const SPARK_HIVE_TABLE_REGEX = /Provider:\s*hive/; -export const SANITIZE_QUERY_REGEX = /\s+/g; -export const SPARK_TIMESTAMP_DATATYPE = 'timestamp'; -export const SPARK_STRING_DATATYPE = 'string'; - -export const ACCELERATION_INDEX_TYPES = [ - { label: 'Skipping Index', value: 'skipping' }, - { label: 'Covering Index', value: 'covering' }, - { label: 'Materialized View', value: 'materialized' }, -]; - -export const ACC_INDEX_TYPE_DOCUMENTATION_URL = - 'https://github.com/opensearch-project/opensearch-spark/blob/main/docs/index.md'; -export const ACC_CHECKPOINT_DOCUMENTATION_URL = - 'https://github.com/opensearch-project/opensearch-spark/blob/main/docs/index.md#create-index-options'; - -export const ACCELERATION_INDEX_NAME_INFO = `All OpenSearch acceleration indices have a naming format of pattern: \`prefix__suffix\`. They share a common prefix structure, which is \`flint____\`. Additionally, they may have a suffix that varies based on the index type. -##### Skipping Index -- For 'Skipping' indices, a fixed index name 'skipping' is used, and this name cannot be modified by the user. The suffix added to this type is \`_index\`. - - An example of a 'Skipping' index name would be: \`flint_mydatasource_mydb_mytable_skipping_index\`. -##### Covering Index -- 'Covering' indices allow users to specify their index name. The suffix added to this type is \`_index\`. - - For instance, a 'Covering' index name could be: \`flint_mydatasource_mydb_mytable_myindexname_index\`. -##### Materialized View Index -- 'Materialized View' indices also enable users to define their index name, but they do not have a suffix. - - An example of a 'Materialized View' index name might look like: \`flint_mydatasource_mydb_mytable_myindexname\`. -##### Note: -- All user given index names must be in lowercase letters, numbers and underscore. Spaces, commas, and characters -, :, ", *, +, /, \, |, ?, #, >, or < are not allowed. - `; - -export const SKIPPING_INDEX_ACCELERATION_METHODS = [ - { value: 'PARTITION', text: 'Partition' }, - { value: 'VALUE_SET', text: 'Value Set' }, - { value: 'MIN_MAX', text: 'Min Max' }, - { value: 'BLOOM_FILTER', text: 'Bloom Filter' }, -]; - -export const ACCELERATION_AGGREGRATION_FUNCTIONS = [ - { label: 'window.start' }, - { label: 'count' }, - { label: 'sum' }, - { label: 'avg' }, - { label: 'max' }, - { label: 'min' }, -]; - -export const SPARK_PARTITION_INFO = `# Partition Information`; -export const OBS_DEFAULT_CLUSTER = 'observability-default'; // prefix key for generating data source id for default cluster in data selector -export const OBS_S3_DATA_SOURCE = 'observability-s3'; // prefix key for generating data source id for s3 data sources in data selector -export const S3_DATA_SOURCE_GROUP_DISPLAY_NAME = 'Amazon S3'; // display group name for Amazon-managed-s3 data sources in data selector -export const S3_DATA_SOURCE_GROUP_SPARK_DISPLAY_NAME = 'Spark'; // display group name for OpenSearch-spark-s3 data sources in data selector -export const SECURITY_DASHBOARDS_LOGOUT_URL = '/logout'; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/hooks/direct_query_hook.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/hooks/direct_query_hook.tsx deleted file mode 100644 index a2b05f47e9ee..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/hooks/direct_query_hook.tsx +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { useEffect, useState } from 'react'; -import { HttpStart, NotificationsStart } from 'opensearch-dashboards/public'; -import { ASYNC_POLLING_INTERVAL } from '../constants'; -import { DirectQueryLoadingStatus, DirectQueryRequest } from '../types'; -import { getAsyncSessionId, setAsyncSessionId } from '../utils/query_session_utils'; -import { get as getObjValue, formatError } from '../utils/shared'; -import { usePolling } from '../utils/use_polling'; -import { SQLService } from '../requests/sql'; - -export const useDirectQuery = ( - http: HttpStart, - notifications: NotificationsStart, - dataSourceMDSId?: string -) => { - const sqlService = new SQLService(http); - const [loadStatus, setLoadStatus] = useState( - DirectQueryLoadingStatus.SCHEDULED - ); - - const { - data: pollingResult, - loading: _pollingLoading, - error: pollingError, - startPolling, - stopPolling: stopLoading, - } = usePolling((params) => { - return sqlService.fetchWithJobId(params, dataSourceMDSId || ''); - }, ASYNC_POLLING_INTERVAL); - - const startLoading = (requestPayload: DirectQueryRequest) => { - setLoadStatus(DirectQueryLoadingStatus.SCHEDULED); - - const sessionId = getAsyncSessionId(requestPayload.datasource); - if (sessionId) { - requestPayload = { ...requestPayload, sessionId }; - } - - sqlService - .fetch(requestPayload, dataSourceMDSId) - .then((result) => { - setAsyncSessionId(requestPayload.datasource, getObjValue(result, 'sessionId', null)); - if (result.queryId) { - startPolling({ - queryId: result.queryId, - }); - } else { - // eslint-disable-next-line no-console - console.error('No query id found in response'); - setLoadStatus(DirectQueryLoadingStatus.FAILED); - } - }) - .catch((e) => { - setLoadStatus(DirectQueryLoadingStatus.FAILED); - const formattedError = formatError( - '', - 'The query failed to execute and the operation could not be complete.', - e.body?.message - ); - notifications.toasts.addError(formattedError, { - title: 'Query Failed', - }); - // eslint-disable-next-line no-console - console.error(e); - }); - }; - - useEffect(() => { - // cancel direct query - if (!pollingResult) return; - const { status: anyCaseStatus, datarows, error } = pollingResult; - const status = anyCaseStatus?.toLowerCase(); - - if (status === DirectQueryLoadingStatus.SUCCESS || datarows) { - setLoadStatus(status); - stopLoading(); - } else if (status === DirectQueryLoadingStatus.FAILED) { - setLoadStatus(status); - stopLoading(); - const formattedError = formatError( - '', - 'The query failed to execute and the operation could not be complete.', - error - ); - notifications.toasts.addError(formattedError, { - title: 'Query Failed', - }); - } else { - setLoadStatus(status); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [pollingResult, pollingError, stopLoading]); - - return { loadStatus, startLoading, stopLoading, pollingResult }; -}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/index.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/index.tsx deleted file mode 100644 index 771fbd6eef3a..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/index.tsx +++ /dev/null @@ -1,9 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -export * from './catalog_cache'; -export * from './hooks'; -export * from './requests'; -export * from './utils'; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/requests/sql.ts b/src/plugins/data/public/ui/dataset_navigator/lib/requests/sql.ts deleted file mode 100644 index f2c9c30c79b9..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/requests/sql.ts +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { HttpStart } from 'opensearch-dashboards/public'; -import { DirectQueryRequest } from '../types'; - -export class SQLService { - private http: HttpStart; - - constructor(http: HttpStart) { - this.http = http; - } - - fetch = async ( - params: DirectQueryRequest, - dataSourceMDSId?: string, - errorHandler?: (error: any) => void - ) => { - const query = { - dataSourceMDSId, - }; - return this.http - .post('/api/observability/query/jobs', { - body: JSON.stringify(params), - query, - }) - .catch((error) => { - // eslint-disable-next-line no-console - console.error('fetch error: ', error.body); - if (errorHandler) errorHandler(error); - throw error; - }); - }; - - fetchWithJobId = async ( - params: { queryId: string }, - dataSourceMDSId?: string, - errorHandler?: (error: any) => void - ) => { - return this.http - .get(`/api/observability/query/jobs/${params.queryId}/${dataSourceMDSId ?? ''}`) - .catch((error) => { - // eslint-disable-next-line no-console - console.error('fetch error: ', error.body); - if (errorHandler) errorHandler(error); - throw error; - }); - }; - - deleteWithJobId = async (params: { queryId: string }, errorHandler?: (error: any) => void) => { - return this.http.delete(`/api/observability/query/jobs/${params.queryId}`).catch((error) => { - // eslint-disable-next-line no-console - console.error('delete error: ', error.body); - if (errorHandler) errorHandler(error); - throw error; - }); - }; -} diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/types.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/types.tsx deleted file mode 100644 index 03d844ff5689..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/types.tsx +++ /dev/null @@ -1,330 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { EuiComboBoxOptionOption } from '@elastic/eui'; -import { SimpleDataSet, SimpleObject } from '../../../../common'; - -export enum DirectQueryLoadingStatus { - SUCCESS = 'success', - FAILED = 'failed', - RUNNING = 'running', - SCHEDULED = 'scheduled', - CANCELED = 'canceled', - WAITING = 'waiting', - INITIAL = 'initial', -} - -export interface DirectQueryRequest { - query: string; - lang: string; - datasource: string; - sessionId?: string; -} - -export type AccelerationStatus = 'ACTIVE' | 'INACTIVE'; - -export interface PermissionsConfigurationProps { - roles: Role[]; - selectedRoles: Role[]; - setSelectedRoles: React.Dispatch>; - layout: 'horizontal' | 'vertical'; - hasSecurityAccess: boolean; -} - -export interface TableColumn { - name: string; - dataType: string; -} - -export interface Acceleration { - name: string; - status: AccelerationStatus; - type: string; - database: string; - table: string; - destination: string; - dateCreated: number; - dateUpdated: number; - index: string; - sql: string; -} - -export interface AssociatedObject { - tableName: string; - datasource: string; - id: string; - name: string; - database: string; - type: AssociatedObjectIndexType; - accelerations: CachedAcceleration[] | AssociatedObject; - columns?: CachedColumn[]; -} - -export type Role = EuiComboBoxOptionOption; - -export type DatasourceType = 'S3GLUE' | 'PROMETHEUS'; - -export interface S3GlueProperties { - 'glue.indexstore.opensearch.uri': string; - 'glue.indexstore.opensearch.region': string; -} - -export interface PrometheusProperties { - 'prometheus.uri': string; -} - -export type DatasourceStatus = 'ACTIVE' | 'DISABLED'; - -export interface DatasourceDetails { - allowedRoles: string[]; - name: string; - connector: DatasourceType; - description: string; - properties: S3GlueProperties | PrometheusProperties; - status: DatasourceStatus; -} - -interface AsyncApiDataResponse { - status: string; - schema?: Array<{ name: string; type: string }>; - datarows?: any; - total?: number; - size?: number; - error?: string; -} - -export interface AsyncApiResponse { - data: { - ok: boolean; - resp: AsyncApiDataResponse; - }; -} - -export type PollingCallback = (statusObj: AsyncApiResponse) => void; - -export type AssociatedObjectIndexType = AccelerationIndexType | 'table'; - -export type AccelerationIndexType = 'skipping' | 'covering' | 'materialized'; - -export type LoadCacheType = 'databases' | 'tables' | 'accelerations' | 'tableColumns'; - -export enum CachedDataSourceStatus { - Updated = 'Updated', - Failed = 'Failed', - Empty = 'Empty', -} - -export interface CachedColumn { - fieldName: string; - dataType: string; -} - -export interface CachedDatabase { - name: string; - tables: SimpleObject[]; - lastUpdated: string; // date string in UTC format - status: CachedDataSourceStatus; -} - -export interface CachedDataSource { - name: string; - lastUpdated: string; // date string in UTC format - status: CachedDataSourceStatus; - databases: CachedDatabase[]; - dataSourceMDSId?: string; -} - -export interface DataSourceCacheData { - version: string; - dataSources: CachedDataSource[]; -} - -export interface CachedAcceleration { - flintIndexName: string; - type: AccelerationIndexType; - database: string; - table: string; - indexName: string; - autoRefresh: boolean; - status: string; -} - -export interface CachedAccelerationByDataSource { - name: string; - accelerations: CachedAcceleration[]; - lastUpdated: string; // date string in UTC format - status: CachedDataSourceStatus; - dataSourceMDSId?: string; -} - -export interface AccelerationsCacheData { - version: string; - dataSources: CachedAccelerationByDataSource[]; -} - -export interface PollingSuccessResult { - schema: Array<{ name: string; type: string }>; - datarows: Array>; -} - -export type AsyncPollingResult = PollingSuccessResult | null; - -export type AggregationFunctionType = 'count' | 'sum' | 'avg' | 'max' | 'min' | 'window.start'; - -export interface MaterializedViewColumn { - id: string; - functionName: AggregationFunctionType; - functionParam?: string; - fieldAlias?: string; -} - -export type SkippingIndexAccMethodType = 'PARTITION' | 'VALUE_SET' | 'MIN_MAX' | 'BLOOM_FILTER'; - -export interface SkippingIndexRowType { - id: string; - fieldName: string; - dataType: string; - accelerationMethod: SkippingIndexAccMethodType; -} - -export interface DataTableFieldsType { - id: string; - fieldName: string; - dataType: string; -} - -export interface RefreshIntervalType { - refreshWindow: number; - refreshInterval: string; -} - -export interface WatermarkDelayType { - delayWindow: number; - delayInterval: string; -} - -export interface GroupByTumbleType { - timeField: string; - tumbleWindow: number; - tumbleInterval: string; -} - -export interface MaterializedViewQueryType { - columnsValues: MaterializedViewColumn[]; - groupByTumbleValue: GroupByTumbleType; -} - -export interface FormErrorsType { - dataSourceError: string[]; - databaseError: string[]; - dataTableError: string[]; - skippingIndexError: string[]; - coveringIndexError: string[]; - materializedViewError: string[]; - indexNameError: string[]; - primaryShardsError: string[]; - replicaShardsError: string[]; - refreshIntervalError: string[]; - checkpointLocationError: string[]; - watermarkDelayError: string[]; -} - -export type AccelerationRefreshType = 'autoInterval' | 'manual' | 'manualIncrement'; - -export interface CreateAccelerationForm { - dataSource: string; - database: string; - dataTable: string; - dataTableFields: DataTableFieldsType[]; - accelerationIndexType: AccelerationIndexType; - skippingIndexQueryData: SkippingIndexRowType[]; - coveringIndexQueryData: string[]; - materializedViewQueryData: MaterializedViewQueryType; - accelerationIndexName: string; - primaryShardsCount: number; - replicaShardsCount: number; - refreshType: AccelerationRefreshType; - checkpointLocation: string | undefined; - watermarkDelay: WatermarkDelayType; - refreshIntervalOptions: RefreshIntervalType; - formErrors: FormErrorsType; -} - -export interface LoadCachehookOutput { - loadStatus: DirectQueryLoadingStatus; - startLoading: (params: StartLoadingParams) => void; - stopLoading: () => void; -} - -export interface StartLoadingParams { - dataSourceName: string; - dataSourceMDSId?: string; - databaseName?: string; - tableName?: string; -} - -export interface RenderAccelerationFlyoutParams { - dataSourceName: string; - dataSourceMDSId?: string; - databaseName?: string; - tableName?: string; - handleRefresh?: () => void; -} - -export interface RenderAssociatedObjectsDetailsFlyoutParams { - tableDetail: AssociatedObject; - dataSourceName: string; - handleRefresh?: () => void; - dataSourceMDSId?: string; -} - -export interface RenderAccelerationDetailsFlyoutParams { - acceleration: CachedAcceleration; - dataSourceName: string; - handleRefresh?: () => void; - dataSourceMDSId?: string; -} - -export interface DataSetOption { - id?: string; - name: string; - dataSourceRef?: string; -} - -export interface RecentDataSetOptionsCacheData { - version: string; - recentDataSets: SimpleDataSet[]; -} - -export interface ExternalDataSource { - name: string; - status: string; - dataSourceRef: string; -} - -export interface ExternalDataSourcesCacheData { - version: string; - externalDataSources: ExternalDataSource[]; - lastUpdated: string; - status: CachedDataSourceStatus; -} - -interface DataSourceMeta { - // ref: string; // MDS ID - // dsName?: string; // flint datasource - id: string; - name: string; - type?: string; -} - -export interface DataSet { - id: string | undefined; // index pattern ID, index name, or flintdatasource.database.table - datasource?: DataSourceMeta; - meta?: { - timestampField: string; - mapping?: any; - }; - type?: 'dataSet' | 'temporary'; -} diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_catalog_cache_status.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_catalog_cache_status.ts deleted file mode 100644 index 697852fdd772..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_catalog_cache_status.ts +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -export enum DirectQueryLoadingStatus { - SUCCESS = 'success', - FAILED = 'failed', - RUNNING = 'running', - SCHEDULED = 'scheduled', - CANCELED = 'canceled', - WAITING = 'waiting', - INITIAL = 'initial', -} - -const catalogCacheFetchingStatus = [ - DirectQueryLoadingStatus.RUNNING, - DirectQueryLoadingStatus.WAITING, - DirectQueryLoadingStatus.SCHEDULED, -]; - -export const isCatalogCacheFetching = (...statuses: DirectQueryLoadingStatus[]) => { - return statuses.some((status: DirectQueryLoadingStatus) => - catalogCacheFetchingStatus.includes(status) - ); -}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_data_sources.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_data_sources.ts deleted file mode 100644 index 7a10d7badb58..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_data_sources.ts +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { SavedObjectsClientContract } from 'opensearch-dashboards/public'; -import { SimpleDataSource } from '../../../../../common'; - -export const fetchDataSources = async (client: SavedObjectsClientContract) => { - const resp = await client.find({ - type: 'data-source', - perPage: 10000, - }); - return resp.savedObjects.map((savedObject) => ({ - id: savedObject.id, - name: savedObject.attributes.title, - type: 'data-source', - })) as SimpleDataSource[]; -}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_external_data_sources.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_external_data_sources.ts deleted file mode 100644 index a9272155e602..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_external_data_sources.ts +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { HttpStart } from 'opensearch-dashboards/public'; - -export const fetchIfExternalDataSourcesEnabled = async (http: HttpStart) => { - try { - await http.get('/api/dataconnections'); - return true; - } catch (e) { - return false; - } -}; - -export const fetchExternalDataSources = async (http: HttpStart, connectedClusters: string[]) => { - const results = await Promise.all( - connectedClusters.map(async (cluster) => { - const dataSources = await http.get(`/api/dataconnections/dataSourceMDSId=${cluster}`); - return dataSources - .filter((dataSource) => dataSource.connector === 'S3GLUE') - .map((dataSource) => ({ - name: dataSource.name, - status: dataSource.status, - dataSourceRef: cluster, - })); - }) - ); - - const flattenedResults = results.flat(); - return flattenedResults; -}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_index_patterns.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_index_patterns.ts deleted file mode 100644 index 3f2cd230300e..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_index_patterns.ts +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { SavedObjectsClientContract } from 'opensearch-dashboards/public'; -import { IIndexPattern } from '../.././../..'; -import { SIMPLE_DATA_SOURCE_TYPES, SIMPLE_DATA_SET_TYPES } from '../../../../../common'; - -export const fetchIndexPatterns = async (client: SavedObjectsClientContract, search: string) => { - const resp = await client.find({ - type: 'index-pattern', - fields: ['title', 'timeFieldName', 'references', 'fields'], - search: `${search}*`, - searchFields: ['title'], - perPage: 100, - }); - return resp.savedObjects.map((savedObject) => ({ - id: savedObject.id, - title: savedObject.attributes.title, - timeFieldName: savedObject.attributes.timeFieldName, - fields: savedObject.attributes.fields, - type: SIMPLE_DATA_SET_TYPES.INDEX_PATTERN, - ...(savedObject.references[0] - ? { - dataSourceRef: { - id: savedObject.references[0]?.id, - name: savedObject.references[0]?.name, - type: SIMPLE_DATA_SOURCE_TYPES.DEFAULT, - }, - } - : {}), - })); -}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_indices.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_indices.ts deleted file mode 100644 index ef10c72bc08c..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_indices.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { map } from 'rxjs/operators'; -import { ISearchStart } from '../../../../search'; - -export const fetchIndices = async (search: ISearchStart, dataSourceId?: string) => { - const buildSearchRequest = () => { - const request = { - params: { - ignoreUnavailable: true, - expand_wildcards: 'all', - index: '*', - body: { - size: 0, // no hits - aggs: { - indices: { - terms: { - field: '_index', - size: 100, - }, - }, - }, - }, - }, - dataSourceId, - }; - - return request; - }; - - const searchResponseToArray = (response: any) => { - const { rawResponse } = response; - return rawResponse.aggregations - ? rawResponse.aggregations.indices.buckets.map((bucket: { key: any }) => bucket.key) - : []; - }; - - return search - .getDefaultSearchInterceptor() - .search(buildSearchRequest()) - .pipe(map(searchResponseToArray)) - .toPromise(); -}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/index.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/index.ts deleted file mode 100644 index 7dbe7ec2d4f4..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/utils/index.ts +++ /dev/null @@ -1,13 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -export * from './fetch_catalog_cache_status'; -export * from './fetch_data_sources'; -export * from './fetch_external_data_sources'; -export * from './fetch_index_patterns'; -export * from './fetch_indices'; -export * from './query_session_utils'; -export * from './shared'; -export * from './use_polling'; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/query_session_utils.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/query_session_utils.ts deleted file mode 100644 index beabcb48c197..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/utils/query_session_utils.ts +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { ASYNC_QUERY_SESSION_ID } from '../constants'; - -export const setAsyncSessionId = (dataSource: string, value: string | null) => { - if (value !== null) { - sessionStorage.setItem(`${ASYNC_QUERY_SESSION_ID}_${dataSource}`, value); - } -}; - -export const getAsyncSessionId = (dataSource: string) => { - return sessionStorage.getItem(`${ASYNC_QUERY_SESSION_ID}_${dataSource}`); -}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/shared.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/shared.ts deleted file mode 100644 index 3e4afc94e80b..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/utils/shared.ts +++ /dev/null @@ -1,332 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -/** - * TODO making this method type-safe is nontrivial: if you just define - * `Nested = { [k: string]: Nested | T }` then you can't accumulate because `T` is not `Nested` - * There might be a way to define a recursive type that accumulates cleanly but it's probably not - * worth the effort. - */ - -export function get(obj: Record, path: string, defaultValue?: T): T { - return path.split('.').reduce((acc: any, part: string) => acc && acc[part], obj) || defaultValue; -} - -export function addBackticksIfNeeded(input: string): string { - if (input === undefined) { - return ''; - } - // Check if the string already has backticks - if (input.startsWith('`') && input.endsWith('`')) { - return input; // Return the string as it is - } else { - // Add backticks to the string - return '`' + input + '`'; - } -} - -export function combineSchemaAndDatarows( - schema: Array<{ name: string; type: string }>, - datarows: Array> -): object[] { - const combinedData: object[] = []; - - datarows.forEach((row) => { - const rowData: { [key: string]: string | number | boolean } = {}; - schema.forEach((field, index) => { - rowData[field.name] = row[index]; - }); - combinedData.push(rowData); - }); - - return combinedData; -} - -export const formatError = (name: string, message: string, details: string) => { - return { - name, - message, - body: { - attributes: { - error: { - caused_by: { - type: '', - reason: details, - }, - }, - }, - }, - }; -}; - -// TODO: relocate to a more appropriate location -// Client route -export const PPL_BASE = '/api/ppl'; -export const PPL_SEARCH = '/search'; -export const DSL_BASE = '/api/dsl'; -export const DSL_SEARCH = '/search'; -export const DSL_CAT = '/cat.indices'; -export const DSL_MAPPING = '/indices.getFieldMapping'; -export const DSL_SETTINGS = '/indices.getFieldSettings'; -export const OBSERVABILITY_BASE = '/api/observability'; -export const INTEGRATIONS_BASE = '/api/integrations'; -export const JOBS_BASE = '/query/jobs'; -export const DATACONNECTIONS_BASE = '/api/dataconnections'; -export const EDIT = '/edit'; -export const DATACONNECTIONS_UPDATE_STATUS = '/status'; -export const SECURITY_ROLES = '/api/v1/configuration/roles'; -export const EVENT_ANALYTICS = '/event_analytics'; -export const SAVED_OBJECTS = '/saved_objects'; -export const SAVED_QUERY = '/query'; -export const SAVED_VISUALIZATION = '/vis'; -export const CONSOLE_PROXY = '/api/console/proxy'; -export const SECURITY_PLUGIN_ACCOUNT_API = '/api/v1/configuration/account'; - -// Server route -export const PPL_ENDPOINT = '/_plugins/_ppl'; -export const SQL_ENDPOINT = '/_plugins/_sql'; -export const DSL_ENDPOINT = '/_plugins/_dsl'; -export const DATACONNECTIONS_ENDPOINT = '/_plugins/_query/_datasources'; -export const JOBS_ENDPOINT_BASE = '/_plugins/_async_query'; -export const JOB_RESULT_ENDPOINT = '/result'; - -export const observabilityID = 'observability-logs'; -export const observabilityTitle = 'Observability'; -export const observabilityPluginOrder = 1500; - -export const observabilityApplicationsID = 'observability-applications'; -export const observabilityApplicationsTitle = 'Applications'; -export const observabilityApplicationsPluginOrder = 5090; - -export const observabilityLogsID = 'observability-logs'; -export const observabilityLogsTitle = 'Logs'; -export const observabilityLogsPluginOrder = 5091; - -export const observabilityMetricsID = 'observability-metrics'; -export const observabilityMetricsTitle = 'Metrics'; -export const observabilityMetricsPluginOrder = 5092; - -export const observabilityTracesID = 'observability-traces'; -export const observabilityTracesTitle = 'Traces'; -export const observabilityTracesPluginOrder = 5093; - -export const observabilityNotebookID = 'observability-notebooks'; -export const observabilityNotebookTitle = 'Notebooks'; -export const observabilityNotebookPluginOrder = 5094; - -export const observabilityPanelsID = 'observability-dashboards'; -export const observabilityPanelsTitle = 'Dashboards'; -export const observabilityPanelsPluginOrder = 5095; - -export const observabilityIntegrationsID = 'integrations'; -export const observabilityIntegrationsTitle = 'Integrations'; -export const observabilityIntegrationsPluginOrder = 9020; - -export const observabilityDataConnectionsID = 'datasources'; -export const observabilityDataConnectionsTitle = 'Data sources'; -export const observabilityDataConnectionsPluginOrder = 9030; - -export const queryWorkbenchPluginID = 'opensearch-query-workbench'; -export const queryWorkbenchPluginCheck = 'plugin:queryWorkbenchDashboards'; - -// Shared Constants -export const SQL_DOCUMENTATION_URL = 'https://opensearch.org/docs/latest/search-plugins/sql/index/'; -export const PPL_DOCUMENTATION_URL = - 'https://opensearch.org/docs/latest/search-plugins/sql/ppl/index'; -export const PPL_PATTERNS_DOCUMENTATION_URL = - 'https://github.com/opensearch-project/sql/blob/2.x/docs/user/ppl/cmd/patterns.rst#description'; -export const UI_DATE_FORMAT = 'MM/DD/YYYY hh:mm A'; -export const PPL_DATE_FORMAT = 'YYYY-MM-DD HH:mm:ss.SSSSSS'; -export const OTEL_DATE_FORMAT = 'YYYY-MM-DDTHH:mm:ss'; -export const SPAN_REGEX = /span/; - -export const PROMQL_METRIC_SUBTYPE = 'promqlmetric'; -export const OTEL_METRIC_SUBTYPE = 'openTelemetryMetric'; -export const PPL_METRIC_SUBTYPE = 'metric'; - -export const PPL_SPAN_REGEX = /by\s*span/i; -export const PPL_STATS_REGEX = /\|\s*stats/i; -export const PPL_INDEX_INSERT_POINT_REGEX = /(search source|source|index)\s*=\s*([^|\s]+)(.*)/i; -export const PPL_INDEX_REGEX = /(search source|source|index)\s*=\s*([^|\s]+)/i; -export const PPL_WHERE_CLAUSE_REGEX = /\s*where\s+/i; -export const PPL_NEWLINE_REGEX = /[\n\r]+/g; -export const PPL_DESCRIBE_INDEX_REGEX = /(describe)\s+([^|\s]+)/i; - -// Observability plugin URI -const BASE_OBSERVABILITY_URI = '/_plugins/_observability'; -const BASE_DATACONNECTIONS_URI = '/_plugins/_query/_datasources'; -export const OPENSEARCH_PANELS_API = { - OBJECT: `${BASE_OBSERVABILITY_URI}/object`, -}; -export const OPENSEARCH_DATACONNECTIONS_API = { - DATACONNECTION: `${BASE_DATACONNECTIONS_URI}`, -}; - -// Saved Objects -export const SAVED_OBJECT = '/object'; - -// Color Constants -export const PLOTLY_COLOR = [ - '#3CA1C7', - '#54B399', - '#DB748A', - '#F2BE4B', - '#68CCC2', - '#2A7866', - '#843769', - '#374FB8', - '#BD6F26', - '#4C636F', -]; - -export const LONG_CHART_COLOR = PLOTLY_COLOR[1]; - -export const pageStyles: CSS.Properties = { - float: 'left', - width: '100%', - maxWidth: '1130px', -}; - -export enum VIS_CHART_TYPES { - Bar = 'bar', - HorizontalBar = 'horizontal_bar', - Line = 'line', - Pie = 'pie', - HeatMap = 'heatmap', - Text = 'text', - Histogram = 'histogram', -} - -export const NUMERICAL_FIELDS = ['short', 'integer', 'long', 'float', 'double']; - -export const ENABLED_VIS_TYPES = [ - VIS_CHART_TYPES.Bar, - VIS_CHART_TYPES.HorizontalBar, - VIS_CHART_TYPES.Line, - VIS_CHART_TYPES.Pie, - VIS_CHART_TYPES.HeatMap, - VIS_CHART_TYPES.Text, -]; - -// Live tail constants -export const LIVE_OPTIONS = [ - { - label: '5s', - startTime: 'now-5s', - delayTime: 5000, - }, - { - label: '10s', - startTime: 'now-10s', - delayTime: 10000, - }, - { - label: '30s', - startTime: 'now-30s', - delayTime: 30000, - }, - { - label: '1m', - startTime: 'now-1m', - delayTime: 60000, - }, - { - label: '5m', - startTime: 'now-5m', - delayTime: 60000 * 5, - }, - { - label: '15m', - startTime: 'now-15m', - delayTime: 60000 * 15, - }, - { - label: '30m', - startTime: 'now-30m', - delayTime: 60000 * 30, - }, - { - label: '1h', - startTime: 'now-1h', - delayTime: 60000 * 60, - }, - { - label: '2h', - startTime: 'now-2h', - delayTime: 60000 * 120, - }, -]; - -export const LIVE_END_TIME = 'now'; - -export interface DefaultChartStylesProps { - DefaultModeLine: string; - Interpolation: string; - LineWidth: number; - FillOpacity: number; - MarkerSize: number; - ShowLegend: string; - LegendPosition: string; - LabelAngle: number; - DefaultSortSectors: string; - DefaultModeScatter: string; -} - -export const DEFAULT_CHART_STYLES: DefaultChartStylesProps = { - DefaultModeLine: 'lines+markers', - Interpolation: 'spline', - LineWidth: 0, - FillOpacity: 100, - MarkerSize: 25, - ShowLegend: 'show', - LegendPosition: 'v', - LabelAngle: 0, - DefaultSortSectors: 'largest_to_smallest', - DefaultModeScatter: 'markers', -}; - -export const FILLOPACITY_DIV_FACTOR = 200; -export const SLIDER_MIN_VALUE = 0; -export const SLIDER_MAX_VALUE = 100; -export const SLIDER_STEP = 1; -export const THRESHOLD_LINE_WIDTH = 3; -export const THRESHOLD_LINE_OPACITY = 0.7; -export const MAX_BUCKET_LENGTH = 16; - -export enum BarOrientation { - horizontal = 'h', - vertical = 'v', -} - -export const PLOT_MARGIN = { - l: 30, - r: 5, - b: 30, - t: 50, - pad: 4, -}; - -export const WAITING_TIME_ON_USER_ACTIONS = 300; - -export const VISUALIZATION_ERROR = { - NO_DATA: 'No data found.', - INVALID_DATA: 'Invalid visualization data', - NO_SERIES: 'Add a field to start', - NO_METRIC: 'Invalid Metric MetaData', -}; - -export const S3_DATA_SOURCE_TYPE = 's3glue'; - -export const ASYNC_QUERY_SESSION_ID = 'async-query-session-id'; -export const ASYNC_QUERY_DATASOURCE_CACHE = 'async-query-catalog-cache'; -export const ASYNC_QUERY_ACCELERATIONS_CACHE = 'async-query-acclerations-cache'; - -export const DIRECT_DUMMY_QUERY = 'select 1'; - -export const DEFAULT_START_TIME = 'now-15m'; -export const QUERY_ASSIST_START_TIME = 'now-40y'; -export const QUERY_ASSIST_END_TIME = 'now'; - -export const TIMESTAMP_DATETIME_TYPES = ['date', 'date_nanos']; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/use_polling.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/use_polling.ts deleted file mode 100644 index 74fedd6cf110..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/utils/use_polling.ts +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { useEffect, useRef, useState } from 'react'; - -type FetchFunction = (params?: P) => Promise; - -export interface PollingConfigurations { - tabId: string; -} - -export class UsePolling { - public data: T | null = null; - public error: Error | null = null; - public loading: boolean = true; - private shouldPoll: boolean = false; - private intervalRef?: NodeJS.Timeout; - - constructor( - private fetchFunction: FetchFunction, - private interval: number = 5000, - private onPollingSuccess?: (data: T, configurations: PollingConfigurations) => boolean, - private onPollingError?: (error: Error) => boolean, - private configurations?: PollingConfigurations - ) {} - - async fetchData(params?: P) { - this.loading = true; - try { - const result = await this.fetchFunction(params); - this.data = result; - this.loading = false; - - if (this.onPollingSuccess && this.onPollingSuccess(result, this.configurations!)) { - this.stopPolling(); - } - } catch (err) { - this.error = err as Error; - this.loading = false; - - if (this.onPollingError && this.onPollingError(this.error)) { - this.stopPolling(); - } - } - } - - startPolling(params?: P) { - this.shouldPoll = true; - if (!this.intervalRef) { - this.intervalRef = setInterval(() => { - if (this.shouldPoll) { - this.fetchData(params); - } - }, this.interval); - } - } - - stopPolling() { - this.shouldPoll = false; - if (this.intervalRef) { - clearInterval(this.intervalRef); - this.intervalRef = undefined; - } - } -} - -interface UsePollingReturn { - data: T | null; - loading: boolean; - error: Error | null; - startPolling: (params?: any) => void; - stopPolling: () => void; -} - -export function usePolling( - fetchFunction: FetchFunction, - interval: number = 5000, - onPollingSuccess?: (data: T, configurations: PollingConfigurations) => boolean, - onPollingError?: (error: Error) => boolean, - configurations?: PollingConfigurations -): UsePollingReturn { - const [data, setData] = useState(null); - const [error, setError] = useState(null); - const [loading, setLoading] = useState(true); - const intervalRef = useRef(undefined); - const unmounted = useRef(false); - - const shouldPoll = useRef(false); - - const startPolling = (params?: P) => { - shouldPoll.current = true; - const intervalId = setInterval(() => { - if (shouldPoll.current) { - fetchData(params); - } - }, interval); - intervalRef.current = intervalId; - if (unmounted.current) { - clearInterval(intervalId); - } - }; - - const stopPolling = () => { - shouldPoll.current = false; - clearInterval(intervalRef.current); - }; - - const fetchData = async (params?: P) => { - try { - const result = await fetchFunction(params); - setData(result); - // Check the success condition and stop polling if it's met - if (onPollingSuccess && onPollingSuccess(result, configurations)) { - stopPolling(); - } - } catch (err: unknown) { - setError(err as Error); - - // Check the error condition and stop polling if it's met - if (onPollingError && onPollingError(err as Error)) { - stopPolling(); - } - } finally { - setLoading(false); - } - }; - - useEffect(() => { - return () => { - unmounted.current = true; - }; - }, []); - - return { data, loading, error, startPolling, stopPolling }; -} diff --git a/src/plugins/data/public/ui/index.ts b/src/plugins/data/public/ui/index.ts index 400887e51d57..5483b540d5bf 100644 --- a/src/plugins/data/public/ui/index.ts +++ b/src/plugins/data/public/ui/index.ts @@ -49,4 +49,3 @@ export { } from './query_editor'; export { SearchBar, SearchBarProps, StatefulSearchBarProps } from './search_bar'; export { SuggestionsComponent } from './typeahead'; -export { DataSetNavigator } from './dataset_navigator'; diff --git a/src/plugins/data/public/ui/query_editor/_language_switcher.scss b/src/plugins/data/public/ui/query_editor/_language_switcher.scss new file mode 100644 index 000000000000..176d072c102b --- /dev/null +++ b/src/plugins/data/public/ui/query_editor/_language_switcher.scss @@ -0,0 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +.languageSelect { + max-width: 150px; + transform: translateY(-1px) translateX(-0.5px); +} diff --git a/src/plugins/data/public/ui/query_editor/_query_editor.scss b/src/plugins/data/public/ui/query_editor/_query_editor.scss index ac411b38ab88..8fc81308b533 100644 --- a/src/plugins/data/public/ui/query_editor/_query_editor.scss +++ b/src/plugins/data/public/ui/query_editor/_query_editor.scss @@ -86,12 +86,6 @@ } } -.osdQueryEditor__dataSetNavigatorWrapper { - :first-child { - border-bottom: $euiBorderThin !important; - } -} - @include euiBreakpoint("xs", "s") { .osdQueryEditor--withDatePicker { > :first-child { diff --git a/src/plugins/data/public/ui/query_editor/language_selector.test.tsx b/src/plugins/data/public/ui/query_editor/language_selector.test.tsx index 62c4ebea288f..f61134211a40 100644 --- a/src/plugins/data/public/ui/query_editor/language_selector.test.tsx +++ b/src/plugins/data/public/ui/query_editor/language_selector.test.tsx @@ -8,6 +8,7 @@ import { QueryLanguageSelector } from './language_selector'; import { OpenSearchDashboardsContextProvider } from 'src/plugins/opensearch_dashboards_react/public'; import { coreMock } from '../../../../../core/public/mocks'; import { mountWithIntl } from 'test_utils/enzyme_helpers'; +import { EuiCompressedComboBox } from '@elastic/eui'; import { QueryEnhancement } from '../types'; const startMock = coreMock.createStart(); diff --git a/src/plugins/data/public/ui/query_editor/language_switcher.tsx b/src/plugins/data/public/ui/query_editor/language_switcher.tsx new file mode 100644 index 000000000000..be22ebffd775 --- /dev/null +++ b/src/plugins/data/public/ui/query_editor/language_switcher.tsx @@ -0,0 +1,102 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { EuiComboBox, EuiComboBoxOptionOption, PopoverAnchorPosition } from '@elastic/eui'; +import { i18n } from '@osd/i18n'; +import React from 'react'; +import { getSearchService, getUiService } from '../../services'; + +interface Props { + language: string; + onSelectLanguage: (newLanguage: string) => void; + anchorPosition?: PopoverAnchorPosition; + appName?: string; +} + +function mapExternalLanguageToOptions(language: string) { + return { + label: language, + value: language, + }; +} + +export function QueryLanguageSwitcher(props: Props) { + const dqlLabel = i18n.translate('data.query.queryBar.dqlLanguageName', { + defaultMessage: 'DQL', + }); + const luceneLabel = i18n.translate('data.query.queryBar.luceneLanguageName', { + defaultMessage: 'Lucene', + }); + + const languageOptions: EuiComboBoxOptionOption[] = [ + { + label: dqlLabel, + value: 'kuery', + }, + { + label: luceneLabel, + value: 'lucene', + }, + ]; + + const uiService = getUiService(); + const searchService = getSearchService(); + + const queryEnhancements = uiService.queryEnhancements; + if (uiService.isEnhancementsEnabled) { + queryEnhancements.forEach((enhancement) => { + if ( + enhancement.supportedAppNames && + props.appName && + !enhancement.supportedAppNames.includes(props.appName) + ) + return; + languageOptions.push(mapExternalLanguageToOptions(enhancement.language)); + }); + } + + const selectedLanguage = { + label: + (languageOptions.find( + (option) => (option.value as string).toLowerCase() === props.language.toLowerCase() + )?.label as string) ?? languageOptions[0].label, + }; + + const setSearchEnhance = (queryLanguage: string) => { + if (!uiService.isEnhancementsEnabled) return; + const queryEnhancement = queryEnhancements.get(queryLanguage); + searchService.__enhance({ + searchInterceptor: queryEnhancement + ? queryEnhancement.search + : searchService.getDefaultSearchInterceptor(), + }); + + if (!queryEnhancement) { + searchService.df.clear(); + } + uiService.Settings.setUiOverridesByUserQueryLanguage(queryLanguage); + }; + + const handleLanguageChange = (newLanguage: EuiComboBoxOptionOption[]) => { + const queryLanguage = newLanguage[0].value as string; + props.onSelectLanguage(queryLanguage); + setSearchEnhance(queryLanguage); + }; + + setSearchEnhance(props.language); + + return ( + + ); +} diff --git a/src/plugins/data/public/ui/query_editor/query_editor.tsx b/src/plugins/data/public/ui/query_editor/query_editor.tsx index db4984b637d4..44d000de1e8f 100644 --- a/src/plugins/data/public/ui/query_editor/query_editor.tsx +++ b/src/plugins/data/public/ui/query_editor/query_editor.tsx @@ -3,7 +3,14 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { EuiFlexGroup, EuiFlexItem, htmlIdGenerator, PopoverAnchorPosition } from '@elastic/eui'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiForm, + EuiFormRow, + htmlIdGenerator, + PopoverAnchorPosition, +} from '@elastic/eui'; import classNames from 'classnames'; import { isEqual } from 'lodash'; import React, { Component, createRef, RefObject } from 'react'; @@ -37,7 +44,10 @@ export interface QueryEditorProps { indexPatterns: Array; dataSource?: DataSource; query: Query; - dataSetContainerRef?: React.RefCallback; + container?: HTMLDivElement; + dataSourceContainerRef?: React.RefCallback; + containerRef?: React.RefCallback; + languageSelectorContainerRef?: React.RefCallback; settings: Settings; disableAutoFocus?: boolean; screenTitle?: string; @@ -50,7 +60,7 @@ export interface QueryEditorProps { onChange?: (query: Query, dateRange?: TimeRange) => void; onChangeQueryEditorFocus?: (isFocused: boolean) => void; onSubmit?: (query: Query, dateRange?: TimeRange) => void; - getQueryStringInitialValue?: (language: string, dataSetName?: string) => string; + getQueryStringInitialValue?: (language: string) => string; dataTestSubj?: string; size?: SuggestionsListSize; className?: string; @@ -67,6 +77,8 @@ interface Props extends QueryEditorProps { } interface State { + isDataSourcesVisible: boolean; + isDataSetsVisible: boolean; isSuggestionsVisible: boolean; index: number | null; suggestions: QuerySuggestion[]; @@ -93,6 +105,8 @@ const KEY_CODES = { // eslint-disable-next-line import/no-default-export export default class QueryEditorUI extends Component { public state: State = { + isDataSourcesVisible: false, + isDataSetsVisible: true, isSuggestionsVisible: false, index: null, suggestions: [], @@ -107,6 +121,7 @@ export default class QueryEditorUI extends Component { private persistedLog: PersistedLog | undefined; private abortController?: AbortController; private services = this.props.opensearchDashboards.services; + private componentIsUnmounting = false; private headerRef: RefObject = createRef(); private bannerRef: RefObject = createRef(); private extensionMap = this.props.settings?.getQueryEditorExtensionMap(); @@ -235,6 +250,10 @@ export default class QueryEditorUI extends Component { : undefined; this.onChange(newQuery, dateRange); this.onSubmit(newQuery, dateRange); + this.setState({ + isDataSourcesVisible: enhancement?.searchBar?.showDataSourcesSelector ?? true, + isDataSetsVisible: enhancement?.searchBar?.showDataSetsSelector ?? true, + }); }; private initPersistedLog = () => { @@ -244,6 +263,20 @@ export default class QueryEditorUI extends Component { : getQueryLog(uiSettings, storage, appName, this.props.query.language); }; + private initDataSourcesVisibility = () => { + if (this.componentIsUnmounting) return; + + return this.props.settings.getQueryEnhancements(this.props.query.language)?.searchBar + ?.showDataSourcesSelector; + }; + + private initDataSetsVisibility = () => { + if (this.componentIsUnmounting) return; + + return this.props.settings.getQueryEnhancements(this.props.query.language)?.searchBar + ?.showDataSetsSelector; + }; + public onMouseEnterSuggestion = (index: number) => { this.setState({ index }); }; @@ -258,6 +291,10 @@ export default class QueryEditorUI extends Component { this.initPersistedLog(); // this.fetchIndexPatterns().then(this.updateSuggestions); + this.setState({ + isDataSourcesVisible: this.initDataSourcesVisibility() || true, + isDataSetsVisible: this.initDataSetsVisibility() || true, + }); } public componentDidUpdate(prevProps: Props) { @@ -271,6 +308,7 @@ export default class QueryEditorUI extends Component { public componentWillUnmount() { if (this.abortController) this.abortController.abort(); + this.componentIsUnmounting = true; } handleOnFocus = () => { @@ -393,15 +431,6 @@ export default class QueryEditorUI extends Component { const useQueryEditor = this.props.query.language !== 'kuery' && this.props.query.language !== 'lucene'; - const languageSelector = ( - - ); - return (
@@ -414,9 +443,17 @@ export default class QueryEditorUI extends Component { isCollapsed={!this.state.isCollapsed} /> - -
- + {this.state.isDataSourcesVisible && ( + +
+ + )} + + {this.state.isDataSetsVisible && ( + +
+ + )} {(this.state.isCollapsed || !useQueryEditor) && ( @@ -459,7 +496,14 @@ export default class QueryEditorUI extends Component { )} {!useQueryEditor && ( -
{languageSelector}
+
+ +
)} @@ -513,7 +557,15 @@ export default class QueryEditorUI extends Component { } > - {languageSelector} + + + {this.state.lineCount} {this.state.lineCount === 1 ? 'line' : 'lines'} diff --git a/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx b/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx index 971d13cfc050..a482d7416418 100644 --- a/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx +++ b/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx @@ -39,7 +39,8 @@ const QueryEditor = withOpenSearchDashboards(QueryEditorUI); // @internal export interface QueryEditorTopRowProps { query?: Query; - dataSetContainerRef?: React.RefCallback; + dataSourceContainerRef?: React.RefCallback; + containerRef?: React.RefCallback; settings?: Settings; onSubmit: (payload: { dateRange: TimeRange; query?: Query }) => void; onChange: (payload: { dateRange: TimeRange; query?: Query }) => void; @@ -207,10 +208,11 @@ export default function QueryEditorTopRow(props: QueryEditorTopRowProps) { ) return ''; - const defaultDataSet = indexPatterns[0]; - const dataSet = typeof defaultDataSet === 'string' ? defaultDataSet : defaultDataSet.title; + const defaultDataSource = indexPatterns[0]; + const dataSource = + typeof defaultDataSource === 'string' ? defaultDataSource : defaultDataSource.title; - return input.replace('', dataSet); + return input.replace('', dataSource); } function renderQueryEditor() { @@ -223,7 +225,8 @@ export default function QueryEditorTopRow(props: QueryEditorTopRowProps) { dataSource={props.dataSource} prepend={props.prepend} query={parsedQuery} - dataSetContainerRef={props.dataSetContainerRef} + dataSourceContainerRef={props.dataSourceContainerRef} + containerRef={props.containerRef} settings={props.settings!} screenTitle={props.screenTitle} onChange={onQueryChange} diff --git a/src/plugins/data/public/ui/search_bar/create_search_bar.tsx b/src/plugins/data/public/ui/search_bar/create_search_bar.tsx index d722aeda510a..31f3401dc76f 100644 --- a/src/plugins/data/public/ui/search_bar/create_search_bar.tsx +++ b/src/plugins/data/public/ui/search_bar/create_search_bar.tsx @@ -48,7 +48,8 @@ interface StatefulSearchBarDeps { data: Omit; storage: IStorageWrapper; settings: Settings; - setDataSetContainerRef: (ref: HTMLDivElement | null) => void; + setDataSourceContainerRef: (ref: HTMLDivElement | null) => void; + setContainerRef: (ref: HTMLDivElement | null) => void; } export type StatefulSearchBarProps = SearchBarOwnProps & { @@ -138,7 +139,8 @@ export function createSearchBar({ storage, data, settings, - setDataSetContainerRef, + setDataSourceContainerRef, + setContainerRef, }: StatefulSearchBarDeps) { // App name should come from the core application service. // Until it's available, we'll ask the user to provide it for the pre-wired component. @@ -174,9 +176,15 @@ export function createSearchBar({ notifications: core.notifications, }); - const dataSetContainerRef = useCallback((node) => { + const dataSourceContainerRef = useCallback((node) => { if (node) { - setDataSetContainerRef(node); + setDataSourceContainerRef(node); + } + }, []); + + const containerRef = useCallback((node) => { + if (node) { + setContainerRef(node); } }, []); @@ -220,7 +228,8 @@ export function createSearchBar({ filters={filters} query={query} settings={settings} - dataSetContainerRef={dataSetContainerRef} + dataSourceContainerRef={dataSourceContainerRef} + containerRef={containerRef} onFiltersUpdated={defaultFiltersUpdated(data.query)} onRefreshChange={defaultOnRefreshChange(data.query)} savedQuery={savedQuery} diff --git a/src/plugins/data/public/ui/search_bar/lib/use_dataset_manager.ts b/src/plugins/data/public/ui/search_bar/lib/use_dataset_manager.ts deleted file mode 100644 index 7a92d03e9f33..000000000000 --- a/src/plugins/data/public/ui/search_bar/lib/use_dataset_manager.ts +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { useState, useEffect } from 'react'; -import { Subscription } from 'rxjs'; -import { SimpleDataSet } from '../../../../../data/common'; -import { DataSetContract } from '../../../query'; - -interface UseDataSetManagerProps { - dataSet?: SimpleDataSet; - dataSetManager: DataSetContract; -} - -export const useDataSetManager = (props: UseDataSetManagerProps) => { - const [dataSet, setDataSet] = useState( - props.dataSet || props.dataSetManager.getDataSet() - ); - - useEffect(() => { - const subscriptions = new Subscription(); - - subscriptions.add( - props.dataSetManager.getUpdates$().subscribe({ - next: () => { - const newDataSet = props.dataSetManager.getDataSet(); - setDataSet(newDataSet); - }, - }) - ); - - return () => { - subscriptions.unsubscribe(); - }; - }, [dataSet, props.dataSet, props.dataSetManager]); - - return { dataSet }; -}; diff --git a/src/plugins/data/public/ui/search_bar/search_bar.tsx b/src/plugins/data/public/ui/search_bar/search_bar.tsx index 4dddba69ff91..b2ff6766e81c 100644 --- a/src/plugins/data/public/ui/search_bar/search_bar.tsx +++ b/src/plugins/data/public/ui/search_bar/search_bar.tsx @@ -81,7 +81,8 @@ export interface SearchBarOwnProps { // Query bar - should be in SearchBarInjectedDeps query?: Query; settings?: Settings; - dataSetContainerRef?: React.RefCallback; + dataSourceContainerRef?: React.RefCallback; + containerRef?: React.RefCallback; // Show when user has privileges to save showSaveQuery?: boolean; savedQuery?: SavedQuery; @@ -492,7 +493,8 @@ class SearchBarUI extends Component { queryEditor = ( ; - /** - * @experimental - Subject to change - */ - DataSetNavigator: React.ComponentType; SearchBar: React.ComponentType; SuggestionsComponent: React.ComponentType; - /** - * @experimental - Subject to change - */ Settings: Settings; - dataSetContainer$: Observable; + dataSourceContainer$: Observable; + container$: Observable; } diff --git a/src/plugins/data/public/ui/ui_service.ts b/src/plugins/data/public/ui/ui_service.ts index 4f403597467b..1e0e6be8b78c 100644 --- a/src/plugins/data/public/ui/ui_service.ts +++ b/src/plugins/data/public/ui/ui_service.ts @@ -8,7 +8,6 @@ import { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from 'src/core import { IStorageWrapper } from '../../../opensearch_dashboards_utils/public'; import { ConfigSchema } from '../../config'; import { DataPublicPluginStart } from '../types'; -import { createDataSetNavigator } from './dataset_navigator'; import { createIndexPatternSelect } from './index_pattern_select'; import { QueryEditorExtensionConfig } from './query_editor'; import { createSearchBar } from './search_bar/create_search_bar'; @@ -30,7 +29,8 @@ export class UiService implements Plugin { enhancementsConfig: ConfigSchema['enhancements']; private queryEnhancements: Map = new Map(); private queryEditorExtensionMap: Record = {}; - private dataSetContainer$ = new BehaviorSubject(null); + private dataSourceContainer$ = new BehaviorSubject(null); + private container$ = new BehaviorSubject(null); constructor(initializerContext: PluginInitializerContext) { const { enhancements } = initializerContext.config.get(); @@ -62,8 +62,12 @@ export class UiService implements Plugin { queryEditorExtensionMap: this.queryEditorExtensionMap, }); - const setDataSetContainerRef = (ref: HTMLDivElement | null) => { - this.dataSetContainer$.next(ref); + const setDataSourceContainerRef = (ref: HTMLDivElement | null) => { + this.dataSourceContainer$.next(ref); + }; + + const setContainerRef = (ref: HTMLDivElement | null) => { + this.container$.next(ref); }; const SearchBar = createSearchBar({ @@ -71,20 +75,17 @@ export class UiService implements Plugin { data: dataServices, storage, settings: Settings, - setDataSetContainerRef, + setDataSourceContainerRef, + setContainerRef, }); return { IndexPatternSelect: createIndexPatternSelect(core.savedObjects.client), - DataSetNavigator: createDataSetNavigator( - core.savedObjects.client, - core.http, - dataServices.query.dataSet - ), SearchBar, SuggestionsComponent, Settings, - dataSetContainer$: this.dataSetContainer$, + dataSourceContainer$: this.dataSourceContainer$, + container$: this.container$, }; } diff --git a/src/plugins/data/server/search/search_service.ts b/src/plugins/data/server/search/search_service.ts index 02ca0c30161c..ecc17dbfe71a 100644 --- a/src/plugins/data/server/search/search_service.ts +++ b/src/plugins/data/server/search/search_service.ts @@ -229,10 +229,7 @@ export class SearchService implements Plugin { dataFrame.meta.queryConfig.dataSourceId = dataSource?.id; } this.dfCache.set(dataFrame); - const dataSetName = `${dataFrame.meta?.queryConfig?.dataSourceId ?? ''}.${ - dataFrame.name - }`; - const existingIndexPattern = await scopedIndexPatterns.get(dataSetName, true); + const existingIndexPattern = scopedIndexPatterns.getByTitle(dataFrame.name!, true); const dataSet = await scopedIndexPatterns.create( dataFrameToSpec(dataFrame, existingIndexPattern?.id), !existingIndexPattern?.id @@ -242,6 +239,8 @@ export class SearchService implements Plugin { }, clear: () => { if (this.dfCache.get() === undefined) return; + // name because the id is not unique for temporary index pattern created + scopedIndexPatterns.clearCache(this.dfCache.get()!.name, false); this.dfCache.clear(); }, }; diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index 616be16e9f56..eea1860dc950 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -30,8 +30,6 @@ export const Sidebar: FC = ({ children }) => { }, } = useOpenSearchDashboards(); - const { DataSetNavigator } = ui; - useEffect(() => { const subscriptions = ui.Settings.getEnabledQueryEnhancementsUpdated$().subscribe( (enabledQueryEnhancements) => { @@ -50,17 +48,17 @@ export const Sidebar: FC = ({ children }) => { useEffect(() => { if (!isEnhancementsEnabled) return; - const subscriptions = ui.dataSetContainer$.subscribe((dataSetContainer) => { - if (dataSetContainer === null) return; + const subscriptions = ui.container$.subscribe((container) => { + if (container === null) return; if (containerRef.current) { - setContainerRef(dataSetContainer); + setContainerRef(container); } }); return () => { subscriptions.unsubscribe(); }; - }, [ui.dataSetContainer$, containerRef, setContainerRef, isEnhancementsEnabled]); + }, [ui.container$, containerRef, setContainerRef, isEnhancementsEnabled]); useEffect(() => { let isMounted = true; @@ -136,6 +134,19 @@ export const Sidebar: FC = ({ children }) => { dataSources.dataSourceService.reload(); }, [dataSources.dataSourceService]); + const dataSourceSelector = ( + + ); + return ( { containerRef.current = node; }} > - + {dataSourceSelector} )} {!isEnhancementsEnabled && ( @@ -160,16 +171,7 @@ export const Sidebar: FC = ({ children }) => { color="transparent" className="deSidebar_dataSource" > - + {dataSourceSelector} )} diff --git a/src/plugins/data_explorer/public/index.ts b/src/plugins/data_explorer/public/index.ts index 6b0561261c16..f8adda434ced 100644 --- a/src/plugins/data_explorer/public/index.ts +++ b/src/plugins/data_explorer/public/index.ts @@ -18,5 +18,4 @@ export { useTypedSelector, useTypedDispatch, setIndexPattern, - setDataSet, } from './utils/state_management'; diff --git a/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts b/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts index fa41a29259e3..e9fe84713120 100644 --- a/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts +++ b/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts @@ -5,13 +5,11 @@ import { createSlice, PayloadAction } from '@reduxjs/toolkit'; import { DataExplorerServices } from '../../types'; -import { SimpleDataSet } from '../../../../data/common'; export interface MetadataState { indexPattern?: string; originatingApp?: string; view?: string; - dataSet?: Omit; } const initialState: MetadataState = {}; @@ -42,9 +40,6 @@ export const slice = createSlice({ setIndexPattern: (state, action: PayloadAction) => { state.indexPattern = action.payload; }, - setDataSet: (state, action: PayloadAction>) => { - state.dataSet = action.payload; - }, setOriginatingApp: (state, action: PayloadAction) => { state.originatingApp = action.payload; }, @@ -58,4 +53,4 @@ export const slice = createSlice({ }); export const { reducer } = slice; -export const { setIndexPattern, setDataSet, setOriginatingApp, setView, setState } = slice.actions; +export const { setIndexPattern, setOriginatingApp, setView, setState } = slice.actions; diff --git a/src/plugins/data_explorer/public/utils/state_management/store.ts b/src/plugins/data_explorer/public/utils/state_management/store.ts index 9d320de4b54b..daf0b3d7e369 100644 --- a/src/plugins/data_explorer/public/utils/state_management/store.ts +++ b/src/plugins/data_explorer/public/utils/state_management/store.ts @@ -116,4 +116,4 @@ export type RenderState = Omit; // Remaining state after export type Store = ReturnType; export type AppDispatch = Store['dispatch']; -export { MetadataState, setIndexPattern, setDataSet, setOriginatingApp } from './metadata_slice'; +export { MetadataState, setIndexPattern, setOriginatingApp } from './metadata_slice'; diff --git a/src/plugins/discover/public/application/utils/state_management/index.ts b/src/plugins/discover/public/application/utils/state_management/index.ts index e6df7e4774b8..989b2662f0d4 100644 --- a/src/plugins/discover/public/application/utils/state_management/index.ts +++ b/src/plugins/discover/public/application/utils/state_management/index.ts @@ -7,7 +7,6 @@ import { TypedUseSelectorHook } from 'react-redux'; import { RootState, setIndexPattern as updateIndexPattern, - setDataSet as updateDataSet, useTypedDispatch, useTypedSelector, } from '../../../../../data_explorer/public'; @@ -21,4 +20,4 @@ export interface DiscoverRootState extends RootState { export const useSelector: TypedUseSelectorHook = useTypedSelector; export const useDispatch = useTypedDispatch; -export { updateIndexPattern, updateDataSet }; +export { updateIndexPattern }; diff --git a/src/plugins/discover/public/application/view_components/utils/update_search_source.ts b/src/plugins/discover/public/application/view_components/utils/update_search_source.ts index 05d4a2dbd8b4..a8480fdad18a 100644 --- a/src/plugins/discover/public/application/view_components/utils/update_search_source.ts +++ b/src/plugins/discover/public/application/view_components/utils/update_search_source.ts @@ -30,12 +30,7 @@ export const updateSearchSource = async ({ histogramConfigs, }: Props) => { const { uiSettings, data } = services; - const queryDataSet = data.query.dataSet.getDataSet(); - - let dataSet = - indexPattern.id === queryDataSet?.id - ? await data.indexPatterns.getByTitle(queryDataSet?.title!) - : indexPattern; + let dataSet = indexPattern; const dataFrame = searchSource?.getDataFrame(); if ( searchSource && diff --git a/src/plugins/discover/public/application/view_components/utils/use_search.ts b/src/plugins/discover/public/application/view_components/utils/use_search.ts index 1e40cf40a8a9..8c2ace81b048 100644 --- a/src/plugins/discover/public/application/view_components/utils/use_search.ts +++ b/src/plugins/discover/public/application/view_components/utils/use_search.ts @@ -250,8 +250,7 @@ export const useSearch = (services: DiscoverViewServices) => { timefilter.getFetch$(), timefilter.getTimeUpdate$(), timefilter.getAutoRefreshFetch$(), - data.query.queryString.getUpdates$(), - data.query.dataSet.getUpdates$() + data.query.queryString.getUpdates$() ).pipe(debounceTime(100)); const subscription = fetch$.subscribe(() => { @@ -281,7 +280,6 @@ export const useSearch = (services: DiscoverViewServices) => { fetch, core.fatalErrors, shouldSearchOnPageLoad, - data.query.dataSet, ]); // Get savedSearch if it exists diff --git a/src/plugins/query_enhancements/opensearch_dashboards.json b/src/plugins/query_enhancements/opensearch_dashboards.json index 69d8fd3bd667..b09494aab0ca 100644 --- a/src/plugins/query_enhancements/opensearch_dashboards.json +++ b/src/plugins/query_enhancements/opensearch_dashboards.json @@ -3,7 +3,7 @@ "version": "opensearchDashboards", "server": true, "ui": true, - "requiredPlugins": ["data", "opensearchDashboardsReact", "opensearchDashboardsUtils", "savedObjects", "uiActions"], + "requiredPlugins": ["data", "opensearchDashboardsReact", "opensearchDashboardsUtils", "dataSourceManagement", "savedObjects", "uiActions"], "optionalPlugins": ["dataSource"] } diff --git a/src/plugins/query_enhancements/public/data_source_connection/components/connections_bar.tsx b/src/plugins/query_enhancements/public/data_source_connection/components/connections_bar.tsx new file mode 100644 index 000000000000..3fd592e50b31 --- /dev/null +++ b/src/plugins/query_enhancements/public/data_source_connection/components/connections_bar.tsx @@ -0,0 +1,94 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React, { useEffect, useRef, useState } from 'react'; +import { EuiPortal } from '@elastic/eui'; +import { distinctUntilChanged } from 'rxjs/operators'; +import { ToastsSetup } from 'opensearch-dashboards/public'; +import { DataPublicPluginStart, QueryEditorExtensionDependencies } from '../../../../data/public'; +import { DataSourceSelector } from '../../../../data_source_management/public'; +import { ConnectionsService } from '../services'; + +interface ConnectionsProps { + dependencies: QueryEditorExtensionDependencies; + toasts: ToastsSetup; + connectionsService: ConnectionsService; +} + +export const ConnectionsBar: React.FC = ({ connectionsService, toasts }) => { + const [isDataSourceEnabled, setIsDataSourceEnabled] = useState(false); + const [uiService, setUiService] = useState(undefined); + const containerRef = useRef(null); + + useEffect(() => { + const uiServiceSubscription = connectionsService.getUiService().subscribe(setUiService); + const dataSourceEnabledSubscription = connectionsService + .getIsDataSourceEnabled$() + .subscribe(setIsDataSourceEnabled); + + return () => { + uiServiceSubscription.unsubscribe(); + dataSourceEnabledSubscription.unsubscribe(); + }; + }, [connectionsService]); + + useEffect(() => { + if (!uiService || !isDataSourceEnabled || !containerRef.current) return; + const subscriptions = uiService.dataSourceContainer$.subscribe((container) => { + if (container && containerRef.current) { + container.append(containerRef.current); + } + }); + + return () => subscriptions.unsubscribe(); + }, [uiService, isDataSourceEnabled]); + + useEffect(() => { + const selectedConnectionSubscription = connectionsService + .getSelectedConnection$() + .pipe(distinctUntilChanged()) + .subscribe((connection) => { + if (connection) { + // Assuming setSelectedConnection$ is meant to update some state or perform an action outside this component + connectionsService.setSelectedConnection$(connection); + } + }); + + return () => selectedConnectionSubscription.unsubscribe(); + }, [connectionsService]); + + const handleSelectedConnection = (id: string | undefined) => { + if (!id) { + connectionsService.setSelectedConnection$(undefined); + return; + } + connectionsService.getConnectionById(id).subscribe((connection) => { + connectionsService.setSelectedConnection$(connection); + }); + }; + + return ( + { + containerRef.current = node; + }} + > +
+ + handleSelectedConnection(dataSource[0]?.id || undefined) + } + /> +
+ + ); +}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/hooks/index.tsx b/src/plugins/query_enhancements/public/data_source_connection/components/index.ts similarity index 61% rename from src/plugins/data/public/ui/dataset_navigator/lib/hooks/index.tsx rename to src/plugins/query_enhancements/public/data_source_connection/components/index.ts index 88974a7c9420..1ee969a1d079 100644 --- a/src/plugins/data/public/ui/dataset_navigator/lib/hooks/index.tsx +++ b/src/plugins/query_enhancements/public/data_source_connection/components/index.ts @@ -3,4 +3,4 @@ * SPDX-License-Identifier: Apache-2.0 */ -export * from './direct_query_hook'; +export { ConnectionsBar } from './connections_bar'; diff --git a/src/plugins/query_enhancements/public/data_source_connection/index.ts b/src/plugins/query_enhancements/public/data_source_connection/index.ts new file mode 100644 index 000000000000..e334163d91d4 --- /dev/null +++ b/src/plugins/query_enhancements/public/data_source_connection/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export { createDataSourceConnectionExtension } from './utils'; +export * from './services'; diff --git a/src/plugins/query_enhancements/public/services/connections_service.ts b/src/plugins/query_enhancements/public/data_source_connection/services/connections_service.ts similarity index 95% rename from src/plugins/query_enhancements/public/services/connections_service.ts rename to src/plugins/query_enhancements/public/data_source_connection/services/connections_service.ts index 97a59c2cd94a..6afec4b51a99 100644 --- a/src/plugins/query_enhancements/public/services/connections_service.ts +++ b/src/plugins/query_enhancements/public/data_source_connection/services/connections_service.ts @@ -6,8 +6,8 @@ import { BehaviorSubject, Observable, from } from 'rxjs'; import { DataPublicPluginStart } from 'src/plugins/data/public'; import { CoreStart } from 'opensearch-dashboards/public'; -import { API } from '../../common'; -import { Connection, ConnectionsServiceDeps } from '../types'; +import { API } from '../../../common'; +import { Connection, ConnectionsServiceDeps } from '../../types'; export class ConnectionsService { protected http!: ConnectionsServiceDeps['http']; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/requests/index.tsx b/src/plugins/query_enhancements/public/data_source_connection/services/index.ts similarity index 58% rename from src/plugins/data/public/ui/dataset_navigator/lib/requests/index.tsx rename to src/plugins/query_enhancements/public/data_source_connection/services/index.ts index 3918a896bd0b..08eeda5a7aa1 100644 --- a/src/plugins/data/public/ui/dataset_navigator/lib/requests/index.tsx +++ b/src/plugins/query_enhancements/public/data_source_connection/services/index.ts @@ -3,4 +3,4 @@ * SPDX-License-Identifier: Apache-2.0 */ -export * from './sql'; +export { ConnectionsService } from './connections_service'; diff --git a/src/plugins/query_enhancements/public/data_source_connection/utils/create_extension.tsx b/src/plugins/query_enhancements/public/data_source_connection/utils/create_extension.tsx new file mode 100644 index 000000000000..e5822c4b378e --- /dev/null +++ b/src/plugins/query_enhancements/public/data_source_connection/utils/create_extension.tsx @@ -0,0 +1,34 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react'; +import { ToastsSetup } from 'opensearch-dashboards/public'; +import { QueryEditorExtensionConfig } from '../../../../data/public'; +import { ConfigSchema } from '../../../common/config'; +import { ConnectionsBar } from '../components'; +import { ConnectionsService } from '../services'; + +export const createDataSourceConnectionExtension = ( + connectionsService: ConnectionsService, + toasts: ToastsSetup, + config: ConfigSchema +): QueryEditorExtensionConfig => { + return { + id: 'data-source-connection', + order: 2000, + isEnabled$: (dependencies) => { + return connectionsService.getIsDataSourceEnabled$(); + }, + getComponent: (dependencies) => { + return ( + + ); + }, + }; +}; diff --git a/src/plugins/data/common/data_sets/index.ts b/src/plugins/query_enhancements/public/data_source_connection/utils/index.ts similarity index 70% rename from src/plugins/data/common/data_sets/index.ts rename to src/plugins/query_enhancements/public/data_source_connection/utils/index.ts index 9f269633f307..9eccc9e6f35a 100644 --- a/src/plugins/data/common/data_sets/index.ts +++ b/src/plugins/query_enhancements/public/data_source_connection/utils/index.ts @@ -3,4 +3,4 @@ * SPDX-License-Identifier: Apache-2.0 */ -export * from './types'; +export * from './create_extension'; diff --git a/src/plugins/query_enhancements/public/plugin.tsx b/src/plugins/query_enhancements/public/plugin.tsx index b74c00ced7e0..d65676b70e78 100644 --- a/src/plugins/query_enhancements/public/plugin.tsx +++ b/src/plugins/query_enhancements/public/plugin.tsx @@ -7,9 +7,10 @@ import moment from 'moment'; import { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '../../../core/public'; import { IStorageWrapper, Storage } from '../../opensearch_dashboards_utils/public'; import { ConfigSchema } from '../common/config'; -import { ConnectionsService, setData, setStorage } from './services'; +import { ConnectionsService, createDataSourceConnectionExtension } from './data_source_connection'; import { createQueryAssistExtension } from './query_assist'; -import { PPLSearchInterceptor, SQLSearchInterceptor } from './search'; +import { PPLSearchInterceptor, SQLAsyncSearchInterceptor, SQLSearchInterceptor } from './search'; +import { setData, setStorage } from './services'; import { QueryEnhancementsPluginSetup, QueryEnhancementsPluginSetupDependencies, @@ -43,21 +44,38 @@ export class QueryEnhancementsPlugin http: core.http, }); - const pplSearchInterceptor = new PPLSearchInterceptor({ - toasts: core.notifications.toasts, - http: core.http, - uiSettings: core.uiSettings, - startServices: core.getStartServices(), - usageCollector: data.search.usageCollector, - }); + const pplSearchInterceptor = new PPLSearchInterceptor( + { + toasts: core.notifications.toasts, + http: core.http, + uiSettings: core.uiSettings, + startServices: core.getStartServices(), + usageCollector: data.search.usageCollector, + }, + this.connectionsService + ); - const sqlSearchInterceptor = new SQLSearchInterceptor({ - toasts: core.notifications.toasts, - http: core.http, - uiSettings: core.uiSettings, - startServices: core.getStartServices(), - usageCollector: data.search.usageCollector, - }); + const sqlSearchInterceptor = new SQLSearchInterceptor( + { + toasts: core.notifications.toasts, + http: core.http, + uiSettings: core.uiSettings, + startServices: core.getStartServices(), + usageCollector: data.search.usageCollector, + }, + this.connectionsService + ); + + const sqlAsyncSearchInterceptor = new SQLAsyncSearchInterceptor( + { + toasts: core.notifications.toasts, + http: core.http, + uiSettings: core.uiSettings, + startServices: core.getStartServices(), + usageCollector: data.search.usageCollector, + }, + this.connectionsService + ); data.__enhance({ ui: { @@ -71,7 +89,7 @@ export class QueryEnhancementsPlugin initialTo: moment().add(2, 'days').toISOString(), }, showFilterBar: false, - showDataSetsSelector: true, + showDataSetsSelector: false, showDataSourcesSelector: true, }, fields: { @@ -92,7 +110,7 @@ export class QueryEnhancementsPlugin searchBar: { showDatePicker: false, showFilterBar: false, - showDataSetsSelector: true, + showDataSetsSelector: false, showDataSourcesSelector: true, queryStringInput: { initialValue: 'SELECT * FROM ' }, }, @@ -107,6 +125,29 @@ export class QueryEnhancementsPlugin }, }); + data.__enhance({ + ui: { + query: { + language: 'SQLAsync', + search: sqlAsyncSearchInterceptor, + searchBar: { + showDatePicker: false, + showFilterBar: false, + showDataSetsSelector: false, + showDataSourcesSelector: true, + queryStringInput: { initialValue: 'SHOW DATABASES IN ::mys3::' }, + }, + fields: { + filterable: false, + visualizable: false, + }, + showDocLinks: false, + supportedAppNames: ['discover'], + connectionService: this.connectionsService, + }, + }, + }); + data.__enhance({ ui: { queryEditorExtension: createQueryAssistExtension( @@ -117,6 +158,16 @@ export class QueryEnhancementsPlugin }, }); + data.__enhance({ + ui: { + queryEditorExtension: createDataSourceConnectionExtension( + this.connectionsService, + core.notifications.toasts, + this.config + ), + }, + }); + return {}; } diff --git a/src/plugins/query_enhancements/public/query_assist/components/query_assist_bar.tsx b/src/plugins/query_enhancements/public/query_assist/components/query_assist_bar.tsx index c28c5cb8b0be..e87e74ce2998 100644 --- a/src/plugins/query_enhancements/public/query_assist/components/query_assist_bar.tsx +++ b/src/plugins/query_enhancements/public/query_assist/components/query_assist_bar.tsx @@ -12,7 +12,7 @@ import { } from '../../../../data/public'; import { useOpenSearchDashboards } from '../../../../opensearch_dashboards_react/public'; import { QueryAssistParameters } from '../../../common/query_assist'; -import { ConnectionsService } from '../../services'; +import { ConnectionsService } from '../../data_source_connection'; import { getStorage } from '../../services'; import { useGenerateQuery } from '../hooks'; import { getPersistedLog, ProhibitedQueryError } from '../utils'; @@ -45,7 +45,7 @@ export const QueryAssistBar: React.FC = (props) => { const subscription = props.connectionsService .getSelectedConnection$() .subscribe((connection) => { - dataSourceIdRef.current = connection?.dataSource?.id; + dataSourceIdRef.current = connection?.dataSource.id; }); return () => subscription.unsubscribe(); }, [props.connectionsService]); diff --git a/src/plugins/query_enhancements/public/query_assist/utils/create_extension.tsx b/src/plugins/query_enhancements/public/query_assist/utils/create_extension.tsx index 23611e39501e..e088457a0717 100644 --- a/src/plugins/query_enhancements/public/query_assist/utils/create_extension.tsx +++ b/src/plugins/query_enhancements/public/query_assist/utils/create_extension.tsx @@ -13,7 +13,7 @@ import { } from '../../../../data/public'; import { API } from '../../../common'; import { ConfigSchema } from '../../../common/config'; -import { ConnectionsService } from '../../services'; +import { ConnectionsService } from '../../data_source_connection'; import { QueryAssistBar, QueryAssistBanner } from '../components'; /** @@ -28,7 +28,7 @@ const getAvailableLanguages$ = ( connectionsService.getSelectedConnection$().pipe( distinctUntilChanged(), switchMap(async (connection) => { - const dataSourceId = connection?.dataSource?.id; + const dataSourceId = connection?.dataSource.id; const cached = availableLanguagesByDataSource.get(dataSourceId); if (cached !== undefined) return cached; const languages = await http diff --git a/src/plugins/query_enhancements/public/search/index.ts b/src/plugins/query_enhancements/public/search/index.ts index 624e7cf6e7b5..9835c1345f02 100644 --- a/src/plugins/query_enhancements/public/search/index.ts +++ b/src/plugins/query_enhancements/public/search/index.ts @@ -5,3 +5,4 @@ export { PPLSearchInterceptor } from './ppl_search_interceptor'; export { SQLSearchInterceptor } from './sql_search_interceptor'; +export { SQLAsyncSearchInterceptor } from './sql_async_search_interceptor'; diff --git a/src/plugins/query_enhancements/public/search/ppl_search_interceptor.ts b/src/plugins/query_enhancements/public/search/ppl_search_interceptor.ts index 13d6bc25874c..bca9961fea3b 100644 --- a/src/plugins/query_enhancements/public/search/ppl_search_interceptor.ts +++ b/src/plugins/query_enhancements/public/search/ppl_search_interceptor.ts @@ -5,6 +5,7 @@ import { trimEnd } from 'lodash'; import { Observable, throwError } from 'rxjs'; +import { i18n } from '@osd/i18n'; import { concatMap } from 'rxjs/operators'; import { DataFrameAggConfig, @@ -33,12 +34,16 @@ import { fetchDataFrame, } from '../../common'; import { QueryEnhancementsPluginStartDependencies } from '../types'; +import { ConnectionsService } from '../data_source_connection'; export class PPLSearchInterceptor extends SearchInterceptor { protected queryService!: DataPublicPluginStart['query']; protected aggsService!: DataPublicPluginStart['search']['aggs']; - constructor(deps: SearchInterceptorDeps) { + constructor( + deps: SearchInterceptorDeps, + private readonly connectionsService: ConnectionsService + ) { super(deps); deps.startServices.then(([coreStart, depsStart]) => { @@ -142,23 +147,34 @@ export class PPLSearchInterceptor extends SearchInterceptor { }; const dataFrame = getRawDataFrame(searchRequest); + if (!dataFrame) { + return throwError( + this.handleSearchError( + { + stack: 'DataFrame is not defined', + }, + request, + signal! + ) + ); + } let queryString = dataFrame.meta?.queryConfig?.qs ?? getRawQueryString(searchRequest) ?? ''; dataFrame.meta = { ...dataFrame.meta, - aggConfig: { - ...dataFrame.meta.aggConfig, - ...(this.aggsService.types.get.bind(this) && - getAggConfig(searchRequest, {}, this.aggsService.types.get.bind(this))), - }, queryConfig: { ...dataFrame.meta.queryConfig, - ...(this.queryService.dataSet.getDataSet() && { - dataSourceId: this.queryService.dataSet.getDataSet()?.dataSourceRef?.id, + ...(this.connectionsService.getSelectedConnection() && { + dataSourceId: this.connectionsService.getSelectedConnection()?.id, }), }, }; + const aggConfig = getAggConfig( + searchRequest, + {}, + this.aggsService.types.get.bind(this) + ) as DataFrameAggConfig; if (!dataFrame.schema) { return fetchDataFrame(dfContext, queryString, dataFrame).pipe( @@ -168,14 +184,14 @@ export class PPLSearchInterceptor extends SearchInterceptor { const jsError = new Error(df.error.response); return throwError(jsError); } - const timeField = getTimeField(df, dataFrame.meta?.aggConfig); + const timeField = getTimeField(df, aggConfig); if (timeField) { const timeFilter = getTimeFilter(timeField); const newQuery = insertTimeFilter(queryString, timeFilter); updateDataFrameMeta({ dataFrame: df, qs: newQuery, - aggConfig: dataFrame.meta?.aggConfig, + aggConfig, timeField, timeFilter, getAggQsFn: getAggQsFn.bind(this), @@ -188,14 +204,14 @@ export class PPLSearchInterceptor extends SearchInterceptor { } if (dataFrame.schema) { - const timeField = getTimeField(dataFrame, dataFrame.meta?.aggConfig); + const timeField = getTimeField(dataFrame, aggConfig); if (timeField) { const timeFilter = getTimeFilter(timeField); const newQuery = insertTimeFilter(queryString, timeFilter); updateDataFrameMeta({ dataFrame, qs: newQuery, - aggConfig: dataFrame.meta?.aggConfig, + aggConfig, timeField, timeFilter, getAggQsFn: getAggQsFn.bind(this), diff --git a/src/plugins/query_enhancements/public/search/sql_async_search_interceptor.ts b/src/plugins/query_enhancements/public/search/sql_async_search_interceptor.ts new file mode 100644 index 000000000000..9232ef146cdb --- /dev/null +++ b/src/plugins/query_enhancements/public/search/sql_async_search_interceptor.ts @@ -0,0 +1,137 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { trimEnd } from 'lodash'; +import { BehaviorSubject, Observable, throwError } from 'rxjs'; +import { i18n } from '@osd/i18n'; +import { concatMap, map } from 'rxjs/operators'; +import { + DATA_FRAME_TYPES, + DataPublicPluginStart, + IOpenSearchDashboardsSearchRequest, + IOpenSearchDashboardsSearchResponse, + ISearchOptions, + SearchInterceptor, + SearchInterceptorDeps, +} from '../../../data/public'; +import { getRawDataFrame, getRawQueryString, IDataFrameResponse } from '../../../data/common'; +import { + API, + DataFramePolling, + FetchDataFrameContext, + SEARCH_STRATEGY, + fetchDataFrame, + fetchDataFramePolling, +} from '../../common'; +import { QueryEnhancementsPluginStartDependencies } from '../types'; +import { ConnectionsService } from '../data_source_connection'; + +export class SQLAsyncSearchInterceptor extends SearchInterceptor { + protected queryService!: DataPublicPluginStart['query']; + protected aggsService!: DataPublicPluginStart['search']['aggs']; + protected indexPatterns!: DataPublicPluginStart['indexPatterns']; + protected dataFrame$ = new BehaviorSubject(undefined); + + constructor( + deps: SearchInterceptorDeps, + private readonly connectionsService: ConnectionsService + ) { + super(deps); + + deps.startServices.then(([coreStart, depsStart]) => { + this.queryService = (depsStart as QueryEnhancementsPluginStartDependencies).data.query; + this.aggsService = (depsStart as QueryEnhancementsPluginStartDependencies).data.search.aggs; + }); + } + + protected runSearch( + request: IOpenSearchDashboardsSearchRequest, + signal?: AbortSignal, + strategy?: string + ): Observable { + const { id, ...searchRequest } = request; + const path = trimEnd(API.SQL_ASYNC_SEARCH); + const dfContext: FetchDataFrameContext = { + http: this.deps.http, + path, + signal, + }; + + const dataFrame = getRawDataFrame(searchRequest); + if (!dataFrame) { + return throwError(this.handleSearchError('DataFrame is not defined', request, signal!)); + } + + const queryString = + dataFrame.meta?.queryConfig?.formattedQs() ?? getRawQueryString(searchRequest) ?? ''; + + dataFrame.meta = { + ...dataFrame.meta, + queryConfig: { + ...dataFrame.meta.queryConfig, + ...(this.connectionsService.getSelectedConnection() && + this.connectionsService.getSelectedConnection()?.dataSource && { + dataSourceId: this.connectionsService.getSelectedConnection()?.dataSource.id, + }), + }, + }; + + const onPollingSuccess = (pollingResult: any) => { + if (pollingResult && pollingResult.body.meta.status === 'SUCCESS') { + return false; + } + if (pollingResult && pollingResult.body.meta.status === 'FAILED') { + const jsError = new Error(pollingResult.data.error.response); + this.deps.toasts.addError(jsError, { + title: i18n.translate('queryEnhancements.sqlQueryError', { + defaultMessage: 'Could not complete the SQL async query', + }), + toastMessage: pollingResult.data.error.response, + }); + return false; + } + + this.deps.toasts.addInfo({ + title: i18n.translate('queryEnhancements.sqlQueryPolling', { + defaultMessage: 'Polling query job results...', + }), + }); + + return true; + }; + + const onPollingError = (error: Error) => { + throw new Error(error.message); + }; + + this.deps.toasts.addInfo({ + title: i18n.translate('queryEnhancements.sqlQueryInfo', { + defaultMessage: 'Starting query job...', + }), + }); + return fetchDataFrame(dfContext, queryString, dataFrame).pipe( + concatMap((jobResponse) => { + const df = jobResponse.body; + const dataFramePolling = new DataFramePolling( + () => fetchDataFramePolling(dfContext, df), + 5000, + onPollingSuccess, + onPollingError + ); + return dataFramePolling.fetch().pipe( + map(() => { + const dfPolling = dataFramePolling.data; + dfPolling.type = DATA_FRAME_TYPES.DEFAULT; + return dfPolling; + }) + ); + }) + ); + } + + public search(request: IOpenSearchDashboardsSearchRequest, options: ISearchOptions) { + return this.runSearch(request, options.abortSignal, SEARCH_STRATEGY.SQL_ASYNC); + } +} diff --git a/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts b/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts index de7fb5938d25..5a3b8278c65a 100644 --- a/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts +++ b/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts @@ -6,13 +6,8 @@ import { trimEnd } from 'lodash'; import { Observable, throwError } from 'rxjs'; import { i18n } from '@osd/i18n'; -import { concatMap, map } from 'rxjs/operators'; -import { - DATA_FRAME_TYPES, - getRawDataFrame, - getRawQueryString, - SIMPLE_DATA_SET_TYPES, -} from '../../../data/common'; +import { concatMap } from 'rxjs/operators'; +import { getRawDataFrame, getRawQueryString } from '../../../data/common'; import { DataPublicPluginStart, IOpenSearchDashboardsSearchRequest, @@ -21,21 +16,18 @@ import { SearchInterceptor, SearchInterceptorDeps, } from '../../../data/public'; -import { - API, - DataFramePolling, - FetchDataFrameContext, - SEARCH_STRATEGY, - fetchDataFrame, - fetchDataFramePolling, -} from '../../common'; +import { API, FetchDataFrameContext, SEARCH_STRATEGY, fetchDataFrame } from '../../common'; import { QueryEnhancementsPluginStartDependencies } from '../types'; +import { ConnectionsService } from '../data_source_connection'; export class SQLSearchInterceptor extends SearchInterceptor { protected queryService!: DataPublicPluginStart['query']; protected aggsService!: DataPublicPluginStart['search']['aggs']; - constructor(deps: SearchInterceptorDeps) { + constructor( + deps: SearchInterceptorDeps, + private readonly connectionsService: ConnectionsService + ) { super(deps); deps.startServices.then(([coreStart, depsStart]) => { @@ -57,6 +49,9 @@ export class SQLSearchInterceptor extends SearchInterceptor { }; const dataFrame = getRawDataFrame(searchRequest); + if (!dataFrame) { + return throwError(this.handleSearchError('DataFrame is not defined', request, signal!)); + } const queryString = dataFrame.meta?.queryConfig?.qs ?? getRawQueryString(searchRequest) ?? ''; @@ -64,8 +59,8 @@ export class SQLSearchInterceptor extends SearchInterceptor { ...dataFrame.meta, queryConfig: { ...dataFrame.meta.queryConfig, - ...(this.queryService.dataSet.getDataSet() && { - dataSourceId: this.queryService.dataSet.getDataSet()?.dataSourceRef?.id, + ...(this.connectionsService.getSelectedConnection() && { + dataSourceId: this.connectionsService.getSelectedConnection()?.id, }), }, }; @@ -86,91 +81,7 @@ export class SQLSearchInterceptor extends SearchInterceptor { return fetchDataFrame(dfContext, queryString, dataFrame); } - protected runSearchAsync( - request: IOpenSearchDashboardsSearchRequest, - signal?: AbortSignal, - strategy?: string - ): Observable { - const { id, ...searchRequest } = request; - const path = trimEnd(API.SQL_ASYNC_SEARCH); - const dfContext: FetchDataFrameContext = { - http: this.deps.http, - path, - signal, - }; - - const dataFrame = getRawDataFrame(searchRequest); - if (!dataFrame) { - return throwError(this.handleSearchError('DataFrame is not defined', request, signal!)); - } - - const queryString = getRawQueryString(searchRequest) ?? ''; - - dataFrame.meta = { - ...dataFrame.meta, - queryConfig: { - ...dataFrame.meta.queryConfig, - }, - }; - - const onPollingSuccess = (pollingResult: any) => { - if (pollingResult && pollingResult.body.meta.status === 'SUCCESS') { - return false; - } - if (pollingResult && pollingResult.body.meta.status === 'FAILED') { - const jsError = new Error(pollingResult.data.error.response); - this.deps.toasts.addError(jsError, { - title: i18n.translate('queryEnhancements.sqlQueryError', { - defaultMessage: 'Could not complete the SQL async query', - }), - toastMessage: pollingResult.data.error.response, - }); - return false; - } - - this.deps.toasts.addInfo({ - title: i18n.translate('queryEnhancements.sqlQueryPolling', { - defaultMessage: 'Polling query job results...', - }), - }); - - return true; - }; - - const onPollingError = (error: Error) => { - throw new Error(error.message); - }; - - this.deps.toasts.addInfo({ - title: i18n.translate('queryEnhancements.sqlQueryInfo', { - defaultMessage: 'Starting query job...', - }), - }); - return fetchDataFrame(dfContext, queryString, dataFrame).pipe( - concatMap((jobResponse) => { - const df = jobResponse.body; - const dataFramePolling = new DataFramePolling( - () => fetchDataFramePolling(dfContext, df), - 5000, - onPollingSuccess, - onPollingError - ); - return dataFramePolling.fetch().pipe( - map(() => { - const dfPolling = dataFramePolling.data; - dfPolling.type = DATA_FRAME_TYPES.DEFAULT; - return dfPolling; - }) - ); - }) - ); - } - public search(request: IOpenSearchDashboardsSearchRequest, options: ISearchOptions) { - const dataSet = this.queryService.dataSet.getDataSet(); - if (dataSet?.type === SIMPLE_DATA_SET_TYPES.TEMPORARY_ASYNC) { - return this.runSearchAsync(request, options.abortSignal, SEARCH_STRATEGY.SQL_ASYNC); - } return this.runSearch(request, options.abortSignal, SEARCH_STRATEGY.SQL); } } diff --git a/src/plugins/query_enhancements/public/services.ts b/src/plugins/query_enhancements/public/services.ts new file mode 100644 index 000000000000..d11233be2dca --- /dev/null +++ b/src/plugins/query_enhancements/public/services.ts @@ -0,0 +1,11 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { createGetterSetter } from '../../opensearch_dashboards_utils/common'; +import { IStorageWrapper } from '../../opensearch_dashboards_utils/public'; +import { DataPublicPluginStart } from '../../data/public'; + +export const [getStorage, setStorage] = createGetterSetter('storage'); +export const [getData, setData] = createGetterSetter('data'); diff --git a/src/plugins/query_enhancements/public/services/index.ts b/src/plugins/query_enhancements/public/services/index.ts deleted file mode 100644 index bb0284408faa..000000000000 --- a/src/plugins/query_enhancements/public/services/index.ts +++ /dev/null @@ -1,13 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { createGetterSetter } from '../../../opensearch_dashboards_utils/common'; -import { IStorageWrapper } from '../../../opensearch_dashboards_utils/public'; -import { DataPublicPluginStart } from '../../../data/public'; - -export const [getStorage, setStorage] = createGetterSetter('storage'); -export const [getData, setData] = createGetterSetter('data'); - -export { ConnectionsService } from './connections_service'; diff --git a/src/plugins/query_enhancements/server/routes/data_source_connection/routes.ts b/src/plugins/query_enhancements/server/routes/data_source_connection/routes.ts index 162cc7e8f103..f4fe42779dae 100644 --- a/src/plugins/query_enhancements/server/routes/data_source_connection/routes.ts +++ b/src/plugins/query_enhancements/server/routes/data_source_connection/routes.ts @@ -5,6 +5,7 @@ import { schema } from '@osd/config-schema'; import { IRouter } from 'opensearch-dashboards/server'; +import { DataSourceAttributes } from '../../../../data_source/common/data_sources'; import { API } from '../../../common'; export function registerDataSourceConnectionsRoutes(router: IRouter) { @@ -17,7 +18,7 @@ export function registerDataSourceConnectionsRoutes(router: IRouter) { }, async (context, request, response) => { const fields = ['id', 'title', 'auth.type']; - const resp = await context.core.savedObjects.client.find({ + const resp = await context.core.savedObjects.client.find({ type: 'data-source', fields, perPage: 10000, @@ -37,7 +38,7 @@ export function registerDataSourceConnectionsRoutes(router: IRouter) { }, }, async (context, request, response) => { - const resp = await context.core.savedObjects.client.get( + const resp = await context.core.savedObjects.client.get( 'data-source', request.params.dataSourceId ); diff --git a/src/plugins/query_enhancements/server/search/sql_async_search_strategy.ts b/src/plugins/query_enhancements/server/search/sql_async_search_strategy.ts index 8cd5014335e0..acd0027d0bc1 100644 --- a/src/plugins/query_enhancements/server/search/sql_async_search_strategy.ts +++ b/src/plugins/query_enhancements/server/search/sql_async_search_strategy.ts @@ -55,7 +55,7 @@ export const sqlAsyncSearchStrategyProvider = ( const sessionId = rawResponse.data?.sessionId; const partial: PartialDataFrame = { - ...request.body.df, + name: '', fields: rawResponse?.data?.schema || [], }; const dataFrame = createDataFrame(partial); diff --git a/src/plugins/query_enhancements/server/search/sql_search_strategy.ts b/src/plugins/query_enhancements/server/search/sql_search_strategy.ts index 0a4683567302..c5ebb40f882b 100644 --- a/src/plugins/query_enhancements/server/search/sql_search_strategy.ts +++ b/src/plugins/query_enhancements/server/search/sql_search_strategy.ts @@ -39,7 +39,7 @@ export const sqlSearchStrategyProvider = ( } const partial: PartialDataFrame = { - ...request.body.df, + name: '', fields: rawResponse.data?.schema || [], }; const dataFrame = createDataFrame(partial); diff --git a/src/plugins/query_enhancements/server/types.ts b/src/plugins/query_enhancements/server/types.ts index b6a03b672de9..1ad76c7bbf85 100644 --- a/src/plugins/query_enhancements/server/types.ts +++ b/src/plugins/query_enhancements/server/types.ts @@ -4,7 +4,7 @@ */ import { PluginSetup } from 'src/plugins/data/server'; -import { DataSourcePluginSetup } from 'src/plugins/data_source/server'; +import { DataSourcePluginSetup } from '../../data_source/server'; import { Logger } from '../../../core/server'; import { ConfigSchema } from '../common/config';