Merge branch 'main' into use-case

opensearch-project · Aug 2, 2024 · 511a50a · 511a50a
2 parents 94c8b20 + d09895a
commit 511a50a
Show file tree

Hide file tree

Showing 32 changed files with 374 additions and 386 deletions.
diff --git a/changelogs/fragments/7609.yml b/changelogs/fragments/7609.yml
@@ -0,0 +1,2 @@
+fix:
+- Dataset nav to load external connections and update namespace ([#7609](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/7609))
diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts
@@ -446,9 +446,6 @@ export {
   // for BWC, keeping the old name
   IUiStart as DataPublicPluginStartUi,
   DataSetNavigator,
-  setAsyncSessionId,
-  getAsyncSessionId,
-  setAsyncSessionIdByObj,
 } from './ui';
 
 /**

diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx
@@ -40,13 +40,7 @@ import {
   getSearchService,
   getUiService,
 } from '../../services';
-import {
-  fetchDataSources,
-  fetchIndexPatterns,
-  fetchIndices,
-  isCatalogCacheFetching,
-  fetchIfExternalDataSourcesEnabled,
-} from './lib';
+import { fetchDataSources, fetchIndexPatterns, fetchIndices, isCatalogCacheFetching } from './lib';
 import { useDataSetManager } from '../search_bar/lib/use_dataset_manager';
 import { DataSetContract } from '../../query';
 
@@ -76,7 +70,6 @@ export const DataSetNavigator: React.FC<DataSetNavigatorProps> = ({
   const isInitialized = useRef(false);
   const [isOpen, setIsOpen] = useState(false);
   const [isLoading, setIsLoading] = useState(false);
-  const [isExternalDataSourcesEnabled, setIsExternalDataSourcesEnabled] = useState(false);
   const [dataSources, setDataSources] = useState<SimpleDataSource[]>([]);
   const [externalDataSources, setExternalDataSources] = useState<SimpleDataSource[]>([]);
   const [indexPatterns, setIndexPatterns] = useState<any[]>([]);
@@ -85,6 +78,7 @@ export const DataSetNavigator: React.FC<DataSetNavigatorProps> = ({
   const [selectedDataSetState, setSelectedDataSetState] = useState<
     SelectedDataSetState | undefined
   >(undefined);
+  const isExternalDataSourcesEnabled = externalDataSources.length > 0;
 
   const {
     loadStatus: dataSourcesLoadStatus,
@@ -104,7 +98,7 @@ export const DataSetNavigator: React.FC<DataSetNavigatorProps> = ({
 
   const onRefresh = useCallback(() => {
     if (!isCatalogCacheFetching(dataSourcesLoadStatus) && dataSources.length > 0) {
-      startLoadingDataSources(dataSources.map((dataSource) => dataSource.id));
+      startLoadingDataSources(dataSources);
     }
   }, [dataSourcesLoadStatus, dataSources, startLoadingDataSources]);
 
@@ -158,19 +152,27 @@ export const DataSetNavigator: React.FC<DataSetNavigatorProps> = ({
 
       setIsLoading(true);
       try {
-        const [
-          fetchedIndexPatterns,
-          fetchedDataSources,
-          fetchedIsExternalDataSourcesEnabled,
-        ] = await Promise.all([
+        const [fetchedIndexPatterns, fetchedDataSources] = await Promise.all([
           fetchIndexPatterns(savedObjectsClient!, ''),
           fetchDataSources(savedObjectsClient!),
-          fetchIfExternalDataSourcesEnabled(http!),
         ]);
 
+        const externalDataSourcesCache = CatalogCacheManager.getExternalDataSourcesCache();
+        if (externalDataSourcesCache.status === CachedDataSourceStatus.Updated) {
+          setExternalDataSources(
+            externalDataSourcesCache.dataSources.map((ds) => ({
+              id: ds.dataSourceRef,
+              name: ds.name,
+              type: SIMPLE_DATA_SOURCE_TYPES.EXTERNAL,
+            }))
+          );
+        } else if (fetchedDataSources.length > 0) {
+          setExternalDataSources(await startLoadingDataSources(fetchedDataSources));
+        }
+
         setIndexPatterns(fetchedIndexPatterns);
         setDataSources(fetchedDataSources);
-        setIsExternalDataSourcesEnabled(fetchedIsExternalDataSourcesEnabled);
+
         if (dataSet) {
           setSelectedDataSetState({
             id: dataSet.id,
@@ -206,7 +208,7 @@ export const DataSetNavigator: React.FC<DataSetNavigatorProps> = ({
     const externalDataSourcesCache = CatalogCacheManager.getExternalDataSourcesCache();
     if (status === DirectQueryLoadingStatus.SUCCESS) {
       setExternalDataSources(
-        externalDataSourcesCache.externalDataSources.map((ds) => ({
+        externalDataSourcesCache.dataSources.map((ds) => ({
           id: ds.dataSourceRef,
           name: ds.name,
           type: SIMPLE_DATA_SOURCE_TYPES.EXTERNAL,
@@ -462,25 +464,7 @@ export const DataSetNavigator: React.FC<DataSetNavigatorProps> = ({
                 {
                   name: S3DataSourcesLabel,
                   panel: 4,
-                  onClick: () => {
-                    const externalDataSourcesCache = CatalogCacheManager.getExternalDataSourcesCache();
-                    if (
-                      (externalDataSourcesCache.status === CachedDataSourceStatus.Empty ||
-                        externalDataSourcesCache.status === CachedDataSourceStatus.Failed) &&
-                      !isCatalogCacheFetching(dataSourcesLoadStatus) &&
-                      dataSources.length > 0
-                    ) {
-                      startLoadingDataSources(dataSources.map((dataSource) => dataSource.id));
-                    } else if (externalDataSourcesCache.status === CachedDataSourceStatus.Updated) {
-                      setExternalDataSources(
-                        externalDataSourcesCache.externalDataSources.map((ds) => ({
-                          id: ds.dataSourceRef,
-                          name: ds.name,
-                          type: SIMPLE_DATA_SOURCE_TYPES.EXTERNAL,
-                        }))
-                      );
-                    }
-                  },
+                  onClick: () => {},
                 },
               ]
             : []),
@@ -527,7 +511,7 @@ export const DataSetNavigator: React.FC<DataSetNavigatorProps> = ({
         ),
         items: externalDataSources.map((dataSource) => ({
           name: dataSource.name,
-          onClick: () => handleSelectExternalDataSource(dataSource),
+          onClick: async () => await handleSelectExternalDataSource(dataSource),
           panel: 5,
         })),
         content: isCatalogCacheFetching(dataSourcesLoadStatus) && createLoadingSpinner(),
@@ -652,7 +636,6 @@ export const DataSetNavigator: React.FC<DataSetNavigatorProps> = ({
       databasesLoadStatus,
       cachedTables,
       tablesLoadStatus,
-      startLoadingDataSources,
       handleSelectedDataSet,
       handleSelectedDataSource,
       handleSelectedObject,

diff --git a/src/plugins/data/public/ui/dataset_navigator/index.tsx b/src/plugins/data/public/ui/dataset_navigator/index.tsx
@@ -5,4 +5,3 @@
 
 export { DataSetNavigator, DataSetNavigatorProps } from './dataset_navigator';
 export { createDataSetNavigator } from './create_dataset_navigator';
-export { setAsyncSessionId, getAsyncSessionId, setAsyncSessionIdByObj } from './lib';
diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_loader.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_loader.tsx
@@ -5,7 +5,7 @@
 
 import { useEffect, useRef, useState } from 'react';
 import { HttpStart, NotificationsStart } from 'opensearch-dashboards/public';
-import { ASYNC_POLLING_INTERVAL, SPARK_HIVE_TABLE_REGEX, SPARK_PARTITION_INFO } from '../constants';
+import { ASYNC_QUERY, SPARK } from '../constants';
 import {
   AsyncPollingResult,
   CachedColumn,
@@ -16,12 +16,12 @@ import {
   DirectQueryLoadingStatus,
   DirectQueryRequest,
 } from '../types';
-import { getAsyncSessionId, setAsyncSessionIdByObj } from '../utils/query_session_utils';
+import { SIMPLE_DATA_SOURCE_TYPES, SimpleDataSource } from '../../../../../common';
 import { addBackticksIfNeeded, combineSchemaAndDatarows, formatError } from '../utils/shared';
 import { usePolling } from '../utils/use_polling';
-import { SQLService } from '../requests/sql';
 import { CatalogCacheManager } from './cache_manager';
 import { fetchExternalDataSources } from '../utils';
+import { getUiService } from '../../../../services';
 
 export const updateDatabasesToCache = (
   dataSourceName: string,
@@ -96,10 +96,9 @@ export const updateTablesToCache = (
       );
       return;
     }
-
     const combinedData = combineSchemaAndDatarows(pollingResult.schema, pollingResult.datarows);
     const newTables = combinedData
-      .filter((row: any) => !SPARK_HIVE_TABLE_REGEX.test(row.information))
+      .filter((row: any) => !SPARK.HIVE_TABLE_REGEX.test(row.information))
       .map((row: any) => ({
         name: row.tableName,
       }));
@@ -184,7 +183,7 @@ export const updateTableColumnsToCache = (
 
     const tableColumns: CachedColumn[] = [];
     for (const row of combinedData) {
-      if (row.col_name === SPARK_PARTITION_INFO) {
+      if (row.col_name === SPARK.PARTITION_INFO) {
         break;
       }
       tableColumns.push({
@@ -282,7 +281,6 @@ export const useLoadToCache = (
   http: HttpStart,
   notifications: NotificationsStart
 ) => {
-  const sqlService = new SQLService(http);
   const [currentDataSourceName, setCurrentDataSourceName] = useState('');
   const [currentDatabaseName, setCurrentDatabaseName] = useState<string | undefined>('');
   const [currentTableName, setCurrentTableName] = useState<string | undefined>('');
@@ -298,8 +296,13 @@ export const useLoadToCache = (
     startPolling,
     stopPolling: stopLoading,
   } = usePolling<any, any>((params) => {
-    return sqlService.fetchWithJobId(params, dataSourceMDSClientId.current);
-  }, ASYNC_POLLING_INTERVAL);
+    return http.fetch(`../../api/enhancements/datasource/jobs`, {
+      query: {
+        id: dataSourceMDSClientId.current,
+        queryId: params.queryId,
+      },
+    });
+  }, ASYNC_QUERY.POLLING_INTERVAL);
 
   const onLoadingFailed = () => {
     setLoadStatus(DirectQueryLoadingStatus.FAILED);
@@ -319,6 +322,7 @@ export const useLoadToCache = (
     databaseName,
     tableName,
   }: StartLoadingParams) => {
+    const uiService = getUiService();
     setLoadStatus(DirectQueryLoadingStatus.SCHEDULED);
     setCurrentDataSourceName(dataSourceName);
     setCurrentDatabaseName(databaseName);
@@ -331,14 +335,19 @@ export const useLoadToCache = (
       datasource: dataSourceName,
     };
 
-    const sessionId = getAsyncSessionId(dataSourceName);
+    const sessionId = uiService.Settings.getUserQuerySessionId(dataSourceName);
     if (sessionId) {
       requestPayload = { ...requestPayload, sessionId };
     }
-    await sqlService
-      .fetch(requestPayload, dataSourceMDSId)
+    await http
+      .post(`../../api/enhancements/datasource/jobs`, {
+        body: JSON.stringify(requestPayload),
+        query: {
+          id: dataSourceMDSClientId.current,
+        },
+      })
       .then((result) => {
-        setAsyncSessionIdByObj(dataSourceName, result);
+        uiService.Settings.setUserQuerySessionIdByObj(dataSourceName, result);
         if (result.queryId) {
           startPolling({
             queryId: result.queryId,
@@ -443,7 +452,9 @@ export const useLoadExternalDataSourcesToCache = (
     DirectQueryLoadingStatus.INITIAL
   );
 
-  const loadExternalDataSources = async (connectedClusters: string[]) => {
+  const loadExternalDataSources = async (
+    connectedClusters: SimpleDataSource[]
+  ): Promise<SimpleDataSource[]> => {
     setLoadStatus(DirectQueryLoadingStatus.SCHEDULED);
     CatalogCacheManager.setExternalDataSourcesLoadingStatus(CachedDataSourceStatus.Empty);
 
@@ -452,13 +463,19 @@ export const useLoadExternalDataSourcesToCache = (
       CatalogCacheManager.updateExternalDataSources(externalDataSources);
       setLoadStatus(DirectQueryLoadingStatus.SUCCESS);
       CatalogCacheManager.setExternalDataSourcesLoadingStatus(CachedDataSourceStatus.Updated);
+      return externalDataSources.map((dataSource) => ({
+        id: dataSource.dataSourceRef,
+        name: dataSource.name,
+        type: SIMPLE_DATA_SOURCE_TYPES.EXTERNAL,
+      }));
     } catch (error) {
       setLoadStatus(DirectQueryLoadingStatus.FAILED);
       CatalogCacheManager.setExternalDataSourcesLoadingStatus(CachedDataSourceStatus.Failed);
       notifications.toasts.addError(error, {
         title: 'Failed to load external datasources',
       });
     }
+    return [];
   };
 
   return { loadStatus, loadExternalDataSources };