Lake formation support updates (#1871)

* modified get tables query for iceberg tables Signed-off-by: Amardeepsingh Siglani <amardeep7194@gmail.com> * updated integration setup for security lake Signed-off-by: Amardeepsingh Siglani <amardeep7194@gmail.com> * filter connection type option based on integration workflows Signed-off-by: Amardeepsingh Siglani <amardeep7194@gmail.com> * updated code to use connector type for Security Lake Signed-off-by: Amardeepsingh Siglani <amardeep7194@gmail.com> * fixed logic to select default index type in create acceleration Signed-off-by: Amardeepsingh Siglani <amardeep7194@gmail.com> * updated tests Signed-off-by: Amardeepsingh Siglani <amardeep7194@gmail.com> --------- Signed-off-by: Amardeepsingh Siglani <amardeep7194@gmail.com>
opensearch-project · Jun 7, 2024 · 19d68b2 · 19d68b2
1 parent e5af7eb
commit 19d68b2
Show file tree

Hide file tree

Showing 31 changed files with 531 additions and 540 deletions.
diff --git a/common/constants/data_connections.ts b/common/constants/data_connections.ts
@@ -19,6 +19,7 @@ export const QUERY_ALL = 'query-all';
 export const DatasourceTypeToDisplayName: { [key in DatasourceType]: string } = {
   PROMETHEUS: 'Prometheus',
   S3GLUE: 'Amazon S3',
+  SECURITYLAKE: 'Amazon Security Lake',
 };
 
 export const PrometheusURL = 'Prometheus';

diff --git a/common/types/data_connections.ts b/common/types/data_connections.ts
@@ -47,12 +47,11 @@ export interface AssociatedObject {
 
 export type Role = EuiComboBoxOptionOption;
 
-export type DatasourceType = 'S3GLUE' | 'PROMETHEUS';
+export type DatasourceType = 'S3GLUE' | 'PROMETHEUS' | 'SECURITYLAKE';
 
 export interface S3GlueProperties {
   'glue.indexstore.opensearch.uri': string;
   'glue.indexstore.opensearch.region': string;
-  'glue.lakeformation.enabled'?: boolean;
 }
 
 export interface PrometheusProperties {
@@ -252,10 +251,12 @@ export interface StartLoadingParams {
   dataSourceMDSId?: string;
   databaseName?: string;
   tableName?: string;
+  dataSourceType?: DatasourceType;
 }
 
 export interface RenderAccelerationFlyoutParams {
   dataSource: string;
+  dataSourceType: DatasourceType;
   dataSourceMDSId?: string;
   databaseName?: string;
   tableName?: string;
@@ -265,9 +266,9 @@ export interface RenderAccelerationFlyoutParams {
 export interface RenderAssociatedObjectsDetailsFlyoutParams {
   tableDetail: AssociatedObject;
   dataSourceName: string;
+  dataSourceType: DatasourceType;
   handleRefresh?: () => void;
   dataSourceMDSId?: string;
-  isS3ConnectionWithLakeFormation?: boolean;
 }
 
 export interface RenderAccelerationDetailsFlyoutParams {

diff --git a/common/types/integrations.ts b/common/types/integrations.ts
@@ -0,0 +1,6 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export type IntegrationConnectionType = 's3' | 'index' | 'securityLake';
diff --git a/...nents/datasources/components/__tests__/__snapshots__/associated_objects_tab.test.tsx.snap b/...nents/datasources/components/__tests__/__snapshots__/associated_objects_tab.test.tsx.snap
@@ -319,6 +319,7 @@ exports[`AssociatedObjectsTab Component renders correctly with associated object
               >
                 <CreateAccelerationFlyoutButton
                   dataSourceName="mock_data_source"
+                  dataSourceType="S3GLUE"
                   handleRefresh={[Function]}
                   renderCreateAccelerationFlyout={[MockFunction]}
                 >
@@ -738,9 +739,9 @@ exports[`AssociatedObjectsTab Component renders correctly with associated object
                     },
                   ]
                 }
+                dataSourceType="S3GLUE"
                 datasourceName="mock_data_source"
                 handleRefresh={[Function]}
-                isS3ConnectionWithLakeFormation={false}
               >
                 <EuiInMemoryTable
                   columns={
@@ -5197,6 +5198,7 @@ exports[`AssociatedObjectsTab Component renders tab with no databases or objects
               >
                 <CreateAccelerationFlyoutButton
                   dataSourceName="mock_data_source"
+                  dataSourceType="S3GLUE"
                   handleRefresh={[Function]}
                   renderCreateAccelerationFlyout={[MockFunction]}
                 >

diff --git a/...datasources/components/__tests__/__snapshots__/installed_integrations_table.test.tsx.snap b/...datasources/components/__tests__/__snapshots__/installed_integrations_table.test.tsx.snap
@@ -409,6 +409,7 @@ exports[`Installed Integrations Table test Renders the installed integrations ta
           Array [
             Object {
               "assets": 3,
+              "id": "d5b55c60-e08c-11ee-9c80-ff3b93498fea",
               "locator": Object {
                 "id": "d5b55c60-e08c-11ee-9c80-ff3b93498fea",
                 "name": "aws_waf-sample",
@@ -444,6 +445,7 @@ exports[`Installed Integrations Table test Renders the installed integrations ta
             Array [
               Object {
                 "assets": 3,
+                "id": "d5b55c60-e08c-11ee-9c80-ff3b93498fea",
                 "locator": Object {
                   "id": "d5b55c60-e08c-11ee-9c80-ff3b93498fea",
                   "name": "aws_waf-sample",

diff --git a/public/components/datasources/components/__tests__/acceleration_table.test.tsx b/public/components/datasources/components/__tests__/acceleration_table.test.tsx
@@ -95,7 +95,11 @@ describe('AccelerationTable Component', () => {
 
   it('renders without crashing', () => {
     const wrapper = mount(
-      <AccelerationTable dataSourceName="testDataSource" cacheLoadingHooks={cacheLoadingHooks} />
+      <AccelerationTable
+        dataSourceName="testDataSource"
+        cacheLoadingHooks={cacheLoadingHooks}
+        dataSourceType="S3GLUE"
+      />
     );
     expect(wrapper).toBeDefined();
   });
@@ -111,7 +115,11 @@ describe('AccelerationTable Component', () => {
     let wrapper: ReactWrapper;
     await act(async () => {
       wrapper = mount(
-        <AccelerationTable dataSourceName="testDataSource" cacheLoadingHooks={cacheLoadingHooks} />
+        <AccelerationTable
+          dataSourceName="testDataSource"
+          cacheLoadingHooks={cacheLoadingHooks}
+          dataSourceType="S3GLUE"
+        />
       );
     });
 
@@ -132,7 +140,11 @@ describe('AccelerationTable Component', () => {
     let wrapper: ReactWrapper;
     await act(async () => {
       wrapper = mount(
-        <AccelerationTable dataSourceName="testDataSource" cacheLoadingHooks={cacheLoadingHooks} />
+        <AccelerationTable
+          dataSourceName="testDataSource"
+          cacheLoadingHooks={cacheLoadingHooks}
+          dataSourceType="S3GLUE"
+        />
       );
     });
     wrapper!.update();
@@ -152,7 +164,11 @@ describe('AccelerationTable Component', () => {
     let wrapper: ReactWrapper;
     await act(async () => {
       wrapper = mount(
-        <AccelerationTable dataSourceName="testDataSource" cacheLoadingHooks={cacheLoadingHooks} />
+        <AccelerationTable
+          dataSourceName="testDataSource"
+          cacheLoadingHooks={cacheLoadingHooks}
+          dataSourceType="S3GLUE"
+        />
       );
       await new Promise((resolve) => setTimeout(resolve, 0));
       wrapper!.update();
@@ -172,7 +188,11 @@ describe('AccelerationTable Component', () => {
     let wrapper: ReactWrapper;
     await act(async () => {
       wrapper = mount(
-        <AccelerationTable dataSourceName="testDataSource" cacheLoadingHooks={cacheLoadingHooks} />
+        <AccelerationTable
+          dataSourceName="testDataSource"
+          cacheLoadingHooks={cacheLoadingHooks}
+          dataSourceType="S3GLUE"
+        />
       );
     });
     wrapper!.update();

diff --git a/public/components/datasources/components/manage/accelerations/acceleration_table.tsx b/public/components/datasources/components/manage/accelerations/acceleration_table.tsx
@@ -21,6 +21,7 @@ import React, { useCallback, useEffect, useState } from 'react';
 import {
   CachedAcceleration,
   CachedDataSourceStatus,
+  DatasourceType,
 } from '../../../../../../common/types/data_connections';
 import { DirectQueryLoadingStatus } from '../../../../../../common/types/explorer';
 import { CatalogCacheManager } from '../../../../../framework/catalog_cache/cache_manager';
@@ -45,7 +46,7 @@ import {
 interface AccelerationTableProps {
   dataSourceName: string;
   cacheLoadingHooks: any;
-  isS3ConnectionWithLakeFormation: boolean;
+  dataSourceType: DatasourceType;
 }
 
 interface ModalState {
@@ -56,7 +57,7 @@ interface ModalState {
 export const AccelerationTable = ({
   dataSourceName,
   cacheLoadingHooks,
-  isS3ConnectionWithLakeFormation,
+  dataSourceType,
 }: AccelerationTableProps) => {
   const [accelerations, setAccelerations] = useState<CachedAcceleration[]>([]);
   const [updatedTime, setUpdatedTime] = useState<string>();
@@ -171,6 +172,7 @@ export const AccelerationTable = ({
               <EuiFlexItem grow={false}>
                 <CreateAccelerationFlyoutButton
                   dataSourceName={dataSourceName}
+                  dataSourceType={dataSourceType}
                   renderCreateAccelerationFlyout={renderCreateAccelerationFlyout}
                   handleRefresh={handleRefresh}
                 />
@@ -327,11 +329,12 @@ export const AccelerationTable = ({
     },
   };
 
-  const accelerationTableColumns = !isS3ConnectionWithLakeFormation
-    ? Object.values(accelerationTableColumnsCollection)
-    : Object.entries(accelerationTableColumnsCollection)
-        .filter(([key]) => key !== 'database' && key !== 'table')
-        .map(([_key, val]) => val);
+  const accelerationTableColumns =
+    dataSourceType.toUpperCase() === 'SECURITYLAKE'
+      ? Object.entries(accelerationTableColumnsCollection)
+          .filter(([key]) => key !== 'database' && key !== 'table')
+          .map(([_key, val]) => val)
+      : Object.values(accelerationTableColumnsCollection);
 
   const pagination = {
     initialPageSize: 10,

diff --git a/...e/accelerations/create_accelerations_flyout/create/__tests__/create_acceleration.test.tsx b/...e/accelerations/create_accelerations_flyout/create/__tests__/create_acceleration.test.tsx
@@ -36,7 +36,11 @@ describe('Create acceleration flyout components', () => {
     coreStartMock.http.get = jest.fn().mockResolvedValue(mockDatasourcesQuery);
 
     const wrapper = mount(
-      <CreateAcceleration selectedDatasource={selectedDatasource} resetFlyout={resetFlyout} />
+      <CreateAcceleration
+        selectedDatasource={selectedDatasource}
+        resetFlyout={resetFlyout}
+        selectedDatasourceType="S3GLUE"
+      />
     );
     wrapper.update();
     await waitFor(() => {

diff --git a/...omponents/manage/accelerations/create_accelerations_flyout/create/create_acceleration.tsx b/...omponents/manage/accelerations/create_accelerations_flyout/create/create_acceleration.tsx
@@ -24,6 +24,7 @@ import {
 import {
   CachedTable,
   CreateAccelerationForm,
+  DatasourceType,
 } from '../../../../../../../../common/types/data_connections';
 import { DirectQueryLoadingStatus } from '../../../../../../../../common/types/explorer';
 import { useLoadTableColumnsToCache } from '../../../../../../../framework/catalog_cache/cache_loader';
@@ -39,11 +40,10 @@ import { QueryVisualEditor } from '../visual_editors/query_visual_editor';
 import { CreateAccelerationButton } from './create_acceleration_button';
 import { CreateAccelerationHeader } from './create_acceleration_header';
 import { hasError } from './utils';
-import { DATACONNECTIONS_BASE } from '../../../../../../../../common/constants/shared';
-import { checkIsConnectionWithLakeFormation } from '../../../../../utils/helpers';
 
 export interface CreateAccelerationProps {
   selectedDatasource: string;
+  selectedDatasourceType: DatasourceType;
   resetFlyout: () => void;
   databaseName?: string;
   tableName?: string;
@@ -53,6 +53,7 @@ export interface CreateAccelerationProps {
 
 export const CreateAcceleration = ({
   selectedDatasource,
+  selectedDatasourceType,
   resetFlyout,
   databaseName,
   tableName,
@@ -61,13 +62,12 @@ export const CreateAcceleration = ({
 }: CreateAccelerationProps) => {
   const { setToast } = useToast();
   const http = coreRefs!.http;
-  const [isS3ConnectionWithLakeFormation, setIsS3ConnectionWithLakeFormation] = useState(false);
   const [accelerationFormData, setAccelerationFormData] = useState<CreateAccelerationForm>({
     dataSource: selectedDatasource,
     database: databaseName ?? '',
     dataTable: tableName ?? '',
     dataTableFields: [],
-    accelerationIndexType: 'skipping',
+    accelerationIndexType: selectedDatasourceType === 'SECURITYLAKE' ? 'materialized' : 'skipping',
     skippingIndexQueryData: [],
     coveringIndexQueryData: [],
     materializedViewQueryData: {
@@ -170,16 +170,6 @@ export const CreateAcceleration = ({
     }
   };
 
-  const updateDataSourceConnectionInfo = () => {
-    coreRefs.http!.get(`${DATACONNECTIONS_BASE}/${selectedDatasource}`).then((data: any) => {
-      setIsS3ConnectionWithLakeFormation(checkIsConnectionWithLakeFormation(data));
-    });
-  };
-
-  useEffect(() => {
-    updateDataSourceConnectionInfo();
-  }, [selectedDatasource]);
-
   useEffect(() => {
     if (databaseName !== undefined && tableName !== undefined) {
       initiateColumnLoad(
@@ -245,14 +235,15 @@ export const CreateAcceleration = ({
                 setDataSourceFormData: setAccelerationFormData,
               }}
               selectedDatasource={selectedDatasource}
+              selectedDataSourceType={selectedDatasourceType}
               dataSourcesPreselected={dataSourcesPreselected}
               tableFieldsLoading={tableFieldsLoading}
               dataSourceMDSId={dataSourceMDSId}
             />
             <EuiSpacer size="xxl" />
             <IndexTypeSelector
               accelerationFormData={accelerationFormData}
-              isS3ConnectionWithLakeFormation={isS3ConnectionWithLakeFormation}
+              dataSourceType={selectedDatasourceType}
               setAccelerationFormData={setAccelerationFormData}
               initiateColumnLoad={initiateColumnLoad}
             />

diff --git a/...ccelerations/create_accelerations_flyout/selectors/__tests__/index_type_selector.test.tsx b/...ccelerations/create_accelerations_flyout/selectors/__tests__/index_type_selector.test.tsx
@@ -21,9 +21,9 @@ describe('Index type selector components', () => {
     const wrapper = mount(
       <IndexTypeSelector
         accelerationFormData={accelerationFormData}
+        dataSourceType="S3GLUE"
         setAccelerationFormData={setAccelerationFormData}
         initiateColumnLoad={jest.fn()}
-        loading={false}
       />
     );
     wrapper.update();
@@ -46,9 +46,9 @@ describe('Index type selector components', () => {
     const wrapper = mount(
       <IndexTypeSelector
         accelerationFormData={accelerationFormData}
+        dataSourceType="S3GLUE"
         setAccelerationFormData={setAccelerationFormData}
         initiateColumnLoad={jest.fn()}
-        loading={true}
       />
     );
     wrapper.update();