diff --git a/common/types/data_connections.ts b/common/types/data_connections.ts index a5585aca76..a5c7edf319 100644 --- a/common/types/data_connections.ts +++ b/common/types/data_connections.ts @@ -52,6 +52,7 @@ export type DatasourceType = 'S3GLUE' | 'PROMETHEUS'; export interface S3GlueProperties { 'glue.indexstore.opensearch.uri': string; 'glue.indexstore.opensearch.region': string; + 'glue.lakeformation.enabled': boolean; } export interface PrometheusProperties { diff --git a/public/components/datasources/components/manage/accelerations/acceleration_table.tsx b/public/components/datasources/components/manage/accelerations/acceleration_table.tsx index e3e386d3ad..708acc229a 100644 --- a/public/components/datasources/components/manage/accelerations/acceleration_table.tsx +++ b/public/components/datasources/components/manage/accelerations/acceleration_table.tsx @@ -14,8 +14,8 @@ import { EuiLoadingSpinner, EuiPanel, EuiSpacer, - EuiTableFieldDataColumnType, EuiText, + EuiBasicTableColumn, } from '@elastic/eui'; import React, { useCallback, useEffect, useState } from 'react'; import { @@ -45,6 +45,7 @@ import { interface AccelerationTableProps { dataSourceName: string; cacheLoadingHooks: any; + isS3ConnectionWithLakeFormation: boolean; } interface ModalState { @@ -55,6 +56,7 @@ interface ModalState { export const AccelerationTable = ({ dataSourceName, cacheLoadingHooks, + isS3ConnectionWithLakeFormation, }: AccelerationTableProps) => { const [accelerations, setAccelerations] = useState([]); const [updatedTime, setUpdatedTime] = useState(); @@ -236,8 +238,10 @@ export const AccelerationTable = ({ }, ]; - const accelerationTableColumns = [ - { + const accelerationTableColumnsCollection: { + [columnKey: string]: EuiBasicTableColumn; + } = { + name: { field: 'indexName', name: 'Name', sortable: true, @@ -254,13 +258,13 @@ export const AccelerationTable = ({ ); }, }, - { + status: { field: 'status', name: 'Status', sortable: true, render: (status: string) => , }, - { + type: { field: 'type', name: 'Type', sortable: true, @@ -282,19 +286,19 @@ export const AccelerationTable = ({ return {label}; }, }, - { + database: { field: 'database', name: 'Database', sortable: true, render: (database: string) => {database}, }, - { + table: { field: 'table', name: 'Table', sortable: true, render: (table: string) => {table || '-'}, }, - { + refreshType: { field: 'refreshType', name: 'Refresh Type', sortable: true, @@ -302,7 +306,7 @@ export const AccelerationTable = ({ return {acceleration.autoRefresh ? 'Auto refresh' : 'Manual'}; }, }, - { + flintIndexName: { field: 'flintIndexName', name: 'Destination Index', sortable: true, @@ -313,11 +317,17 @@ export const AccelerationTable = ({ return flintIndexName || '-'; }, }, - { + actions: { name: 'Actions', actions: tableActions, }, - ] as Array>; + }; + + const accelerationTableColumns = !isS3ConnectionWithLakeFormation + ? Object.values(accelerationTableColumnsCollection) + : Object.entries(accelerationTableColumnsCollection) + .filter(([key]) => key !== 'database' && key !== 'table') + .map(([_key, val]) => val); const pagination = { initialPageSize: 10, diff --git a/public/components/datasources/components/manage/associated_objects/associated_objects_details_flyout.tsx b/public/components/datasources/components/manage/associated_objects/associated_objects_details_flyout.tsx index 5cd6414113..67305874ee 100644 --- a/public/components/datasources/components/manage/associated_objects/associated_objects_details_flyout.tsx +++ b/public/components/datasources/components/manage/associated_objects/associated_objects_details_flyout.tsx @@ -5,7 +5,6 @@ import { EuiButton, - EuiButtonEmpty, EuiDescriptionList, EuiDescriptionListDescription, EuiDescriptionListTitle, @@ -15,13 +14,14 @@ import { EuiFlyoutBody, EuiFlyoutHeader, EuiHorizontalRule, - EuiIcon, EuiInMemoryTable, EuiLink, EuiSpacer, EuiTableFieldDataColumnType, EuiText, EuiTitle, + EuiButtonIcon, + EuiToolTip, } from '@elastic/eui'; import { i18n } from '@osd/i18n'; import React, { useEffect, useState } from 'react'; @@ -56,6 +56,7 @@ export interface AssociatedObjectsFlyoutProps { resetFlyout: () => void; handleRefresh?: () => void; dataSourceMDSId?: string; + isS3ConnectionWithLakeFormation?: boolean; } export const AssociatedObjectsDetailsFlyout = ({ @@ -64,6 +65,7 @@ export const AssociatedObjectsDetailsFlyout = ({ resetFlyout, handleRefresh, dataSourceMDSId, + isS3ConnectionWithLakeFormation, }: AssociatedObjectsFlyoutProps) => { const { loadStatus, startLoading } = useLoadTableColumnsToCache(); const [tableColumns, setTableColumns] = useState([]); @@ -72,38 +74,39 @@ export const AssociatedObjectsDetailsFlyout = ({ const DiscoverButton = () => { return ( - { - if (tableDetail.type !== 'table') return; - redirectToExplorerWithDataSrc( - tableDetail.datasource, - DATA_SOURCE_TYPES.S3Glue, - tableDetail.database, - tableDetail.name - ); - resetFlyout(); - }} - > - - + + { + if (tableDetail.type !== 'table') return; + redirectToExplorerWithDataSrc( + tableDetail.datasource, + DATA_SOURCE_TYPES.S3Glue, + tableDetail.database, + tableDetail.name + ); + resetFlyout(); + }} + /> + ); }; + const onCreateAcceleration = () => + renderCreateAccelerationFlyout( + datasourceName, + '', + tableDetail.database, + tableDetail.name, + handleRefresh + ); + const AccelerateButton = () => { return ( - - renderCreateAccelerationFlyout( - datasourceName, - '', - tableDetail.database, - tableDetail.name, - handleRefresh - ) - } - > - - + + + ); }; @@ -129,14 +132,19 @@ export const AssociatedObjectsDetailsFlyout = ({ ); }; - const TableTitleComponent = (titleProps: { title: string }) => { - const { title } = titleProps; + const TableTitleComponent = (props: { + title: string; + description?: string; + horizontalRuleBottom?: boolean; + }) => { + const { title, description, horizontalRuleBottom } = props; return ( <>

{title}

 - + {description && {description}} + {horizontalRuleBottom && } ); }; @@ -295,17 +303,34 @@ export const AssociatedObjectsDetailsFlyout = ({ - + - + - + + + + + {isS3ConnectionWithLakeFormation && ( + + Create acceleration + + )} + {accelerationData.length > 0 ? ( <> - + = (props) => { const { datasource, cacheLoadingHooks, selectedDatabase, setSelectedDatabase } = props; + const isS3ConnectionWithLakeFormation = useMemo( + () => checkIsConnectionWithLakeFormation(datasource), + [datasource] + ); const [isRefreshing, setIsRefreshing] = useState(false); const [lastUpdated, setLastUpdated] = useState(new Date().toLocaleString()); const [isObjectsLoading, setIsObjectsLoading] = useState(false); @@ -76,14 +84,11 @@ export const AssociatedObjectsTab: React.FC = (props) startLoadingAccelerations, } = cacheLoadingHooks; - let lastChecked: boolean; - if (selectedDatabase !== '') { - lastChecked = true; - } else { - lastChecked = false; - } + const lastChecked: boolean = selectedDatabase !== ''; // Get last selected if there is one, set to first option if not - const [databaseSelectorOptions, setDatabaseSelectorOptions] = useState( + const [databaseSelectorOptions, setDatabaseSelectorOptions] = useState< + Array> + >( cachedDatabases.map((database, index) => { return { label: database.name, @@ -107,11 +112,15 @@ export const AssociatedObjectsTab: React.FC = (props) const AssociatedObjectsHeader = () => { const panelTitle = i18n.translate('datasources.associatedObjectsTab.panelTitle', { - defaultMessage: ASSC_OBJ_PANEL_TITLE, + defaultMessage: isS3ConnectionWithLakeFormation + ? ASSC_OBJ_PANEL_TITLE_FOR_S3_WITH_LAKE_FORMATION + : ASSC_OBJ_PANEL_TITLE, }); const panelDescription = i18n.translate('datasources.associatedObjectsTab.panelDescription', { - defaultMessage: ASSC_OBJ_PANEL_DESCRIPTION, + defaultMessage: isS3ConnectionWithLakeFormation + ? ASSC_OBJ_PANEL_DESCRIPTION_FOR_S3_WITH_LAKE_FORMATION + : ASSC_OBJ_PANEL_DESCRIPTION, }); const LastUpdatedText = () => { @@ -147,13 +156,15 @@ export const AssociatedObjectsTab: React.FC = (props) onClick={onRefreshButtonClick} /> - - - + {!isS3ConnectionWithLakeFormation && ( + + + + )} ); }; @@ -325,24 +336,28 @@ export const AssociatedObjectsTab: React.FC = (props) columns: table.columns, }; }); - const accelerationObjects: AssociatedObject[] = cachedAccelerations - .filter((acceleration: CachedAcceleration) => acceleration.database === selectedDatabase) - .map((acceleration: CachedAcceleration) => ({ - tableName: acceleration.table, - datasource: datasource.name, - id: acceleration.indexName, - name: getAccelerationName(acceleration), - database: acceleration.database, - type: ACCELERATION_INDEX_TYPES.find((accelType) => accelType.value === acceleration.type)! - .value as AssociatedObjectIndexType, - accelerations: - acceleration.type === 'covering' || acceleration.type === 'skipping' - ? tableObjects.find( - (tableObject: AssociatedObject) => tableObject.name === acceleration.table - ) - : [], - columns: undefined, - })); + // For data connections using lake formation we don't want to show accelerations, so we simply assign empty array + const accelerationObjects: AssociatedObject[] = isS3ConnectionWithLakeFormation + ? [] + : cachedAccelerations + .filter((acceleration: CachedAcceleration) => acceleration.database === selectedDatabase) + .map((acceleration: CachedAcceleration) => ({ + tableName: acceleration.table, + datasource: datasource.name, + id: acceleration.indexName, + name: getAccelerationName(acceleration), + database: acceleration.database, + type: ACCELERATION_INDEX_TYPES.find( + (accelType) => accelType.value === acceleration.type + )!.value as AssociatedObjectIndexType, + accelerations: + acceleration.type === 'covering' || acceleration.type === 'skipping' + ? tableObjects.find( + (tableObject: AssociatedObject) => tableObject.name === acceleration.table + ) || [] + : [], + columns: undefined, + })); setAssociatedObjects([...tableObjects, ...accelerationObjects]); }, [selectedDatabase, cachedTables, cachedAccelerations]); @@ -351,8 +366,12 @@ export const AssociatedObjectsTab: React.FC = (props) return ( <> - - + {!isS3ConnectionWithLakeFormation && ( + <> + + + + )} @@ -400,6 +419,7 @@ export const AssociatedObjectsTab: React.FC = (props) datasourceName={datasource.name} associatedObjects={associatedObjects} cachedAccelerations={cachedAccelerations} + isS3ConnectionWithLakeFormation={isS3ConnectionWithLakeFormation} handleRefresh={onRefreshButtonClick} /> ) : ( diff --git a/public/components/datasources/components/manage/associated_objects/modules/associated_objects_table.tsx b/public/components/datasources/components/manage/associated_objects/modules/associated_objects_table.tsx index 3371797fe7..56d7c0a207 100644 --- a/public/components/datasources/components/manage/associated_objects/modules/associated_objects_table.tsx +++ b/public/components/datasources/components/manage/associated_objects/modules/associated_objects_table.tsx @@ -23,6 +23,7 @@ import { } from '../../../../../../plugin'; import { ASSC_OBJ_TABLE_ACC_COLUMN_NAME, + ASSC_OBJ_TABLE_FOR_S3_WITH_LAKE_FORMATION_SEARCH_HINT, ASSC_OBJ_TABLE_SEARCH_HINT, ASSC_OBJ_TABLE_SUBJ, redirectToExplorerOSIdx, @@ -38,6 +39,7 @@ interface AssociatedObjectsTableProps { datasourceName: string; associatedObjects: AssociatedObject[]; cachedAccelerations: CachedAcceleration[]; + isS3ConnectionWithLakeFormation: boolean; handleRefresh: () => void; } @@ -53,8 +55,13 @@ interface AssociatedTableFilter { value: string; } -export const AssociatedObjectsTable = (props: AssociatedObjectsTableProps) => { - const { datasourceName, associatedObjects, cachedAccelerations, handleRefresh } = props; +export const AssociatedObjectsTable = ({ + datasourceName, + associatedObjects, + cachedAccelerations, + isS3ConnectionWithLakeFormation, + handleRefresh, +}: AssociatedObjectsTableProps) => { const [accelerationFilterOptions, setAccelerationFilterOptions] = useState([]); const [filteredObjects, setFilteredObjects] = useState([]); @@ -62,7 +69,7 @@ export const AssociatedObjectsTable = (props: AssociatedObjectsTableProps) => { { field: 'name', name: i18n.translate('datasources.associatedObjectsTab.column.name', { - defaultMessage: 'Name', + defaultMessage: isS3ConnectionWithLakeFormation ? 'Table' : 'Name', }), sortable: true, 'data-test-subj': 'nameCell', @@ -70,7 +77,13 @@ export const AssociatedObjectsTable = (props: AssociatedObjectsTableProps) => { { if (item.type === 'table') { - renderAssociatedObjectsDetailsFlyout(item, datasourceName, handleRefresh); + renderAssociatedObjectsDetailsFlyout( + item, + datasourceName, + handleRefresh, + undefined, + isS3ConnectionWithLakeFormation + ); } else { const acceleration = cachedAccelerations.find((acc) => acc.indexName === item.id); if (acceleration) { @@ -83,21 +96,10 @@ export const AssociatedObjectsTable = (props: AssociatedObjectsTableProps) => { ), }, - { - field: 'type', - name: i18n.translate('datasources.associatedObjectsTab.column.type', { - defaultMessage: 'Type', - }), - sortable: true, - render: (type) => { - if (type === 'table') return 'Table'; - return ACCELERATION_INDEX_TYPES.find((accType) => type === accType.value)!.label; - }, - }, { field: 'accelerations', name: i18n.translate('datasources.associatedObjectsTab.column.accelerations', { - defaultMessage: 'Associations', + defaultMessage: isS3ConnectionWithLakeFormation ? 'Accelerations' : 'Associations', }), sortable: true, render: (accelerations: CachedAcceleration[] | AssociatedObject, obj: AssociatedObject) => { @@ -119,7 +121,13 @@ export const AssociatedObjectsTable = (props: AssociatedObjectsTableProps) => { return ( - renderAssociatedObjectsDetailsFlyout(obj, datasourceName, handleRefresh) + renderAssociatedObjectsDetailsFlyout( + obj, + datasourceName, + handleRefresh, + undefined, + isS3ConnectionWithLakeFormation + ) } > View all {accelerations.length} @@ -129,7 +137,13 @@ export const AssociatedObjectsTable = (props: AssociatedObjectsTableProps) => { return ( - renderAssociatedObjectsDetailsFlyout(accelerations, datasourceName, handleRefresh) + renderAssociatedObjectsDetailsFlyout( + accelerations, + datasourceName, + handleRefresh, + undefined, + isS3ConnectionWithLakeFormation + ) } > {accelerations.name} @@ -143,6 +157,22 @@ export const AssociatedObjectsTable = (props: AssociatedObjectsTableProps) => { defaultMessage: 'Actions', }), actions: [ + { + name: i18n.translate('datasources.associatedObjectsTab.action.accelerate.name', { + defaultMessage: 'Accelerate', + }), + description: i18n.translate( + 'datasources.associatedObjectsTab.action.accelerate.description', + { + defaultMessage: 'Accelerate this object', + } + ), + type: 'icon', + icon: 'bolt', + available: (item: AssociatedObject) => item.type === 'table', + onClick: (item: AssociatedObject) => + renderCreateAccelerationFlyout(datasourceName, item.database, item.name, handleRefresh), + }, { name: i18n.translate('datasources.associatedObjectsTab.action.discover.name', { defaultMessage: 'Discover', @@ -172,26 +202,24 @@ export const AssociatedObjectsTable = (props: AssociatedObjectsTableProps) => { } }, }, - { - name: i18n.translate('datasources.associatedObjectsTab.action.accelerate.name', { - defaultMessage: 'Accelerate', - }), - description: i18n.translate( - 'datasources.associatedObjectsTab.action.accelerate.description', - { - defaultMessage: 'Accelerate this object', - } - ), - type: 'icon', - icon: 'bolt', - available: (item: AssociatedObject) => item.type === 'table', - onClick: (item: AssociatedObject) => - renderCreateAccelerationFlyout(datasourceName, item.database, item.name, handleRefresh), - }, ], }, ] as Array>; + if (!isS3ConnectionWithLakeFormation) { + columns.splice(1, 0, { + field: 'type', + name: i18n.translate('datasources.associatedObjectsTab.column.type', { + defaultMessage: 'Type', + }), + sortable: true, + render: (type) => { + if (type === 'table') return 'Table'; + return ACCELERATION_INDEX_TYPES.find((accType) => type === accType.value)!.label; + }, + }); + } + const onSearchChange = ({ query, error }) => { if (error) { console.log('Search error:', error); @@ -238,7 +266,9 @@ export const AssociatedObjectsTable = (props: AssociatedObjectsTableProps) => { filters: searchFilters, box: { incremental: true, - placeholder: ASSC_OBJ_TABLE_SEARCH_HINT, + placeholder: isS3ConnectionWithLakeFormation + ? ASSC_OBJ_TABLE_FOR_S3_WITH_LAKE_FORMATION_SEARCH_HINT + : ASSC_OBJ_TABLE_SEARCH_HINT, schema: { fields: { name: { type: 'string' }, database: { type: 'string' } }, }, diff --git a/public/components/datasources/components/manage/associated_objects/utils/associated_objects_tab_utils.tsx b/public/components/datasources/components/manage/associated_objects/utils/associated_objects_tab_utils.tsx index 63ad3261ee..d6a1a4dcc8 100644 --- a/public/components/datasources/components/manage/associated_objects/utils/associated_objects_tab_utils.tsx +++ b/public/components/datasources/components/manage/associated_objects/utils/associated_objects_tab_utils.tsx @@ -18,10 +18,17 @@ export const ASSC_OBJ_TABLE_ACC_COLUMN_NAME = 'accelerations'; export const ASSC_OBJ_TABLE_SEARCH_HINT = 'Search for objects'; +export const ASSC_OBJ_TABLE_FOR_S3_WITH_LAKE_FORMATION_SEARCH_HINT = 'Search for tables'; + export const ASSC_OBJ_PANEL_TITLE = 'Associated objects'; +export const ASSC_OBJ_PANEL_TITLE_FOR_S3_WITH_LAKE_FORMATION = 'Tables'; + export const ASSC_OBJ_PANEL_DESCRIPTION = 'Manage objects associated with this data sources.'; +export const ASSC_OBJ_PANEL_DESCRIPTION_FOR_S3_WITH_LAKE_FORMATION = + 'Security Lake tables partially accelerated with skipping indexes.'; + export const ASSC_OBJ_NO_DATA_TITLE = 'You have no associated objects'; export const ASSC_OBJ_NO_DATA_DESCRIPTION = diff --git a/public/components/datasources/components/manage/data_connection.tsx b/public/components/datasources/components/manage/data_connection.tsx index 1a67284127..f9427c4816 100644 --- a/public/components/datasources/components/manage/data_connection.tsx +++ b/public/components/datasources/components/manage/data_connection.tsx @@ -18,6 +18,7 @@ import { EuiTabbedContent, EuiText, EuiTitle, + EuiButtonEmpty, } from '@elastic/eui'; import _ from 'lodash'; import React, { useEffect, useState } from 'react'; @@ -47,6 +48,7 @@ import { InstallIntegrationFlyout, InstalledIntegrationsTable, } from './integrations/installed_integrations_table'; +import { checkIsConnectionWithLakeFormation } from '../../utils/helpers'; const renderCreateAccelerationFlyout = getRenderCreateAccelerationFlyout(); @@ -60,9 +62,14 @@ export const DataConnection = (props: { dataSource: string }) => { properties: { 'prometheus.uri': 'placeholder' }, status: 'ACTIVE', }); + const [isS3ConnectionWithLakeFormation, setIsS3ConnectionWithLakeFormation] = useState( + false + ); const [hasAccess, setHasAccess] = useState(true); const { http, chrome, application } = coreRefs; const [selectedDatabase, setSelectedDatabase] = useState(''); + const accessControlTabId = 'access_control'; + const [selectedTabId, setSelectedTabId] = useState(accessControlTabId); const { loadStatus: databasesLoadStatus, @@ -89,6 +96,10 @@ export const DataConnection = (props: { dataSource: string }) => { const [refreshIntegrationsFlag, setRefreshIntegrationsFlag] = useState(false); const refreshInstances = () => setRefreshIntegrationsFlag((prev) => !prev); + useEffect(() => { + setIsS3ConnectionWithLakeFormation(checkIsConnectionWithLakeFormation(datasourceDetails)); + }, [datasourceDetails]); + useEffect(() => { const searchDataSourcePattern = new RegExp( `flint_${_.escapeRegExp(datasourceDetails.name)}_default_.*` @@ -119,6 +130,7 @@ export const DataConnection = (props: { dataSource: string }) => { closeFlyout={() => setShowIntegrationsFlyout(false)} datasourceType={datasourceDetails.connector} datasourceName={datasourceDetails.name} + isS3ConnectionWithLakeFormation={isS3ConnectionWithLakeFormation} /> ) : null; @@ -130,19 +142,45 @@ export const DataConnection = (props: { dataSource: string }) => { redirectToExplorerS3(dataSource); }; + const dataSourceCardsText = isS3ConnectionWithLakeFormation + ? { + integrationCard: { + title: 'Add dashbaords and queries', + description: 'Gain instant insights with preconfigured log dashboards and queries', + childTitle: 'Add dashbaords and queries from integrations', + }, + queryCard: { + title: 'Query data', + description: 'Turn your data into actionable insights', + childTitle: 'Query in Log Explorer', + }, + } + : { + integrationCard: { + title: 'Configure Integrations', + description: 'Connect to common application log types using integrations', + childTitle: 'Add Integrations', + }, + queryCard: { + title: 'Query data', + description: 'Uncover insights from your data or better understand it', + childTitle: 'Query in Log Explorer', + }, + }; + const DefaultDatasourceCards = () => { return ( } - title={'Configure Integrations'} - description="Connect to common application log types using integrations" + title={dataSourceCardsText.integrationCard.title} + description={dataSourceCardsText.integrationCard.description} onClick={onclickIntegrationsCard} selectable={{ onClick: onclickIntegrationsCard, isDisabled: false, - children: 'Add Integrations', + children: dataSourceCardsText.integrationCard.childTitle, }} /> @@ -162,13 +200,13 @@ export const DataConnection = (props: { dataSource: string }) => { } - title={'Query data'} - description="Uncover insights from your data or better understand it" + title={dataSourceCardsText.queryCard.title} + description={dataSourceCardsText.queryCard.description} onClick={onclickDiscoverCard} selectable={{ onClick: onclickDiscoverCard, isDisabled: false, - children: 'Query in Observability Logs', + children: dataSourceCardsText.queryCard.childTitle, }} /> @@ -211,7 +249,7 @@ export const DataConnection = (props: { dataSource: string }) => { const genericTabs = [ { - id: 'access_control', + id: accessControlTabId, name: 'Access control', disabled: false, content: ( @@ -231,7 +269,7 @@ export const DataConnection = (props: { dataSource: string }) => { ? [ { id: 'associated_objects', - name: 'Associated Objects', + name: isS3ConnectionWithLakeFormation ? 'Tables' : 'Associated Objects', disabled: false, content: ( { ), }, @@ -262,6 +301,7 @@ export const DataConnection = (props: { dataSource: string }) => { integrations={dataSourceIntegrations} datasourceType={datasourceDetails.connector} datasourceName={datasourceDetails.name} + isS3ConnectionWithLakeFormation={isS3ConnectionWithLakeFormation} refreshInstances={refreshInstances} /> ), @@ -309,15 +349,23 @@ export const DataConnection = (props: { dataSource: string }) => { - + Query Access + - Query Access {datasourceDetails.allowedRoles.length > 0 ? `Restricted to ${datasourceDetails.allowedRoles.join(', ')}` : 'Admin only'} + + setSelectedTabId(accessControlTabId)} + > + Edit + + @@ -368,6 +416,8 @@ export const DataConnection = (props: { dataSource: string }) => { return ; case 'PROMETHEUS': return ; + default: + return null; } }; @@ -410,7 +460,11 @@ export const DataConnection = (props: { dataSource: string }) => { > - + id === selectedTabId)} + onTabClick={({ id }) => setSelectedTabId(id)} + /> )} diff --git a/public/components/datasources/components/manage/integrations/installed_integrations_table.tsx b/public/components/datasources/components/manage/integrations/installed_integrations_table.tsx index d2d0879fb1..3b4e55b3c5 100644 --- a/public/components/datasources/components/manage/integrations/installed_integrations_table.tsx +++ b/public/components/datasources/components/manage/integrations/installed_integrations_table.tsx @@ -123,17 +123,21 @@ const NoInstalledIntegrations = ({ toggleFlyout }: { toggleFlyout: () => void }) ); }; +export interface InstallIntegrationFlyoutProps { + datasourceType: DatasourceType; + datasourceName: string; + isS3ConnectionWithLakeFormation?: boolean; + closeFlyout: () => void; + refreshInstances: () => void; +} + export const InstallIntegrationFlyout = ({ - closeFlyout, datasourceType, datasourceName, + isS3ConnectionWithLakeFormation, + closeFlyout, refreshInstances, -}: { - closeFlyout: () => void; - datasourceType: DatasourceType; - datasourceName: string; - refreshInstances: () => void; -}) => { +}: InstallIntegrationFlyoutProps) => { const [availableIntegrations, setAvailableIntegrations] = useState({ hits: [], } as AvailableIntegrationsList); @@ -181,6 +185,9 @@ export const InstallIntegrationFlyout = ({ ? { name: datasourceName, type: 's3', + properties: { + lakeFormationEnabled: isS3ConnectionWithLakeFormation, + }, } : undefined } @@ -201,11 +208,13 @@ export const InstalledIntegrationsTable = ({ integrations, datasourceType, datasourceName, + isS3ConnectionWithLakeFormation, refreshInstances, }: { integrations: IntegrationInstanceResult[]; datasourceType: DatasourceType; datasourceName: string; + isS3ConnectionWithLakeFormation?: boolean; refreshInstances: () => void; }) => { const [query, setQuery] = useState(''); @@ -256,6 +265,7 @@ export const InstalledIntegrationsTable = ({ closeFlyout={() => setShowAvailableFlyout(false)} datasourceType={datasourceType} datasourceName={datasourceName} + isS3ConnectionWithLakeFormation={isS3ConnectionWithLakeFormation} refreshInstances={refreshInstances} /> ) : null} diff --git a/public/components/datasources/components/manage/manage_data_connections_table.tsx b/public/components/datasources/components/manage/manage_data_connections_table.tsx index 2a95458dde..533c466683 100644 --- a/public/components/datasources/components/manage/manage_data_connections_table.tsx +++ b/public/components/datasources/components/manage/manage_data_connections_table.tsx @@ -38,11 +38,13 @@ import { DataConnectionsDescription } from './manage_data_connections_descriptio import { getRenderCreateAccelerationFlyout } from '../../../../../public/plugin'; import { InstallIntegrationFlyout } from './integrations/installed_integrations_table'; import { redirectToExplorerS3 } from './associated_objects/utils/associated_objects_tab_utils'; +import { checkIsConnectionWithLakeFormation } from '../../utils/helpers'; interface DataConnection { connectionType: DatasourceType; name: string; dsStatus: DatasourceStatus; + isConnectionWithLakeFormation: boolean; } export const ManageDataConnectionsTable = (props: HomeProps) => { @@ -76,13 +78,18 @@ export const ManageDataConnectionsTable = (props: HomeProps) => { http! .get(`${DATACONNECTIONS_BASE}`) .then((res: DatasourceDetails[]) => { - const dataConnections = res.map((dataConnection: DatasourceDetails) => { - return { - name: dataConnection.name, - connectionType: dataConnection.connector, - dsStatus: dataConnection.status, - }; - }); + const dataConnections: DataConnection[] = res.map( + (dataSourceDetails: DatasourceDetails): DataConnection => { + const { name, status, connector } = dataSourceDetails; + + return { + name, + connectionType: connector, + dsStatus: status, + isConnectionWithLakeFormation: checkIsConnectionWithLakeFormation(dataSourceDetails), + }; + } + ); setData(dataConnections); }) .catch((err) => { @@ -163,6 +170,7 @@ export const ManageDataConnectionsTable = (props: HomeProps) => { closeFlyout={() => setShowIntegrationsFlyout(false)} datasourceType={datasource.connectionType} datasourceName={datasource.name} + isS3ConnectionWithLakeFormation={datasource.isConnectionWithLakeFormation} /> ); setShowIntegrationsFlyout(true); @@ -212,9 +220,24 @@ export const ManageDataConnectionsTable = (props: HomeProps) => { ), }, + { + name: 'Type', + render: (connection: DataConnection) => { + switch (connection.connectionType) { + case 'PROMETHEUS': + return 'Prometheus'; + case 'S3GLUE': + return connection.isConnectionWithLakeFormation + ? 'Amazon S3 with Lake Formation' + : 'Amazon S3 with AWS Glue Data Catalog'; + default: + return '-'; + } + }, + }, { field: 'status', - name: 'Status', + name: 'Connection status', sortable: true, truncateText: true, render: (value, record: DataConnection) => @@ -237,12 +260,17 @@ export const ManageDataConnectionsTable = (props: HomeProps) => { }, }; - const entries = data.map((dataconnection: DataConnection) => { - const name = dataconnection.name; - const connectionType = dataconnection.connectionType; - const dsStatus = dataconnection.dsStatus; - return { connectionType, name, dsStatus, data: { name, connectionType } }; - }); + const entries = data.map( + ({ name, connectionType, dsStatus, isConnectionWithLakeFormation }: DataConnection) => { + return { + connectionType, + name, + dsStatus, + data: { name, connectionType }, + isConnectionWithLakeFormation, + }; + } + ); const renderCreateAccelerationFlyout = getRenderCreateAccelerationFlyout(); diff --git a/public/components/datasources/utils/helpers.ts b/public/components/datasources/utils/helpers.ts new file mode 100644 index 0000000000..58ef207ad2 --- /dev/null +++ b/public/components/datasources/utils/helpers.ts @@ -0,0 +1,14 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { S3GlueProperties } from 'common/types/data_connections'; +import { DatasourceDetails } from '../components/manage/data_connection'; + +export function checkIsConnectionWithLakeFormation({ + connector, + properties, +}: DatasourceDetails): boolean { + return connector === 'S3GLUE' && (properties as S3GlueProperties)['glue.lakeformation.enabled']; +} diff --git a/public/components/event_analytics/explorer/accelerate_callout.tsx b/public/components/event_analytics/explorer/accelerate_callout.tsx new file mode 100644 index 0000000000..81c4daff74 --- /dev/null +++ b/public/components/event_analytics/explorer/accelerate_callout.tsx @@ -0,0 +1,38 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react'; +import { EuiCallOut, EuiButtonEmpty, EuiLink, EuiSpacer } from '@elastic/eui'; + +interface AccelerateCalloutProps { + onCreateAcceleration: () => void; +} + +export const AccelerateCallout = ({ onCreateAcceleration }: AccelerateCalloutProps) => ( + <> + + + + Security Lake tables include acceleration with skipping index, but the query performance + can be further improved with other types of accelerations. + + Create acceleration +  or  + + Learn more + + + } + /> + + +); diff --git a/public/components/event_analytics/explorer/datasources/datasources_selection.tsx b/public/components/event_analytics/explorer/datasources/datasources_selection.tsx index 96560e1ffd..a73ced052c 100644 --- a/public/components/event_analytics/explorer/datasources/datasources_selection.tsx +++ b/public/components/event_analytics/explorer/datasources/datasources_selection.tsx @@ -11,7 +11,6 @@ import { DataSource, DataSourceGroup, DataSourceSelectable, - DataSourceType, } from '../../../../../../../src/plugins/data/public'; import { DATA_SOURCE_NAME_URL_PARAM_KEY, @@ -114,7 +113,7 @@ export const DataSourceSelection = ({ tabId }: { tabId: string }) => { const dispatch = useDispatch(); const routerContext = useContext(LogExplorerRouterContext); const explorerSearchMetadata = useSelector(selectSearchMetaData)[tabId]; - const [activeDataSources, setActiveDataSources] = useState([]); + const [activeDataSources, setActiveDataSources] = useState([]); const [dataSourceOptionList, setDataSourceOptionList] = useState([]); const [selectedSources, setSelectedSources] = useState( getMatchedOption( @@ -190,7 +189,7 @@ export const DataSourceSelection = ({ tabId }: { tabId: string }) => { * Subscribe to data source updates and manage the active data sources state. */ useEffect(() => { - const subscription = dataSources.dataSourceService + const subscription = dataSources?.dataSourceService .getDataSources$() .subscribe((currentDataSources: DataSource[]) => { // temporary solution for 2.11 to render OpenSearch / default cluster for observability @@ -299,8 +298,8 @@ export const DataSourceSelection = ({ tabId }: { tabId: string }) => { }, [dataSourceOptionList]); const onRefresh = useCallback(() => { - dataSources.dataSourceService.reload(); - }, [dataSources.dataSourceService]); + dataSources?.dataSourceService.reload(); + }, [dataSources?.dataSourceService]); return ( void; +} + +interface TableItemType { + name: string; + type: string; +} + +export const TablesFlyout = ({ dataSourceName, resetFlyout }: TablesFlyoutProps) => { + const { setToast } = useToast(); + const { + loadStatus: databasesLoadStatus, + startLoading: startLoadingDatabases, + } = useLoadDatabasesToCache(); + const { loadStatus: tablesLoadStatus, startLoading: startLoadingTables } = useLoadTablesToCache(); + + const [selectedDatabase, setSelectedDatabase] = useState(''); + const [isObjectsLoading, setIsObjectsLoading] = useState(false); + const [cachedDatabases, setCachedDatabases] = useState([]); + const [cachedTables, setCachedTables] = useState([]); + const [tables, setTables] = useState([]); + const [isFirstTimeLoading, setIsFirstTimeLoading] = useState(true); + const [databasesLoadFailed, setDatabasesLoadFailed] = useState(false); + const [associatedObjectsLoadFailed, setAssociatedObjectsLoadFailed] = useState(false); + + const lastChecked: boolean = selectedDatabase !== ''; + + // Get last selected if there is one, set to first option if not + const [databaseSelectorOptions, setDatabaseSelectorOptions] = useState< + Array> + >( + cachedDatabases.map((database, index) => { + return { + label: database.name, + checked: lastChecked + ? database.name === selectedDatabase + ? 'on' + : index === 0 + ? 'on' + : undefined + : undefined, + }; + }) + ); + + // Load databases if empty or retrieve from cache if updated + useEffect(() => { + if (dataSourceName) { + const datasourceCache = CatalogCacheManager.getOrCreateDataSource(dataSourceName); + if ( + (datasourceCache.status === CachedDataSourceStatus.Empty || + datasourceCache.status === CachedDataSourceStatus.Failed) && + !isCatalogCacheFetching(databasesLoadStatus) + ) { + startLoadingDatabases({ dataSourceName }); + } else if (datasourceCache.status === CachedDataSourceStatus.Updated) { + setCachedDatabases(datasourceCache.databases); + setIsFirstTimeLoading(false); + } + } + }, [dataSourceName]); + + // Retrieve from cache upon load success + useEffect(() => { + const status = databasesLoadStatus.toLowerCase(); + const datasourceCache = CatalogCacheManager.getOrCreateDataSource(dataSourceName); + if (status === DirectQueryLoadingStatus.SUCCESS) { + setCachedDatabases(datasourceCache.databases); + setIsFirstTimeLoading(false); + } else if ( + status === DirectQueryLoadingStatus.FAILED || + status === DirectQueryLoadingStatus.CANCELED + ) { + setDatabasesLoadFailed(true); + setIsFirstTimeLoading(false); + } + }, [dataSourceName, databasesLoadStatus]); + + const handleObjectsLoad = (databaseCache: CachedDatabase) => { + if (databaseCache.status === CachedDataSourceStatus.Updated) { + setIsObjectsLoading(false); + } + }; + + // Load tables if empty or retrieve from cache if not + useEffect(() => { + if (dataSourceName && selectedDatabase) { + let databaseCache; + try { + databaseCache = CatalogCacheManager.getDatabase(dataSourceName, selectedDatabase); + } catch (error) { + console.error(error); + setToast('Your cache is outdated, refresh databases and tables', 'warning'); + return; + } + if ( + (databaseCache.status === CachedDataSourceStatus.Empty || + databaseCache.status === CachedDataSourceStatus.Failed) && + !isCatalogCacheFetching(tablesLoadStatus) + ) { + startLoadingTables({ dataSourceName, databaseName: selectedDatabase }); + setIsObjectsLoading(true); + } else if (databaseCache.status === CachedDataSourceStatus.Updated) { + setCachedTables(databaseCache.tables); + } + } + }, [dataSourceName, selectedDatabase, databaseSelectorOptions]); + + // Retrieve from tables cache upon load success + useEffect(() => { + if (dataSourceName && selectedDatabase) { + const tablesStatus = tablesLoadStatus.toLowerCase(); + let databaseCache; + try { + databaseCache = CatalogCacheManager.getDatabase(dataSourceName, selectedDatabase); + } catch (error) { + console.error(error); + setToast('Your cache is outdated, refresh databases and tables', 'warning'); + return; + } + + if (tablesStatus === DirectQueryLoadingStatus.SUCCESS) { + setCachedTables(databaseCache.tables); + } else if ( + tablesStatus === DirectQueryLoadingStatus.FAILED || + tablesStatus === DirectQueryLoadingStatus.CANCELED + ) { + setAssociatedObjectsLoadFailed(true); + setIsObjectsLoading(false); + } + handleObjectsLoad(databaseCache); + } + }, [dataSourceName, selectedDatabase, tablesLoadStatus]); + + useEffect(() => { + setDatabaseSelectorOptions( + cachedDatabases.map((database, index) => { + if (selectedDatabase) { + return { + label: database.name, + checked: database.name === selectedDatabase ? 'on' : undefined, + }; + } + return { + label: database.name, + checked: index === 0 ? 'on' : undefined, + }; + }) + ); + }, [cachedDatabases]); + + useEffect(() => { + setSelectedDatabase((prevState) => { + const select = databaseSelectorOptions.find((option) => option.checked === 'on')?.label; + if (select) { + return select; + } + return prevState; + }); + }, [databaseSelectorOptions]); + + useEffect(() => { + const tableObjects: TableItemType[] = cachedTables.map((table: CachedTable) => { + return { + name: table.name, + type: 'table', + }; + }); + setTables(tableObjects); + }, [selectedDatabase, cachedTables]); + + const columns: Array> = [ + { + field: 'name', + name: 'Name', + sortable: true, + }, + { + field: 'type', + name: 'Type', + sortable: true, + }, + { + actions: [ + { + name: 'Add to query', + description: 'Add table to the query.', + render: ({ name }: TableItemType) => { + return ( + + {(copy) => ( + + )} + + ); + }, + }, + ], + }, + ]; + + const tableSearch = { + box: { + incremental: true, + placeholder: 'Search', + schema: { + fields: { name: { type: 'string' }, database: { type: 'string' } }, + }, + }, + }; + + return ( + + + +

{dataSourceName}

+ + + + +

Databases

+ + + {isFirstTimeLoading ? ( + + ) : databasesLoadFailed ? ( + + ) : ( + <> + {cachedDatabases.length === 0 ? ( + + ) : ( + <> + + setDatabaseSelectorOptions(newOptions)} + > + {(list, search) => ( + <> + {search} + {list} + + )} + + + +

Tables

+ + + {isFirstTimeLoading || isObjectsLoading ? ( + + ) : associatedObjectsLoadFailed ? ( + + ) : ( + <> + {cachedTables.length > 0 ? ( + + ) : ( + + )} + + )} + + )} + + )} + + + + + + Close + + + + + + ); +}; diff --git a/public/components/event_analytics/explorer/direct_query_running.tsx b/public/components/event_analytics/explorer/direct_query_running.tsx index 33ad428a6c..136c685d1a 100644 --- a/public/components/event_analytics/explorer/direct_query_running.tsx +++ b/public/components/event_analytics/explorer/direct_query_running.tsx @@ -3,7 +3,15 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { EuiButton, EuiEmptyPrompt, EuiProgress, EuiSpacer, EuiText } from '@elastic/eui'; +import { + EuiButton, + EuiEmptyPrompt, + EuiProgress, + EuiSpacer, + EuiText, + EuiFlexGroup, + EuiFlexItem, +} from '@elastic/eui'; import React from 'react'; import { useDispatch, useSelector } from 'react-redux'; import { DirectQueryLoadingStatus } from '../../../../common/types/explorer'; @@ -13,8 +21,19 @@ import { selectSearchMetaData, update as updateSearchMetaData, } from '../redux/slices/search_meta_data_slice'; +import { AccelerateCallout } from './accelerate_callout'; + +interface DirectQueryRunningProps { + tabId: string; + isS3ConnectionWithLakeFormation: boolean; + onCreateAcceleration: () => void; +} -export const DirectQueryRunning = ({ tabId }: { tabId: string }) => { +export const DirectQueryRunning = ({ + tabId, + isS3ConnectionWithLakeFormation, + onCreateAcceleration, +}: DirectQueryRunningProps) => { const explorerSearchMeta = useSelector(selectSearchMetaData)[tabId] || {}; const dispatch = useDispatch(); const sqlService = new SQLService(coreRefs.http); @@ -39,20 +58,36 @@ export const DirectQueryRunning = ({ tabId }: { tabId: string }) => { }; return ( - } - title={

Query Processing

} - body={ - <> - - Status: {explorerSearchMeta.status ?? DirectQueryLoadingStatus.SCHEDULED} - - - - Cancel - - - } - /> + <> + {isS3ConnectionWithLakeFormation && ( + + )} + } + title={

Query Processing

} + body={ + <> + + Status: {explorerSearchMeta.status ?? DirectQueryLoadingStatus.SCHEDULED} + + + + + + Cancel + + + {isS3ConnectionWithLakeFormation && ( + + + Create acceleration + + + )} + + + } + /> + ); }; diff --git a/public/components/event_analytics/explorer/explorer.tsx b/public/components/event_analytics/explorer/explorer.tsx index c20fea643a..71e1a303cf 100644 --- a/public/components/event_analytics/explorer/explorer.tsx +++ b/public/components/event_analytics/explorer/explorer.tsx @@ -62,6 +62,7 @@ import { } from '../../../../common/constants/explorer'; import { QUERY_ASSIST_API } from '../../../../common/constants/query_assist'; import { + DATACONNECTIONS_BASE, LIVE_END_TIME, LIVE_OPTIONS, PPL_DESCRIBE_INDEX_REGEX, @@ -135,6 +136,12 @@ import { TimechartHeader } from './timechart_header'; import { ExplorerVisualizations } from './visualizations'; import { CountDistribution } from './visualizations/count_distribution'; import { DirectQueryVisualization } from './visualizations/direct_query_vis'; +import { + getRenderCreateAccelerationFlyout, + getRenderLogExplorerTablesFlyout, +} from '../../../plugin'; +import { S3GlueProperties } from '../../../../common/types/data_connections'; +import { AccelerateCallout } from './accelerate_callout'; export const Explorer = ({ pplService, @@ -206,6 +213,11 @@ export const Explorer = ({ const [liveTimestamp, setLiveTimestamp] = useState(DATE_PICKER_FORMAT); const [triggerAvailability, setTriggerAvailability] = useState(false); const [isQueryRunning, setIsQueryRunning] = useState(false); + const dataSourceName = explorerSearchMeta?.datasources[0]?.label; + const renderTablesFlyout = getRenderLogExplorerTablesFlyout(); + const renderCreateAccelerationFlyout = getRenderCreateAccelerationFlyout(); + const [isS3ConnectionWithLakeFormation, setIsS3ConnectionWithLakeFormation] = useState(false); + const onCreateAcceleration = () => renderCreateAccelerationFlyout(dataSourceName); const currentPluggable = useMemo(() => { return explorerSearchMeta.datasources?.[0]?.type ? dataSourcePluggables[explorerSearchMeta?.datasources[0]?.type] @@ -334,6 +346,18 @@ export const Explorer = ({ } }, []); + const updateDataSourceConnectionInfo = () => { + coreRefs.http!.get(`${DATACONNECTIONS_BASE}/${dataSourceName}`).then((data: any) => { + setIsS3ConnectionWithLakeFormation( + !!(data.properties as S3GlueProperties)['glue.lakeformation.enabled'] + ); + }); + }; + + useEffect(() => { + updateDataSourceConnectionInfo(); + }, [dataSourceName]); + const getErrorHandler = (title: string) => { return (error: any) => { if (coreRefs.summarizeEnabled) return; @@ -552,6 +576,11 @@ export const Explorer = ({
{explorerData && !isEmpty(explorerData.jsonData) ? ( + {isS3ConnectionWithLakeFormation && ( + + + + )} {showTimeBasedComponents && ( <> @@ -693,6 +722,7 @@ export const Explorer = ({ query, isLiveTailOnRef.current, isQueryRunning, + isS3ConnectionWithLakeFormation, ]); const visualizationSettings = !isEmpty(userVizConfigs[curVisId]) @@ -742,6 +772,8 @@ export const Explorer = ({ currentDataSource={ explorerSearchMeta.datasources ? explorerSearchMeta.datasources?.[0]?.label : '' } + isS3ConnectionWithLakeFormation={isS3ConnectionWithLakeFormation} + onCreateAcceleration={onCreateAcceleration} /> ); }, [ @@ -752,6 +784,8 @@ export const Explorer = ({ explorerData, visualizations, explorerSearchMeta.datasources, + isS3ConnectionWithLakeFormation, + onCreateAcceleration, ]); const contentTabs = [ @@ -1065,8 +1099,25 @@ export const Explorer = ({ isAppAnalytics={appLogEvents} pplService={pplService} /> + {isS3ConnectionWithLakeFormation && ( + <> + { + renderTablesFlyout(dataSourceName); + }} + > + View databases and tables + + + + )} {explorerSearchMeta.isPolling ? ( - + ) : ( { // get the queries isLoaded, if it exists AND is true = show no res @@ -66,6 +67,8 @@ export const NoResults = ({ tabId }: any) => { } } + const renderTablesFlyout = getRenderLogExplorerTablesFlyout(); + const S3Callouts = () => { return ( @@ -96,9 +99,7 @@ export const NoResults = ({ tabId }: any) => { To start exploring this datasource, enter a query or{' '} { - coreRefs?.application!.navigateToApp('datasources', { - path: `#/manage/${datasourceName}`, - }); + renderTablesFlyout(datasourceName); }} > view databases and tables. diff --git a/public/components/event_analytics/explorer/visualizations/direct_query_vis.tsx b/public/components/event_analytics/explorer/visualizations/direct_query_vis.tsx index a1314d6f26..36871f7b1e 100644 --- a/public/components/event_analytics/explorer/visualizations/direct_query_vis.tsx +++ b/public/components/event_analytics/explorer/visualizations/direct_query_vis.tsx @@ -3,7 +3,15 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { EuiCallOut, EuiFlexGroup, EuiFlexItem, EuiLink, EuiPage, EuiText } from '@elastic/eui'; +import { + EuiCallOut, + EuiFlexGroup, + EuiFlexItem, + EuiLink, + EuiPage, + EuiText, + EuiButton, +} from '@elastic/eui'; import React from 'react'; import { FormattedMessage, I18nProvider } from '@osd/i18n/react'; import { queryWorkbenchPluginID } from '../../../../../common/constants/shared'; @@ -11,56 +19,75 @@ import { coreRefs } from '../../../../framework/core_refs'; interface DirectQueryVisualizationProps { currentDataSource: string; + isS3ConnectionWithLakeFormation: boolean; + onCreateAcceleration: () => void; } -export const DirectQueryVisualization = ({ currentDataSource }: DirectQueryVisualizationProps) => { +export const DirectQueryVisualization = ({ + currentDataSource, + isS3ConnectionWithLakeFormation, + onCreateAcceleration, +}: DirectQueryVisualizationProps) => { return ( - - - - } - color="danger" - iconType="alert" - > -

- - coreRefs?.application!.navigateToApp(queryWorkbenchPluginID, { - path: `#/accelerate/${currentDataSource}`, - }) - } - > + {isS3ConnectionWithLakeFormation ? ( + +

+ Create acceleration for the table to visualize or select accelerated data. Contact + your administrator to accelerate data by creating a materialized view or covering + index. +

+ + Create acceleration + + + ) : ( + + + + } + color="danger" + iconType="alert" + > +

+ + coreRefs?.application!.navigateToApp(queryWorkbenchPluginID, { + path: `#/accelerate/${currentDataSource}`, + }) + } + > + + +

+ + + + +

- -

- - - - -

+

-

- - - - + + + + )} ); diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index e521e3b74b..89a74aaaa9 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -41,6 +41,7 @@ export interface IntegrationConfigProps { integration: IntegrationConfig; setupCallout: SetupCallout; lockConnectionType?: boolean; + isS3ConnectionWithLakeFormation?: boolean; } type SetupCallout = { show: true; title: string; color?: Color; text?: string } | { show: false }; @@ -330,6 +331,9 @@ export function SetupIntegrationForm({ forceConnection?: { name: string; type: string; + properties?: { + lakeFormationEnabled?: boolean; + }; }; setIsInstalling?: (isInstalling: boolean, success?: boolean) => void; }) { @@ -381,6 +385,7 @@ export function SetupIntegrationForm({ integration={template} setupCallout={setupCallout} lockConnectionType={forceConnection !== undefined} + isS3ConnectionWithLakeFormation={forceConnection?.properties?.lakeFormationEnabled} /> )} @@ -411,6 +416,7 @@ export function SetupIntegrationForm({ integration={template} setupCallout={setupCallout} lockConnectionType={forceConnection !== undefined} + isS3ConnectionWithLakeFormation={forceConnection?.properties?.lakeFormationEnabled} /> )} diff --git a/public/components/integrations/components/setup_integration_inputs.tsx b/public/components/integrations/components/setup_integration_inputs.tsx index 77ff20963f..5ad176869a 100644 --- a/public/components/integrations/components/setup_integration_inputs.tsx +++ b/public/components/integrations/components/setup_integration_inputs.tsx @@ -14,6 +14,7 @@ import { EuiSpacer, EuiText, EuiTitle, + EuiHorizontalRule, } from '@elastic/eui'; import React, { useState, useEffect } from 'react'; import { coreRefs } from '../../../framework/core_refs'; @@ -111,17 +112,20 @@ export function SetupWorkflowSelector({ const cards = integration.workflows.map((workflow) => { return ( - toggleWorkflow(workflow.name)} - > - {workflow.description} - + <> + toggleWorkflow(workflow.name)} + > + {workflow.description} + + + ); }); @@ -132,14 +136,16 @@ export function IntegrationDetailsInputs({ config, updateConfig, integration, + isS3ConnectionWithLakeFormation, }: { config: IntegrationSetupInputs; updateConfig: (updates: Partial) => void; integration: IntegrationConfig; + isS3ConnectionWithLakeFormation?: boolean; }) { return ( @@ -241,51 +247,59 @@ export function IntegrationQueryInputs({ config, updateConfig, integration, + isS3ConnectionWithLakeFormation, }: { config: IntegrationSetupInputs; updateConfig: (updates: Partial) => void; integration: IntegrationConfig; + isS3ConnectionWithLakeFormation?: boolean; }) { const [isBucketBlurred, setIsBucketBlurred] = useState(false); const [isCheckpointBlurred, setIsCheckpointBlurred] = useState(false); return ( <> + {!isS3ConnectionWithLakeFormation && ( + <> + + { + updateConfig({ connectionTableName: evt.target.value }); + }} + isInvalid={config.connectionTableName.length === 0} + /> + + + updateConfig({ connectionLocation: event.target.value })} + placeholder="s3://" + isInvalid={isBucketBlurred && !config.connectionLocation.startsWith('s3://')} + onBlur={() => { + setIsBucketBlurred(true); + }} + /> + + + )} - { - updateConfig({ connectionTableName: evt.target.value }); - }} - isInvalid={config.connectionTableName.length === 0} - /> - - - updateConfig({ connectionLocation: event.target.value })} - placeholder="s3://" - isInvalid={isBucketBlurred && !config.connectionLocation.startsWith('s3://')} - onBlur={() => { - setIsBucketBlurred(true); - }} - /> - - + const integrationFormWithLakeFormation = ( + <> + +

Add integration

+ + + {setupCallout.show ? ( + <> + +

{setupCallout.text}

+ + + + ) : null} + + + {config.connectionType === 's3' ? ( + <> + +

Integration data location

+ + + + {integration.workflows ? ( + <> + + +

Included resources

+ + + +

+ This integration offers resources compatible with your data source. These can + include dashboards, visualizations, indexes, and queries. Select at least one of + the following options. +

+ + + + + + ) : null} + {/* Bottom bar will overlap content if there isn't some space at the end */} + + + + ) : null} + + ); + + const integrationFormWithoutLakeFormation = ( + <>

Set Up Integration

 @@ -431,6 +506,14 @@ export function SetupIntegrationFormInputs({ ) : null} + + ); + + return ( + + {isS3ConnectionWithLakeFormation + ? integrationFormWithLakeFormation + : integrationFormWithoutLakeFormation} ); } diff --git a/public/framework/core_refs.ts b/public/framework/core_refs.ts index b78240e01b..59315c2fa7 100644 --- a/public/framework/core_refs.ts +++ b/public/framework/core_refs.ts @@ -16,6 +16,7 @@ import { import { DashboardStart } from '../../../../src/plugins/dashboard/public'; import { DataSourcePluginStart } from '../../../../src/plugins/data_source/public'; import PPLService from '../services/requests/ppl'; +import { DataSourceStart } from '../../../../src/plugins/data/public/data_sources/datasource_services/types'; class CoreRefs { private static _instance: CoreRefs; @@ -34,6 +35,7 @@ class CoreRefs { public dashboardProviders?: unknown; public overlays?: OverlayStart; public dataSource?: DataSourcePluginStart; + public dataSources?: DataSourceStart; private constructor() { // ... } diff --git a/public/framework/datasources/obs_opensearch_datasource.ts b/public/framework/datasources/obs_opensearch_datasource.ts index e7bfe6eeb4..64404fd847 100644 --- a/public/framework/datasources/obs_opensearch_datasource.ts +++ b/public/framework/datasources/obs_opensearch_datasource.ts @@ -18,7 +18,7 @@ export class ObservabilityDefaultDataSource extends DataSource { @@ -26,6 +26,6 @@ export class ObservabilityDefaultDataSource extends DataSource void, - dataSourceMDSId?: string + dataSourceMDSId?: string, + isS3ConnectionWithLakeFormation?: boolean ) => void >('renderAssociatedObjectsDetailsFlyout'); @@ -149,6 +151,11 @@ export const [ ) => void >('renderCreateAccelerationFlyout'); +export const [ + getRenderLogExplorerTablesFlyout, + setRenderLogExplorerTablesFlyout, +] = createGetterSetter<(dataSourceName: string) => void>('renderLogExplorerTablesFlyout'); + export class ObservabilityPlugin implements Plugin { @@ -497,7 +504,8 @@ export class ObservabilityPlugin tableDetail: AssociatedObject, datasourceName: string, handleRefresh?: () => void, - dataSourceMDSId?: string + dataSourceMDSId?: string, + isS3ConnectionWithLakeFormation?: boolean ) => { const associatedObjectsDetailsFlyout = core.overlays.openFlyout( toMountPoint( @@ -507,6 +515,7 @@ export class ObservabilityPlugin resetFlyout={() => associatedObjectsDetailsFlyout.close()} handleRefresh={handleRefresh} dataSourceMDSId={dataSourceMDSId} + isS3ConnectionWithLakeFormation={isS3ConnectionWithLakeFormation} /> ) ); @@ -535,6 +544,18 @@ export class ObservabilityPlugin }; setRenderCreateAccelerationFlyout(renderCreateAccelerationFlyout); + const renderLogExplorerTablesFlyout = (dataSourceName: string) => { + const createLogExplorerTablesFlyout = core.overlays.openFlyout( + toMountPoint( + createLogExplorerTablesFlyout.close()} + /> + ) + ); + }; + setRenderLogExplorerTablesFlyout(renderLogExplorerTablesFlyout); + const CatalogCacheManagerInstance = CatalogCacheManager; const useLoadDatabasesToCacheHook = useLoadDatabasesToCache; const useLoadTablesToCacheHook = useLoadTablesToCache;