From 88970fbf8d1386bcb46d0a4adca69416715d0bd2 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 18 Oct 2023 14:38:15 -0700 Subject: [PATCH] Add materlized views, manual refresh option and minor fixes (#159) (#161) * enable materialized views, update acc btn fill * update acceleration links * add manual refresh option, update tests * update copy for datasources * fix redirection issue --------- (cherry picked from commit 43b085bb40d10d7a502c631bd7e1176f3d0d0086) Signed-off-by: Shenoy Pratik Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- common/constants/index.ts | 8 +- common/types/index.ts | 6 +- public/components/Main/main.tsx | 14 ++- public/components/SQLPage/SQLPage.tsx | 18 +++- .../create_acceleration.test.tsx.snap | 65 +++++++++++--- .../create_acceleration_header.test.tsx.snap | 2 +- .../index_setting_options.test.tsx.snap | 87 ++++++++++++++++--- .../index_type_selector.test.tsx.snap | 4 +- .../source_selector.test.tsx.snap | 4 +- .../selectors/index_setting_options.tsx | 83 +++++++++++------- .../selectors/source_selector.tsx | 2 +- .../__tests__/query_builder.test.tsx | 7 ++ .../visual_editors/query_builder.tsx | 8 +- test/mocks/accelerationMock.ts | 20 ++++- 14 files changed, 251 insertions(+), 77 deletions(-) diff --git a/common/constants/index.ts b/common/constants/index.ts index c7ed1d43..ea25495a 100644 --- a/common/constants/index.ts +++ b/common/constants/index.ts @@ -5,8 +5,10 @@ export const PLUGIN_ID = 'queryWorkbenchDashboards'; export const PLUGIN_NAME = 'Query Workbench'; -export const OPENSEARCH_ACC_DOCUMENTATION_URL = 'https://opensearch.org/docs/latest'; -export const ACC_INDEX_TYPE_DOCUMENTATION_URL = 'https://opensearch.org/docs/latest'; +export const OPENSEARCH_ACC_DOCUMENTATION_URL = + 'https://opensearch.org/docs/latest/dashboards/management/accelerate-external-data/'; +export const ACC_INDEX_TYPE_DOCUMENTATION_URL = + 'https://github.com/opensearch-project/opensearch-spark/blob/main/docs/index.md'; export const SKIPPING_INDEX = `skipping_index`; export const ON_LOAD_QUERY = `SHOW tables LIKE '%';`; @@ -37,7 +39,7 @@ OPTIONS ( export const ACCELERATION_INDEX_TYPES = [ { label: 'Skipping Index', value: 'skipping' }, { label: 'Covering Index', value: 'covering' }, - // { label: 'Materialized View', value: 'materialized' }, Hidden Option -> Until opensearch-spark feature is ready + { label: 'Materialized View', value: 'materialized' }, ]; export const ACCELERATION_AGGREGRATION_FUNCTIONS = [ diff --git a/common/types/index.ts b/common/types/index.ts index c96bb147..e669ce49 100644 --- a/common/types/index.ts +++ b/common/types/index.ts @@ -59,6 +59,8 @@ export interface FormErrorsType { checkpointLocationError: string[]; } +export type AccelerationRefreshType = 'auto' | 'interval' | 'manual'; + export interface CreateAccelerationForm { dataSource: string; database: string; @@ -71,10 +73,10 @@ export interface CreateAccelerationForm { accelerationIndexName: string; primaryShardsCount: number; replicaShardsCount: number; - refreshType: 'interval' | 'auto'; + refreshType: AccelerationRefreshType; checkpointLocation: string | undefined; refreshIntervalOptions: RefreshIntervalType; formErrors: FormErrorsType; } -export type AsyncQueryLoadingStatus = "SUCCESS" | "FAILED" | "RUNNING" | "SCHEDULED" | "CANCELED" \ No newline at end of file +export type AsyncQueryLoadingStatus = 'SUCCESS' | 'FAILED' | 'RUNNING' | 'SCHEDULED' | 'CANCELED'; diff --git a/public/components/Main/main.tsx b/public/components/Main/main.tsx index 86f24463..ae28ff67 100644 --- a/public/components/Main/main.tsx +++ b/public/components/Main/main.tsx @@ -109,6 +109,7 @@ interface MainState { asyncLoading: boolean; asyncLoadingStatus: AsyncQueryLoadingStatus; asyncJobId: string; + isAccelerationFlyoutOpened: boolean; } const SUCCESS_MESSAGE = 'Success'; @@ -246,6 +247,7 @@ export class Main extends React.Component { asyncLoading: false, asyncLoadingStatus: 'SUCCESS', asyncJobId: '', + isAccelerationFlyoutOpened: false, }; this.httpClient = this.props.httpClient; this.updateSQLQueries = _.debounce(this.updateSQLQueries, 250).bind(this); @@ -789,7 +791,7 @@ export class Main extends React.Component { }); } - handleDataSelect = (selectedItems: []) => { + handleDataSelect = (selectedItems: EuiComboBoxOptionOption[]) => { if (selectedItems[0].label !== 'OpenSearch' && this.state.language === 'SQL') { this.updateSQLQueries(''); } @@ -798,6 +800,12 @@ export class Main extends React.Component { }); }; + setIsAccelerationFlyoutOpened = (value: boolean) => { + this.setState({ + isAccelerationFlyoutOpened: value, + }); + }; + render() { let page; let link; @@ -819,6 +827,10 @@ export class Main extends React.Component { updateSQLQueries={this.updateSQLQueries} selectedDatasource={this.state.selectedDatasource} asyncLoading={this.state.asyncLoading} + openAccelerationFlyout={ + this.props.isAccelerationFlyoutOpen && !this.state.isAccelerationFlyoutOpened + } + setIsAccelerationFlyoutOpened={this.setIsAccelerationFlyoutOpened} /> ); link = 'https://opensearch.org/docs/latest/search-plugins/sql/index/'; diff --git a/public/components/SQLPage/SQLPage.tsx b/public/components/SQLPage/SQLPage.tsx index b70029e6..29311eb7 100644 --- a/public/components/SQLPage/SQLPage.tsx +++ b/public/components/SQLPage/SQLPage.tsx @@ -37,6 +37,8 @@ interface SQLPageProps { sqlTranslations: ResponseDetail[]; selectedDatasource: EuiComboBoxOptionOption[]; asyncLoading: boolean; + openAccelerationFlyout: boolean; + setIsAccelerationFlyoutOpened: (value: boolean) => void; } interface SQLPageState { @@ -80,6 +82,21 @@ export class SQLPage extends React.Component { }); }; + componentDidUpdate(prevProps: SQLPageProps) { + const { selectedDatasource, openAccelerationFlyout } = this.props; + const prevDataSource = prevProps.selectedDatasource[0].label; + const currentDataSource = selectedDatasource[0].label; + + if ( + currentDataSource !== prevDataSource && + currentDataSource !== 'OpenSearch' && + openAccelerationFlyout + ) { + this.setAccelerationFlyout(); + this.props.setIsAccelerationFlyoutOpened(true); + } + } + render() { const closeModal = () => this.setIsModalVisible(false); const showModal = () => this.setIsModalVisible(true); @@ -187,7 +204,6 @@ export class SQLPage extends React.Component { this.props.selectedDatasource[0].label !== 'OpenSearch' && ( @@ -132,7 +132,7 @@ Array [
- Select data connection where the data you want to accelerate resides. + Select the data source to accelerate data from. External data sources may take time to load.
- Refresh by interval + Auto refresh by interval + +
+
+ +
+
@@ -698,7 +718,7 @@ Array [ class="euiFormHelpText euiFormRow__text" id="some_html_id-help-0" > - Specify how often the index should refresh, which publishes the most recent changes and make them available for search. Default is set to auto refresh when data at the source changes. + Specify how often the index should refresh, which publishes the most recent changes and make them available for search. @@ -1181,7 +1201,7 @@ Array [ @@ -1243,7 +1263,7 @@ Array [
- Select data connection where the data you want to accelerate resides. + Select the data source to accelerate data from. External data sources may take time to load.
,
- Refresh by interval + Auto refresh by interval + +
+
+ +
+
@@ -1929,7 +1970,7 @@ Array [ className="euiFormHelpText euiFormRow__text" id="some_html_id-help-0" > - Specify how often the index should refresh, which publishes the most recent changes and make them available for search. Default is set to auto refresh when data at the source changes. + Specify how often the index should refresh, which publishes the most recent changes and make them available for search. , diff --git a/public/components/acceleration/create/__tests__/__snapshots__/create_acceleration_header.test.tsx.snap b/public/components/acceleration/create/__tests__/__snapshots__/create_acceleration_header.test.tsx.snap index 1c099f4b..e722b161 100644 --- a/public/components/acceleration/create/__tests__/__snapshots__/create_acceleration_header.test.tsx.snap +++ b/public/components/acceleration/create/__tests__/__snapshots__/create_acceleration_header.test.tsx.snap @@ -29,7 +29,7 @@ exports[`Acceleration header renders acceleration flyout header 1`] = ` diff --git a/public/components/acceleration/selectors/__tests__/__snapshots__/index_setting_options.test.tsx.snap b/public/components/acceleration/selectors/__tests__/__snapshots__/index_setting_options.test.tsx.snap index 49459fd5..ea5c51e3 100644 --- a/public/components/acceleration/selectors/__tests__/__snapshots__/index_setting_options.test.tsx.snap +++ b/public/components/acceleration/selectors/__tests__/__snapshots__/index_setting_options.test.tsx.snap @@ -32,7 +32,7 @@ Array [ > @@ -297,7 +297,7 @@ Array [ className="euiRadio__label" htmlFor="refresh-option-1" > - Auto Refresh + Auto refresh
- Refresh by interval + Auto refresh by interval + +
+
+ +
+
@@ -326,7 +347,7 @@ Array [ className="euiFormHelpText euiFormRow__text" id="some_html_id-help-0" > - Specify how often the index should refresh, which publishes the most recent changes and make them available for search. Default is set to auto refresh when data at the source changes. + Specify how often the index should refresh, which publishes the most recent changes and make them available for search. , @@ -411,7 +432,7 @@ Array [ > @@ -684,7 +705,7 @@ Array [ className="euiRadio__label" htmlFor="refresh-option-1" > - Auto Refresh + Auto refresh
- Refresh by interval + Auto refresh by interval + +
+
+ +
+
@@ -713,7 +755,7 @@ Array [ className="euiFormHelpText euiFormRow__text" id="some_html_id-help-0" > - Specify how often the index should refresh, which publishes the most recent changes and make them available for search. Default is set to auto refresh when data at the source changes. + Specify how often the index should refresh, which publishes the most recent changes and make them available for search. , @@ -798,7 +840,7 @@ Array [ > @@ -1071,7 +1113,7 @@ Array [ className="euiRadio__label" htmlFor="refresh-option-1" > - Auto Refresh + Auto refresh
- Refresh by interval + Auto refresh by interval + +
+
+ +
+
@@ -1100,7 +1163,7 @@ Array [ className="euiFormHelpText euiFormRow__text" id="some_html_id-help-0" > - Specify how often the index should refresh, which publishes the most recent changes and make them available for search. Default is set to auto refresh when data at the source changes. + Specify how often the index should refresh, which publishes the most recent changes and make them available for search. , diff --git a/public/components/acceleration/selectors/__tests__/__snapshots__/index_type_selector.test.tsx.snap b/public/components/acceleration/selectors/__tests__/__snapshots__/index_type_selector.test.tsx.snap index 6a41fd85..aa035061 100644 --- a/public/components/acceleration/selectors/__tests__/__snapshots__/index_type_selector.test.tsx.snap +++ b/public/components/acceleration/selectors/__tests__/__snapshots__/index_type_selector.test.tsx.snap @@ -20,7 +20,7 @@ exports[`Index type selector components renders type selector with default optio > @@ -170,7 +170,7 @@ exports[`Index type selector components renders type selector with different opt > diff --git a/public/components/acceleration/selectors/__tests__/__snapshots__/source_selector.test.tsx.snap b/public/components/acceleration/selectors/__tests__/__snapshots__/source_selector.test.tsx.snap index 47b7da24..23129c8e 100644 --- a/public/components/acceleration/selectors/__tests__/__snapshots__/source_selector.test.tsx.snap +++ b/public/components/acceleration/selectors/__tests__/__snapshots__/source_selector.test.tsx.snap @@ -19,7 +19,7 @@ Array [
- Select data connection where the data you want to accelerate resides. + Select the data source to accelerate data from. External data sources may take time to load.
,
- Select data connection where the data you want to accelerate resides. + Select the data source to accelerate data from. External data sources may take time to load.
,
{ const autoRefreshId = 'refresh-option-1'; const intervalRefreshId = 'refresh-option-2'; + const manualRefreshId = 'refresh-option-3'; const refreshOptions = [ { id: autoRefreshId, - label: 'Auto Refresh', + label: 'Auto refresh', }, { id: intervalRefreshId, - label: 'Refresh by interval', + label: 'Auto refresh by interval', + }, + { + id: manualRefreshId, + label: 'Manual refresh', }, ]; @@ -70,9 +75,21 @@ export const IndexSettingOptions = ({ }; const onChangeRefreshType = (optionId: React.SetStateAction) => { + let refreshOption: AccelerationRefreshType = 'auto'; + switch (optionId) { + case autoRefreshId: + refreshOption = 'auto'; + break; + case intervalRefreshId: + refreshOption = 'interval'; + break; + case manualRefreshId: + refreshOption = 'manual'; + break; + } setAccelerationFormData({ ...accelerationFormData, - refreshType: optionId === autoRefreshId ? 'auto' : 'interval', + refreshType: refreshOption, }); setRefreshTypeSelected(optionId); }; @@ -166,7 +183,7 @@ export const IndexSettingOptions = ({ )} - - { - setAccelerationFormData( - producer((accData) => { - accData.formErrors.checkpointLocationError = validateCheckpointLocation( - accData.accelerationIndexType, - e.target.value - ); - }) - ); - }} - /> - + error={accelerationFormData.formErrors.checkpointLocationError} + > + { + setAccelerationFormData( + producer((accData) => { + accData.formErrors.checkpointLocationError = validateCheckpointLocation( + accData.accelerationIndexType, + e.target.value + ); + }) + ); + }} + /> + + )} ); }; diff --git a/public/components/acceleration/selectors/source_selector.tsx b/public/components/acceleration/selectors/source_selector.tsx index ea365876..d558e1dd 100644 --- a/public/components/acceleration/selectors/source_selector.tsx +++ b/public/components/acceleration/selectors/source_selector.tsx @@ -114,7 +114,7 @@ export const AccelerationDataSourceSelector = ({ - Select data connection where the data you want to accelerate resides. + Select the data source to accelerate data from. External data sources may take time to load. { expect(indexOptions).toEqual(indexOptionsMockResult3); }); + it('should build index options with manual refresh', () => { + const indexOptions = buildIndexOptions(indexOptionsMock4); + expect(indexOptions).toEqual(indexOptionsMockResult4); + }); + describe('skippingIndexQueryBuilder', () => { it('should build skipping index query as expected with interval refresh', () => { const result = skippingIndexQueryBuilder(skippingIndexBuilderMock1); diff --git a/public/components/acceleration/visual_editors/query_builder.tsx b/public/components/acceleration/visual_editors/query_builder.tsx index b4537f0e..83f8f7af 100644 --- a/public/components/acceleration/visual_editors/query_builder.tsx +++ b/public/components/acceleration/visual_editors/query_builder.tsx @@ -27,7 +27,7 @@ export const buildIndexOptions = (accelerationformData: CreateAccelerationForm) ); // Add auto refresh option - indexOptions.push(`auto_refresh = ${refreshType === 'auto'}`); + indexOptions.push(`auto_refresh = ${['auto', 'interval'].includes(refreshType)}`); // Add refresh interval option if (refreshType === 'interval') { @@ -63,7 +63,7 @@ const buildSkippingIndexColumns = (skippingIndexQueryData: SkippingIndexRowType[ * field2 PARTITION, * field3 MIN_MAX, * ) WITH ( - * auto_refresh = false, + * auto_refresh = true, * refresh_interval = '1 minute', * checkpoint_location = 's3://test/', * index_settings = '{"number_of_shards":9,"number_of_replicas":2}' @@ -95,7 +95,7 @@ const buildCoveringIndexColumns = (coveringIndexQueryData: string[]) => { * field2, * field3, * ) WITH ( - * auto_refresh = false, + * auto_refresh = true, * refresh_interval = '1 minute', * checkpoint_location = 's3://test/', * index_settings = '{"number_of_shards":9,"number_of_replicas":2}' @@ -146,7 +146,7 @@ const buildTumbleValue = (GroupByTumbleValue: GroupByTumbleType) => { * sum(field2), * avg(field3) as average * WITH ( - * auto_refresh = false, + * auto_refresh = true, * refresh_interval = '1 minute', * checkpoint_location = 's3://test/', * index_settings = '{"number_of_shards":9,"number_of_replicas":2}' diff --git a/test/mocks/accelerationMock.ts b/test/mocks/accelerationMock.ts index 3cc0c2ae..82d4d1de 100644 --- a/test/mocks/accelerationMock.ts +++ b/test/mocks/accelerationMock.ts @@ -148,7 +148,7 @@ export const indexOptionsMock2: CreateAccelerationForm = { export const indexOptionsMockResult2 = `WITH ( index_settings = '{"number_of_shards":3,"number_of_replicas":2}', -auto_refresh = false, +auto_refresh = true, refresh_interval = '10 minutes' )`; @@ -166,6 +166,18 @@ auto_refresh = true, checkpoint_location = 's3://path/to/checkpoint' )`; +export const indexOptionsMock4: CreateAccelerationForm = { + ...createAccelerationEmptyDataMock, + primaryShardsCount: 3, + replicaShardsCount: 2, + refreshType: 'manual', +}; + +export const indexOptionsMockResult4 = `WITH ( +index_settings = '{"number_of_shards":3,"number_of_replicas":2}', +auto_refresh = false +)`; + export const skippingIndexBuilderMock1: CreateAccelerationForm = { ...createAccelerationEmptyDataMock, dataSource: 'datasource', @@ -208,7 +220,7 @@ ON datasource.database.table ( field3 MIN_MAX ) WITH ( index_settings = '{"number_of_shards":9,"number_of_replicas":2}', -auto_refresh = false, +auto_refresh = true, refresh_interval = '1 minute', checkpoint_location = 's3://test/' )`; @@ -265,7 +277,7 @@ ON datasource.database.table ( field3 ) WITH ( index_settings = '{"number_of_shards":9,"number_of_replicas":2}', -auto_refresh = false, +auto_refresh = true, refresh_interval = '1 minute', checkpoint_location = 's3://test/' )`; @@ -331,7 +343,7 @@ FROM datasource.database.table GROUP BY TUMBLE (timestamp, '1 minute') WITH ( index_settings = '{"number_of_shards":9,"number_of_replicas":2}', -auto_refresh = false, +auto_refresh = true, refresh_interval = '1 minute', checkpoint_location = 's3://test/' )`;