From 0625e52746e74104f5d59a0410f909988a41e376 Mon Sep 17 00:00:00 2001 From: Mahad Zaryab Date: Mon, 19 Aug 2024 19:36:13 -0400 Subject: [PATCH 01/10] Add Pagination And Parrallelism Config For SAPODataSource --- internal/service/appflow/flow.go | 64 ++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/internal/service/appflow/flow.go b/internal/service/appflow/flow.go index 1766f8238cba..fe56cfb29f28 100644 --- a/internal/service/appflow/flow.go +++ b/internal/service/appflow/flow.go @@ -962,6 +962,34 @@ func resourceFlow() *schema.Resource { Required: true, ValidateFunc: validation.All(validation.StringMatch(regexache.MustCompile(`\S+`), "must not contain any whitespace characters"), validation.StringLenBetween(1, 512)), }, + "pagination_config": { + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "max_page_size": { + Type: schema.TypeInt, + Required: true, + ValidateFunc: validation.IntBetween(1, 10000), + }, + }, + }, + }, + "parallelism_config": { + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "max_page_size": { + Type: schema.TypeInt, + Required: true, + ValidateFunc: validation.IntBetween(1, 10), + }, + }, + }, + }, }, }, }, @@ -2345,6 +2373,34 @@ func expandSalesforceSourceProperties(tfMap map[string]interface{}) *types.Sales return a } +func expandSAPODataPaginationConfigProperties(tfMap map[string]interface{}) *types.SAPODataPaginationConfig { + if tfMap == nil { + return nil + } + + a := &types.SAPODataPaginationConfig{} + + if v, ok := tfMap["max_page_size"].(int); ok && v != 0 { + a.MaxPageSize = aws.Int32(int32(v)) + } + + return a +} + +func expandSAPODataParallelismConfigProperties(tfMap map[string]interface{}) *types.SAPODataParallelismConfig { + if tfMap == nil { + return nil + } + + a := &types.SAPODataParallelismConfig{} + + if v, ok := tfMap["max_parallelism"].(int); ok && v != 0 { + a.MaxParallelism = aws.Int32(int32(v)) + } + + return a +} + func expandSAPODataSourceProperties(tfMap map[string]interface{}) *types.SAPODataSourceProperties { if tfMap == nil { return nil @@ -2356,6 +2412,14 @@ func expandSAPODataSourceProperties(tfMap map[string]interface{}) *types.SAPODat a.ObjectPath = aws.String(v) } + if v, ok := tfMap["pagination_config"].([]interface{}); ok && len(v) > 0 && v[0] != nil { + a.PaginationConfig = expandSAPODataPaginationConfigProperties(v[0].(map[string]interface{})) + } + + if v, ok := tfMap["parallelism_config"].([]interface{}); ok && len(v) > 0 && v[0] != nil { + a.ParallelismConfig = expandSAPODataParallelismConfigProperties(v[0].(map[string]interface{})) + } + return a } From 7cf9a2cae53f09e493e563750dac9494a2fb934d Mon Sep 17 00:00:00 2001 From: Mahad Zaryab Date: Mon, 19 Aug 2024 19:45:23 -0400 Subject: [PATCH 02/10] Add Logic For Flattening SAPO Data Source Properties --- internal/service/appflow/flow.go | 36 ++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/internal/service/appflow/flow.go b/internal/service/appflow/flow.go index fe56cfb29f28..9229db309b68 100644 --- a/internal/service/appflow/flow.go +++ b/internal/service/appflow/flow.go @@ -3546,6 +3546,34 @@ func flattenSalesforceSourceProperties(salesforceSourceProperties *types.Salesfo return m } +func flattenSAPODataPaginationConfigProperties(sapoDataPaginationConfig *types.SAPODataPaginationConfig) map[string]interface{} { + if sapoDataPaginationConfig == nil { + return nil + } + + m := map[string]interface{}{} + + if v := sapoDataPaginationConfig.MaxPageSize; v != nil { + m["max_page_size"] = aws.ToInt32(v) + } + + return m +} + +func flattenSAPODataParallelismConfigProperties(sapoDataParallelismConfig *types.SAPODataParallelismConfig) map[string]interface{} { + if sapoDataParallelismConfig == nil { + return nil + } + + m := map[string]interface{}{} + + if v := sapoDataParallelismConfig.MaxParallelism; v != nil { + m["max_parallelism"] = aws.ToInt32(v) + } + + return m +} + func flattenSAPODataSourceProperties(sapoDataSourceProperties *types.SAPODataSourceProperties) map[string]interface{} { if sapoDataSourceProperties == nil { return nil @@ -3557,6 +3585,14 @@ func flattenSAPODataSourceProperties(sapoDataSourceProperties *types.SAPODataSou m["object_path"] = aws.ToString(v) } + if v := sapoDataSourceProperties.PaginationConfig; v != nil { + m["pagination_config"] = flattenSAPODataPaginationConfigProperties(v) + } + + if v := sapoDataSourceProperties.ParallelismConfig; v != nil { + m["pagination_config"] = flattenSAPODataParallelismConfigProperties(v) + } + return m } From 5ecf0fd386b4ad57b8929e870919385cde3ef906 Mon Sep 17 00:00:00 2001 From: Mahad Zaryab Date: Mon, 19 Aug 2024 21:12:30 -0400 Subject: [PATCH 03/10] Update Documentation --- website/docs/cdktf/typescript/r/appflow_flow.html.markdown | 2 ++ 1 file changed, 2 insertions(+) diff --git a/website/docs/cdktf/typescript/r/appflow_flow.html.markdown b/website/docs/cdktf/typescript/r/appflow_flow.html.markdown index b3cdef6bd05b..eea59a299485 100644 --- a/website/docs/cdktf/typescript/r/appflow_flow.html.markdown +++ b/website/docs/cdktf/typescript/r/appflow_flow.html.markdown @@ -365,6 +365,8 @@ Amplitude, Datadog, Dynatrace, Google Analytics, Infor Nexus, Marketo, ServiceNo ##### SAPOData Source Properties * `objectPath` - (Required) Object path specified in the SAPOData flow source. +* `paginationConfig` - (Optional) Sets the page size for each concurrent process that transfers OData records from your SAP instance. +* `parallelismConfig` - (Optional) Sets the number of concurrent processes that transfers OData records from your SAP instance. ##### Veeva Source Properties From 5e2ca5fcb09ec27c71b86d431bcd88a96e4834a1 Mon Sep 17 00:00:00 2001 From: Mahad Zaryab Date: Mon, 19 Aug 2024 21:23:18 -0400 Subject: [PATCH 04/10] Add Changelog --- .changelog/38932.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .changelog/38932.txt diff --git a/.changelog/38932.txt b/.changelog/38932.txt new file mode 100644 index 000000000000..c1cc8c06b714 --- /dev/null +++ b/.changelog/38932.txt @@ -0,0 +1,3 @@ +```release-note:enhancement +resource/aws_eip: Add support for pagination and parallelism properties for appflow_flow +``` \ No newline at end of file From 121b555afb0b615adc43fb906343f1e7878b3bca Mon Sep 17 00:00:00 2001 From: Mahad Zaryab Date: Mon, 19 Aug 2024 21:24:42 -0400 Subject: [PATCH 05/10] Fix Changelog Entry --- .changelog/38932.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changelog/38932.txt b/.changelog/38932.txt index c1cc8c06b714..b4951d97db5c 100644 --- a/.changelog/38932.txt +++ b/.changelog/38932.txt @@ -1,3 +1,3 @@ ```release-note:enhancement -resource/aws_eip: Add support for pagination and parallelism properties for appflow_flow +resource/aws_appflow_flow: Add support for pagination and parallelism properties ``` \ No newline at end of file From def3b3631f2ce82364a1e53c34f22fc1a1d337be Mon Sep 17 00:00:00 2001 From: Adrian Johnson Date: Tue, 15 Oct 2024 11:08:14 -0500 Subject: [PATCH 06/10] chore: revert changed to cdktf docs that are automatically generated --- website/docs/cdktf/typescript/r/appflow_flow.html.markdown | 2 -- 1 file changed, 2 deletions(-) diff --git a/website/docs/cdktf/typescript/r/appflow_flow.html.markdown b/website/docs/cdktf/typescript/r/appflow_flow.html.markdown index 48c3761f9e42..c125a611dd92 100644 --- a/website/docs/cdktf/typescript/r/appflow_flow.html.markdown +++ b/website/docs/cdktf/typescript/r/appflow_flow.html.markdown @@ -365,8 +365,6 @@ Amplitude, Datadog, Dynatrace, Google Analytics, Infor Nexus, Marketo, ServiceNo ##### SAPOData Source Properties * `objectPath` - (Required) Object path specified in the SAPOData flow source. -* `paginationConfig` - (Optional) Sets the page size for each concurrent process that transfers OData records from your SAP instance. -* `parallelismConfig` - (Optional) Sets the number of concurrent processes that transfers OData records from your SAP instance. ##### Veeva Source Properties From 67c7cf2713910702bc3fd84feee383593e6adf07 Mon Sep 17 00:00:00 2001 From: Adrian Johnson Date: Tue, 15 Oct 2024 11:14:22 -0500 Subject: [PATCH 07/10] aws_appflow_flow: update documentation --- website/docs/r/appflow_flow.html.markdown | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/website/docs/r/appflow_flow.html.markdown b/website/docs/r/appflow_flow.html.markdown index fa590de27f88..aff5093fe700 100644 --- a/website/docs/r/appflow_flow.html.markdown +++ b/website/docs/r/appflow_flow.html.markdown @@ -324,6 +324,10 @@ Amplitude, Datadog, Dynatrace, Google Analytics, Infor Nexus, Marketo, ServiceNo ##### SAPOData Source Properties * `object_path` - (Required) Object path specified in the SAPOData flow source. +* `pagination_config` - (Optional) Sets the page size for each concurrent process that transfers OData records from your SAP instance. + * `max_page_size` - (Optional) he maximum number of records that Amazon AppFlow receives in each page of the response from your SAP application. +* `parallelism_config` - (Optional) Sets the number of concurrent processes that transfers OData records from your SAP instance. + * `max_parallelism` - (Optional) The maximum number of processes that Amazon AppFlow runs at the same time when it retrieves your data from your SAP application. ##### Veeva Source Properties From 14bfe4356993621ca2b43beeefe7d4e2af686916 Mon Sep 17 00:00:00 2001 From: Adrian Johnson Date: Tue, 15 Oct 2024 11:45:05 -0500 Subject: [PATCH 08/10] aws_appflow_flow: return slices on flatteners to match schema --- internal/service/appflow/flow.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/service/appflow/flow.go b/internal/service/appflow/flow.go index 56da23e68b5b..3a91897a6cef 100644 --- a/internal/service/appflow/flow.go +++ b/internal/service/appflow/flow.go @@ -3546,32 +3546,32 @@ func flattenSalesforceSourceProperties(salesforceSourceProperties *types.Salesfo return m } -func flattenSAPODataPaginationConfigProperties(sapoDataPaginationConfig *types.SAPODataPaginationConfig) map[string]interface{} { +func flattenSAPODataPaginationConfigProperties(sapoDataPaginationConfig *types.SAPODataPaginationConfig) []any { if sapoDataPaginationConfig == nil { return nil } - m := map[string]interface{}{} + m := map[string]any{} if v := sapoDataPaginationConfig.MaxPageSize; v != nil { m["max_page_size"] = aws.ToInt32(v) } - return m + return []any{m} } -func flattenSAPODataParallelismConfigProperties(sapoDataParallelismConfig *types.SAPODataParallelismConfig) map[string]interface{} { +func flattenSAPODataParallelismConfigProperties(sapoDataParallelismConfig *types.SAPODataParallelismConfig) []any { if sapoDataParallelismConfig == nil { return nil } - m := map[string]interface{}{} + m := map[string]any{} if v := sapoDataParallelismConfig.MaxParallelism; v != nil { m["max_parallelism"] = aws.ToInt32(v) } - return m + return []any{m} } func flattenSAPODataSourceProperties(sapoDataSourceProperties *types.SAPODataSourceProperties) map[string]interface{} { From 975330807c409174e038bd0f339203afb718d7fd Mon Sep 17 00:00:00 2001 From: Adrian Johnson Date: Tue, 15 Oct 2024 11:55:27 -0500 Subject: [PATCH 09/10] tweak CHANGELOG entry --- .changelog/38932.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changelog/38932.txt b/.changelog/38932.txt index b4951d97db5c..9b4ac721f239 100644 --- a/.changelog/38932.txt +++ b/.changelog/38932.txt @@ -1,3 +1,3 @@ ```release-note:enhancement -resource/aws_appflow_flow: Add support for pagination and parallelism properties +resource/aws_appflow_flow: Add `source_flow_config.source_connector_properties.sapo_data.pagination_config` and `source_flow_config.source_connector_properties.sapo_data.parallelism_config` attributes ``` \ No newline at end of file From 57ed0030b45a2e91f0e1d2e575cccf979c764fb4 Mon Sep 17 00:00:00 2001 From: Adrian Johnson Date: Tue, 15 Oct 2024 12:03:09 -0500 Subject: [PATCH 10/10] chore: documentation linting --- website/docs/r/appflow_flow.html.markdown | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/docs/r/appflow_flow.html.markdown b/website/docs/r/appflow_flow.html.markdown index aff5093fe700..7d5bb0a24dd0 100644 --- a/website/docs/r/appflow_flow.html.markdown +++ b/website/docs/r/appflow_flow.html.markdown @@ -325,9 +325,9 @@ Amplitude, Datadog, Dynatrace, Google Analytics, Infor Nexus, Marketo, ServiceNo * `object_path` - (Required) Object path specified in the SAPOData flow source. * `pagination_config` - (Optional) Sets the page size for each concurrent process that transfers OData records from your SAP instance. - * `max_page_size` - (Optional) he maximum number of records that Amazon AppFlow receives in each page of the response from your SAP application. + * `max_page_size` - (Optional) he maximum number of records that Amazon AppFlow receives in each page of the response from your SAP application. * `parallelism_config` - (Optional) Sets the number of concurrent processes that transfers OData records from your SAP instance. - * `max_parallelism` - (Optional) The maximum number of processes that Amazon AppFlow runs at the same time when it retrieves your data from your SAP application. + * `max_parallelism` - (Optional) The maximum number of processes that Amazon AppFlow runs at the same time when it retrieves your data from your SAP application. ##### Veeva Source Properties