From bc630e2b76f14e68b8b049948b95bb6c6f27115e Mon Sep 17 00:00:00 2001 From: Leonardo Drigo Date: Thu, 9 Nov 2023 14:24:53 -0300 Subject: [PATCH 1/3] creating new resource to use as default endpoint for DLTs in DMS --- modules/endpoints_s3_dlt/README.md | 43 ++++++++++++++++++++++++ modules/endpoints_s3_dlt/main.tf | 30 +++++++++++++++++ modules/endpoints_s3_dlt/output.tf | 19 +++++++++++ modules/endpoints_s3_dlt/variables.tf | 47 +++++++++++++++++++++++++++ 4 files changed, 139 insertions(+) create mode 100644 modules/endpoints_s3_dlt/README.md create mode 100644 modules/endpoints_s3_dlt/main.tf create mode 100644 modules/endpoints_s3_dlt/output.tf create mode 100644 modules/endpoints_s3_dlt/variables.tf diff --git a/modules/endpoints_s3_dlt/README.md b/modules/endpoints_s3_dlt/README.md new file mode 100644 index 0000000..2c4c6d6 --- /dev/null +++ b/modules/endpoints_s3_dlt/README.md @@ -0,0 +1,43 @@ + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_dms_s3_endpoint.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/dms_s3_endpoint) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [certificate\_arn](#input\_certificate\_arn) | (Optional, Default: empty string) The Amazon Resource Name (ARN) for the certificate. | `string` | `""` | no | +| [create](#input\_create) | (Opcional) Used to create the Data Migration Service | `bool` | `false` | no | +| [endpoint\_id](#input\_endpoint\_id) | (Required) The database endpoint identifier. | `string` | `""` | no | +| [endpoint\_type](#input\_endpoint\_type) | (Required) The type of endpoint. Can be one of source \| target | `string` | `"source"` | no | +| [kms\_key\_arn](#input\_kms\_key\_arn) | (Required when engine\_name is mongodb, optional otherwise) | `string` | `""` | no | +| [s3\_settings](#input\_s3\_settings) | (Optional) Configuration block with S3 settings. Detailed below. | `any` | `{}` | no | +| [ssl\_mode](#input\_ssl\_mode) | Optional, Default: none) The SSL mode to use for the connection. Can be one of none \| require \| verify-ca \| verify-full | `string` | `"none"` | no | +| [tags](#input\_tags) | (Optional) A map of tags to assign to the resource. | `map(string)` | `{}` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [certificate\_arn](#output\_certificate\_arn) | Certificate ARN | +| [endpoint\_arn](#output\_endpoint\_arn) | Endpoint ARN | +| [kms\_key\_arn](#output\_kms\_key\_arn) | KMS key ARN | +| [ssl\_mode](#output\_ssl\_mode) | SSL mode | + \ No newline at end of file diff --git a/modules/endpoints_s3_dlt/main.tf b/modules/endpoints_s3_dlt/main.tf new file mode 100644 index 0000000..5e4a192 --- /dev/null +++ b/modules/endpoints_s3_dlt/main.tf @@ -0,0 +1,30 @@ +resource "aws_dms_dlt_s3_endpoint" "this" { + count = var.create ? 1 : 0 + + certificate_arn = var.certificate_arn + endpoint_type = var.endpoint_type + endpoint_id = "${var.endpoint_id}-${var.endpoint_type}" + ssl_mode = var.ssl_mode + kms_key_arn = var.kms_key_arn + + bucket_name = var.s3_settings[0]["bucket_name"] + + bucket_folder = lookup(var.s3_settings[0], "bucket_folder", null) + external_table_definition = lookup(var.s3_settings[0], "external_table_definition", null) + service_access_role_arn = lookup(var.s3_settings[0], "service_access_role_arn", null) + compression_type = lookup(var.s3_settings[0], "compression_type", "NONE") + csv_delimiter = lookup(var.s3_settings[0], "csv_delimiter", ",") + csv_row_delimiter = lookup(var.s3_settings[0], "csv_row_delimiter", null) + date_partition_enabled = lookup(var.s3_settings[0], "date_partition_enabled", false) + data_format = lookup(var.s3_settings[0], "data_format", "csv") + date_partition_delimiter = lookup(var.s3_settings[0], "date_partition_delimiter", "NONE") + date_partition_sequence = lookup(var.s3_settings[0], "date_partition_sequence", "YYYYMMDD") + timestamp_column_name = lookup(var.s3_settings[0], "timestamp_column_name", "dmsTimestamp") + include_op_for_full_load = lookup(var.s3_settings[0], "include_op_for_full_load", false) + parquet_timestamp_in_millisecond = lookup(var.s3_settings[0], "parquet_timestamp_in_millisecond", false) + parquet_version = lookup(var.s3_settings[0], "parquet_version", "parquet-1-0") + enable_statistics = lookup(var.s3_settings[0], "enable_statistics", true) + preserve_transactions = lookup(var.s3_settings[0], "preserve_transactions", false) + + tags = var.tags +} diff --git a/modules/endpoints_s3_dlt/output.tf b/modules/endpoints_s3_dlt/output.tf new file mode 100644 index 0000000..549985f --- /dev/null +++ b/modules/endpoints_s3_dlt/output.tf @@ -0,0 +1,19 @@ +output "endpoint_arn" { + description = "Endpoint ARN" + value = aws_dms_s3_endpoint.this[0].endpoint_arn +} + +output "certificate_arn" { + description = "Certificate ARN" + value = aws_dms_s3_endpoint.this[0].certificate_arn +} + +output "kms_key_arn" { + description = "KMS key ARN" + value = aws_dms_s3_endpoint.this[0].kms_key_arn +} + +output "ssl_mode" { + description = "SSL mode" + value = aws_dms_s3_endpoint.this[0].ssl_mode +} diff --git a/modules/endpoints_s3_dlt/variables.tf b/modules/endpoints_s3_dlt/variables.tf new file mode 100644 index 0000000..fe004de --- /dev/null +++ b/modules/endpoints_s3_dlt/variables.tf @@ -0,0 +1,47 @@ +variable "create" { + description = "(Opcional) Used to create the Data Migration Service" + type = bool + default = false +} + +variable "certificate_arn" { + description = "(Optional, Default: empty string) The Amazon Resource Name (ARN) for the certificate." + type = string + default = "" +} + +variable "endpoint_type" { + description = "(Required) The type of endpoint. Can be one of source | target" + type = string + default = "source" +} + +variable "endpoint_id" { + description = "(Required) The database endpoint identifier." + type = string + default = "" +} + +variable "ssl_mode" { + description = "Optional, Default: none) The SSL mode to use for the connection. Can be one of none | require | verify-ca | verify-full" + type = string + default = "none" +} + +variable "kms_key_arn" { + description = "(Required when engine_name is mongodb, optional otherwise)" + type = string + default = "" +} + +variable "s3_settings" { + description = "(Optional) Configuration block with S3 settings. Detailed below." + type = any + default = {} +} + +variable "tags" { + description = "(Optional) A map of tags to assign to the resource." + type = map(string) + default = {} +} From 9e920789df066deed68427036c9f5ef253c9a1f2 Mon Sep 17 00:00:00 2001 From: Leonardo Drigo Date: Thu, 9 Nov 2023 14:54:33 -0300 Subject: [PATCH 2/3] setting new endpoint resource in output and readme --- modules/endpoints_s3_dlt/README.md | 2 +- modules/endpoints_s3_dlt/output.tf | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/endpoints_s3_dlt/README.md b/modules/endpoints_s3_dlt/README.md index 2c4c6d6..80513a0 100644 --- a/modules/endpoints_s3_dlt/README.md +++ b/modules/endpoints_s3_dlt/README.md @@ -17,7 +17,7 @@ No modules. | Name | Type | |------|------| -| [aws_dms_s3_endpoint.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/dms_s3_endpoint) | resource | +| [aws_dms_dlt_s3_endpoint.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/dms_s3_endpoint) | resource | ## Inputs diff --git a/modules/endpoints_s3_dlt/output.tf b/modules/endpoints_s3_dlt/output.tf index 549985f..f0c9884 100644 --- a/modules/endpoints_s3_dlt/output.tf +++ b/modules/endpoints_s3_dlt/output.tf @@ -1,19 +1,19 @@ output "endpoint_arn" { description = "Endpoint ARN" - value = aws_dms_s3_endpoint.this[0].endpoint_arn + value = aws_dms_dlt_s3_endpoint.this[0].endpoint_arn } output "certificate_arn" { description = "Certificate ARN" - value = aws_dms_s3_endpoint.this[0].certificate_arn + value = aws_dms_dlt_s3_endpoint.this[0].certificate_arn } output "kms_key_arn" { description = "KMS key ARN" - value = aws_dms_s3_endpoint.this[0].kms_key_arn + value = aws_dms_dlt_s3_endpoint.this[0].kms_key_arn } output "ssl_mode" { description = "SSL mode" - value = aws_dms_s3_endpoint.this[0].ssl_mode + value = aws_dms_dlt_s3_endpoint.this[0].ssl_mode } From 1afef3bc4a0743ee6dd8abcf49ae5f8c91a73755 Mon Sep 17 00:00:00 2001 From: Leonardo Drigo Date: Thu, 9 Nov 2023 15:16:47 -0300 Subject: [PATCH 3/3] fixing resource name --- modules/endpoints_s3_dlt/README.md | 2 +- modules/endpoints_s3_dlt/main.tf | 2 +- modules/endpoints_s3_dlt/output.tf | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/endpoints_s3_dlt/README.md b/modules/endpoints_s3_dlt/README.md index 80513a0..2c4c6d6 100644 --- a/modules/endpoints_s3_dlt/README.md +++ b/modules/endpoints_s3_dlt/README.md @@ -17,7 +17,7 @@ No modules. | Name | Type | |------|------| -| [aws_dms_dlt_s3_endpoint.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/dms_s3_endpoint) | resource | +| [aws_dms_s3_endpoint.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/dms_s3_endpoint) | resource | ## Inputs diff --git a/modules/endpoints_s3_dlt/main.tf b/modules/endpoints_s3_dlt/main.tf index 5e4a192..ea47409 100644 --- a/modules/endpoints_s3_dlt/main.tf +++ b/modules/endpoints_s3_dlt/main.tf @@ -1,4 +1,4 @@ -resource "aws_dms_dlt_s3_endpoint" "this" { +resource "aws_dms_s3_endpoint" "this" { count = var.create ? 1 : 0 certificate_arn = var.certificate_arn diff --git a/modules/endpoints_s3_dlt/output.tf b/modules/endpoints_s3_dlt/output.tf index f0c9884..549985f 100644 --- a/modules/endpoints_s3_dlt/output.tf +++ b/modules/endpoints_s3_dlt/output.tf @@ -1,19 +1,19 @@ output "endpoint_arn" { description = "Endpoint ARN" - value = aws_dms_dlt_s3_endpoint.this[0].endpoint_arn + value = aws_dms_s3_endpoint.this[0].endpoint_arn } output "certificate_arn" { description = "Certificate ARN" - value = aws_dms_dlt_s3_endpoint.this[0].certificate_arn + value = aws_dms_s3_endpoint.this[0].certificate_arn } output "kms_key_arn" { description = "KMS key ARN" - value = aws_dms_dlt_s3_endpoint.this[0].kms_key_arn + value = aws_dms_s3_endpoint.this[0].kms_key_arn } output "ssl_mode" { description = "SSL mode" - value = aws_dms_dlt_s3_endpoint.this[0].ssl_mode + value = aws_dms_s3_endpoint.this[0].ssl_mode }