feat: optional sql server change tables and transaction logs (GoogleC…

…loudPlatform#11247)
pjotrekk · Aug 23, 2024 · 4c3e8a6 · 4c3e8a6
1 parent 723d418
commit 4c3e8a6
Show file tree

Hide file tree

Showing 5 changed files with 243 additions and 0 deletions.
diff --git a/mmv1/products/datastream/Stream.yaml b/mmv1/products/datastream/Stream.yaml
@@ -123,6 +123,23 @@ examples:
       stream_id: 'stream'
     test_vars_overrides:
       deletion_protection: 'false'
+  - !ruby/object:Provider::Terraform::Examples
+    name: 'datastream_stream_sql_server_change_tables'
+    primary_resource_id: 'default'
+    # Requires SQLServer Configuration
+    skip_test: true
+    vars:
+      database_name: 'db'
+      database_password: 'password'
+      database_user: 'user'
+      deletion_protection: 'true'
+      destination_connection_profile_id: 'destination-profile'
+      source_connection_profile_id: 'source-profile'
+      sql_server_name: 'sql-server'
+      sql_server_root_password: 'root-password'
+      stream_id: 'stream'
+    test_vars_overrides:
+      deletion_protection: 'false'
   - !ruby/object:Provider::Terraform::Examples
     name: 'datastream_stream_postgresql_bigquery_dataset_id'
     primary_resource_id: 'default'
@@ -1002,6 +1019,24 @@ properties:
             default_from_api: true
             validation: !ruby/object:Provider::Terraform::Validation
               function: 'validation.IntAtLeast(0)'
+          - !ruby/object:Api::Type::NestedObject
+            name: 'transactionLogs'
+            allow_empty_object: true
+            send_empty_value: true
+            conflicts:
+              - source_config.0.sql_server_source_config.change_tables
+            description: |
+              CDC reader reads from transaction logs.
+            properties: []
+          - !ruby/object:Api::Type::NestedObject
+            name: 'changeTables'
+            allow_empty_object: true
+            send_empty_value: true
+            conflicts:
+              - source_config.0.sql_server_source_config.transaction_logs
+            description: |
+              CDC reader reads from change tables.
+            properties: []
   - !ruby/object:Api::Type::NestedObject
     name: 'destinationConfig'
     required: true

diff --git a/mmv1/templates/terraform/examples/datastream_stream_sql_server.tf.erb b/mmv1/templates/terraform/examples/datastream_stream_sql_server.tf.erb
@@ -83,6 +83,7 @@ resource "google_datastream_stream" "default" {
                     }
                 }
             }
+            transaction_logs {}
         }
     }
 

diff --git a/mmv1/templates/terraform/examples/datastream_stream_sql_server_change_tables.tf.erb b/mmv1/templates/terraform/examples/datastream_stream_sql_server_change_tables.tf.erb
@@ -0,0 +1,103 @@
+resource "google_sql_database_instance" "instance" {
+    name                = "<%= ctx[:vars]['sql_server_name'] %>"
+    database_version    = "SQLSERVER_2019_STANDARD"
+    region              = "us-central1"
+    root_password       = "<%= ctx[:vars]['sql_server_root_password'] %>"
+    deletion_protection = "<%= ctx[:vars]['deletion_protection'] %>"
+
+    settings {
+        tier = "db-custom-2-4096"
+        ip_configuration {
+            // Datastream IPs will vary by region.
+            // https://cloud.google.com/datastream/docs/ip-allowlists-and-regions
+            authorized_networks {
+                value = "34.71.242.81"
+            }
+
+            authorized_networks {
+                value = "34.72.28.29"
+            }
+
+            authorized_networks {
+                value = "34.67.6.157"
+            }
+
+            authorized_networks {
+                value = "34.67.234.134"
+            }
+
+            authorized_networks {
+                value = "34.72.239.218"
+            }
+        }
+    }
+}
+
+resource "google_sql_database" "db" {
+    name       = "<%= ctx[:vars]['database_name'] %>"
+    instance   = google_sql_database_instance.instance.name
+    depends_on = [google_sql_user.user]
+}
+
+resource "google_sql_user" "user" {
+    name     = "<%= ctx[:vars]['database_user'] %>"
+    instance = google_sql_database_instance.instance.name
+    password = "<%= ctx[:vars]['database_password'] %>"
+}
+
+resource "google_datastream_connection_profile" "source" {
+    display_name          = "SQL Server Source"
+    location              = "us-central1"
+    connection_profile_id = "<%= ctx[:vars]['source_connection_profile_id'] %>"
+
+    sql_server_profile {
+        hostname = google_sql_database_instance.instance.public_ip_address
+        port     = 1433
+        username = google_sql_user.user.name
+        password = google_sql_user.user.password
+        database = google_sql_database.db.name
+    }
+}
+
+resource "google_datastream_connection_profile" "destination" {
+    display_name          = "BigQuery Destination"
+    location              = "us-central1"
+    connection_profile_id = "<%= ctx[:vars]['destination_connection_profile_id'] %>"
+
+    bigquery_profile {}
+}
+
+resource "google_datastream_stream" "default" {
+    display_name = "SQL Server to BigQuery"
+    location     = "us-central1"
+    stream_id    = "<%= ctx[:vars]['stream_id'] %>"
+
+    source_config {
+        source_connection_profile = google_datastream_connection_profile.source.id
+        sql_server_source_config {
+            include_objects {
+                schemas {
+                    schema = "schema"
+                    tables {
+                        table = "table"
+                    }
+                }
+            }
+            change_tables {}
+        }
+    }
+
+    destination_config {
+        destination_connection_profile = google_datastream_connection_profile.destination.id
+        bigquery_destination_config {
+            data_freshness = "900s"
+            source_hierarchy_datasets {
+                dataset_template {
+                    location = "us-central1"
+                }
+            }
+        }
+    }
+
+    backfill_none {}
+}
diff --git a/mmv1/templates/terraform/examples/go/datastream_stream_sql_server.tf.tmpl b/mmv1/templates/terraform/examples/go/datastream_stream_sql_server.tf.tmpl
@@ -83,6 +83,7 @@ resource "google_datastream_stream" "default" {
                     }
                 }
             }
+            transaction_logs {}
         }
     }
 

diff --git a/mmv1/templates/terraform/examples/go/datastream_stream_sql_server_change_tables.tf.tmpl b/mmv1/templates/terraform/examples/go/datastream_stream_sql_server_change_tables.tf.tmpl
@@ -0,0 +1,103 @@
+resource "google_sql_database_instance" "instance" {
+    name                = "{{index $.Vars "sql_server_name"}}"
+    database_version    = "SQLSERVER_2019_STANDARD"
+    region              = "us-central1"
+    root_password       = "{{index $.Vars "sql_server_root_password"}}"
+    deletion_protection = "{{index $.Vars "deletion_protection"}}"
+
+    settings {
+        tier = "db-custom-2-4096"
+        ip_configuration {
+            // Datastream IPs will vary by region.
+            // https://cloud.google.com/datastream/docs/ip-allowlists-and-regions
+            authorized_networks {
+                value = "34.71.242.81"
+            }
+
+            authorized_networks {
+                value = "34.72.28.29"
+            }
+
+            authorized_networks {
+                value = "34.67.6.157"
+            }
+
+            authorized_networks {
+                value = "34.67.234.134"
+            }
+
+            authorized_networks {
+                value = "34.72.239.218"
+            }
+        }
+    }
+}
+
+resource "google_sql_database" "db" {
+    name       = "{{index $.Vars "database_name"}}"
+    instance   = google_sql_database_instance.instance.name
+    depends_on = [google_sql_user.user]
+}
+
+resource "google_sql_user" "user" {
+    name     = "{{index $.Vars "database_user"}}"
+    instance = google_sql_database_instance.instance.name
+    password = "{{index $.Vars "database_password"}}"
+}
+
+resource "google_datastream_connection_profile" "source" {
+    display_name          = "SQL Server Source"
+    location              = "us-central1"
+    connection_profile_id = "{{index $.Vars "source_connection_profile_id"}}"
+
+    sql_server_profile {
+        hostname = google_sql_database_instance.instance.public_ip_address
+        port     = 1433
+        username = google_sql_user.user.name
+        password = google_sql_user.user.password
+        database = google_sql_database.db.name
+    }
+}
+
+resource "google_datastream_connection_profile" "destination" {
+    display_name          = "BigQuery Destination"
+    location              = "us-central1"
+    connection_profile_id = "{{index $.Vars "destination_connection_profile_id"}}"
+
+    bigquery_profile {}
+}
+
+resource "google_datastream_stream" "default" {
+    display_name = "SQL Server to BigQuery"
+    location     = "us-central1"
+    stream_id    = "{{index $.Vars "stream_id"}}"
+
+    source_config {
+        source_connection_profile = google_datastream_connection_profile.source.id
+        sql_server_source_config {
+            include_objects {
+                schemas {
+                    schema = "schema"
+                    tables {
+                        table = "table"
+                    }
+                }
+            }
+            change_tables {}
+        }
+    }
+
+    destination_config {
+        destination_connection_profile = google_datastream_connection_profile.destination.id
+        bigquery_destination_config {
+            data_freshness = "900s"
+            source_hierarchy_datasets {
+                dataset_template {
+                    location = "us-central1"
+                }
+            }
+        }
+    }
+
+    backfill_none {}
+}
-Original file line number
+Diff line change
@@ Expand Up / @@ -83,6 +83,7 @@ resource "google_datastream_stream" "default" { @@
                         }
                     }
                 }
+                transaction_logs {}
             }
         }
@@ Expand Down @@