From 0cb996cba9865e8ddc06bae505e28b70b51e6fe9 Mon Sep 17 00:00:00 2001 From: sagi yosefia <7414322+sagiyosefia@users.noreply.github.com> Date: Wed, 22 Nov 2023 16:32:09 +0200 Subject: [PATCH 1/3] add included table (#9499) Co-authored-by: Sagi Yosefia --- .../templates/terraform/examples/datastream_stream_full.tf.erb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb b/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb index e1c05fa1ac66..4c57f84110fd 100644 --- a/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb +++ b/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb @@ -138,6 +138,9 @@ resource "google_datastream_stream" "<%= ctx[:primary_resource_id] %>" { ordinal_position = 0 } } + mysql_tables { + table = "includedTable_2" + } } } exclude_objects { From fe2040c449bf2070c1b7c8a60bbb7c4a75d19369 Mon Sep 17 00:00:00 2001 From: sagi yosefia <7414322+sagiyosefia@users.noreply.github.com> Date: Wed, 22 Nov 2023 21:44:51 +0200 Subject: [PATCH 2/3] Fix Datastream private connectivity docs (#9455) Co-authored-by: Sagi Yosefia --- .../datastream/ConnectionProfile.yaml | 19 +++- mmv1/products/datastream/Stream.yaml | 7 +- ...profile_bigquery_private_connection.tf.erb | 30 ------- ...ofile_postgresql_private_connection.tf.erb | 86 +++++++++++++++++++ 4 files changed, 107 insertions(+), 35 deletions(-) delete mode 100644 mmv1/templates/terraform/examples/datastream_connection_profile_bigquery_private_connection.tf.erb create mode 100644 mmv1/templates/terraform/examples/datastream_connection_profile_postgresql_private_connection.tf.erb diff --git a/mmv1/products/datastream/ConnectionProfile.yaml b/mmv1/products/datastream/ConnectionProfile.yaml index 3a8a4d94879f..9dcb84d8186c 100644 --- a/mmv1/products/datastream/ConnectionProfile.yaml +++ b/mmv1/products/datastream/ConnectionProfile.yaml @@ -36,12 +36,21 @@ examples: vars: connection_profile_id: 'my-profile' - !ruby/object:Provider::Terraform::Examples - name: 'datastream_connection_profile_bigquery_private_connection' + name: 'datastream_connection_profile_postgresql_private_connection' + pull_external: true primary_resource_id: 'default' + # Random provider + skip_vcr: true vars: private_connection_id: 'my-connection' connection_profile_id: 'my-profile' network_name: 'my-network' + database_instance_name: 'my-instance' + deletion_protection: 'true' + test_vars_overrides: + deletion_protection: 'false' + oics_vars_overrides: + deletion_protection: 'false' - !ruby/object:Provider::Terraform::Examples name: 'datastream_connection_profile_full' primary_resource_id: @@ -57,9 +66,11 @@ examples: vars: connection_profile_id: 'my-profile' database_instance_name: 'my-instance' - deletion_protection: - 'true' - # This is covered by the update tests. + deletion_protection: 'true' + test_vars_overrides: + deletion_protection: 'false' + oics_vars_overrides: + deletion_protection: 'false' skip_test: true parameters: - !ruby/object:Api::Type::String diff --git a/mmv1/products/datastream/Stream.yaml b/mmv1/products/datastream/Stream.yaml index a88ed1eeb73c..3168af7640ff 100644 --- a/mmv1/products/datastream/Stream.yaml +++ b/mmv1/products/datastream/Stream.yaml @@ -67,6 +67,8 @@ examples: destination_connection_profile_id: 'destination-profile' test_vars_overrides: deletion_protection: 'false' + oics_vars_overrides: + deletion_protection: 'false' - !ruby/object:Provider::Terraform::Examples name: 'datastream_stream_full' pull_external: true @@ -87,6 +89,8 @@ examples: test_vars_overrides: deletion_protection: 'false' stream_cmek: 'acctest.BootstrapKMSKeyInLocation(t, "us-central1").CryptoKey.Name' + oics_vars_overrides: + deletion_protection: 'false' - !ruby/object:Provider::Terraform::Examples name: 'datastream_stream_postgresql' primary_resource_id: 'default' @@ -136,7 +140,8 @@ examples: deletion_protection: 'false' bigquery_destination_table_kms_key_name: 'acctest.BootstrapKMSKeyInLocation(t, "us-central1").CryptoKey.Name' - + oics_vars_overrides: + deletion_protection: 'false' parameters: - !ruby/object:Api::Type::String name: streamId diff --git a/mmv1/templates/terraform/examples/datastream_connection_profile_bigquery_private_connection.tf.erb b/mmv1/templates/terraform/examples/datastream_connection_profile_bigquery_private_connection.tf.erb deleted file mode 100644 index 05222e1a84da..000000000000 --- a/mmv1/templates/terraform/examples/datastream_connection_profile_bigquery_private_connection.tf.erb +++ /dev/null @@ -1,30 +0,0 @@ -resource "google_datastream_private_connection" "private_connection" { - display_name = "Connection profile" - location = "us-central1" - private_connection_id = "<%= ctx[:vars]['private_connection_id'] %>" - - labels = { - key = "value" - } - - vpc_peering_config { - vpc = google_compute_network.default.id - subnet = "10.0.0.0/29" - } -} - -resource "google_compute_network" "default" { - name = "<%= ctx[:vars]['network_name'] %>" -} - -resource "google_datastream_connection_profile" "<%= ctx[:primary_resource_id] %>" { - display_name = "Connection profile" - location = "us-central1" - connection_profile_id = "<%= ctx[:vars]['connection_profile_id'] %>" - - bigquery_profile {} - - private_connectivity { - private_connection = google_datastream_private_connection.private_connection.id - } -} diff --git a/mmv1/templates/terraform/examples/datastream_connection_profile_postgresql_private_connection.tf.erb b/mmv1/templates/terraform/examples/datastream_connection_profile_postgresql_private_connection.tf.erb new file mode 100644 index 000000000000..5df86ff5990e --- /dev/null +++ b/mmv1/templates/terraform/examples/datastream_connection_profile_postgresql_private_connection.tf.erb @@ -0,0 +1,86 @@ +resource "google_datastream_private_connection" "private_connection" { + display_name = "Connection profile" + location = "us-central1" + private_connection_id = "<%= ctx[:vars]['private_connection_id'] %>" + + labels = { + key = "value" + } + + vpc_peering_config { + vpc = google_compute_network.default.id + subnet = "10.0.0.0/29" + } +} + +resource "google_compute_network" "default" { + name = "<%= ctx[:vars]['network_name'] %>" +} + +resource "google_sql_database_instance" "instance" { + name = "<%= ctx[:vars]['database_instance_name'] %>" + database_version = "POSTGRES_14" + region = "us-central1" + settings { + tier = "db-f1-micro" + + ip_configuration { + + // Datastream IPs will vary by region. + authorized_networks { + value = "34.71.242.81" + } + + authorized_networks { + value = "34.72.28.29" + } + + authorized_networks { + value = "34.67.6.157" + } + + authorized_networks { + value = "34.67.234.134" + } + + authorized_networks { + value = "34.72.239.218" + } + } + } + + deletion_protection = "<%= ctx[:vars]['deletion_protection'] %>" +} + +resource "google_sql_database" "db" { + instance = google_sql_database_instance.instance.name + name = "db" +} + +resource "random_password" "pwd" { + length = 16 + special = false +} + +resource "google_sql_user" "user" { + name = "user" + instance = google_sql_database_instance.instance.name + password = random_password.pwd.result +} + +resource "google_datastream_connection_profile" "<%= ctx[:primary_resource_id] %>" { + display_name = "Connection profile" + location = "us-central1" + connection_profile_id = "<%= ctx[:vars]['connection_profile_id'] %>" + + postgresql_profile { + hostname = google_sql_database_instance.instance.public_ip_address + username = google_sql_user.user.name + password = google_sql_user.user.password + database = google_sql_database.db.name + } + + private_connectivity { + private_connection = google_datastream_private_connection.private_connection.id + } +} From 5365d2d0a4aa41fccdc9fde47bbe860da0e78cf3 Mon Sep 17 00:00:00 2001 From: Scott Suarez Date: Wed, 22 Nov 2023 11:49:28 -0800 Subject: [PATCH 3/3] Clean up the alternative route for base branch building.. (#9479) --- .../downstream-builder/generate_downstream.sh | 99 ++++--------------- 1 file changed, 19 insertions(+), 80 deletions(-) diff --git a/.ci/scripts/build-environment/downstream-builder/generate_downstream.sh b/.ci/scripts/build-environment/downstream-builder/generate_downstream.sh index 68d26e190dde..8165e3ea8e24 100755 --- a/.ci/scripts/build-environment/downstream-builder/generate_downstream.sh +++ b/.ci/scripts/build-environment/downstream-builder/generate_downstream.sh @@ -98,90 +98,29 @@ elif [ "$COMMAND" == "downstream" ]; then COMMIT_MESSAGE="$ORIGINAL_MESSAGE$NEWLINE[upstream:$REFERENCE]" fi -if [ "$COMMAND" == "base" ]; then - pushd mmv1 - if [ "$REPO" == "terraform" ]; then - pushd $LOCAL_PATH - go mod download - find . -type f -not -wholename "./.git*" -not -wholename "./.changelog*" -not -name ".travis.yml" -not -name ".golangci.yml" -not -name "CHANGELOG.md" -not -name "CHANGELOG_v*.md" -not -name "GNUmakefile" -not -name "docscheck.sh" -not -name "LICENSE" -not -name "README.md" -not -wholename "./examples*" -not -name ".go-version" -not -name ".hashibot.hcl" -print0 | xargs -0 git rm - popd - fi - if [ "$REPO" == "terraform-google-conversion" ]; then - # Generate tfplan2cai - pushd $LOCAL_PATH - # clear out the templates as they are copied during - # generation from mmv1/third_party/validator/tests/data - rm -rf ./tfplan2cai/testdata/templates/ - rm -rf ./tfplan2cai/testdata/generatedconvert/ - rm -rf ./tfplan2cai/converters/google/provider - rm -rf ./tfplan2cai/converters/google/resources - find ./tfplan2cai/test/** -type f -exec git rm {} \; - popd - - bundle exec compiler.rb -a -e terraform -f validator -o $LOCAL_PATH/tfplan2cai -v $VERSION - - # Generate cai2hcl - pushd $LOCAL_PATH - rm -rf ./cai2hcl/* - popd - - bundle exec compiler.rb -a -e terraform -f tgc_cai2hcl -o $LOCAL_PATH/cai2hcl -v $VERSION - - if [ "$COMMAND" == "downstream" ]; then - pushd $LOCAL_PATH - go get -d github.com/hashicorp/terraform-provider-google-beta@$BASE_BRANCH - go mod tidy - set +e - make build - set -e - popd - fi - elif [ "$REPO" == "tf-oics" ]; then - # use terraform generator with oics override - bundle exec compiler.rb -a -e terraform -f oics -o $LOCAL_PATH -v $VERSION - else - if [ "$REPO" == "terraform" ]; then - if [ "$VERSION" == "ga" ]; then - bundle exec compiler.rb -a -e $REPO -o $LOCAL_PATH -v $VERSION --no-docs - bundle exec compiler.rb -a -e $REPO -o $LOCAL_PATH -v beta --no-code - else - bundle exec compiler.rb -a -e $REPO -o $LOCAL_PATH -v $VERSION - fi - pushd ../ - make tpgtools OUTPUT_PATH=$LOCAL_PATH VERSION=$VERSION - - # Only generate TeamCity-related file for TPG and TPGB - if [ "$VERSION" == "ga" ] || [ "$VERSION" == "beta" ]; then - make teamcity-servicemap-generate OUTPUT_PATH=$LOCAL_PATH VERSION=$VERSION - fi - popd - fi - fi - popd -else - if [ "$REPO" == "terraform-google-conversion" ]; then - make clean-tgc OUTPUT_PATH="$LOCAL_PATH" - make tgc OUTPUT_PATH="$LOCAL_PATH" - - if [ "$COMMAND" == "downstream" ]; then - pushd $LOCAL_PATH - go get -d github.com/hashicorp/terraform-provider-google-beta@$BASE_BRANCH - go mod tidy - set +e - make build - set -e - popd - fi - elif [ "$REPO" == "tf-oics" ]; then - # use terraform generator with oics override - make tf-oics OUTPUT_PATH="$LOCAL_PATH" - elif [ "$REPO" == "terraform" ]; then - make clean-provider OUTPUT_PATH="$LOCAL_PATH" - make provider OUTPUT_PATH="$LOCAL_PATH" VERSION=$VERSION +if [ "$REPO" == "terraform-google-conversion" ]; then + make clean-tgc OUTPUT_PATH="$LOCAL_PATH" + make tgc OUTPUT_PATH="$LOCAL_PATH" + + if [ "$COMMAND" == "downstream" ]; then + pushd $LOCAL_PATH + go get -d github.com/hashicorp/terraform-provider-google-beta@$BASE_BRANCH + go mod tidy + set +e + make build + set -e + popd fi +elif [ "$REPO" == "tf-oics" ]; then + # use terraform generator with oics override + make tf-oics OUTPUT_PATH="$LOCAL_PATH" +elif [ "$REPO" == "terraform" ]; then + make clean-provider OUTPUT_PATH="$LOCAL_PATH" + make provider OUTPUT_PATH="$LOCAL_PATH" VERSION=$VERSION fi + pushd $LOCAL_PATH git config --local user.name "Modular Magician"