use {product}.googleapis.com endpoints (#3755) (#2396)

* use {product}.googleapis.com endpoints

* use actual correct urls

* fix zone data source test

* fix network peering tests

* possibly fix deleting default network

Signed-off-by: Modular Magician <magic-modules@google.com>

.changelog/3755.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+all: updated base urls for compute, dns, storage, and bigquery APIs to their recommended endpoints
+```

google-beta/config.go

@@ -245,7 +245,7 @@ var AccessContextManagerDefaultBasePath = "https://accesscontextmanager.googleap
 var ActiveDirectoryDefaultBasePath = "https://managedidentities.googleapis.com/v1/"
 var AppEngineDefaultBasePath = "https://appengine.googleapis.com/v1/"
 var ArtifactRegistryDefaultBasePath = "https://artifactregistry.googleapis.com/v1beta1/"
-var BigQueryDefaultBasePath = "https://www.googleapis.com/bigquery/v2/"
+var BigQueryDefaultBasePath = "https://bigquery.googleapis.com/bigquery/v2/"
 var BigqueryConnectionDefaultBasePath = "https://bigqueryconnection.googleapis.com/v1beta1/"
 var BigqueryDataTransferDefaultBasePath = "https://bigquerydatatransfer.googleapis.com/v1/"
 var BigqueryReservationDefaultBasePath = "https://bigqueryreservation.googleapis.com/v1beta1/"
@@ -260,15 +260,15 @@ var CloudIotDefaultBasePath = "https://cloudiot.googleapis.com/v1/"
 var CloudRunDefaultBasePath = "https://{{location}}-run.googleapis.com/"
 var CloudSchedulerDefaultBasePath = "https://cloudscheduler.googleapis.com/v1/"
 var CloudTasksDefaultBasePath = "https://cloudtasks.googleapis.com/v2/"
-var ComputeDefaultBasePath = "https://www.googleapis.com/compute/beta/"
+var ComputeDefaultBasePath = "https://compute.googleapis.com/compute/beta/"
 var ContainerAnalysisDefaultBasePath = "https://containeranalysis.googleapis.com/v1beta1/"
 var DataCatalogDefaultBasePath = "https://datacatalog.googleapis.com/v1/"
 var DataFusionDefaultBasePath = "https://datafusion.googleapis.com/v1beta1/"
 var DataprocDefaultBasePath = "https://dataproc.googleapis.com/v1beta2/"
 var DatastoreDefaultBasePath = "https://datastore.googleapis.com/v1/"
 var DeploymentManagerDefaultBasePath = "https://www.googleapis.com/deploymentmanager/v2/"
 var DialogflowDefaultBasePath = "https://dialogflow.googleapis.com/v2/"
-var DNSDefaultBasePath = "https://www.googleapis.com/dns/v1beta2/"
+var DNSDefaultBasePath = "https://dns.googleapis.com/dns/v1beta2/"
 var FilestoreDefaultBasePath = "https://file.googleapis.com/v1beta1/"
 var FirebaseDefaultBasePath = "https://firebase.googleapis.com/v1beta1/"
 var FirestoreDefaultBasePath = "https://firestore.googleapis.com/v1/"
@@ -298,7 +298,7 @@ var ServiceUsageDefaultBasePath = "https://serviceusage.googleapis.com/v1beta1/"
 var SourceRepoDefaultBasePath = "https://sourcerepo.googleapis.com/v1/"
 var SpannerDefaultBasePath = "https://spanner.googleapis.com/v1/"
 var SQLDefaultBasePath = "https://sqladmin.googleapis.com/sql/v1beta4/"
-var StorageDefaultBasePath = "https://www.googleapis.com/storage/v1/"
+var StorageDefaultBasePath = "https://storage.googleapis.com/storage/v1/"
 var TPUDefaultBasePath = "https://tpu.googleapis.com/v1/"
 var VPCAccessDefaultBasePath = "https://vpcaccess.googleapis.com/v1/"
 

google-beta/data_source_google_compute_zones.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"log"
 	"sort"
+	"strings"
 	"time"
 
 	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
@@ -51,30 +52,28 @@ func dataSourceGoogleComputeZonesRead(d *schema.ResourceData, meta interface{})
 		return err
 	}
 
-	// we want to share exactly the same base path as the compute client or the
-	// region string may mismatch, giving us no results
-	// note that the client's BasePath includes a `projects/` suffix, so that'll
-	// need to be added to the URL below if the source changes
-	computeClientBasePath := config.clientCompute.BasePath
-
-	regionUrl, err := replaceVars(d, config, fmt.Sprintf("%s%s/regions/%s", computeClientBasePath, project, region))
-	if err != nil {
-		return err
-	}
-	filter := fmt.Sprintf("(region eq %s)", regionUrl)
-
+	filter := ""
 	if s, ok := d.GetOk("status"); ok {
 		filter += fmt.Sprintf(" (status eq %s)", s)
 	}
 
-	call := config.clientCompute.Zones.List(project).Filter(filter)
+	zones := []string{}
+	err = config.clientCompute.Zones.List(project).Filter(filter).Pages(config.context, func(zl *compute.ZoneList) error {
+		for _, zone := range zl.Items {
+			// We have no way to guarantee a specific base path for the region, but the built-in API-level filtering
+			// only lets us query on exact matches, so we do our own filtering here.
+			if strings.HasSuffix(zone.Region, "/"+region) {
+				zones = append(zones, zone.Name)
+			}
+		}
+		return nil
+	})
 
-	resp, err := call.Do()
 	if err != nil {
 		return err
 	}
 
-	zones := flattenZones(resp.Items)
+	sort.Strings(zones)
 	log.Printf("[DEBUG] Received Google Compute Zones: %q", zones)
 
 	d.Set("names", zones)
@@ -84,12 +83,3 @@ func dataSourceGoogleComputeZonesRead(d *schema.ResourceData, meta interface{})
 
 	return nil
 }
-
-func flattenZones(zones []*compute.Zone) []string {
-	result := make([]string, len(zones))
-	for i, zone := range zones {
-		result[i] = zone.Name
-	}
-	sort.Strings(result)
-	return result
-}

google-beta/data_source_google_compute_zones_test.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"strconv"
+	"strings"
 	"testing"
 
 	"github.com/hashicorp/terraform-plugin-sdk/helper/resource"
@@ -18,7 +19,7 @@ func TestAccComputeZones_basic(t *testing.T) {
 		Providers: testAccProviders,
 		Steps: []resource.TestStep{
 			{
-				Config: testAccCheckGoogleComputeZonesConfig,
+				Config: testAccComputeZones_basic,
 				Check: resource.ComposeTestCheckFunc(
 					testAccCheckGoogleComputeZonesMeta("data.google_compute_zones.available"),
 				),
@@ -27,6 +28,24 @@ func TestAccComputeZones_basic(t *testing.T) {
 	})
 }
 
+func TestAccComputeZones_filter(t *testing.T) {
+	t.Parallel()
+	region := "us-central1"
+
+	vcrTest(t, resource.TestCase{
+		PreCheck:  func() { testAccPreCheck(t) },
+		Providers: testAccProviders,
+		Steps: []resource.TestStep{
+			{
+				Config: testAccComputeZones_filter(region),
+				Check: resource.ComposeTestCheckFunc(
+					testAccCheckGoogleComputeZonesRegion("data.google_compute_zones.available", region),
+				),
+			},
+		},
+	})
+}
+
 func testAccCheckGoogleComputeZonesMeta(n string) resource.TestCheckFunc {
 	return func(s *terraform.State) error {
 		rs, ok := s.RootModule().Resources[n]
@@ -67,6 +86,51 @@ func testAccCheckGoogleComputeZonesMeta(n string) resource.TestCheckFunc {
 	}
 }
 
-var testAccCheckGoogleComputeZonesConfig = `
+func testAccCheckGoogleComputeZonesRegion(n, region string) resource.TestCheckFunc {
+	return func(s *terraform.State) error {
+		rs, ok := s.RootModule().Resources[n]
+		if !ok {
+			return fmt.Errorf("Can't find zones data source: %s", n)
+		}
+
+		if rs.Primary.ID == "" {
+			return errors.New("zones data source ID not set.")
+		}
+
+		count, ok := rs.Primary.Attributes["names.#"]
+		if !ok {
+			return errors.New("can't find 'names' attribute")
+		}
+
+		noOfNames, err := strconv.Atoi(count)
+		if err != nil {
+			return errors.New("failed to read number of zones")
+		}
+
+		for i := 0; i < noOfNames; i++ {
+			idx := "names." + strconv.Itoa(i)
+			v, ok := rs.Primary.Attributes[idx]
+			if !ok {
+				return fmt.Errorf("zone list is corrupt (%q not found), this is definitely a bug", idx)
+			}
+			if !strings.Contains(v, region) {
+				return fmt.Errorf("zone name %q does not contain region %q", v, region)
+			}
+		}
+
+		return nil
+	}
+}
+
+var testAccComputeZones_basic = `
 data "google_compute_zones" "available" {}
 `
+
+func testAccComputeZones_filter(region string) string {
+	return fmt.Sprintf(`
+data "google_compute_zones" "available" {
+  region = "%s"
+  status = "UP"
+}
+`, region)
+}

google-beta/resource_compute_address_sweeper_test.go

@@ -61,7 +61,7 @@ func testSweepComputeAddress(region string) error {
 		},
 	}
 
-	listTemplate := strings.Split("https://www.googleapis.com/compute/beta/projects/{{project}}/regions/{{region}}/addresses", "?")[0]
+	listTemplate := strings.Split("https://compute.googleapis.com/compute/beta/projects/{{project}}/regions/{{region}}/addresses", "?")[0]
 	listUrl, err := replaceVars(d, config, listTemplate)
 	if err != nil {
 		log.Printf("[INFO][SWEEPER_LOG] error preparing sweeper list url: %s", err)
@@ -99,7 +99,7 @@ func testSweepComputeAddress(region string) error {
 			continue
 		}
 
-		deleteTemplate := "https://www.googleapis.com/compute/beta/projects/{{project}}/regions/{{region}}/addresses/{{name}}"
+		deleteTemplate := "https://compute.googleapis.com/compute/beta/projects/{{project}}/regions/{{region}}/addresses/{{name}}"
 		deleteUrl, err := replaceVars(d, config, deleteTemplate)
 		if err != nil {
 			log.Printf("[INFO][SWEEPER_LOG] error preparing delete url: %s", err)

google-beta/resource_compute_autoscaler_sweeper_test.go

@@ -61,7 +61,7 @@ func testSweepComputeAutoscaler(region string) error {
 		},
 	}
 
-	listTemplate := strings.Split("https://www.googleapis.com/compute/beta/projects/{{project}}/aggregated/autoscalers", "?")[0]
+	listTemplate := strings.Split("https://compute.googleapis.com/compute/beta/projects/{{project}}/aggregated/autoscalers", "?")[0]
 	listUrl, err := replaceVars(d, config, listTemplate)
 	if err != nil {
 		log.Printf("[INFO][SWEEPER_LOG] error preparing sweeper list url: %s", err)
@@ -110,7 +110,7 @@ func testSweepComputeAutoscaler(region string) error {
 			continue
 		}
 
-		deleteTemplate := "https://www.googleapis.com/compute/beta/projects/{{project}}/zones/{{zone}}/autoscalers/{{name}}"
+		deleteTemplate := "https://compute.googleapis.com/compute/beta/projects/{{project}}/zones/{{zone}}/autoscalers/{{name}}"
 		if obj["zone"] == nil {
 			log.Printf("[INFO][SWEEPER_LOG] %s resource zone was nil", resourceName)
 			return nil

google-beta/resource_compute_backend_bucket_signed_url_key_sweeper_test.go

@@ -61,7 +61,7 @@ func testSweepComputeBackendBucketSignedUrlKey(region string) error {
 		},
 	}
 
-	listTemplate := strings.Split("https://www.googleapis.com/compute/beta/projects/{{project}}/global/backendBuckets/{{backend_bucket}}", "?")[0]
+	listTemplate := strings.Split("https://compute.googleapis.com/compute/beta/projects/{{project}}/global/backendBuckets/{{backend_bucket}}", "?")[0]
 	listUrl, err := replaceVars(d, config, listTemplate)
 	if err != nil {
 		log.Printf("[INFO][SWEEPER_LOG] error preparing sweeper list url: %s", err)
@@ -99,7 +99,7 @@ func testSweepComputeBackendBucketSignedUrlKey(region string) error {
 			continue
 		}
 
-		deleteTemplate := "https://www.googleapis.com/compute/beta/projects/{{project}}/global/backendBuckets/{{backend_bucket}}/deleteSignedUrlKey?keyName={{name}}"
+		deleteTemplate := "https://compute.googleapis.com/compute/beta/projects/{{project}}/global/backendBuckets/{{backend_bucket}}/deleteSignedUrlKey?keyName={{name}}"
 		deleteUrl, err := replaceVars(d, config, deleteTemplate)
 		if err != nil {
 			log.Printf("[INFO][SWEEPER_LOG] error preparing delete url: %s", err)

google-beta/resource_compute_backend_bucket_sweeper_test.go

@@ -61,7 +61,7 @@ func testSweepComputeBackendBucket(region string) error {
 		},
 	}
 
-	listTemplate := strings.Split("https://www.googleapis.com/compute/beta/projects/{{project}}/global/backendBuckets", "?")[0]
+	listTemplate := strings.Split("https://compute.googleapis.com/compute/beta/projects/{{project}}/global/backendBuckets", "?")[0]
 	listUrl, err := replaceVars(d, config, listTemplate)
 	if err != nil {
 		log.Printf("[INFO][SWEEPER_LOG] error preparing sweeper list url: %s", err)
@@ -99,7 +99,7 @@ func testSweepComputeBackendBucket(region string) error {
 			continue
 		}
 
-		deleteTemplate := "https://www.googleapis.com/compute/beta/projects/{{project}}/global/backendBuckets/{{name}}"
+		deleteTemplate := "https://compute.googleapis.com/compute/beta/projects/{{project}}/global/backendBuckets/{{name}}"
 		deleteUrl, err := replaceVars(d, config, deleteTemplate)
 		if err != nil {
 			log.Printf("[INFO][SWEEPER_LOG] error preparing delete url: %s", err)

google-beta/resource_compute_backend_service_signed_url_key_sweeper_test.go

@@ -61,7 +61,7 @@ func testSweepComputeBackendServiceSignedUrlKey(region string) error {
 		},
 	}
 
-	listTemplate := strings.Split("https://www.googleapis.com/compute/beta/projects/{{project}}/global/backendServices/{{backend_service}}", "?")[0]
+	listTemplate := strings.Split("https://compute.googleapis.com/compute/beta/projects/{{project}}/global/backendServices/{{backend_service}}", "?")[0]
 	listUrl, err := replaceVars(d, config, listTemplate)
 	if err != nil {
 		log.Printf("[INFO][SWEEPER_LOG] error preparing sweeper list url: %s", err)
@@ -99,7 +99,7 @@ func testSweepComputeBackendServiceSignedUrlKey(region string) error {
 			continue
 		}
 
-		deleteTemplate := "https://www.googleapis.com/compute/beta/projects/{{project}}/global/backendServices/{{backend_service}}/deleteSignedUrlKey?keyName={{name}}"
+		deleteTemplate := "https://compute.googleapis.com/compute/beta/projects/{{project}}/global/backendServices/{{backend_service}}/deleteSignedUrlKey?keyName={{name}}"
 		deleteUrl, err := replaceVars(d, config, deleteTemplate)
 		if err != nil {
 			log.Printf("[INFO][SWEEPER_LOG] error preparing delete url: %s", err)

google-beta/resource_compute_backend_service_sweeper_test.go

@@ -61,7 +61,7 @@ func testSweepComputeBackendService(region string) error {
 		},
 	}
 
-	listTemplate := strings.Split("https://www.googleapis.com/compute/beta/projects/{{project}}/global/backendServices", "?")[0]
+	listTemplate := strings.Split("https://compute.googleapis.com/compute/beta/projects/{{project}}/global/backendServices", "?")[0]
 	listUrl, err := replaceVars(d, config, listTemplate)
 	if err != nil {
 		log.Printf("[INFO][SWEEPER_LOG] error preparing sweeper list url: %s", err)
@@ -99,7 +99,7 @@ func testSweepComputeBackendService(region string) error {
 			continue
 		}
 
-		deleteTemplate := "https://www.googleapis.com/compute/beta/projects/{{project}}/global/backendServices/{{name}}"
+		deleteTemplate := "https://compute.googleapis.com/compute/beta/projects/{{project}}/global/backendServices/{{name}}"
 		deleteUrl, err := replaceVars(d, config, deleteTemplate)
 		if err != nil {
 			log.Printf("[INFO][SWEEPER_LOG] error preparing delete url: %s", err)

google-beta/resource_compute_external_vpn_gateway_sweeper_test.go

@@ -61,7 +61,7 @@ func testSweepComputeExternalVpnGateway(region string) error {
 		},
 	}
 
-	listTemplate := strings.Split("https://www.googleapis.com/compute/beta/projects/{{project}}/global/externalVpnGateways", "?")[0]
+	listTemplate := strings.Split("https://compute.googleapis.com/compute/beta/projects/{{project}}/global/externalVpnGateways", "?")[0]
 	listUrl, err := replaceVars(d, config, listTemplate)
 	if err != nil {
 		log.Printf("[INFO][SWEEPER_LOG] error preparing sweeper list url: %s", err)
@@ -99,7 +99,7 @@ func testSweepComputeExternalVpnGateway(region string) error {
 			continue
 		}
 
-		deleteTemplate := "https://www.googleapis.com/compute/beta/projects/{{project}}/global/externalVpnGateways/{{name}}"
+		deleteTemplate := "https://compute.googleapis.com/compute/beta/projects/{{project}}/global/externalVpnGateways/{{name}}"
 		deleteUrl, err := replaceVars(d, config, deleteTemplate)
 		if err != nil {
 			log.Printf("[INFO][SWEEPER_LOG] error preparing delete url: %s", err)

google-beta/resource_compute_firewall_sweeper_test.go

@@ -61,7 +61,7 @@ func testSweepComputeFirewall(region string) error {
 		},
 	}
 
-	listTemplate := strings.Split("https://www.googleapis.com/compute/beta/projects/{{project}}/global/firewalls", "?")[0]
+	listTemplate := strings.Split("https://compute.googleapis.com/compute/beta/projects/{{project}}/global/firewalls", "?")[0]
 	listUrl, err := replaceVars(d, config, listTemplate)
 	if err != nil {
 		log.Printf("[INFO][SWEEPER_LOG] error preparing sweeper list url: %s", err)
@@ -99,7 +99,7 @@ func testSweepComputeFirewall(region string) error {
 			continue
 		}
 
-		deleteTemplate := "https://www.googleapis.com/compute/beta/projects/{{project}}/global/firewalls/{{name}}"
+		deleteTemplate := "https://compute.googleapis.com/compute/beta/projects/{{project}}/global/firewalls/{{name}}"
 		deleteUrl, err := replaceVars(d, config, deleteTemplate)
 		if err != nil {
 			log.Printf("[INFO][SWEEPER_LOG] error preparing delete url: %s", err)

google-beta/resource_compute_forwarding_rule_sweeper_test.go

@@ -61,7 +61,7 @@ func testSweepComputeForwardingRule(region string) error {
 		},
 	}
 
-	listTemplate := strings.Split("https://www.googleapis.com/compute/beta/projects/{{project}}/regions/{{region}}/forwardingRules", "?")[0]
+	listTemplate := strings.Split("https://compute.googleapis.com/compute/beta/projects/{{project}}/regions/{{region}}/forwardingRules", "?")[0]
 	listUrl, err := replaceVars(d, config, listTemplate)
 	if err != nil {
 		log.Printf("[INFO][SWEEPER_LOG] error preparing sweeper list url: %s", err)
@@ -99,7 +99,7 @@ func testSweepComputeForwardingRule(region string) error {
 			continue
 		}
 
-		deleteTemplate := "https://www.googleapis.com/compute/beta/projects/{{project}}/regions/{{region}}/forwardingRules/{{name}}"
+		deleteTemplate := "https://compute.googleapis.com/compute/beta/projects/{{project}}/regions/{{region}}/forwardingRules/{{name}}"
 		deleteUrl, err := replaceVars(d, config, deleteTemplate)
 		if err != nil {
 			log.Printf("[INFO][SWEEPER_LOG] error preparing delete url: %s", err)

google-beta/resource_compute_global_address_sweeper_test.go

@@ -61,7 +61,7 @@ func testSweepComputeGlobalAddress(region string) error {
 		},
 	}
 
-	listTemplate := strings.Split("https://www.googleapis.com/compute/beta/projects/{{project}}/global/addresses", "?")[0]
+	listTemplate := strings.Split("https://compute.googleapis.com/compute/beta/projects/{{project}}/global/addresses", "?")[0]
 	listUrl, err := replaceVars(d, config, listTemplate)
 	if err != nil {
 		log.Printf("[INFO][SWEEPER_LOG] error preparing sweeper list url: %s", err)
@@ -99,7 +99,7 @@ func testSweepComputeGlobalAddress(region string) error {
 			continue
 		}
 
-		deleteTemplate := "https://www.googleapis.com/compute/beta/projects/{{project}}/global/addresses/{{name}}"
+		deleteTemplate := "https://compute.googleapis.com/compute/beta/projects/{{project}}/global/addresses/{{name}}"
 		deleteUrl, err := replaceVars(d, config, deleteTemplate)
 		if err != nil {
 			log.Printf("[INFO][SWEEPER_LOG] error preparing delete url: %s", err)

google-beta/resource_compute_global_forwarding_rule_sweeper_test.go

@@ -61,7 +61,7 @@ func testSweepComputeGlobalForwardingRule(region string) error {
 		},
 	}
 
-	listTemplate := strings.Split("https://www.googleapis.com/compute/beta/projects/{{project}}/global/forwardingRules", "?")[0]
+	listTemplate := strings.Split("https://compute.googleapis.com/compute/beta/projects/{{project}}/global/forwardingRules", "?")[0]
 	listUrl, err := replaceVars(d, config, listTemplate)
 	if err != nil {
 		log.Printf("[INFO][SWEEPER_LOG] error preparing sweeper list url: %s", err)
@@ -99,7 +99,7 @@ func testSweepComputeGlobalForwardingRule(region string) error {
 			continue
 		}
 
-		deleteTemplate := "https://www.googleapis.com/compute/beta/projects/{{project}}/global/forwardingRules/{{name}}"
+		deleteTemplate := "https://compute.googleapis.com/compute/beta/projects/{{project}}/global/forwardingRules/{{name}}"
 		deleteUrl, err := replaceVars(d, config, deleteTemplate)
 		if err != nil {
 			log.Printf("[INFO][SWEEPER_LOG] error preparing delete url: %s", err)