Merge branch 'GoogleCloudPlatform:main' into main

.ci/gcb-generate-diffs-new.yml

@@ -65,104 +65,117 @@ steps:
           - head/$_HEAD_BRANCH
 
     - name: 'gcr.io/graphite-docker-images/build-environment'
-      entrypoint: '/workspace/.ci/scripts/build-environment/downstream-builder/generate_downstream.sh'
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
+      id: build-magician-binary
+      waitFor: ["merged"]
+
+    - name: 'gcr.io/graphite-docker-images/build-environment'
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       id: tpg-head
       secretEnv: ["GITHUB_TOKEN"]
-      waitFor: ["merged"]
+      waitFor: ["build-magician-binary"]
       env:
         - BASE_BRANCH=$_BASE_BRANCH
       args:
+          - 'generate-downstream'
           - 'head'
           - 'terraform'
           - 'ga'
           - $_PR_NUMBER
 
     - name: 'gcr.io/graphite-docker-images/build-environment'
-      entrypoint: '/workspace/.ci/scripts/build-environment/downstream-builder/generate_downstream.sh'
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       id: tpg-base
       secretEnv: ["GITHUB_TOKEN"]
-      waitFor: ["merged"]
+      waitFor: ["build-magician-binary"]
       env:
         - BASE_BRANCH=$_BASE_BRANCH
       args:
+          - 'generate-downstream'
           - 'base'
           - 'terraform'
           - 'ga'
           - $_PR_NUMBER
 
     - name: 'gcr.io/graphite-docker-images/build-environment'
-      entrypoint: '/workspace/.ci/scripts/build-environment/downstream-builder/generate_downstream.sh'
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       secretEnv: ["GITHUB_TOKEN"]
       id: tpgb-head
-      waitFor: ["merged"]
+      waitFor: ["build-magician-binary"]
       env:
         - BASE_BRANCH=$_BASE_BRANCH
       args:
+          - 'generate-downstream'
           - 'head'
           - 'terraform'
           - 'beta'
           - $_PR_NUMBER
 
     - name: 'gcr.io/graphite-docker-images/build-environment'
-      entrypoint: '/workspace/.ci/scripts/build-environment/downstream-builder/generate_downstream.sh'
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       id: tpgb-base
       secretEnv: ["GITHUB_TOKEN"]
-      waitFor: ["merged"]
+      waitFor: ["build-magician-binary"]
       env:
         - BASE_BRANCH=$_BASE_BRANCH
       args:
+          - 'generate-downstream'
           - 'base'
           - 'terraform'
           - 'beta'
           - $_PR_NUMBER
 
     - name: 'gcr.io/graphite-docker-images/build-environment'
-      entrypoint: '/workspace/.ci/scripts/build-environment/downstream-builder/generate_downstream.sh'
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       id: tgc-head
       secretEnv: ["GITHUB_TOKEN"]
-      waitFor: ["merged"]
+      waitFor: ["build-magician-binary"]
       env:
         - BASE_BRANCH=$_BASE_BRANCH
       args:
+          - 'generate-downstream'
           - 'head'
           - 'terraform-google-conversion'
           - 'beta'
           - $_PR_NUMBER
 
     - name: 'gcr.io/graphite-docker-images/build-environment'
-      entrypoint: '/workspace/.ci/scripts/build-environment/downstream-builder/generate_downstream.sh'
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       id: tgc-base
       secretEnv: ["GITHUB_TOKEN"]
-      waitFor: ["merged"]
+      waitFor: ["build-magician-binary"]
       env:
         - BASE_BRANCH=$_BASE_BRANCH
       args:
+          - 'generate-downstream'
           - 'base'
           - 'terraform-google-conversion'
           - 'beta'
           - $_PR_NUMBER
 
     - name: 'gcr.io/graphite-docker-images/build-environment'
-      entrypoint: '/workspace/.ci/scripts/build-environment/downstream-builder/generate_downstream.sh'
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       id: tf-oics-head
       secretEnv: ["GITHUB_TOKEN"]
-      waitFor: ["merged"]
+      waitFor: ["build-magician-binary"]
       env:
         - BASE_BRANCH=$_BASE_BRANCH
       args:
+          - 'generate-downstream'
           - 'head'
           - 'tf-oics'
           - 'beta'
           - $_PR_NUMBER
 
     - name: 'gcr.io/graphite-docker-images/build-environment'
-      entrypoint: '/workspace/.ci/scripts/build-environment/downstream-builder/generate_downstream.sh'
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       id: tf-oics-base
       secretEnv: ["GITHUB_TOKEN"]
-      waitFor: ["merged"]
+      waitFor: ["build-magician-binary"]
       env:
         - BASE_BRANCH=$_BASE_BRANCH
       args:
+          - 'generate-downstream'
           - 'base'
           - 'tf-oics'
           - 'beta'
@@ -240,17 +253,21 @@ steps:
 
     - name: 'gcr.io/graphite-docker-images/go-plus'
       id: gcb-tpg-vcr-test
-      entrypoint: '/workspace/.ci/scripts/go-plus/gcb-terraform-vcr-tester/test_terraform_vcr.sh'
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       secretEnv: ["GITHUB_TOKEN", "GOOGLE_BILLING_ACCOUNT", "GOOGLE_CUST_ID", "GOOGLE_FIRESTORE_PROJECT", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION", "GOOGLE_TPU_V2_VM_RUNTIME_VERSION"]
       waitFor: ["diff"]
       env:
         - BASE_BRANCH=$_BASE_BRANCH
+        - "GOOGLE_REGION=us-central1"
+        - "GOOGLE_ZONE=us-central1-a"
+        - "USER=magician"
       args:
-          - $_PR_NUMBER
-          - $COMMIT_SHA
-          - $BUILD_ID
-          - $PROJECT_ID
-          - "22"  # Build step
+        - 'test-terraform-vcr'
+        - $_PR_NUMBER
+        - $COMMIT_SHA
+        - $BUILD_ID
+        - $PROJECT_ID
+        - "22"  # Build step
 
     - name: 'gcr.io/graphite-docker-images/go-plus'
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'

.ci/gcb-push-downstream.yml

@@ -24,6 +24,11 @@ steps:
           - checkout
           - $COMMIT_SHA
 
+    - name: 'gcr.io/graphite-docker-images/build-environment'
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
+      id: build-magician-binary
+      waitFor: ["checkout"]
+
     # TPG
     - name: 'gcr.io/graphite-docker-images/bash-plus'
       entrypoint: '/workspace/.ci/scripts/bash-plus/downstream-waiter/wait_for_commit.sh'
@@ -36,13 +41,14 @@ steps:
           - $COMMIT_SHA
 
     - name: 'gcr.io/graphite-docker-images/build-environment'
-      entrypoint: '/workspace/.ci/scripts/build-environment/downstream-builder/generate_downstream.sh'
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       secretEnv: ["GITHUB_TOKEN"]
       id: tpg-push
-      waitFor: ["tpg-sync"]
+      waitFor: ["tpg-sync", "build-magician-binary"]
       env:
         - BASE_BRANCH=$BRANCH_NAME
       args:
+          - 'generate-downstream'
           - 'downstream'
           - 'terraform'
           - 'ga'
@@ -73,13 +79,14 @@ steps:
           - $COMMIT_SHA
 
     - name: 'gcr.io/graphite-docker-images/build-environment'
-      entrypoint: '/workspace/.ci/scripts/build-environment/downstream-builder/generate_downstream.sh'
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       secretEnv: ["GITHUB_TOKEN"]
       id: tpgb-push
-      waitFor: ["tpgb-sync"]
+      waitFor: ["tpgb-sync", "build-magician-binary"]
       env:
         - BASE_BRANCH=$BRANCH_NAME
       args:
+          - 'generate-downstream'
           - 'downstream'
           - 'terraform'
           - 'beta'
@@ -110,13 +117,14 @@ steps:
           - $COMMIT_SHA
 
     - name: 'gcr.io/graphite-docker-images/build-environment'
-      entrypoint: '/workspace/.ci/scripts/build-environment/downstream-builder/generate_downstream.sh'
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       secretEnv: ["GITHUB_TOKEN"]
       id: tgc-push
       waitFor: ["tgc-sync", "tpgb-push"]
       env:
         - BASE_BRANCH=$BRANCH_NAME
       args:
+          - 'generate-downstream'
           - 'downstream'
           - 'terraform-google-conversion'
           - 'beta'
@@ -147,13 +155,14 @@ steps:
           - $COMMIT_SHA
 
     - name: 'gcr.io/graphite-docker-images/build-environment'
-      entrypoint: '/workspace/.ci/scripts/build-environment/downstream-builder/generate_downstream.sh'
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       secretEnv: ["GITHUB_TOKEN"]
       id: tf-oics-push
-      waitFor: ["tf-oics-sync"]
+      waitFor: ["tf-oics-sync", "build-magician-binary"]
       env:
         - BASE_BRANCH=$BRANCH_NAME
       args:
+          - 'generate-downstream'
           - 'downstream'
           - 'tf-oics'
           - 'beta'

.ci/gcb-vcr-nightly.yml

@@ -3,7 +3,7 @@ steps:
     - name: 'gcr.io/graphite-docker-images/go-plus'
       id: gcb-vcr-nightly
       entrypoint: '/workspace/.ci/scripts/go-plus/vcr-cassette-update/vcr_cassette_update.sh'
-      secretEnv: ["GITHUB_TOKEN", "GOOGLE_BILLING_ACCOUNT", "GOOGLE_CUST_ID", "GOOGLE_FIRESTORE_PROJECT", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION", "GOOGLE_TPU_V2_VM_RUNTIME_VERSION"]
+      secretEnv: ["GOOGLE_BILLING_ACCOUNT", "GOOGLE_CUST_ID", "GOOGLE_FIRESTORE_PROJECT", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION", "GOOGLE_TPU_V2_VM_RUNTIME_VERSION"]
       args:
           - $BUILD_ID
 
@@ -15,8 +15,6 @@ options:
 logsBucket: 'gs://cloudbuild-vcr-nightly-logs'
 availableSecrets:
   secretManager:
-    - versionName: projects/673497134629/secrets/github-magician-token/versions/latest
-      env: GITHUB_TOKEN
     - versionName: projects/673497134629/secrets/ci-test-billing-account/versions/latest
       env: GOOGLE_BILLING_ACCOUNT
     - versionName: projects/673497134629/secrets/ci-test-cust-id/versions/latest

.ci/infra/terraform/main.tf

@@ -196,6 +196,7 @@ module "project-services" {
     "cloudidentity.googleapis.com",
     "cloudiot.googleapis.com",
     "cloudkms.googleapis.com",
+    "cloudquotas.googleapis.com",
     "cloudresourcemanager.googleapis.com",
     "cloudscheduler.googleapis.com",
     "cloudtasks.googleapis.com",
@@ -222,6 +223,7 @@ module "project-services" {
     "datastream.googleapis.com",
     "deploymentmanager.googleapis.com",
     "dialogflow.googleapis.com",
+    "discoveryengine.googleapis.com",
     "dlp.googleapis.com",
     "dns.googleapis.com",
     "documentai.googleapis.com",
@@ -275,6 +277,7 @@ module "project-services" {
     "orgpolicy.googleapis.com",
     "osconfig.googleapis.com",
     "oslogin.googleapis.com",
+    "parallelstore.googleapis.com",
     "privateca.googleapis.com",
     "pubsub.googleapis.com",
     "pubsublite.googleapis.com",
@@ -291,6 +294,7 @@ module "project-services" {
     "securesourcemanager.googleapis.com",
     "securetoken.googleapis.com",
     "securitycenter.googleapis.com",
+    "securityposture.googleapis.com",
     "serviceconsumermanagement.googleapis.com",
     "servicecontrol.googleapis.com",
     "servicedirectory.googleapis.com",

.ci/magician/cmd/check_cassettes.go

@@ -11,7 +11,7 @@ import (
 	"github.com/spf13/cobra"
 )
 
-var environmentVariables = [...]string{
+var ccEnvironmentVariables = [...]string{
 	"COMMIT_SHA",
 	"GITHUB_TOKEN",
 	"GOCACHE",
@@ -42,12 +42,12 @@ var checkCassettesCmd = &cobra.Command{
 	VCR cassettes using the newly built beta provider.
 
 	The following environment variables are expected:
-` + listEnvironmentVariables() + `
+` + listCCEnvironmentVariables() + `
 
 	It prints a list of tests that failed in replaying mode along with all test output.`,
 	Run: func(cmd *cobra.Command, args []string) {
-		env := make(map[string]string, len(environmentVariables))
-		for _, ev := range environmentVariables {
+		env := make(map[string]string, len(ccEnvironmentVariables))
+		for _, ev := range ccEnvironmentVariables {
 			val, ok := os.LookupEnv(ev)
 			if !ok {
 				fmt.Printf("Did not provide %s environment variable\n", ev)
@@ -64,25 +64,25 @@ var checkCassettesCmd = &cobra.Command{
 
 		ctlr := source.NewController(env["GOPATH"], "modular-magician", env["GITHUB_TOKEN"], rnr)
 
-		t, err := vcr.NewTester(env, rnr)
+		vt, err := vcr.NewTester(env, rnr)
 		if err != nil {
 			fmt.Println("Error creating VCR tester: ", err)
 			os.Exit(1)
 		}
-		execCheckCassettes(env["COMMIT_SHA"], t, ctlr)
+		execCheckCassettes(env["COMMIT_SHA"], vt, ctlr)
 	},
 }
 
-func listEnvironmentVariables() string {
+func listCCEnvironmentVariables() string {
 	var result string
-	for i, ev := range environmentVariables {
+	for i, ev := range ccEnvironmentVariables {
 		result += fmt.Sprintf("\t%2d. %s\n", i+1, ev)
 	}
 	return result
 }
 
-func execCheckCassettes(commit string, t vcr.Tester, ctlr *source.Controller) {
-	if err := t.FetchCassettes(provider.Beta); err != nil {
+func execCheckCassettes(commit string, vt *vcr.Tester, ctlr *source.Controller) {
+	if err := vt.FetchCassettes(provider.Beta, "main", ""); err != nil {
 		fmt.Println("Error fetching cassettes: ", err)
 		os.Exit(1)
 	}
@@ -96,19 +96,22 @@ func execCheckCassettes(commit string, t vcr.Tester, ctlr *source.Controller) {
 		fmt.Println("Error cloning provider: ", err)
 		os.Exit(1)
 	}
-	t.SetRepoPath(provider.Beta, providerRepo.Path)
+	vt.SetRepoPath(provider.Beta, providerRepo.Path)
 
-	result, err := t.Run(vcr.Replaying, provider.Beta)
+	result, err := vt.Run(vcr.Replaying, provider.Beta, nil)
 	if err != nil {
 		fmt.Println("Error running VCR: ", err)
+	}
+	if err := vt.UploadLogs("vcr-check-cassettes", "", "", false, false, vcr.Replaying, provider.Beta); err != nil {
+		fmt.Println("Error uploading logs: ", err)
 		os.Exit(1)
 	}
 	fmt.Println(len(result.FailedTests), " failed tests: ", result.FailedTests)
 	// TODO(trodge) report these failures to bigquery
 	fmt.Println(len(result.PassedTests), " passed tests: ", result.PassedTests)
 	fmt.Println(len(result.SkippedTests), " skipped tests: ", result.SkippedTests)
 
-	if err := t.Cleanup(); err != nil {
+	if err := vt.Cleanup(); err != nil {
 		fmt.Println("Error cleaning up vcr tester: ", err)
 		os.Exit(1)
 	}