add integration tests for validator to pipeline (GoogleCloudPlatform#…

…5498)

.ci/containers/terraform-validator-tester-integration/Dockerfile

@@ -0,0 +1,20 @@
+from golang:1.16-stretch as resource
+SHELL ["/bin/bash", "-c"]
+# Set up Github SSH cloning.
+RUN ssh-keyscan github.com >> /known_hosts
+RUN echo "UserKnownHostsFile /known_hosts" >> /etc/ssh/ssh_config
+
+RUN apt-get update
+RUN apt-get install -y git jq unzip
+
+ADD test_terraform_validator_integration.sh /test_terraform_validator_integration.sh
+ENTRYPOINT ["/test_terraform_validator_integration.sh"]
+
+RUN wget https://releases.hashicorp.com/terraform/0.13.7/terraform_0.13.7_linux_amd64.zip
+RUN unzip terraform_0.13.7_linux_amd64.zip && rm terraform_0.13.7_linux_amd64.zip
+RUN mkdir /terraform
+RUN mv terraform /terraform/0.13.7
+
+RUN wget https://releases.hashicorp.com/terraform/0.12.31/terraform_0.12.31_linux_amd64.zip
+RUN unzip terraform_0.12.31_linux_amd64.zip && rm terraform_0.12.31_linux_amd64.zip
+RUN mv terraform /terraform/0.12.31

.ci/containers/terraform-validator-tester-integration/test_terraform_validator_integration.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+
+set -e
+
+pr_number=$1
+mm_commit_sha=$2
+build_id=$3
+project_id=$4
+build_step=$5
+github_username=modular-magician
+gh_repo=terraform-validator
+
+
+new_branch="auto-pr-$pr_number"
+git_remote=https://$github_username:$GITHUB_TOKEN@github.com/$github_username/$gh_repo
+local_path=$GOPATH/src/github.com/GoogleCloudPlatform/$gh_repo
+mkdir -p "$(dirname $local_path)"
+git clone $git_remote $local_path --branch $new_branch --depth 2
+pushd $local_path
+
+# Only skip tests if we can tell for sure that no go files were changed
+echo "Checking for modified go files"
+# get the names of changed files and look for go files
+# (ignoring "no matches found" errors from grep)
+gofiles=$(git diff --name-only HEAD~1 | { grep "\.go$" || test $? = 1; })
+if [[ -z $gofiles ]]; then
+    echo "Skipping tests: No go files changed"
+    exit 0
+else
+    echo "Running tests: Go files changed"
+fi
+
+
+TERRAFORM_BINARY=/terraform/$TERRAFORM_VERSION
+if test -f "$TERRAFORM_BINARY"; then
+    echo "terraform binary $TERRAFORM_BINARY exists on container"
+    echo setting terraform to version $TERRAFORM_VERSION
+    alias terraform /terraform/$TERRAFORM_VERSION
+    terraform version
+else
+    echo "terraform binary $TERRAFORM_BINARY does not exist."
+    echo "exiting ..."
+	  state="failure"
+    post_body=$( jq -n \
+      --arg context "terraform-validator-test-integration-${TERRAFORM_VERSION}" \
+      --arg target_url "https://console.cloud.google.com/cloud-build/builds;region=global/${build_id};step=${build_step}?project=${project_id}" \
+      --arg state "$state" \
+      '{context: $context, target_url: $target_url, state: $state}')
+    exit 0
+fi
+
+post_body=$( jq -n \
+	--arg context "terraform-validator-test-integration-${TERRAFORM_VERSION}" \
+	--arg target_url "https://console.cloud.google.com/cloud-build/builds;region=global/${build_id};step=${build_step}?project=${project_id}" \
+	--arg state "pending" \
+	'{context: $context, target_url: $target_url, state: $state}')
+
+curl \
+  -X POST \
+  -u "$github_username:$GITHUB_TOKEN" \
+  -H "Accept: application/vnd.github.v3+json" \
+  "https://api.github.com/repos/GoogleCloudPlatform/magic-modules/statuses/$mm_commit_sha" \
+  -d "$post_body"
+
+set +e
+
+make build
+make test-integration
+exit_code=$?
+
+set -e
+
+if [ $exit_code -ne 0 ]; then
+	state="failure"
+else
+	state="success"
+fi
+
+post_body=$( jq -n \
+	--arg context "terraform-validator-test-integration-${TERRAFORM_VERSION}" \
+	--arg target_url "https://console.cloud.google.com/cloud-build/builds;region=global/${build_id};step=${build_step}?project=${project_id}" \
+	--arg state "${state}" \
+	'{context: $context, target_url: $target_url, state: $state}')
+
+curl \
+  -X POST \
+  -u "$github_username:$GITHUB_TOKEN" \
+  -H "Accept: application/vnd.github.v3+json" \
+  "https://api.github.com/repos/GoogleCloudPlatform/magic-modules/statuses/$mm_commit_sha" \
+  -d "$post_body"

.ci/gcb-generate-diffs.yml

@@ -182,6 +182,34 @@ steps:
           - $PROJECT_ID
           - "20"  # Build step
 
+    - name: 'gcr.io/graphite-docker-images/terraform-validator-tester-integration'
+      id: tfv-test-integration-0.12.31
+      secretEnv: ["GITHUB_TOKEN"]
+      waitFor: ["tfv-head", "tfv-base"]
+      env:
+        - TERRAFORM_VERSION=0.12.31
+        - TEST_PROJECT=$_VALIDATOR_TEST_PROJECT
+      args:
+          - $_PR_NUMBER
+          - $COMMIT_SHA
+          - $BUILD_ID
+          - $PROJECT_ID
+          - "21"  # Build step
+
+    - name: 'gcr.io/graphite-docker-images/terraform-validator-tester-integration'
+      id: tfv-test-integration-0.13.7
+      secretEnv: ["GITHUB_TOKEN"]
+      waitFor: ["tfv-head", "tfv-base"]
+      env:
+        - TERRAFORM_VERSION=0.13.7
+        - TEST_PROJECT=$_VALIDATOR_TEST_PROJECT
+      args:
+          - $_PR_NUMBER
+          - $COMMIT_SHA
+          - $BUILD_ID
+          - $PROJECT_ID
+          - "22"  # Build step
+
     - name: 'gcr.io/graphite-docker-images/terraform-tester'
       id: tpgb-test
       secretEnv: ["GITHUB_TOKEN"]
@@ -193,7 +221,7 @@ steps:
           - $BUILD_ID
           - $PROJECT_ID
           - GoogleCloudPlatform/magic-modules
-          - "21"  # Build step
+          - "23"  # Build step
 
     - name: 'gcr.io/graphite-docker-images/terraform-tester'
       id: tpg-test
@@ -206,7 +234,7 @@ steps:
           - $BUILD_ID
           - $PROJECT_ID
           - GoogleCloudPlatform/magic-modules
-          - "22"  # Build step
+          - "24"  # Build step
 
     - name: 'gcr.io/graphite-docker-images/terraform-vcr-tester'
       id: tpg-vcr-test