Create better diff messages for TF 3.0.0 commits. (#2370)

Merged PR #2370.

.ci/ci.yml.tmpl

@@ -156,8 +156,6 @@ jobs:
             # pushing this anywhere - but it lets us reuse all the other
             # generation stuff.
           - aggregate:
-            - get: patches
-              resource: merged-prs
               # consumes: magic-modules (detached HEAD)
               # produces: magic-modules-branched (new branch, with submodule)
             - task: branch-magic-modules
@@ -166,6 +164,7 @@ jobs:
                 GH_TOKEN: ((github-account.password))
                 CREDS: ((repo-key.private_key))
                 ALL_SUBMODULES: {{' '.join(vars.all_submodules)}}
+                INCLUDE_PREVIOUS: true
             - put: magic-modules-3.0-prs
               params:
                 status: pending
@@ -189,22 +188,34 @@ jobs:
 
                 - put: {{v.short_name}}-intermediate
                   params:
-                      repository: terraform-diff/{{k}}
+                      repository: terraform-diff/new
                       branch_file: magic-modules-branched/branchname
                       force: true
                   get_params:
                       skip_clone: true
 
-                - put: magic-modules-3.0-prs
+                - put: {{v.short_name}}-intermediate
                   params:
-                      status: success
-                      path: magic-modules
-                      comment: terraform-diff/{{v.short_name}}_comment.txt
+                      repository: terraform-diff/old
+                      branch_file: magic-modules-previous/branchname
+                      force: true
                   get_params:
                       skip_clone: true
 
 {% endfor %}
 
+          - task: create-message
+            file: magic-modules-branched/.ci/magic-modules/create-diff-message.yml
+
+          - put: magic-modules-3.0-prs
+            params:
+                status: success
+                path: magic-modules
+                comment: message/message.txt
+            get_params:
+                skip_clone: true
+
+
 
     - name: mm-generate
       plan:

.ci/magic-modules/branch-magic-modules.sh

@@ -3,10 +3,6 @@ set -e
 set -x
 
 pushd "magic-modules"
-# "codegen-pr" vs "codegen-sha" is *LOAD-BEARING*.  Don't change
-# them (or introduce other options) unless you also change the
-# logic in create-or-update-pr - because we decide whether to
-# create or to update by which one of these we're prefixed by.
 export GH_TOKEN
 if PR_ID=$(git config --get pullrequest.id) &&
   [ -z "$USE_SHA" ] &&
@@ -32,3 +28,16 @@ chmod 400 ~/github_private_key
 ssh-agent bash -c "ssh-add ~/github_private_key; git submodule update --remote --init $ALL_SUBMODULES"
 
 cp -r ./ ../magic-modules-branched/
+
+if [ "true" == "$INCLUDE_PREVIOUS" ] ; then
+    # Since this is fetched after a merge commit, HEAD~ is
+    # the newest commit on the branch being merged into.
+    git reset --hard HEAD~
+    BRANCH="$BRANCH-previous"
+    git checkout -B "$BRANCH"
+    # ./branchname is intentionally never committed - it isn't necessary once
+    # this output is no longer available.
+    echo "$BRANCH" > ./branchname
+    ssh-agent bash -c "ssh-add ~/github_private_key; git submodule update --remote --init $ALL_SUBMODULES"
+    cp -r ./ ../magic-modules-previous/
+fi

.ci/magic-modules/branch.yml

@@ -17,12 +17,14 @@ inputs:
 
 outputs:
     - name: magic-modules-branched
+    - name: magic-modules-previous
 
 params:
   USE_SHA: ""
   GH_TOKEN: ""
   CREDS: ""
   ALL_SUBMODULES: ""
+  INCLUDE_PREVIOUS: ""
 
 run:
     path: magic-modules/.ci/magic-modules/branch-magic-modules.sh

.ci/magic-modules/create-diff-message.sh

@@ -0,0 +1,11 @@
+#! /bin/bash
+
+pushd magic-modules-branched
+
+BRANCH_NAME=$(cat branchname)
+{
+    echo "## 3.0.0 diff report as of $(git rev-parse HEAD^2)";
+    echo "[TPG Diff](https://github.com/modular-magician/terraform-provider-google/compare/$BRANCH_NAME-previous..$BRANCH_NAME)";
+    echo "[TPGB Diff](https://github.com/modular-magician/terraform-provider-google-beta/compare/$BRANCH_NAME-previous..$BRANCH_NAME)";
+    echo "[Mapper Diff](https://github.com/modular-magician/terraform-google-conversion/compare/$BRANCH_NAME-previous..$BRANCH_NAME)";
+} > ../message/message.txt

.ci/magic-modules/create-diff-message.yml

@@ -0,0 +1,19 @@
+---
+platform: linux
+
+image_resource:
+    type: docker-image
+    source:
+        # This task requires a container with 'git', 'python', and the pip
+        # package 'pygithub'.
+        repository: gcr.io/magic-modules/python
+        tag: '1.0'
+
+inputs:
+    - name: magic-modules-branched
+
+outputs:
+    - name: message
+
+run:
+    path: magic-modules-branched/.ci/magic-modules/create-diff-message.sh

.ci/magic-modules/diff-terraform.sh

@@ -1,8 +1,5 @@
 #!/bin/bash
 
-# This script takes in 'magic-modules-branched', a git repo tracking the head of a PR against magic-modules.
-# It outputs "terraform-diff/${SHORT_NAME}_comment.txt"
-
 # The vast majority of this file is a direct copy of generate-terraform.sh.  We could factor out all that
 # code into a shared library, but I don't think we need to do that.  This is an inherently temporary file,
 # until TPG 3.0.0 is released, which is in the relatively near future.  The cost of the copy is that
@@ -13,78 +10,77 @@
 set -x
 set -e
 source "$(dirname "$0")/helpers.sh"
-PATCH_DIR="$(pwd)/patches"
 
 # Create $GOPATH structure - in order to successfully run Terraform codegen, we need to run
 # it with a correctly-set-up $GOPATH.  It calls out to `goimports`, which means that
 # we need to have all the dependencies correctly downloaded.
 export GOPATH="${PWD}/go"
 mkdir -p "${GOPATH}/src/github.com/$GITHUB_ORG"
 
-pushd magic-modules-branched
-ln -s "${PWD}/build/$SHORT_NAME/" "${GOPATH}/src/github.com/$GITHUB_ORG/$PROVIDER_NAME"
-popd
-
-pushd "${GOPATH}/src/github.com/$GITHUB_ORG/$PROVIDER_NAME"
-
-# Other orgs are not fully-generated.  This may be transitional - if this causes pain,
-# try vendoring into third-party, as with TPG and TPGB.
-if [ "$GITHUB_ORG" = "terraform-providers" ]; then
-    # This line removes every file which is not specified here.
-    # If you add files to Terraform which are not generated, you have to add them here.
-    # It uses the somewhat obtuse 'find' command.  To explain:
-    # "find .": all files and directories recursively under the current directory, subject to matchers.
-    # "-type f": all regular real files, i.e. not directories.
-    # "-not": do the opposite of the next thing, always used with another matcher.
-    # "-wholename": entire relative path - including directory names - matches following wildcard.
-    # "-name": filename alone matches following string.  e.g. -name README.md matches ./README.md *and* ./foo/bar/README.md
-    # "-exec": for each file found, execute the command following until the literal ';'
-    find . -type f -not -wholename "./.git*" -not -wholename "./vendor*" -not -name ".travis.yml" -not -name ".golangci.yml" -not -name "CHANGELOG.md" -not -name GNUmakefile -not -name LICENSE -not -name README.md -not -wholename "./examples*" -not -name "go.mod" -not -name "go.sum" -not -name "staticcheck.conf" -not -name  ".hashibot.hcl"  -exec git rm {} \;
-fi
-
-popd
-
-pushd magic-modules-branched
-
-# Choose the author of the most recent commit as the downstream author
-# Note that we don't use the last submitted commit, we use the primary GH email
-# of the GH PR submitted. If they've enabled a private email, we'll actually
-# use their GH noreply email which isn't compatible with CLAs.
-COMMIT_AUTHOR="$(git log --pretty="%an <%ae>" -n1 HEAD)"
-
-if [ -n "$OVERRIDE_PROVIDER" ] && [ "$OVERRIDE_PROVIDER" != "null" ]; then
-  bundle exec compiler -a -e terraform -f "$OVERRIDE_PROVIDER" -o "${GOPATH}/src/github.com/$GITHUB_ORG/$PROVIDER_NAME/"
-else
-  bundle exec compiler -a -e terraform -o "${GOPATH}/src/github.com/$GITHUB_ORG/$PROVIDER_NAME/" -v "$VERSION"
-fi
-
-TERRAFORM_COMMIT_MSG="$(cat .git/title)"
-BRANCH_NAME="$(cat branchname)"
-
-pushd "build/$SHORT_NAME"
-
-# These config entries will set the "committer".
-git config --global user.email "magic-modules@google.com"
-git config --global user.name "Modular Magician"
-
-git add -A
-
-git commit -m "$TERRAFORM_COMMIT_MSG" --author="$COMMIT_AUTHOR" || true  # don't crash if no changes
-git checkout -B "$BRANCH_NAME"
-
-OLD_COMMIT_SHA="$(git merge-base HEAD master)"
-apply_patches "$PATCH_DIR/$GITHUB_ORG/$PROVIDER_NAME" "$TERRAFORM_COMMIT_MSG" "$COMMIT_AUTHOR" "master"
-
-NEWLINE=$'\n'
-MESSAGE="## $PROVIDER_NAME diff report$NEWLINE"
-if [ "$(git rev-parse HEAD)" != "$OLD_COMMIT_SHA" ]; then
-    MESSAGE="${MESSAGE}[View Diff](https://github.com/$GITHUB_ORG/$PROVIDER_NAME/compare/$OLD_COMMIT_SHA...modular-magician:$BRANCH_NAME)"
-else
-    MESSAGE="${MESSAGE}No diff generated."
-fi
-
-popd
-popd
-
-echo "$MESSAGE" > "terraform-diff/${SHORT_NAME}_comment.txt"
-git clone "magic-modules-branched/build/$SHORT_NAME" "./terraform-diff/$VERSION"
+for mm_dir in magic-modules-branched magic-modules-previous; do
+
+    pushd $mm_dir
+    # delete the symlink if it exists
+    rm "${GOPATH}/src/github.com/$GITHUB_ORG/$PROVIDER_NAME" || true
+    ln -s "${PWD}/build/$SHORT_NAME/" "${GOPATH}/src/github.com/$GITHUB_ORG/$PROVIDER_NAME"
+    popd
+
+    pushd "${GOPATH}/src/github.com/$GITHUB_ORG/$PROVIDER_NAME"
+
+    # Other orgs are not fully-generated.  This may be transitional - if this causes pain,
+    # try vendoring into third-party, as with TPG and TPGB.
+    if [ "$GITHUB_ORG" = "terraform-providers" ]; then
+        # This line removes every file which is not specified here.
+        # If you add files to Terraform which are not generated, you have to add them here.
+        # It uses the somewhat obtuse 'find' command.  To explain:
+        # "find .": all files and directories recursively under the current directory, subject to matchers.
+        # "-type f": all regular real files, i.e. not directories.
+        # "-not": do the opposite of the next thing, always used with another matcher.
+        # "-wholename": entire relative path - including directory names - matches following wildcard.
+        # "-name": filename alone matches following string.  e.g. -name README.md matches ./README.md *and* ./foo/bar/README.md
+        # "-exec": for each file found, execute the command following until the literal ';'
+        find . -type f -not -wholename "./.git*" -not -wholename "./vendor*" -not -name ".travis.yml" -not -name ".golangci.yml" -not -name "CHANGELOG.md" -not -name GNUmakefile -not -name LICENSE -not -name README.md -not -wholename "./examples*" -not -name "go.mod" -not -name "go.sum" -not -name "staticcheck.conf" -not -name  ".hashibot.hcl"  -exec git rm {} \;
+    fi
+
+    popd
+
+    pushd $mm_dir
+
+    # Choose the author of the most recent commit as the downstream author
+    # Note that we don't use the last submitted commit, we use the primary GH email
+    # of the GH PR submitted. If they've enabled a private email, we'll actually
+    # use their GH noreply email which isn't compatible with CLAs.
+    COMMIT_AUTHOR="$(git log --pretty="%an <%ae>" -n1 HEAD)"
+
+    if [ -n "$OVERRIDE_PROVIDER" ] && [ "$OVERRIDE_PROVIDER" != "null" ]; then
+      bundle exec compiler -a -e terraform -f "$OVERRIDE_PROVIDER" -o "${GOPATH}/src/github.com/$GITHUB_ORG/$PROVIDER_NAME/"
+    else
+      bundle exec compiler -a -e terraform -o "${GOPATH}/src/github.com/$GITHUB_ORG/$PROVIDER_NAME/" -v "$VERSION"
+    fi
+
+    if [ "$mm_dir" == "magic-modules-branched" ] ; then
+        TERRAFORM_COMMIT_MSG="$(cat .git/title)"
+    else
+        TERRAFORM_COMMIT_MSG="Old generated base as of $(git rev-parse HEAD)."
+    fi
+
+    BRANCH_NAME="$(cat branchname)"
+
+    pushd "build/$SHORT_NAME"
+
+    # These config entries will set the "committer".
+    git config --global user.email "magic-modules@google.com"
+    git config --global user.name "Modular Magician"
+
+    git add -A
+
+    git commit -m "$TERRAFORM_COMMIT_MSG" --author="$COMMIT_AUTHOR" || true  # don't crash if no changes
+    git checkout -B "$BRANCH_NAME"
+
+    popd
+    popd
+
+done
+
+git clone "magic-modules-branched/build/$SHORT_NAME" "./terraform-diff/new"
+git clone "magic-modules-previous/build/$SHORT_NAME" "./terraform-diff/old"

.ci/magic-modules/diff-terraform.yml

@@ -1,6 +1,7 @@
 ---
-# This file takes two inputs: magic-modules-branched in detached-HEAD state, and the list of patches.
-# It spits out "terraform-diff/pr_comment.txt", which contains the markdown-format diff.
+# This file takes two inputs: magic-modules-branched in detached-HEAD state, and magic-modules-previous.
+# It spits out "terraform-diff/comment.txt", which contains the markdown-format diff, as well as
+# "terraform-diff/old" and "terraform-diff/new".
 platform: linux
 
 image_resource:
@@ -11,7 +12,7 @@ image_resource:
 
 inputs:
     - name: magic-modules-branched
-    - name: patches
+    - name: magic-modules-previous
 
 outputs:
     - name: terraform-diff