diff --git a/.ci/ci.yml.tmpl b/.ci/ci.yml.tmpl index 270361c68bbe..016e4b36e791 100644 --- a/.ci/ci.yml.tmpl +++ b/.ci/ci.yml.tmpl @@ -156,8 +156,6 @@ jobs: # pushing this anywhere - but it lets us reuse all the other # generation stuff. - aggregate: - - get: patches - resource: merged-prs # consumes: magic-modules (detached HEAD) # produces: magic-modules-branched (new branch, with submodule) - task: branch-magic-modules @@ -166,6 +164,7 @@ jobs: GH_TOKEN: ((github-account.password)) CREDS: ((repo-key.private_key)) ALL_SUBMODULES: {{' '.join(vars.all_submodules)}} + INCLUDE_PREVIOUS: true - put: magic-modules-3.0-prs params: status: pending @@ -189,22 +188,34 @@ jobs: - put: {{v.short_name}}-intermediate params: - repository: terraform-diff/{{k}} + repository: terraform-diff/new branch_file: magic-modules-branched/branchname force: true get_params: skip_clone: true - - put: magic-modules-3.0-prs + - put: {{v.short_name}}-intermediate params: - status: success - path: magic-modules - comment: terraform-diff/{{v.short_name}}_comment.txt + repository: terraform-diff/old + branch_file: magic-modules-previous/branchname + force: true get_params: skip_clone: true {% endfor %} + - task: create-message + file: magic-modules-branched/.ci/magic-modules/create-diff-message.yml + + - put: magic-modules-3.0-prs + params: + status: success + path: magic-modules + comment: message/message.txt + get_params: + skip_clone: true + + - name: mm-generate plan: diff --git a/.ci/magic-modules/branch-magic-modules.sh b/.ci/magic-modules/branch-magic-modules.sh index f7360aa883c4..61530090444a 100755 --- a/.ci/magic-modules/branch-magic-modules.sh +++ b/.ci/magic-modules/branch-magic-modules.sh @@ -3,10 +3,6 @@ set -e set -x pushd "magic-modules" -# "codegen-pr" vs "codegen-sha" is *LOAD-BEARING*. Don't change -# them (or introduce other options) unless you also change the -# logic in create-or-update-pr - because we decide whether to -# create or to update by which one of these we're prefixed by. export GH_TOKEN if PR_ID=$(git config --get pullrequest.id) && [ -z "$USE_SHA" ] && @@ -32,3 +28,16 @@ chmod 400 ~/github_private_key ssh-agent bash -c "ssh-add ~/github_private_key; git submodule update --remote --init $ALL_SUBMODULES" cp -r ./ ../magic-modules-branched/ + +if [ "true" == "$INCLUDE_PREVIOUS" ] ; then + # Since this is fetched after a merge commit, HEAD~ is + # the newest commit on the branch being merged into. + git reset --hard HEAD~ + BRANCH="$BRANCH-previous" + git checkout -B "$BRANCH" + # ./branchname is intentionally never committed - it isn't necessary once + # this output is no longer available. + echo "$BRANCH" > ./branchname + ssh-agent bash -c "ssh-add ~/github_private_key; git submodule update --remote --init $ALL_SUBMODULES" + cp -r ./ ../magic-modules-previous/ +fi diff --git a/.ci/magic-modules/branch.yml b/.ci/magic-modules/branch.yml index b6ae2f045d17..4bf6450772c7 100644 --- a/.ci/magic-modules/branch.yml +++ b/.ci/magic-modules/branch.yml @@ -17,12 +17,14 @@ inputs: outputs: - name: magic-modules-branched + - name: magic-modules-previous params: USE_SHA: "" GH_TOKEN: "" CREDS: "" ALL_SUBMODULES: "" + INCLUDE_PREVIOUS: "" run: path: magic-modules/.ci/magic-modules/branch-magic-modules.sh diff --git a/.ci/magic-modules/create-diff-message.sh b/.ci/magic-modules/create-diff-message.sh new file mode 100755 index 000000000000..75ba4275f77b --- /dev/null +++ b/.ci/magic-modules/create-diff-message.sh @@ -0,0 +1,11 @@ +#! /bin/bash + +pushd magic-modules-branched + +BRANCH_NAME=$(cat branchname) +{ + echo "## 3.0.0 diff report as of $(git rev-parse HEAD^2)"; + echo "[TPG Diff](https://github.com/modular-magician/terraform-provider-google/compare/$BRANCH_NAME-previous..$BRANCH_NAME)"; + echo "[TPGB Diff](https://github.com/modular-magician/terraform-provider-google-beta/compare/$BRANCH_NAME-previous..$BRANCH_NAME)"; + echo "[Mapper Diff](https://github.com/modular-magician/terraform-google-conversion/compare/$BRANCH_NAME-previous..$BRANCH_NAME)"; +} > ../message/message.txt diff --git a/.ci/magic-modules/create-diff-message.yml b/.ci/magic-modules/create-diff-message.yml new file mode 100644 index 000000000000..1545d789e7be --- /dev/null +++ b/.ci/magic-modules/create-diff-message.yml @@ -0,0 +1,19 @@ +--- +platform: linux + +image_resource: + type: docker-image + source: + # This task requires a container with 'git', 'python', and the pip + # package 'pygithub'. + repository: gcr.io/magic-modules/python + tag: '1.0' + +inputs: + - name: magic-modules-branched + +outputs: + - name: message + +run: + path: magic-modules-branched/.ci/magic-modules/create-diff-message.sh diff --git a/.ci/magic-modules/diff-terraform.sh b/.ci/magic-modules/diff-terraform.sh index 3cfff0de44dd..acc398f625ac 100755 --- a/.ci/magic-modules/diff-terraform.sh +++ b/.ci/magic-modules/diff-terraform.sh @@ -1,8 +1,5 @@ #!/bin/bash -# This script takes in 'magic-modules-branched', a git repo tracking the head of a PR against magic-modules. -# It outputs "terraform-diff/${SHORT_NAME}_comment.txt" - # The vast majority of this file is a direct copy of generate-terraform.sh. We could factor out all that # code into a shared library, but I don't think we need to do that. This is an inherently temporary file, # until TPG 3.0.0 is released, which is in the relatively near future. The cost of the copy is that @@ -13,7 +10,6 @@ set -x set -e source "$(dirname "$0")/helpers.sh" -PATCH_DIR="$(pwd)/patches" # Create $GOPATH structure - in order to successfully run Terraform codegen, we need to run # it with a correctly-set-up $GOPATH. It calls out to `goimports`, which means that @@ -21,70 +17,70 @@ PATCH_DIR="$(pwd)/patches" export GOPATH="${PWD}/go" mkdir -p "${GOPATH}/src/github.com/$GITHUB_ORG" -pushd magic-modules-branched -ln -s "${PWD}/build/$SHORT_NAME/" "${GOPATH}/src/github.com/$GITHUB_ORG/$PROVIDER_NAME" -popd - -pushd "${GOPATH}/src/github.com/$GITHUB_ORG/$PROVIDER_NAME" - -# Other orgs are not fully-generated. This may be transitional - if this causes pain, -# try vendoring into third-party, as with TPG and TPGB. -if [ "$GITHUB_ORG" = "terraform-providers" ]; then - # This line removes every file which is not specified here. - # If you add files to Terraform which are not generated, you have to add them here. - # It uses the somewhat obtuse 'find' command. To explain: - # "find .": all files and directories recursively under the current directory, subject to matchers. - # "-type f": all regular real files, i.e. not directories. - # "-not": do the opposite of the next thing, always used with another matcher. - # "-wholename": entire relative path - including directory names - matches following wildcard. - # "-name": filename alone matches following string. e.g. -name README.md matches ./README.md *and* ./foo/bar/README.md - # "-exec": for each file found, execute the command following until the literal ';' - find . -type f -not -wholename "./.git*" -not -wholename "./vendor*" -not -name ".travis.yml" -not -name ".golangci.yml" -not -name "CHANGELOG.md" -not -name GNUmakefile -not -name LICENSE -not -name README.md -not -wholename "./examples*" -not -name "go.mod" -not -name "go.sum" -not -name "staticcheck.conf" -not -name ".hashibot.hcl" -exec git rm {} \; -fi - -popd - -pushd magic-modules-branched - -# Choose the author of the most recent commit as the downstream author -# Note that we don't use the last submitted commit, we use the primary GH email -# of the GH PR submitted. If they've enabled a private email, we'll actually -# use their GH noreply email which isn't compatible with CLAs. -COMMIT_AUTHOR="$(git log --pretty="%an <%ae>" -n1 HEAD)" - -if [ -n "$OVERRIDE_PROVIDER" ] && [ "$OVERRIDE_PROVIDER" != "null" ]; then - bundle exec compiler -a -e terraform -f "$OVERRIDE_PROVIDER" -o "${GOPATH}/src/github.com/$GITHUB_ORG/$PROVIDER_NAME/" -else - bundle exec compiler -a -e terraform -o "${GOPATH}/src/github.com/$GITHUB_ORG/$PROVIDER_NAME/" -v "$VERSION" -fi - -TERRAFORM_COMMIT_MSG="$(cat .git/title)" -BRANCH_NAME="$(cat branchname)" - -pushd "build/$SHORT_NAME" - -# These config entries will set the "committer". -git config --global user.email "magic-modules@google.com" -git config --global user.name "Modular Magician" - -git add -A - -git commit -m "$TERRAFORM_COMMIT_MSG" --author="$COMMIT_AUTHOR" || true # don't crash if no changes -git checkout -B "$BRANCH_NAME" - -OLD_COMMIT_SHA="$(git merge-base HEAD master)" -apply_patches "$PATCH_DIR/$GITHUB_ORG/$PROVIDER_NAME" "$TERRAFORM_COMMIT_MSG" "$COMMIT_AUTHOR" "master" - -NEWLINE=$'\n' -MESSAGE="## $PROVIDER_NAME diff report$NEWLINE" -if [ "$(git rev-parse HEAD)" != "$OLD_COMMIT_SHA" ]; then - MESSAGE="${MESSAGE}[View Diff](https://github.com/$GITHUB_ORG/$PROVIDER_NAME/compare/$OLD_COMMIT_SHA...modular-magician:$BRANCH_NAME)" -else - MESSAGE="${MESSAGE}No diff generated." -fi - -popd -popd - -echo "$MESSAGE" > "terraform-diff/${SHORT_NAME}_comment.txt" -git clone "magic-modules-branched/build/$SHORT_NAME" "./terraform-diff/$VERSION" +for mm_dir in magic-modules-branched magic-modules-previous; do + + pushd $mm_dir + # delete the symlink if it exists + rm "${GOPATH}/src/github.com/$GITHUB_ORG/$PROVIDER_NAME" || true + ln -s "${PWD}/build/$SHORT_NAME/" "${GOPATH}/src/github.com/$GITHUB_ORG/$PROVIDER_NAME" + popd + + pushd "${GOPATH}/src/github.com/$GITHUB_ORG/$PROVIDER_NAME" + + # Other orgs are not fully-generated. This may be transitional - if this causes pain, + # try vendoring into third-party, as with TPG and TPGB. + if [ "$GITHUB_ORG" = "terraform-providers" ]; then + # This line removes every file which is not specified here. + # If you add files to Terraform which are not generated, you have to add them here. + # It uses the somewhat obtuse 'find' command. To explain: + # "find .": all files and directories recursively under the current directory, subject to matchers. + # "-type f": all regular real files, i.e. not directories. + # "-not": do the opposite of the next thing, always used with another matcher. + # "-wholename": entire relative path - including directory names - matches following wildcard. + # "-name": filename alone matches following string. e.g. -name README.md matches ./README.md *and* ./foo/bar/README.md + # "-exec": for each file found, execute the command following until the literal ';' + find . -type f -not -wholename "./.git*" -not -wholename "./vendor*" -not -name ".travis.yml" -not -name ".golangci.yml" -not -name "CHANGELOG.md" -not -name GNUmakefile -not -name LICENSE -not -name README.md -not -wholename "./examples*" -not -name "go.mod" -not -name "go.sum" -not -name "staticcheck.conf" -not -name ".hashibot.hcl" -exec git rm {} \; + fi + + popd + + pushd $mm_dir + + # Choose the author of the most recent commit as the downstream author + # Note that we don't use the last submitted commit, we use the primary GH email + # of the GH PR submitted. If they've enabled a private email, we'll actually + # use their GH noreply email which isn't compatible with CLAs. + COMMIT_AUTHOR="$(git log --pretty="%an <%ae>" -n1 HEAD)" + + if [ -n "$OVERRIDE_PROVIDER" ] && [ "$OVERRIDE_PROVIDER" != "null" ]; then + bundle exec compiler -a -e terraform -f "$OVERRIDE_PROVIDER" -o "${GOPATH}/src/github.com/$GITHUB_ORG/$PROVIDER_NAME/" + else + bundle exec compiler -a -e terraform -o "${GOPATH}/src/github.com/$GITHUB_ORG/$PROVIDER_NAME/" -v "$VERSION" + fi + + if [ "$mm_dir" == "magic-modules-branched" ] ; then + TERRAFORM_COMMIT_MSG="$(cat .git/title)" + else + TERRAFORM_COMMIT_MSG="Old generated base as of $(git rev-parse HEAD)." + fi + + BRANCH_NAME="$(cat branchname)" + + pushd "build/$SHORT_NAME" + + # These config entries will set the "committer". + git config --global user.email "magic-modules@google.com" + git config --global user.name "Modular Magician" + + git add -A + + git commit -m "$TERRAFORM_COMMIT_MSG" --author="$COMMIT_AUTHOR" || true # don't crash if no changes + git checkout -B "$BRANCH_NAME" + + popd + popd + +done + +git clone "magic-modules-branched/build/$SHORT_NAME" "./terraform-diff/new" +git clone "magic-modules-previous/build/$SHORT_NAME" "./terraform-diff/old" diff --git a/.ci/magic-modules/diff-terraform.yml b/.ci/magic-modules/diff-terraform.yml index fffa607e375d..578f3eec1457 100644 --- a/.ci/magic-modules/diff-terraform.yml +++ b/.ci/magic-modules/diff-terraform.yml @@ -1,6 +1,7 @@ --- -# This file takes two inputs: magic-modules-branched in detached-HEAD state, and the list of patches. -# It spits out "terraform-diff/pr_comment.txt", which contains the markdown-format diff. +# This file takes two inputs: magic-modules-branched in detached-HEAD state, and magic-modules-previous. +# It spits out "terraform-diff/comment.txt", which contains the markdown-format diff, as well as +# "terraform-diff/old" and "terraform-diff/new". platform: linux image_resource: @@ -11,7 +12,7 @@ image_resource: inputs: - name: magic-modules-branched - - name: patches + - name: magic-modules-previous outputs: - name: terraform-diff diff --git a/build/terraform b/build/terraform index 4c8a5b142b7d..f1ca4b0a018a 160000 --- a/build/terraform +++ b/build/terraform @@ -1 +1 @@ -Subproject commit 4c8a5b142b7ddd732e845c604a3cbac0f9d29d80 +Subproject commit f1ca4b0a018a6585fb56dfb7a0845b12199b01f3 diff --git a/build/terraform-beta b/build/terraform-beta index dbac3a4d4149..4dbd5e9e55e0 160000 --- a/build/terraform-beta +++ b/build/terraform-beta @@ -1 +1 @@ -Subproject commit dbac3a4d4149ea37a80a767d55b9a32426869eae +Subproject commit 4dbd5e9e55e06c46ffea9aed26cd3d573be6f771