Merge branch 'gke-release-channel' of github.com:chrissng/magic-modul…

…es into gke-release-channel

.ci/RELEASE_NOTES_GUIDE.md

@@ -127,6 +127,7 @@ Don’t write notes like:
 ## Headings
 
 Release notes should be formatted with one of the following headings:
+- `release-note:enhancement`
 - `release-note:bug`
 - `release-note:note`
 - `release-note:new-resource`

.ci/acceptance-tests/ansible-integration.sh

@@ -8,26 +8,32 @@ echo "${ANSIBLE_TEMPLATE}" > /tmp/ansible-template.ini
 set -e
 set -x
 
-# Get the newest version of Ansible from the PR
+# Install ansible from source
+git clone https://github.com/ansible/ansible.git
+pushd ansible
+pip install -r requirements.txt
+source hacking/env-setup
+popd
+
+# Clone ansible_collections_google because submodules
+# break collections
+git clone https://github.com/ansible/ansible_collections_google.git
+
+# Build newest modules
 pushd magic-modules-gcp
 bundle install
-for i in $(find products/ -name 'ansible.yaml' -printf '%h\n');
-do
-  bundle exec compiler -p $i -e ansible -o "build/ansible/"
-done
+bundle exec compiler -a -e ansible -o ../ansible_collections_google
 popd
 
-# Go to the newly-compiled version of Ansible
-pushd magic-modules-gcp/build/ansible
+# Install collection
+pushd ansible_collections_google
+ansible-galaxy collection build .
+ansible-galaxy collection install *.gz
+popd
 
 # Setup Cloud configuration template with variables
-cp /tmp/ansible-template.ini test/integration/cloud-config-gcp.ini
-
-# Install dependencies for ansible
-pip install -r requirements.txt
-
-# Setup ansible
-source hacking/env-setup
+pushd ~/.ansible/collections/ansible_collections/google/cloud
+cp /tmp/ansible-template.ini tests/integration/cloud-config-gcp.ini
 
 # Run ansible
-ansible-test integration -v --allow-unsupported --continue-on-error $(find test/integration/targets -name "gcp*" -type d -printf "%P ")
+ansible-test integration -v --allow-unsupported --continue-on-error $(find tests/integration/targets -name "gcp*" -type d -printf "%P ")

.ci/containers/hub/Dockerfile

@@ -1,4 +1,4 @@
-from gcr.io/magic-modules/go-ruby:1.11.5-2.6.0
+from gcr.io/magic-modules/go-ruby-python:1.11.5-2.6.0-2.7-v4
 
 RUN apt-get update
 RUN apt-get install -y ca-certificates

.ci/magic-modules/release-ansible.sh

@@ -15,16 +15,13 @@ get_all_modules() {
   file_name=$remote_name
   ssh-agent bash -c "ssh-add ~/github_private_key; git fetch $remote_name"
   git checkout $remote_name/devel
-  git ls-files -- plugins/modules/gcp_* | cut -d/ -f 6 | cut -d. -f 1 > $file_name
+  git ls-files -- lib/ansible/modules/cloud/google/gcp_* | cut -d/ -f 6 | cut -d. -f 1 > $file_name
 
   for i in "${ignored_modules[@]}"; do
     sed -i "/$i/d" $file_name
   done
 }
 
-# Clone ansible/ansible
-git clone git@github.com:modular-magician/ansible.git
-
 # Install dependencies for Template Generator
 pushd "magic-modules-gcp"
 bundle install
@@ -40,6 +37,9 @@ set -x
 chmod 400 ~/github_private_key
 popd
 
+# Clone ansible/ansible
+ssh-agent bash -c "ssh-add ~/github_private_key; git clone git@github.com:modular-magician/ansible.git"
+
 # Setup Git config and remotes.
 pushd "ansible"
 git config --global user.email "magic-modules@google.com"
@@ -63,7 +63,7 @@ popd
 pushd "ansible"
 git add lib/ansible/modules/cloud/google/gcp_* test/integration/targets/gcp_*
 git commit -m "Migrating code from collection"
-ssh-agent bash -c "ssh-add ~/github_private_key; git push origin devel"
+ssh-agent bash -c "ssh-add ~/github_private_key; git push magician devel"
 
 set -e
 
@@ -101,7 +101,7 @@ for filename in mm-bug*; do
   git commit -m "Bug fixes for GCP modules"
 
   # Create a PR message + save to file
-  ruby ../../tools/ansible-pr/generate_template.rb > bug_fixes$filename
+  ruby ../magic-modules-gcp/tools/ansible-pr/generate_template.rb > bug_fixes$filename
 
   # Create PR
   ssh-agent bash -c "ssh-add ~/github_private_key; git push origin bug_fixes$filename --force"
@@ -129,7 +129,7 @@ while read module; do
   # Create a PR message + save to file
   set +e
   git commit -m "New Module: $module"
-  ruby ../../tools/ansible-pr/generate_template.rb --new-module-name $module > $module
+  ruby ../magic-modules-gcp/tools/ansible-pr/generate_template.rb --new-module-name $module > $module
 
   # Create PR
   ssh-agent bash -c "ssh-add ~/github_private_key; git push origin $module --force"

.ci/magic-modules/release-ansible.yml

@@ -7,7 +7,7 @@ image_resource:
     type: docker-image
     source:
         repository: gcr.io/magic-modules/hub
-        tag: '1.1'
+        tag: '1.2'
 
 inputs:
     - name: magic-modules-gcp

.github/PULL_REQUEST_TEMPLATE.md

@@ -13,6 +13,7 @@ this PR, but make sure not to indent notes and to leave newlines between
 code blocks for Markdown's sake.
 
 For Terraform PRs, we use the following "release-note:" headings
+    - release-note:enhancement
     - release-note:bug
     - release-note:note
     - release-note:new-resource
@@ -25,4 +26,4 @@ For Terraform PRs, we use the following "release-note:" headings
 
 ```release-note:REPLACEME
 
-```
+```

api/resource/iam_policy.rb

@@ -54,6 +54,13 @@ class IamPolicy < Api::Object
       # compareSelfLinkOrResourceName
       attr_reader :custom_diff_suppress
 
+      # Some resources (IAP) use fields named differently from the parent resource.
+      # We need to use the parent's attributes to create an IAM policy, but they may not be
+      # named as the IAM IAM resource expects.
+      # This allows us to specify a file (relative to MM root) containing a partial terraform
+      # config with the test/example attributes of the IAM resource.
+      attr_reader :example_config_body
+
       def validate
         super
 
@@ -64,6 +71,10 @@ def validate
         check :allowed_iam_role, type: String, default: 'roles/viewer'
         check :parent_resource_attribute, type: String, default: 'id'
         check :test_project_name, type: String
+        check(
+          :example_config_body,
+          type: String, default: 'templates/terraform/iam/iam_attributes.tf.erb'
+        )
       end
     end
   end

products/appengine/api.yaml

@@ -440,3 +440,9 @@ objects:
             name: 'shell'
             description: |
               The format should be a shell command that can be fed to bash -c.
+      - !ruby/object:Api::Type::String
+        name: 'instanceClass'
+        description: |
+          Instance class that is used to run this version. Valid values are
+          AutomaticScaling F1, F2, F4, F4_1G
+          (Only AutomaticScaling is supported at the moment)

products/appengine/terraform.yaml

@@ -37,8 +37,12 @@ overrides: !ruby/object:Overrides::ResourceOverrides
         name: 'noop_on_destroy'
         description: |
           If set to `true`, the application version will not be deleted.
+      - !ruby/object:Provider::Terraform::VirtualFields
+        name: 'delete_service_on_destroy'
+        description: |
+          If set to `true`, the service will be deleted if it is the last version.    
     custom_code: !ruby/object:Provider::Terraform::CustomCode
-      custom_delete: templates/terraform/custom_delete/noop_on_destroy_appengine_version.go.erb
+      custom_delete: templates/terraform/custom_delete/appversion_delete.go.erb
       test_check_destroy: templates/terraform/custom_check_destroy/appengine_version.go.erb
     properties:
       id: !ruby/object:Overrides::Terraform::PropertyOverride
@@ -51,6 +55,8 @@ overrides: !ruby/object:Overrides::ResourceOverrides
         ignore_read: true
       threadsafe: !ruby/object:Overrides::Terraform::PropertyOverride
         ignore_read: true
+      instanceClass: !ruby/object:Overrides::Terraform::PropertyOverride
+        ignore_read: true    
     examples:
       - !ruby/object:Provider::Terraform::Examples
         name: "app_engine_standard_app_version"

products/bigquery/api.yaml

@@ -215,6 +215,20 @@ objects:
           The default value is multi-regional location `US`.
           Changing this forces a new resource to be created.
         default_value: US
+      - !ruby/object:Api::Type::NestedObject
+        name: 'defaultEncryptionConfiguration'
+        description: |
+          The default encryption key for all tables in the dataset. Once this property is set,
+          all newly-created partitioned tables in the dataset will have encryption key set to
+          this value, unless table creation request (or query) overrides the key.
+        properties:
+          - !ruby/object:Api::Type::String
+            name: 'kmsKeyName'
+            required: true
+            description: |
+              Describes the Cloud KMS encryption key that will be used to protect destination
+              BigQuery table. The BigQuery Service Account associated with your project requires
+              access to this encryption key.
   - !ruby/object:Api::Resource
     name: 'Table'
     kind: 'bigquery#table'

products/bigquery/terraform.yaml

@@ -24,6 +24,14 @@ overrides: !ruby/object:Overrides::ResourceOverrides
         primary_resource_id: "dataset"
         vars:
           dataset_id: "example_dataset"
+      - !ruby/object:Provider::Terraform::Examples
+        name: "bigquery_dataset_cmek"
+        skip_test: true
+        primary_resource_id: "dataset"
+        vars:
+          dataset_id: "example_dataset"
+          key_name: "example-key"
+          keyring_name: "example-keyring"
     virtual_fields:
       - !ruby/object:Provider::Terraform::VirtualFields
         name: 'delete_contents_on_destroy'

products/cloudbuild/api.yaml

@@ -18,6 +18,14 @@ versions:
   - !ruby/object:Api::Product::Version
     name: ga
     base_url: https://cloudbuild.googleapis.com/v1/
+  - !ruby/object:Api::Product::Version
+    name: beta
+    # This is correct even though it's the same base_url as v1.
+    # Some features in this API are beta, even though they're all
+    # released as v1.  This is not ideal, but it's not something
+    # we can fix - we just want to protect our users from APIs that
+    # might change unexpectedly.
+    base_url: https://cloudbuild.googleapis.com/v1/
 scopes:
   - https://www.googleapis.com/auth/cloud-platform
 apis_required:
@@ -132,6 +140,51 @@ objects:
             name: 'commitSha'
             description: |
               Explicit commit SHA to build. Exactly one of a branch name, tag, or commit SHA must be provided.
+      - !ruby/object:Api::Type::NestedObject
+        name: 'github'
+        description: |
+            Describes the configuration of a trigger that creates a build whenever a GitHub event is received.
+        min_version: beta
+        properties:
+          - !ruby/object:Api::Type::String
+            name: 'owner'
+            description: |
+                Owner of the repository. For example: The owner for
+                https://github.com/googlecloudplatform/cloud-builders is "googlecloudplatform".
+          - !ruby/object:Api::Type::String
+            name: 'name'
+            description: |
+                Name of the repository. For example: The name for
+                https://github.com/googlecloudplatform/cloud-builders is "cloud-builders".
+          - !ruby/object:Api::Type::NestedObject
+            name: 'pullRequest'
+            description: |
+                filter to match changes in pull requests.  Specify only one of pullRequest or push.
+            properties:
+              - !ruby/object:Api::Type::String
+                name: 'branch'
+                description: |
+                    Regex of branches to match.
+              - !ruby/object:Api::Type::Enum
+                name: 'commentControl'
+                description: |
+                    Whether to block builds on a "/gcbrun" comment from a repository owner or collaborator.
+                values:
+                  - :COMMENTS_DISABLED
+                  - :COMMENTS_ENABLED
+          - !ruby/object:Api::Type::NestedObject
+            name: 'push'
+            description: |
+                filter to match changes in refs, like branches or tags.  Specify only one of pullRequest or push.
+            properties:
+              - !ruby/object:Api::Type::String
+                name: 'branch'
+                description: |
+                    Regex of branches to match.  Specify only one of branch or tag.
+              - !ruby/object:Api::Type::String
+                name: 'tag'
+                description: |
+                    Regex of tags to match.  Specify only one of branch or tag.
       - !ruby/object:Api::Type::NestedObject
         name: 'build'
         description: |

products/cloudrun/terraform.yaml

@@ -24,7 +24,7 @@ overrides: !ruby/object:Overrides::ResourceOverrides
         test_env_vars:
           namespace: :PROJECT_NAME
     custom_code: !ruby/object:Provider::Terraform::CustomCode
-      encoder: 'templates/terraform/encoders/cloud_run.go.erb'
+      encoder: 'templates/terraform/encoders/cloud_run_domain_mapping.go.erb'
       decoder: 'templates/terraform/decoders/cloud_run.go.erb'
     properties:
       name: !ruby/object:Overrides::Terraform::PropertyOverride
@@ -53,7 +53,7 @@ overrides: !ruby/object:Overrides::ResourceOverrides
         test_env_vars:
           namespace: :PROJECT_NAME
     custom_code: !ruby/object:Provider::Terraform::CustomCode
-      encoder: 'templates/terraform/encoders/cloud_run.go.erb'
+      encoder: 'templates/terraform/encoders/cloud_run_service.go.erb'
       decoder: 'templates/terraform/decoders/cloud_run.go.erb'
     properties:
       name: !ruby/object:Overrides::Terraform::PropertyOverride

products/compute/ansible.yaml

@@ -33,12 +33,6 @@ datasources: !ruby/object:Overrides::ResourceOverrides
     exclude: true
   NetworkEndpoint: !ruby/object:Overrides::Ansible::ResourceOverride
     exclude: true
-  NetworkEndpointGroup: !ruby/object:Overrides::Ansible::ResourceOverride
-    exclude: true
-  NodeGroup: !ruby/object:Overrides::Ansible::ResourceOverride
-    exclude: true
-  RegionBackendService: !ruby/object:Overrides::Ansible::ResourceOverride
-    exclude: true
   Region: !ruby/object:Overrides::Ansible::ResourceOverride
     exclude: true
   RegionDiskType: !ruby/object:Overrides::Ansible::ResourceOverride
@@ -91,8 +85,6 @@ overrides: !ruby/object:Overrides::ResourceOverrides
       timeoutSec: !ruby/object:Overrides::Ansible::PropertyOverride
         aliases:
           - timeout_seconds
-  RegionBackendService: !ruby/object:Overrides::Ansible::ResourceOverride
-    exclude: true
   DiskResourcePolicyAttachment: !ruby/object:Overrides::Ansible::ResourceOverride
     exclude: true
   Disk: !ruby/object:Overrides::Ansible::ResourceOverride
@@ -201,6 +193,11 @@ overrides: !ruby/object:Overrides::ResourceOverrides
       timeoutSec: !ruby/object:Overrides::Ansible::PropertyOverride
         aliases:
           - timeout_seconds
+  Image: !ruby/object:Overrides::Ansible::ResourceOverride
+    properties:
+      guestOsFeatures.type: !ruby/object:Overrides::Ansible::PropertyOverride
+        # Changing this description because of line length issues.
+        description: The type of supported feature.
   InstanceGroup: !ruby/object:Overrides::Ansible::ResourceOverride
     properties:
       instances: !ruby/object:Overrides::Ansible::PropertyOverride
@@ -302,10 +299,6 @@ overrides: !ruby/object:Overrides::ResourceOverrides
     exclude: true
   NetworkEndpoint: !ruby/object:Overrides::Ansible::ResourceOverride
     exclude: true
-  NetworkEndpointGroup: !ruby/object:Overrides::Ansible::ResourceOverride
-    exclude: true
-  NodeGroup: !ruby/object:Overrides::Ansible::ResourceOverride
-    exclude: true
   Region: !ruby/object:Overrides::Ansible::ResourceOverride
     exclude: true
   RegionDiskType: !ruby/object:Overrides::Ansible::ResourceOverride