Merge branch 'master' into service-account-description

.ci/RELEASE_NOTES_GUIDE.md

@@ -127,6 +127,7 @@ Don’t write notes like:
 ## Headings
 
 Release notes should be formatted with one of the following headings:
+- `release-note:enhancement`
 - `release-note:bug`
 - `release-note:note`
 - `release-note:new-resource`

.ci/changelog.tmpl

@@ -0,0 +1,66 @@
+{{- $notes := newStringList -}}
+{{- $unknown := newStringList -}}
+{{- $breaking := newStringList -}}
+{{- $deprecations := newStringList -}}
+{{- $features := newStringList -}}
+{{- $improvements := newStringList -}}
+{{- $bugs := newStringList -}}
+{{- range . -}}
+	{{if eq "note" .Type -}}
+		{{$notes = append $notes (renderReleaseNote .) -}}
+	{{else if eq "breaking-change" .Type -}}
+		{{$breaking = append $breaking (renderReleaseNote .) -}}
+	{{else if eq "deprecation" .Type -}}
+		{{$deprecations = append $deprecations (renderReleaseNote .) -}}
+	{{else if or (eq "new-resource" .Type) (eq "new-datasource" .Type) (eq "feature" .Type) -}}
+		{{$features = append $features (renderReleaseNote .) -}}
+	{{else if or (eq "improvement" .Type) (eq "enhancement" .Type) -}}
+		{{$improvements = append $improvements (renderReleaseNote .) -}}
+	{{ else if eq "bug" .Type -}}
+		{{$bugs = append $bugs (renderReleaseNote .) -}}
+	{{ else -}}
+		{{$unknown = append $unknown (renderReleaseNote .) -}}
+	{{end -}}
+{{- end -}}
+{{- if gt (len $unknown) 0 -}}
+UNKNOWN CHANGELOG TYPE:
+{{range $unknown | sortAlpha -}}
+* {{. }}
+{{- end -}}
+{{- end -}}
+{{- if gt (len $notes) 0 -}}
+NOTES:
+{{range $notes | sortAlpha -}}
+* {{. }}
+{{- end -}}
+{{- end -}}
+{{- if gt (len $deprecations) 0 -}}
+DEPRECATIONS:
+{{range $deprecations | sortAlpha -}}
+* {{. }}
+{{- end -}}
+{{- end -}}
+{{- if gt (len $breaking) 0 -}}
+BREAKING CHANGES:
+{{range $breaking | sortAlpha -}}
+* {{. }}
+{{- end -}}
+{{- end -}}
+{{- if gt (len $features) 0}}
+FEATURES:
+{{range $features | sortAlpha -}}
+* {{. }}
+{{- end -}}
+{{- end -}}
+{{- if gt (len $improvements) 0}}
+IMPROVEMENTS:
+{{range $improvements | sortAlpha -}}
+* {{. }}
+{{- end -}}
+{{- end -}}
+{{- if gt (len $bugs) 0}}
+BUG FIXES:
+{{range $bugs | sortAlpha -}}
+* {{. }}
+{{- end -}}
+{{- end -}}

.ci/ci.yml.tmpl

@@ -183,29 +183,30 @@ jobs:
                     GITHUB_ORG: {{v.github_org}}
                     OVERRIDE_PROVIDER: {{v.override_provider}}
 
-                - task: test-{{v.short_name}}
-                  file: magic-modules-branched/.ci/unit-tests/tf-3.yml
-                  timeout: 30m
-                  params:
-                      PROVIDER_NAME: {{v.provider_name}}
-                      TEST_DIR: {{v.test_dir}}
-
                 - put: {{v.short_name}}-intermediate
                   params:
-                      repository: terraform-diff/new
+                      repository: terraform-diff/{{k}}/new
                       branch_file: magic-modules-branched/branchname
                       force: true
                   get_params:
                       skip_clone: true
 
                 - put: {{v.short_name}}-intermediate
                   params:
-                      repository: terraform-diff/old
+                      repository: terraform-diff/{{k}}/old
                       branch_file: magic-modules-previous/branchname
                       force: true
                   get_params:
                       skip_clone: true
 
+                - task: test-{{v.short_name}}
+                  file: magic-modules-branched/.ci/unit-tests/tf-3.yml
+                  timeout: 30m
+                  params:
+                      PROVIDER_NAME: {{v.provider_name}}
+                      TEST_DIR: {{v.test_dir}}
+                      SUBDIR: {{k}}
+
 {% endfor %}
 
             on_failure:

.ci/magic-modules/diff-terraform.sh

@@ -82,5 +82,7 @@ for mm_dir in magic-modules-branched magic-modules-previous; do
 
 done
 
-git clone "magic-modules-branched/build/$SHORT_NAME" "./terraform-diff/new"
-git clone "magic-modules-previous/build/$SHORT_NAME" "./terraform-diff/old"
+mkdir "./terraform-diff/$VERSION"
+
+git clone "magic-modules-branched/build/$SHORT_NAME" "./terraform-diff/$VERSION/new"
+git clone "magic-modules-previous/build/$SHORT_NAME" "./terraform-diff/$VERSION/old"

.ci/magic-modules/vars/validator_handwritten_files.txt

@@ -1,7 +1,9 @@
 resource_compute_instance.go
+resource_container_cluster.go
+resource_container_node_pool.go
 resource_google_project.go
 resource_sql_database_instance.go.erb
 resource_storage_bucket.go
 iam_folder.go
 iam_organization.go
-iam_project.go
+iam_project.go

.ci/release-note.tmpl

@@ -0,0 +1 @@
+{{if eq "new-resource" .Type}}**New Resource:** {{else if eq "new-data-source" .Type}}**New Data Source:** {{ end }}{{.Text }} ([#{{- .PRNumber -}}]({{- .PRURL -}}))

.ci/unit-tests/tf-3.sh

@@ -9,7 +9,7 @@ set -x
 
 # Create GOPATH structure
 mkdir -p "${GOPATH}/src/github.com/terraform-providers"
-ln -s "${PWD}/terraform-diff/new" "${GOPATH}/src/github.com/terraform-providers/$PROVIDER_NAME"
+ln -s "${PWD}/terraform-diff/${SUBDIR}/new" "${GOPATH}/src/github.com/terraform-providers/$PROVIDER_NAME"
 
 cd "${GOPATH}/src/github.com/terraform-providers/$PROVIDER_NAME"
 

.ci/unit-tests/tf-3.yml

@@ -12,3 +12,4 @@ run:
 params:
   PROVIDER_NAME: ""
   TEST_DIR: ""
+  SUBDIR: ""

.github/PULL_REQUEST_TEMPLATE.md

@@ -13,6 +13,7 @@ this PR, but make sure not to indent notes and to leave newlines between
 code blocks for Markdown's sake.
 
 For Terraform PRs, we use the following "release-note:" headings
+    - release-note:enhancement
     - release-note:bug
     - release-note:note
     - release-note:new-resource
@@ -25,4 +26,4 @@ For Terraform PRs, we use the following "release-note:" headings
 
 ```release-note:REPLACEME
 
-```
+```

README.md

@@ -26,7 +26,7 @@ features/tools:
 * Terraform in Cloud Shell
 
 Importantly, Magic Modules *isn't* full code generation. Every change is made
-manually; more than a code generator, Magic Modules is a force multiplier for 
+manually; more than a code generator, Magic Modules is a force multiplier for
 development. While many Magic Modules resources are defined exactly based on the
 GCP API, we use Magic Modules to preemptively solve issues across each tool by
 encoding our field-tested learnings from other tools in those definitions. In
@@ -61,6 +61,8 @@ To get started, you'll need:
   * You can use `rbenv` to manage your Ruby version(s)
 * [`Bundler`](https://github.com/bundler/bundler)
   * This can be installed with `gem install bundler`
+* If you are getting "Too many open files" ulimit needs to be raised.
+  * Mac OSX: `ulimit -n 1000`
 
 ### Preparing Magic Modules / One-time setup
 

api/resource/iam_policy.rb

@@ -54,6 +54,13 @@ class IamPolicy < Api::Object
       # compareSelfLinkOrResourceName
       attr_reader :custom_diff_suppress
 
+      # Some resources (IAP) use fields named differently from the parent resource.
+      # We need to use the parent's attributes to create an IAM policy, but they may not be
+      # named as the IAM IAM resource expects.
+      # This allows us to specify a file (relative to MM root) containing a partial terraform
+      # config with the test/example attributes of the IAM resource.
+      attr_reader :example_config_body
+
       def validate
         super
 
@@ -64,6 +71,10 @@ def validate
         check :allowed_iam_role, type: String, default: 'roles/viewer'
         check :parent_resource_attribute, type: String, default: 'id'
         check :test_project_name, type: String
+        check(
+          :example_config_body,
+          type: String, default: 'templates/terraform/iam/iam_attributes.tf.erb'
+        )
       end
     end
   end

api/type.rb

@@ -35,8 +35,20 @@ module Fields
       attr_reader :input # If set to true value is used only on creation
       attr_reader :url_param_only # If, true will not be send in request body
       attr_reader :required
+
       attr_reader :update_verb
       attr_reader :update_url
+      # Some updates only allow updating certain fields at once (generally each
+      # top-level field can be updated one-at-a-time). If this is set, we group
+      # fields to update by (verb, url, fingerprint, id) instead of just
+      # (verb, url, fingerprint), to allow multiple fields to reuse the same
+      # endpoints.
+      attr_reader :update_id
+      # THe fingerprint value required to update this field. Downstreams should
+      # GET the resource and parse the fingerprint value while doing each update
+      # call. This ensures we can supply the fingerprint to each distinct
+      # request.
+      attr_reader :fingerprint_name
       # If true, we will include the empty value in requests made including
       # this attribute (both creates and updates).  This rarely needs to be
       # set to true, and corresponds to both the "NullFields" and
@@ -91,6 +103,8 @@ def validate
                           default: @__resource&.update_verb
 
       check :update_url, type: ::String
+      check :update_id, type: ::String
+      check :fingerprint_name, type: ::String
       check :pattern, type: ::String
 
       check_default_value_property

products/appengine/ansible.yaml

@@ -20,6 +20,8 @@ datasources: !ruby/object:Overrides::ResourceOverrides
   DomainMapping: !ruby/object:Overrides::Ansible::ResourceOverride
     exclude: true
 overrides: !ruby/object:Overrides::ResourceOverrides
+  ApplicationUrlDispatchRules: !ruby/object:Overrides::Ansible::ResourceOverride
+    exclude: true
   FirewallRule: !ruby/object:Overrides::Ansible::ResourceOverride
     # Because of lack of state, Ansible treats identity
     # differently than Terraform.

products/appengine/api.yaml

@@ -440,3 +440,66 @@ objects:
             name: 'shell'
             description: |
               The format should be a shell command that can be fed to bash -c.
+      - !ruby/object:Api::Type::String
+        name: 'instanceClass'
+        description: |
+          Instance class that is used to run this version. Valid values are
+          AutomaticScaling F1, F2, F4, F4_1G
+          (Only AutomaticScaling is supported at the moment)
+  - !ruby/object:Api::Resource
+    name: 'ApplicationUrlDispatchRules'
+    description: |
+      Rules to match an HTTP request and dispatch that request to a service.
+    base_url: 'apps/{{project}}'
+    create_url: 'apps/{{project}}?updateMask=dispatch_rules'
+    create_verb: :PATCH
+    delete_url: 'apps/{{project}}?updateMask=dispatch_rules'
+    delete_verb: :PATCH
+    update_url: 'apps/{{project}}?updateMask=dispatch_rules'
+    update_verb: :PATCH
+    references: !ruby/object:Api::Resource::ReferenceLinks
+      api: 'https://cloud.google.com/appengine/docs/admin-api/reference/rest/v1/apps#UrlDispatchRule'
+    async: !ruby/object:Api::Async
+      operation: !ruby/object:Api::Async::Operation
+        path: 'name'
+        base_url: '{{op_id}}'
+        wait_ms: 1000
+      result: !ruby/object:Api::Async::Result
+        path: 'response'
+      status: !ruby/object:Api::Async::Status
+        path: 'status'
+        complete: 'DONE'
+        allowed:
+          - 'PENDING'
+          - 'RUNNING'
+          - 'DONE'
+      error: !ruby/object:Api::Async::Error
+        path: 'error/errors'
+        message: 'message'
+    properties:
+      - !ruby/object:Api::Type::Array   
+        name: 'dispatchRules'
+        required: true
+        description: |
+          Rules to match an HTTP request and dispatch that request to a service.
+        item_type: !ruby/object:Api::Type::NestedObject
+          properties:
+            - !ruby/object:Api::Type::String
+              name: 'domain'
+              default_value: '*'
+              description: |
+                Domain name to match against. The wildcard "*" is supported if specified before a period: "*.".
+                Defaults to matching all domains: "*".
+            - !ruby/object:Api::Type::String
+              name: 'path'
+              description: |
+                Pathname within the host. Must start with a "/". A single "*" can be included at the end of the path.
+                The sum of the lengths of the domain and path may not exceed 100 characters.
+              required: true
+            - !ruby/object:Api::Type::String
+              name: 'service'
+              description: |
+                Pathname within the host. Must start with a "/". A single "*" can be included at the end of the path.
+                The sum of the lengths of the domain and path may not exceed 100 characters.
+              required: true
+

products/appengine/inspec.yaml

@@ -14,6 +14,8 @@
 --- !ruby/object:Provider::Inspec::Config
 legacy_name: appengine
 overrides: !ruby/object:Overrides::ResourceOverrides
+  ApplicationUrlDispatchRules: !ruby/object:Overrides::Inspec::ResourceOverride
+    exclude: true
   DomainMapping: !ruby/object:Overrides::Inspec::ResourceOverride
     exclude: true
   FirewallRule: !ruby/object:Overrides::Inspec::ResourceOverride