Only compile Terraform's versioned files once per run on MM. (#660)

Merged PR #660.

compiler.rb

@@ -143,8 +143,10 @@
   provider.generate output_path, types_to_generate, version
 end
 
-# In order to only copy files once per provider this must be called outside of the products loop
-# This will get called with the value from the final iteration of the loop
+# In order to only copy/compile files once per provider this must be called outside
+# of the products loop. This will get called with the provider from the final iteration
+# of the loop
 provider&.copy_common_files(output_path, version)
+provider&.compile_common_files(output_path, version)
 
 # rubocop:enable Metrics/BlockLength

coverage/.last_run.json

@@ -1,5 +1,5 @@
 {
   "result": {
-    "covered_percent": 78.81
+    "covered_percent": 77.85
   }
 }

products/binaryauthorization/terraform.yaml

@@ -108,4 +108,4 @@ files: !ruby/object:Provider::Config::Files
   # These files have templating (ERB) code that will be run.
   # This is usually to add licensing info, autogeneration notices, etc.
   compile:
-<%= lines(indent(compile('provider/terraform/common~compile.yaml'), 4)) -%>
+<%= lines(indent(compile('provider/terraform/product~compile.yaml'), 4)) -%>

products/compute/ansible.yaml

@@ -347,5 +347,5 @@ overrides: !ruby/object:Provider::ResourceOverrides
     exclude: true
 files: !ruby/object:Provider::Config::Files
   compile:
-<%= lines(indent(compile('provider/ansible/common~compile.yaml'), 4)) -%>
+<%= lines(indent(compile('provider/ansible/product~compile.yaml'), 4)) -%>
 tests: !ruby/object:Api::Resource::HashArray

products/compute/terraform.yaml

@@ -967,4 +967,4 @@ files: !ruby/object:Provider::Config::Files
   # These files have templating (ERB) code that will be run.
   # This is usually to add licensing info, autogeneration notices, etc.
   compile:
-<%= lines(indent(compile('provider/terraform/common~compile.yaml'), 4)) -%>
+<%= lines(indent(compile('provider/terraform/product~compile.yaml'), 4)) -%>

products/container/ansible.yaml

@@ -47,5 +47,5 @@ overrides: !ruby/object:Provider::ResourceOverrides
       - 'products/container/helpers/python/provider_node_pool.py'
 files: !ruby/object:Provider::Config::Files
   compile:
-<%= lines(indent(compile('provider/ansible/common~compile.yaml'), 4)) -%>
+<%= lines(indent(compile('provider/ansible/product~compile.yaml'), 4)) -%>
 tests: !ruby/object:Api::Resource::HashArray

products/containeranalysis/terraform.yaml

@@ -34,4 +34,4 @@ files: !ruby/object:Provider::Config::Files
   # These files have templating (ERB) code that will be run.
   # This is usually to add licensing info, autogeneration notices, etc.
   compile:
-<%= lines(indent(compile('provider/terraform/common~compile.yaml'), 4)) -%>
+<%= lines(indent(compile('provider/terraform/product~compile.yaml'), 4)) -%>

products/dns/ansible.yaml

@@ -93,5 +93,5 @@ overrides: !ruby/object:Provider::ResourceOverrides
     exclude: true
 files: !ruby/object:Provider::Config::Files
   compile:
-<%= lines(indent(compile('provider/ansible/common~compile.yaml'), 4)) -%>
+<%= lines(indent(compile('provider/ansible/product~compile.yaml'), 4)) -%>
 tests: !ruby/object:Api::Resource::HashArray

products/filestore/terraform.yaml

@@ -40,4 +40,4 @@ files: !ruby/object:Provider::Config::Files
   # These files have templating (ERB) code that will be run.
   # This is usually to add licensing info, autogeneration notices, etc.
   compile:
-<%= lines(indent(compile('provider/terraform/common~compile.yaml'), 4)) -%>
+<%= lines(indent(compile('provider/terraform/product~compile.yaml'), 4)) -%>

products/monitoring/terraform.yaml

@@ -88,4 +88,4 @@ files: !ruby/object:Provider::Config::Files
   # These files have templating (ERB) code that will be run.
   # This is usually to add licensing info, autogeneration notices, etc.
   compile:
-<%= lines(indent(compile('provider/terraform/common~compile.yaml'), 4)) -%>
+<%= lines(indent(compile('provider/terraform/product~compile.yaml'), 4)) -%>

products/pubsub/ansible.yaml

@@ -50,5 +50,5 @@ overrides: !ruby/object:Provider::ResourceOverrides
       - 'products/pubsub/helpers/python/provider_subscription.py'
 files: !ruby/object:Provider::Config::Files
   compile:
-<%= lines(indent(compile('provider/ansible/common~compile.yaml'), 4)) -%>
+<%= lines(indent(compile('provider/ansible/product~compile.yaml'), 4)) -%>
 tests: !ruby/object:Api::Resource::HashArray

products/redis/terraform.yaml

@@ -59,4 +59,4 @@ files: !ruby/object:Provider::Config::Files
   # These files have templating (ERB) code that will be run.
   # This is usually to add licensing info, autogeneration notices, etc.
   compile:
-<%= lines(indent(compile('provider/terraform/common~compile.yaml'), 4)) -%>
+<%= lines(indent(compile('provider/terraform/product~compile.yaml'), 4)) -%>

products/resourcemanager/terraform.yaml

@@ -40,4 +40,4 @@ files: !ruby/object:Provider::Config::Files
   # These files have templating (ERB) code that will be run.
   # This is usually to add licensing info, autogeneration notices, etc.
   compile:
-<%= lines(indent(compile('provider/terraform/common~compile.yaml'), 4)) -%>
+<%= lines(indent(compile('provider/terraform/product~compile.yaml'), 4)) -%>

products/spanner/ansible.yaml

@@ -54,5 +54,5 @@ overrides: !ruby/object:Provider::ResourceOverrides
     exclude: true
 files: !ruby/object:Provider::Config::Files
   compile:
-<%= lines(indent(compile('provider/ansible/common~compile.yaml'), 4)) -%>
+<%= lines(indent(compile('provider/ansible/product~compile.yaml'), 4)) -%>
 tests: !ruby/object:Api::Resource::HashArray

products/sql/ansible.yaml

@@ -64,5 +64,5 @@ overrides: !ruby/object:Provider::ResourceOverrides
 files: !ruby/object:Provider::Config::Files
   copy:
   compile:
-<%= lines(indent(compile('provider/ansible/common~compile.yaml'), 4)) -%>
+<%= lines(indent(compile('provider/ansible/product~compile.yaml'), 4)) -%>
 tests: !ruby/object:Api::Resource::HashArray

products/storage/ansible.yaml

@@ -44,5 +44,5 @@ overrides: !ruby/object:Provider::ResourceOverrides
     exclude: true
 files: !ruby/object:Provider::Config::Files
   compile:
-<%= lines(indent(compile('provider/ansible/common~compile.yaml'), 4)) -%>
+<%= lines(indent(compile('provider/ansible/product~compile.yaml'), 4)) -%>
 tests: !ruby/object:Api::Resource::HashArray

products/storage/terraform.yaml

@@ -57,4 +57,4 @@ files: !ruby/object:Provider::Config::Files
   # These files have templating (ERB) code that will be run.
   # This is usually to add licensing info, autogeneration notices, etc.
   compile:
-<%= lines(indent(compile('provider/terraform/common~compile.yaml'), 4)) -%>
+<%= lines(indent(compile('provider/terraform/product~compile.yaml'), 4)) -%>

provider/core.rb

@@ -58,6 +58,9 @@ def generate(output_folder, types, version_name)
       # Compilation has to be the last step, as some files (e.g.
       # CONTRIBUTING.md) may depend on the list of all files previously copied
       # or compiled.
+      # common-compile.yaml is a special file that will get compiled by the last product
+      # used in a single invocation of the compiled. It should not contain product-specific
+      # information; instead, it should be run-specific such as the version to compile at.
       compile_files(output_folder, version_name) \
         unless @config.files.nil? || @config.files.compile.nil?
 
@@ -77,11 +80,21 @@ def copy_files(output_folder)
     def copy_common_files(output_folder, _version_name = nil)
       provider_name = self.class.name.split('::').last.downcase
       return unless File.exist?("provider/#{provider_name}/common~copy.yaml")
-      files = YAML.safe_load(compile("provider/#{provider_name}/common~copy.yaml"))
 
+      Google::LOGGER.info "Copying common files for #{provider_name}"
+      files = YAML.safe_load(compile("provider/#{provider_name}/common~copy.yaml"))
       copy_file_list(output_folder, files)
     end
 
+    def compile_common_files(output_folder, version_name = nil)
+      provider_name = self.class.name.split('::').last.downcase
+      return unless File.exist?("provider/#{provider_name}/common~compile.yaml")
+
+      Google::LOGGER.info "Compiling common files for #{provider_name}"
+      files = YAML.safe_load(compile("provider/#{provider_name}/common~compile.yaml"))
+      compile_file_list(output_folder, files, version: version_name)
+    end
+
     def copy_file_list(output_folder, files)
       files.each do |target, source|
         target_file = File.join(output_folder, target)

provider/terraform/common~compile.yaml

@@ -12,9 +12,7 @@
 # limitations under the License.
 # These files contains code that needs to be compiled before being delivered to
 # the final module tree structure:
-<% dir = _version_name == 'beta' ? 'google-beta' : 'google' -%>
-'<%= dir -%>/provider_<%= api.prefix[1..-1] -%>_gen.go': 'templates/terraform/provider_gen.erb'
-
+<% dir = version_name == 'beta' ? 'google-beta' : 'google' -%>
 <%  Dir["provider/terraform/tests/*.go.erb"].each do |file_path|
     fname = file_path.split('/')[-1]
 -%>

provider/terraform/product~compile.yaml

@@ -0,0 +1,16 @@
+# Copyright 2018 Google Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# These files contains code that needs to be compiled before being delivered to
+# the final module tree structure:
+<% dir = _version_name == 'beta' ? 'google-beta' : 'google' -%>
+'<%= dir -%>/provider_<%= api.prefix[1..-1] -%>_gen.go': 'templates/terraform/provider_gen.erb'