Interpolation updates

.circleci/config.yml

@@ -8,8 +8,8 @@ jobs:
       - image: circleci/ruby:2.3.6
 
     environment:
-      COVALENCE_VERSION: 0.7.6
-      TERRAFORM_VERSION: 0.11.3
+      COVALENCE_VERSION: 0.7.7
+      TERRAFORM_VERSION: 0.11.7
 
     steps:
       - checkout

CHANGELOG.md

@@ -1,7 +1,19 @@
 ## Unreleased
-* Add support for Terraform environments
+* Add support for Terraform workspaces
 * Add ability to toggle primary state store
 
+## 0.7.7 (May 6, 2018)
+BACKWARDS INCOMPATIBILITIES:
+- Versions of Terraform prior to v0.11.4 are no longer supported.
+
+IMPROVEMENTS:
+- Extended shell interpolation for input values to support nesting and escaping
+- Improved support for Terraform plugin caching
+- Replaced deprecated `-force` option for Terraform `destroy` task with `-auto-approve=true`.
+
+FIXES:
+- Updated input processing to support nested complex types properly.
+
 ## 0.7.6 (December 6, 2017)
 IMPROVEMENTS:
 - Terraform init failures were not being reported.  Update spec tests to catch errors with 'terraform init'.

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    covalence (0.7.6)
+    covalence (0.7.7)
       activemodel (~> 4.2.6)
       activesupport (~> 4.2.6)
       aws-sdk (~> 2.9.5)

lib/covalence.rb

@@ -11,7 +11,7 @@
 module Covalence
   # Configurable constants
   #TODO: look into how WORKSPACE is being used, maybe this can just be an internal ROOT and make CONFIG not depend on WORKSPACE
-  WORKSPACE = File.absolute_path((ENV['COVALENCE_WORKSPACE'] || '.'))
+  WORKSPACE = File.absolute_path(ENV['COVALENCE_WORKSPACE'] || '.')
   CONFIG = File.join(WORKSPACE, (ENV['COVALENCE_CONFIG'] || 'covalence.yaml'))
   # TODO: could use better naming
   PACKER = File.absolute_path(File.join(WORKSPACE, (ENV['COVALENCE_PACKER_DIR'] || '.')))
@@ -22,6 +22,7 @@ module Covalence
 
   TERRAFORM_CMD = ENV['TERRAFORM_CMD'] || "terraform"
   TERRAFORM_VERSION = ENV['TERRAFORM_VERSION'] || `#{TERRAFORM_CMD} version`.split("\n", 2)[0].gsub('Terraform v','')
+  TERRAFORM_PLUGIN_CACHE = File.absolute_path("#{ENV['TF_PLUGIN_CACHE_DIR']}/linux_amd64" || "#{ENV['HOME']}/.terraform.d/plugin-cache/linus_amd64")
 
   PACKER_CMD = ENV['PACKER_CMD'] || "packer"
 

lib/covalence/core/cli_wrappers/terraform_cli.rb

@@ -41,8 +41,14 @@ def self.terraform_check_style(path)
     end
 
     def self.terraform_init(path='', args: '', ignore_exitcode: false)
-      output = PopenWrapper.run([
-        Covalence::TERRAFORM_CMD, "init", "-get=false", "-input=false"],
+      if ENV['TF_PLUGIN_LOCAL'] == 'true'
+        cmd = [Covalence::TERRAFORM_CMD, "init", "-get=false", "-input=false", "-plugin-dir=#{Covalence::TERRAFORM_PLUGIN_CACHE}"]
+      else
+        cmd = [Covalence::TERRAFORM_CMD, "init", "-get=false", "-input=false"]
+      end
+
+      output = PopenWrapper.run(
+        cmd,
         path,
         args,
         ignore_exitcode: ignore_exitcode)

lib/covalence/core/entities/input.rb

@@ -2,7 +2,8 @@
 require 'active_support/core_ext/string/inflections'
 require 'active_support/core_ext/hash'
 require 'active_model'
-require 'open3'
+
+require_relative '../../helpers/shell_interpolation'
 
 module Covalence
   class Input
@@ -24,33 +25,7 @@ def value
     end
 
     def to_command_option
-      parsed_value = value()
-
-      if parsed_value.nil?
-        "#{name} = \"\""
-
-      elsif parsed_value.is_a?(Hash)
-        config = "#{name} = {\n"
-        parsed_value.each do |k,v|
-          config += "  \"#{k}\" = \"#{v}\"\n"
-        end
-        config += "}"
-
-      elsif parsed_value.is_a?(Array)
-        config = "#{name} = [\n"
-        parsed_value.each do |v|
-          config += "  \"#{v}\",\n"
-        end
-        config += "]"
-
-      elsif parsed_value.start_with?("$(")
-        Covalence::LOGGER.info "Evaluating interpolated value: \"#{parsed_value}\""
-        interpolated_value = Open3.capture2e(ENV, "echo \"#{parsed_value}\"")[0].chomp
-        "#{name} = \"#{interpolated_value}\""
-
-      else
-        "#{name} = \"#{parsed_value}\""
-      end
+      "#{name} = #{parse_input(value())}"
     end
 
     private
@@ -72,6 +47,40 @@ def get_value(input)
       end
     end
 
+    def parse_array(input)
+      config = "[\n"
+      input.each do |v|
+        config += "  #{parse_input(v)},\n"
+      end
+      config += "]"
+    end
+
+    def parse_hash(input)
+      config = "{\n"
+      input.each do |k,v|
+        config += "  \"#{k}\" = #{parse_input(v)}\n"
+      end
+      config += "}"
+    end
+
+    def parse_input(input)
+      if input.nil?
+        "\"\""
+
+      elsif input.is_a?(Hash)
+        parse_hash(input)
+
+      elsif input.is_a?(Array)
+        parse_array(input)
+
+      elsif input.to_s.include?("$(")
+        "\"#{Covalence::Helpers::ShellInterpolation.parse_shell(input)}\""
+
+      else
+        "\"#{input}\""
+      end
+    end
+
     # :reek:FeatureEnvy
     def parse_type(input)
       if input.stringify_keys.has_key?('type')

lib/covalence/core/entities/stack.rb

@@ -53,8 +53,9 @@ def materialize_cmd_inputs
         inputs.each do |name, input|
           config[name] = input.value
         end
-        logger.info "\nStack inputs:\n\n#{JSON.generate(config)}"
-        File.open('covalence-inputs.json','w') {|f| f.write(JSON.generate(config))}
+        config_json = JSON.generate(config)
+        logger.info "\nStack inputs:\n\n#{config_json}"
+        File.open('covalence-inputs.json','w') {|f| f.write(config_json)}
       end
     end
 

lib/covalence/core/services/terraform_stack_tasks.rb

@@ -163,7 +163,7 @@ def context_destroy(*additional_args)
 
           stack.materialize_cmd_inputs
           args = collect_args("-input=false",
-                              "-force",
+                              "-auto-approve=true",
                               stack.args,
                               additional_args,
                               "-var-file=covalence-inputs.tfvars")

lib/covalence/core/state_stores/consul.rb

@@ -2,6 +2,8 @@
 require 'rest-client'
 require 'base64'
 
+require_relative '../../helpers/shell_interpolation'
+
 module Covalence
   module Consul
 
@@ -99,6 +101,7 @@ def self.get_state_store(params)
 
       params.delete('name')
       params.each do |k,v|
+        v = Covalence::Helpers::ShellInterpolation.parse_shell(v) if v.include?("$(")
         config += "    #{k} = \"#{v}\"\n"
       end
 

lib/covalence/core/state_stores/s3.rb

@@ -1,6 +1,8 @@
 require 'json'
 require 'aws-sdk'
 
+require_relative '../../helpers/shell_interpolation'
+
 module Covalence
   module S3
 
@@ -70,14 +72,26 @@ def self.get_state_store(params)
         raise "Missing '#{param}' store parameter" unless params.has_key?(param)
       end
 
-      config = <<-CONF
+      if params.has_key?('key')
+        config = <<-CONF
+terraform {
+  backend "s3" {
+CONF
+
+      else
+        config = <<-CONF
 terraform {
   backend "s3" {
     key = "#{params['name']}/terraform.tfstate"
 CONF
 
+        Covalence::LOGGER.debug "'key' parameter not specified. Inferring value from 'name' parameter."
+      end
+
       params.delete('name')
+
       params.each do |k,v|
+        v = Covalence::Helpers::ShellInterpolation.parse_shell(v) if v.to_s.include?("$(")
         config += "    #{k} = \"#{v}\"\n"
       end
 

lib/covalence/helpers/shell_interpolation.rb

@@ -0,0 +1,28 @@
+require 'open3'
+
+module Covalence
+  module Helpers
+    class ShellInterpolation
+
+      def self.parse_shell(input)
+        Covalence::LOGGER.info "Evaluating requested interpolation: \"#{input}\""
+        matches = input.scan(/.?\$\([^)]*\)/)
+
+        Covalence::LOGGER.debug "matches: #{matches}"
+        matches.each do |cmd|
+          if cmd[0] != "\\"
+            cmd = cmd[1..-1] unless cmd[0] == "$"
+            interpolated_value = Open3.capture2e(ENV, "echo \"#{cmd}\"")[0].chomp
+            input = input.gsub(cmd, interpolated_value)
+            Covalence::LOGGER.debug "updated value: #{input}"
+          else
+            input = input.gsub(cmd, cmd[1..-1])
+          end
+        end
+        Covalence::LOGGER.info "Interpolated value: \"#{input}\""
+        return input
+      end
+
+    end
+  end
+end

lib/covalence/version.rb

@@ -1,3 +1,3 @@
 module Covalence
-  VERSION = "0.7.6"
+  VERSION = "0.7.7"
 end

spec/core/entities/input_spec.rb

@@ -26,23 +26,47 @@ module Covalence
         it { expect(input.value).to eq(raw_value) }
       end
 
+      context "simple list with nested complex types" do
+        let(:raw_value) { ["foo", ["bar"], {"foo"=>"bar"}] }
+
+        it { expect(input.value).to eq(raw_value) }
+      end
+
+      context "simple map with nested complex types" do
+        let(:raw_value) { {"foo"=>["bar"], "bar"=>{"foo"=>"bar"}} }
+
+        it { expect(input.value).to eq(raw_value) }
+      end
+
       context "complex string" do
         let(:raw_value) { {"type"=>"string","value"=>"test"} }
 
         it { expect(input.value).to eq("test") }
       end
 
       context "complex list" do
-        let(:raw_value) { {"type"=>"string","value"=>["test"]} }
+        let(:raw_value) { {"type"=>"list","value"=>["test"]} }
 
         it { expect(input.value).to eq(["test"]) }
       end
 
+      context "complex list with nested complex types" do
+        let(:raw_value) { {"type"=>"list", "value"=>["foo", ["bar"], {"foo"=>"bar"}]} }
+
+        it { expect(input.value).to eq(["foo", ["bar"], {"foo"=>"bar"}]) }
+      end
+
       context "complex map" do
-        let(:raw_value) { {"type"=>"string","value"=>{"foo"=>"bar"}} }
+        let(:raw_value) { {"type"=>"map","value"=>{"foo"=>"bar"}} }
 
         it { expect(input.value).to eq({"foo"=>"bar"}) }
       end
+
+      context "complex map with nested complex types" do
+        let(:raw_value) { {"type"=>"map", "value"=>{"foo"=>["bar"], "bar"=>{"foo"=>"bar"}}} }
+
+        it { expect(input.value).to eq({"foo"=>["bar"], "bar"=>{"foo"=>"bar"}}) }
+      end
     end
 
     context "with remote input" do
@@ -112,6 +136,24 @@ module Covalence
         it { expect(input.to_command_option).to eq("input = \"#{`pwd`.chomp}\"") }
       end
 
+      context "with nested interpolated shell value" do
+        let(:raw_value) { "this-is-a-test-$(pwd)" }
+
+        it { expect(input.to_command_option).to eq("input = \"this-is-a-test-#{`pwd`.chomp}\"") }
+      end
+
+      context "with nested interpolated shell values" do
+        let(:raw_value) { "this-is-a-test-$(pwd)-and-$(date)" }
+
+        it { expect(input.to_command_option).to eq("input = \"this-is-a-test-#{`pwd`.chomp}-and-#{`date`.chomp}\"") }
+      end
+
+      context "with nested interpolated shell values with escapes" do
+        let(:raw_value) { "this-is-a-test-\\$(pwd)-and-$(date)" }
+
+        it { expect(input.to_command_option).to eq("input = \"this-is-a-test-$(pwd)-and-#{`date`.chomp}\"") }
+      end
+
       context "all other values" do
         it { expect(input.to_command_option).to eq("input = \"test\"") }
       end

spec/core/entities/state_store_spec.rb

@@ -35,6 +35,18 @@ module Covalence
         expect(S3).to receive(:get_state_store).with({"name" => "example/state_store"})
         state_store.get_config
       end
+
+      it "does return a state configuration" do
+        state_store = Fabricate(:state_store, params: { bucket: "test", name: "example/state_store", foo: "bar" })
+
+        expect(state_store.get_config).to eq("terraform {\n  backend \"s3\" {\n    key = \"example/state_store/terraform.tfstate\"\n    bucket = \"test\"\n    foo = \"bar\"\n  }\n}\n")
+      end
+
+      it "does process shell interpolations" do
+        state_store = Fabricate(:state_store, params: { bucket: "test", name: "example/state_store", path: "$(pwd)" })
+
+        expect(state_store.get_config).to eq("terraform {\n  backend \"s3\" {\n    key = \"example/state_store/terraform.tfstate\"\n    bucket = \"test\"\n    path = \"#{`pwd`.chomp}\"\n  }\n}\n")
+      end
     end
 
     describe '#params' do

spec/rake/environment_tasks_spec.rb

@@ -387,7 +387,7 @@ module Covalence
           "-input=false",
           "-no-color",
           "-target=\"module.az0\"",
-          "-force",
+          "-auto-approve=true",
           "-var-file=covalence-inputs.tfvars"
         )))
         subject.invoke
@@ -639,7 +639,7 @@ module Covalence
       it "executes a destroy" do
         expect(TerraformCli).to receive(:terraform_destroy).with(hash_including(args: [
           "-input=false",
-          "-force",
+          "-auto-approve=true",
           "-no-color",
           "-target=\"module.az1\"",
           "-target=\"module.common.aws_eip.myapp\"",
@@ -898,7 +898,7 @@ module Covalence
       it "executes a destroy" do
         expect(TerraformCli).to receive(:terraform_destroy).with(hash_including(args: [
           "-input=false",
-          "-force",
+          "-auto-approve=true",
           "-var-file=covalence-inputs.tfvars"
         ]))
         subject.invoke