Merge pull request #59 from unifio/workspace-support

Terraform workspace support

.circleci/config.yml

@@ -8,7 +8,7 @@ jobs:
       - image: circleci/ruby:2.3.6
 
     environment:
-      COVALENCE_VERSION: 0.7.7
+      COVALENCE_VERSION: 0.7.8
       TERRAFORM_VERSION: 0.11.7
 
     steps:

CHANGELOG.md

@@ -1,6 +1,11 @@
 ## Unreleased
-* Add support for Terraform workspaces
 * Add ability to toggle primary state store
+* Add environment variable interpolation for input variables
+* Add support for sops encryption / decryption of hierarchy data
+
+## 0.7.8 (May 9, 2018)
+IMPROVEMENTS:
+- Added support for Terraform workspaces
 
 ## 0.7.7 (May 6, 2018)
 BACKWARDS INCOMPATIBILITIES:
@@ -13,6 +18,7 @@ IMPROVEMENTS:
 
 FIXES:
 - Updated input processing to support nested complex types properly.
+- Updated input processing to properly handle non-string values for non-complex input types.
 
 ## 0.7.6 (December 6, 2017)
 IMPROVEMENTS:

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    covalence (0.7.7)
+    covalence (0.7.8)
       activemodel (~> 4.2.6)
       activesupport (~> 4.2.6)
       aws-sdk (~> 2.9.5)
@@ -18,25 +18,25 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (4.2.8)
-      activesupport (= 4.2.8)
+    activemodel (4.2.10)
+      activesupport (= 4.2.10)
       builder (~> 3.1)
-    activesupport (4.2.8)
+    activesupport (4.2.10)
       i18n (~> 0.7)
       minitest (~> 5.1)
       thread_safe (~> 0.3, >= 0.3.4)
       tzinfo (~> 1.1)
-    addressable (2.5.1)
-      public_suffix (~> 2.0, >= 2.0.2)
+    addressable (2.5.2)
+      public_suffix (>= 2.0.2, < 4.0)
     awesome_print (1.7.0)
-    aws-sdk (2.9.18)
-      aws-sdk-resources (= 2.9.18)
-    aws-sdk-core (2.9.18)
+    aws-sdk (2.9.44)
+      aws-sdk-resources (= 2.9.44)
+    aws-sdk-core (2.9.44)
       aws-sigv4 (~> 1.0)
       jmespath (~> 1.0)
-    aws-sdk-resources (2.9.18)
-      aws-sdk-core (= 2.9.18)
-    aws-sigv4 (1.0.0)
+    aws-sdk-resources (2.9.44)
+      aws-sdk-core (= 2.9.44)
+    aws-sigv4 (1.0.2)
     axiom-types (0.1.1)
       descendants_tracker (~> 0.0.4)
       ice_nine (~> 0.11.0)
@@ -50,14 +50,15 @@ GEM
       rspec (>= 2.14, < 4)
     coercible (1.0.0)
       descendants_tracker (~> 0.0.1)
+    concurrent-ruby (1.0.5)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
     deep_merge (1.0.1)
     descendants_tracker (0.0.4)
       thread_safe (~> 0.3, >= 0.3.1)
     diff-lcs (1.3)
     docile (1.1.5)
-    domain_name (0.5.20170404)
+    domain_name (0.5.20180417)
       unf (>= 0.0.5, < 1.0.0)
     dotenv (2.1.2)
     equalizer (0.0.11)
@@ -70,48 +71,49 @@ GEM
       multi_json (~> 1.10)
       netrc (~> 0.10.0)
       thor (>= 0.14.0, < 1.0.0.pre)
-    hashdiff (0.3.4)
-    hiera (3.3.1)
+    hashdiff (0.3.7)
+    hiera (3.3.3)
     highline (1.6.21)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
-    i18n (0.8.1)
+    i18n (0.9.5)
+      concurrent-ruby (~> 1.0)
     ice_nine (0.11.2)
-    jmespath (1.3.1)
+    jmespath (1.4.0)
     json (1.8.6)
     mime-types (3.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2016.0521)
-    minitest (5.10.2)
-    multi_json (1.12.1)
+    minitest (5.11.3)
+    multi_json (1.13.1)
     multipart-post (2.0.0)
     net-scp (1.2.1)
       net-ssh (>= 2.6.5)
-    net-ssh (4.1.0)
+    net-ssh (4.2.0)
     net-telnet (0.1.1)
     netrc (0.10.3)
-    public_suffix (2.0.5)
-    rake (12.0.0)
+    public_suffix (3.0.2)
+    rake (12.3.1)
     rest-client (2.0.2)
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
-    rspec (3.6.0)
-      rspec-core (~> 3.6.0)
-      rspec-expectations (~> 3.6.0)
-      rspec-mocks (~> 3.6.0)
-    rspec-core (3.6.0)
-      rspec-support (~> 3.6.0)
-    rspec-expectations (3.6.0)
+    rspec (3.7.0)
+      rspec-core (~> 3.7.0)
+      rspec-expectations (~> 3.7.0)
+      rspec-mocks (~> 3.7.0)
+    rspec-core (3.7.1)
+      rspec-support (~> 3.7.0)
+    rspec-expectations (3.7.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.6.0)
+      rspec-support (~> 3.7.0)
     rspec-its (1.2.0)
       rspec-core (>= 3.0.0)
       rspec-expectations (>= 3.0.0)
-    rspec-mocks (3.6.0)
+    rspec-mocks (3.7.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.6.0)
-    rspec-support (3.6.0)
+      rspec-support (~> 3.7.0)
+    rspec-support (3.7.1)
     safe_yaml (1.0.4)
     semantic (1.4.1)
     serverspec (2.36.1)
@@ -124,20 +126,20 @@ GEM
       docile (~> 1.1.0)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
-    simplecov-html (0.10.0)
+    simplecov-html (0.10.2)
     slop (4.4.3)
-    specinfra (2.67.10)
+    specinfra (2.73.3)
       net-scp
       net-ssh (>= 2.7, < 5.0)
       net-telnet
       sfl
-    thor (0.19.4)
+    thor (0.20.0)
     thread_safe (0.3.6)
-    tzinfo (1.2.3)
+    tzinfo (1.2.5)
       thread_safe (~> 0.1)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.4)
+    unf_ext (0.0.7.5)
     virtus (1.0.5)
       axiom-types (~> 0.1)
       coercible (~> 1.0)
@@ -166,4 +168,4 @@ DEPENDENCIES
   webmock (~> 2.0.3)
 
 BUNDLED WITH
-   1.13.6
+   1.16.1

README.md

@@ -190,6 +190,20 @@ vpc::state:                                           # Terraform backend config
       address: 'consul.example.com:8500'
       name: "%{environment}/%{stack}"
 
+# Workspace
+vpc::workspace: 'blue'                                # Terraform workspace configuration. The key is
+                                                      # prepended with the stack name, as the backend
+                                                      # configuration is stack specific.
+
+## Dependencies
+vpc::deps:                                            # List of paths to files or directories outside
+                                                      # of the module directory that are to be copied
+                                                      # into the working directory of the module during
+                                                      # Covalence execution. An example of this would be
+                                                      # an SSH key for cloning a private Terraform
+                                                      # module.
+  - '.ssh'
+
 # Execution targets
 terraform::vpc::targets:                              # Resource targeting. The key is prepended with
                                                       # the module name, as target assignment can be

lib/covalence/core/cli_wrappers/terraform.yml

@@ -29,6 +29,11 @@ commands:
     #rm:
     #show:
   #taint:
-  #untaint:
   validate:
+  #untaint:
+  workspace:
+    list:
+    select:
+    new:
+    delete:
   version:

lib/covalence/core/cli_wrappers/terraform_cli.rb

@@ -55,6 +55,18 @@ def self.terraform_init(path='', args: '', ignore_exitcode: false)
       (output == 0)
     end
 
+    def self.terraform_workspace(workspace, path='', args: '', ignore_exitcode: false)
+      cmd = [Covalence::TERRAFORM_CMD, "workspace", "new", workspace]
+
+      output = PopenWrapper.run(
+        cmd,
+        path,
+        args,
+        ignore_exitcode: ignore_exitcode)
+
+      (output == 0)
+    end
+
     def self.terraform_output(output_var, args: '')
       raise "TODO: implement me"
     end

lib/covalence/core/entities/stack.rb

@@ -22,6 +22,7 @@ class Stack
     attribute :contexts, Array[Context]
     attribute :inputs, Hash[String => Input]
     attribute :args, String
+    attribute :workspace, String
 
     validates! :type, inclusion: {
       in: %w(terraform packer)

lib/covalence/core/entities/state_store.rb

@@ -15,6 +15,7 @@ class StateStore
 
     attribute :params, Hash, :writer => :private
     attribute :backend, Object, :writer => :private
+    attribute :workspace_enabled, Boolean
 
     validate :validate_params_has_name,
       :backend_has_state_store
@@ -40,7 +41,7 @@ def backend=(backend_name)
     end
 
     def get_config
-      backend::get_state_store(@params)
+      backend::get_state_store(@params, @workspace_enabled)
     end
 
     private

lib/covalence/core/repositories/stack_repository.rb

@@ -38,7 +38,8 @@ def self.populate(data_store, stack)
 
       stack.dependencies = lookup_dependencies(stack_data_store, stack.name)
       stack.packer_template = lookup_packer_template(stack_data_store, stack.name)
-      stack.state_stores = StateStoreRepository.query_by_stack_name(stack_data_store, stack.name, stack.type)
+      stack.workspace = lookup_workspace(stack_data_store, stack.name)
+      stack.state_stores = StateStoreRepository.query_by_stack_name(stack_data_store, stack.name, stack.workspace, stack.type)
       stack.contexts = ContextRepository.query_by_namespace(stack_data_store, shared_namespace, stack.type)
       stack.inputs = InputRepository.query_by_namespace(stack_data_store, shared_namespace, stack.type)
       stack.args = find_args_by_namespace(stack_data_store, shared_namespace)
@@ -58,6 +59,12 @@ def lookup_shared_namespace(data_store, stack_name)
         data_store.lookup("#{stack_name}::module", stack_name)
       end
 
+      def lookup_workspace(data_store, stack_name)
+        wrkspc = data_store.lookup("#{stack_name}::workspace", "")
+        wrkspc = Covalence::Helpers::ShellInterpolation.parse_shell(wrkspc) if wrkspc.to_s.include?("$(")
+        return wrkspc
+      end
+
       # maybe arg_string instead of args
       def find_args_by_namespace(data_store, namespace)
         data_store.lookup("#{namespace}::args", "")

lib/covalence/core/repositories/state_store_repository.rb

@@ -5,23 +5,24 @@ module Covalence
   # todo: monitor behavior forking to determine when the split the class
   class StateStoreRepository
     class << self
-      def query_by_stack_name(data_store, stack_name, tool)
+      def query_by_stack_name(data_store, stack_name, stack_workspace, tool)
         if tool == 'terraform'
-          query_tool_by_stack_name(data_store, stack_name)
+          query_tool_by_stack_name(data_store, stack_name, stack_workspace)
         else
           return nil
         end
       end
 
       private
 
-      def query_tool_by_stack_name(data_store, stack_name)
+      def query_tool_by_stack_name(data_store, stack_name, stack_workspace)
         stores = data_store.lookup("#{stack_name}::state", [])
         raise "State store array cannot be empty" if stores.blank?
         stores.map do |store|
           StateStore.new(
             backend: store.keys.first,
-            params: store.values.first
+            params: store.values.first,
+            workspace_enabled: !stack_workspace.empty?
           )
         end
       end

lib/covalence/core/services/terraform_stack_tasks.rb

@@ -54,6 +54,8 @@ def stack_refresh
         Dir.chdir(tmpdir) do
           logger.info "In #{tmpdir}:"
 
+          TerraformCli.terraform_workspace(@stack.workspace) unless stack.workspace.to_s.empty?
+
           stack.materialize_state_inputs
           TerraformCli.terraform_get(@path)
           TerraformCli.terraform_init
@@ -70,6 +72,8 @@ def stack_sync
         Dir.chdir(tmpdir) do
           logger.info "In #{tmpdir}:"
 
+          TerraformCli.terraform_workspace(@stack.workspace) unless stack.workspace.to_s.empty?
+
           stack.materialize_state_inputs
           TerraformCli.terraform_get(@path)
           TerraformCli.terraform_init
@@ -89,6 +93,8 @@ def context_plan(*additional_args)
         Dir.chdir(tmpdir) do
           logger.info "In #{tmpdir}:"
 
+          TerraformCli.terraform_workspace(@stack.workspace) unless stack.workspace.to_s.empty?
+
           stack.materialize_state_inputs
           TerraformCli.terraform_get(@path)
           TerraformCli.terraform_init
@@ -111,6 +117,8 @@ def context_plan_destroy(*additional_args)
         Dir.chdir(tmpdir) do
           logger.info "In #{tmpdir}:"
 
+          TerraformCli.terraform_workspace(@stack.workspace) unless stack.workspace.to_s.empty?
+
           stack.materialize_state_inputs
           TerraformCli.terraform_get(@path)
           TerraformCli.terraform_init
@@ -134,6 +142,8 @@ def context_apply(*additional_args)
         Dir.chdir(tmpdir) do
           logger.info "In #{tmpdir}:"
 
+          TerraformCli.terraform_workspace(@stack.workspace) unless stack.workspace.to_s.empty?
+
           stack.materialize_state_inputs
           TerraformCli.terraform_get(@path)
           TerraformCli.terraform_init
@@ -157,6 +167,8 @@ def context_destroy(*additional_args)
         Dir.chdir(tmpdir) do
           logger.info "In #{tmpdir}:"
 
+          TerraformCli.terraform_workspace(@stack.workspace) unless stack.workspace.to_s.empty?
+
           stack.materialize_state_inputs
           TerraformCli.terraform_get(@path)
           TerraformCli.terraform_init

lib/covalence/core/state_stores/atlas.rb

@@ -86,7 +86,7 @@ def self.get_output(name, stack)
       end
     end
 
-    def self.get_state_store(params)
+    def self.get_state_store(params, workspace_enabled=false)
       raise "State store parameters must be a Hash" unless params.is_a?(Hash)
       raise "Missing 'name' store parameter" unless params.has_key? 'name'
 

lib/covalence/core/state_stores/consul.rb

@@ -83,7 +83,7 @@ def self.get_output(name, stack)
     end
 
     # Return configuration for remote state store.
-    def self.get_state_store(params)
+    def self.get_state_store(params, workspace_enabled=false)
       raise "State store parameters must be a Hash" unless params.is_a?(Hash)
       required_params = [
         'access_token',