Allow lookup of hiera keys #264

.rubocop_todo.yml

@@ -316,11 +316,6 @@ Style/BlockDelimiters:
 Style/ClassAndModuleChildren:
   Exclude:
     - 'test/channels/application_cable/connection_test.rb'
-    - 'test/models/hiera_data/config_test.rb'
-    - 'test/models/hiera_data/data_file_test.rb'
-    - 'test/models/hiera_data/git_repo_test.rb'
-    - 'test/models/hiera_data/interpolation_test.rb'
-    - 'test/models/hiera_data/yaml_file_test.rb'
 
 # Offense count: 1
 # This cop supports safe auto-correction (--auto-correct).

Gemfile

@@ -35,6 +35,7 @@ gem 'breadcrumbs_on_rails'
 gem 'cancancan'
 gem 'ruby-saml'
 gem 'diffy'
+gem 'deep_merge', require: "deep_merge/core"
 
 # To use retry middleware with Faraday v2.0+
 gem 'faraday-retry'

Gemfile.lock

@@ -483,6 +483,7 @@ DEPENDENCIES
   capybara (>= 2.15)
   dartsass-rails
   dartsass-sprockets
+  deep_merge
   diffy
   factory_bot_rails
   faker

app/models/hiera_data.rb

@@ -124,15 +124,23 @@ def encrypt_value(hierarchy_name, value)
     token.to_encrypted format: :string
   end
 
-  def lookup_options(facts)
-    result = {}
-    config.hierarchies.each do |hierarchy|
-      hierarchy.resolved_paths(facts:).each do |path|
-        file = DataFile.new(path: hierarchy.datadir(facts:).join(path))
-        result = (file["lookup_options"] || {}).merge(result)
-      end
+  def lookup_options_for(key, facts: {})
+    candidates = lookup_for(facts)
+                 .lookup("lookup_options", merge_strategy: :hash)
+    merge = extract_merge_value(key, candidates)
+    case merge
+    when String
+      merge
+    when Hash
+      merge["strategy"]
+    else
+      "first"
     end
-    result
+  end
+
+  def lookup(key, facts: {})
+    merge_strategy = lookup_options_for(key, facts:).to_sym
+    lookup_for(facts).lookup(key, merge_strategy:)
   end
 
   private
@@ -145,5 +153,26 @@ def config
   def find_hierarchy(name)
     config.hierarchies.find { |h| h.name == name }
   end
+
+  def lookup_for(facts)
+    @cached_lookups ||= {}
+    @cached_lookups[facts] ||= begin
+      hashes = config.hierarchies.flat_map do |hierarchy|
+        hierarchy.resolved_paths(facts:).map do |path|
+          DataFile.new(path: hierarchy.datadir(facts:).join(path))
+                  .content
+        end
+      end
+      Lookup.new(hashes.compact)
+    end
+  end
+
+  def extract_merge_value(key, candidates)
+    result = candidates&.dig(key)
+    result ||= candidates&.select { |k, _v| k[0] == "^" }
+                         &.find { |k, _v| key.match(Regexp.new(k)) }
+                         &.last
+    result&.dig("merge")
+  end
 end
 # rubocop:enable Metrics/ClassLength

app/models/hiera_data/data_file.rb

@@ -2,7 +2,8 @@ class HieraData
   class DataFile
     attr_reader :path, :file
 
-    delegate :exist?, :writable?, :keys, :content_for_key, :[],
+    delegate :content, :exist?, :writable?, :keys,
+      :content_for_key, :[],
       to: :file
 
     def initialize(path:, facts: {}, type: :yaml)

app/models/hiera_data/lookup.rb

@@ -0,0 +1,60 @@
+class HieraData
+  class Lookup
+    def initialize(hashes)
+      @hashes = hashes
+    end
+
+    def lookup(key, merge_strategy: :first)
+      applicable_values = extract_key_from_hashes(key)
+      merge_values(applicable_values, merge_strategy)
+    end
+
+    private
+
+    def extract_key_from_hashes(key)
+      @hashes
+        .select { |h| h.key?(key) }
+        .pluck(key)
+    end
+
+    def merge_values(values, strategy)
+      case strategy
+      when :first
+        values.first
+      when :unique
+        unique_array_merge(values)
+      when :hash
+        flat_hash_merge(values)
+      when :deep
+        deep_merge(values)
+      end
+    end
+
+    def unique_array_merge(values)
+      values.inject do |memo, value|
+        memo ||= []
+        raise Hdm::Error, "Merge strategy `unique` is not applicable to a hash." if value.is_a?(Hash)
+
+        memo + Array(value)
+      end.uniq
+    end
+
+    def flat_hash_merge(values)
+      values.inject do |memo, value|
+        memo ||= {}
+        raise Hdm::Error, "Merge strategy `hash` can only be used with hashes." unless value.is_a?(Hash)
+
+        value.merge(memo)
+      end
+    end
+
+    def deep_merge(values)
+      values.inject do |memo, value|
+        memo ||= {}
+        raise Hdm::Error, "Merge strategy `deep` can only be used with hashes and arrays." unless value.is_a?(Hash) || value.is_a?(Array)
+
+        DeepMerge.deep_merge!(memo, value, merge_hash_arrays: true)
+      end
+    end
+  end
+end

app/models/key.rb

@@ -39,21 +39,6 @@ def to_s
   private
 
   def load_lookup_options(node)
-    lookup_option_candidates = hiera_data.lookup_options(node.facts)
-    result = lookup_option_candidates.find do |key, options|
-      key == name
-    end
-    result ||= lookup_option_candidates.find do |key, options|
-      name.match(Regexp.new("\\A#{key}\\z"))
-    end
-    merge = result&.last&.dig("merge")
-    case merge
-    when String
-      merge
-    when Hash
-      merge["stategy"]
-    else
-      "first"
-    end
+    hiera_data.lookup_options_for(name, facts: node.facts)
   end
 end

test/fixtures/files/puppet/environments/lookup_tests/data/common.yaml

@@ -0,0 +1,29 @@
+---
+hdm_integer: 93
+hdm_unique_array:
+  - common
+hdm_duplicates_in_array:
+  - node
+  - zone
+  - common
+hdm_nested_array:
+  - from_common
+  - - nested
+    - list
+hdm_simple_hash:
+  origin: common
+  added_by_common: common
+hdm_nested_hash:
+  origin: common
+  nested_hash:
+    integer: 32
+    added_by_common: common
+lookup_options:
+  '^hdm_duplicates':
+    merge: hash
+  hdm_duplicates_in_array:
+    merge: unique
+  hash_syntax:
+    merge:
+      strategy: deep
+

test/fixtures/files/puppet/environments/lookup_tests/data/nodes/lookup.betadots.training.yaml

@@ -0,0 +1,16 @@
+---
+hdm_integer: 23
+hdm_unique_array:
+  - node
+hdm_duplicates_in_array:
+  - node
+hdm_nested_array:
+  - duplicate 
+  - nested_hash:
+      node_integer: 2
+hdm_simple_hash:
+  origin: node
+hdm_nested_hash:
+  origin: node
+  nested_hash:
+    integer: 5

test/fixtures/files/puppet/environments/lookup_tests/data/role/test_role.yaml

@@ -0,0 +1,18 @@
+---
+hdm_integer: 42
+hdm_string: "role"
+hdm_unique_array:
+  - role
+hdm_duplicates_in_array:
+  - node
+  - role
+hdm_nested_array:
+  - from_role
+  - nested_hash:
+      role_integer: 2
+hdm_simple_hash:
+  origin: role
+  added_by_role: role
+hdm_nested_hash:
+  nested_hash:
+    added_by_role: role

test/fixtures/files/puppet/environments/lookup_tests/data/zone/test_zone.yaml

@@ -0,0 +1,15 @@
+---
+hdm_unique_array:
+  - zone
+hdm_duplicates_in_array:
+  - node
+  - zone
+hdm_nested_array:
+  - duplicate 
+hdm_simple_hash:
+  added_by_zone: zone
+hdm_nested_hash:
+  origin: zone
+  nested_hash:
+    integer: 44
+    added_by_zone: zone

test/fixtures/files/puppet/environments/lookup_tests/hiera.yaml

@@ -0,0 +1,16 @@
+version: 5
+defaults:
+  datadir: data
+  data_hash: yaml_data
+
+hierarchy:
+  - name: "Host specific"
+    path: "nodes/%{::facts.fqdn}.yaml"
+
+  - name: "Role / zone specific" # Uses custom facts.
+    paths:
+      - "role/%{::facts.role}.yaml"
+      - "zone/%{::facts.zone}.yaml"
+
+  - name: "Global data"
+    path: "common.yaml"

test/fixtures/files/puppet/nodes/lookup.betadots.training_facts.yaml

@@ -0,0 +1,5 @@
+---
+fqdn: 'lookup.betadots.training'
+role: 'test_role'
+environment: 'lookup_tests'
+zone: 'test_zone'

test/models/environment_test.rb

@@ -9,6 +9,7 @@ class EnvironmentTest < ActiveSupport::TestCase
       eyaml
       globs
       hdm
+      lookup_tests
       minimal
       multiple_hierarchies
       no_config