Graph refresh skeletal precreate

app/models/manager_refresh/inventory_collection.rb

@@ -73,10 +73,10 @@ class InventoryCollection
     attr_accessor :parent_inventory_collections
 
     attr_reader :model_class, :strategy, :attributes_blacklist, :attributes_whitelist, :custom_save_block, :parent,
-                :internal_attributes, :delete_method, :dependency_attributes, :manager_ref,
+                :internal_attributes, :delete_method, :dependency_attributes, :manager_ref, :create_only,
                 :association, :complete, :update_only, :transitive_dependency_attributes, :check_changed, :arel,
                 :inventory_object_attributes, :name, :saver_strategy, :manager_uuids, :builder_params,
-                :skeletal_manager_uuids, :targeted_arel, :targeted, :manager_ref_allowed_nil, :use_ar_object,
+                :targeted_arel, :targeted, :manager_ref_allowed_nil, :use_ar_object,
                 :created_records, :updated_records, :deleted_records,
                 :custom_reconnect_block, :batch_extra_attributes, :references_storage
 
@@ -107,8 +107,10 @@ class InventoryCollection
              :find_by,
              :lazy_find,
              :lazy_find_by,
+             :named_ref,
              :primary_index,
              :reindex_secondary_indexes!,
+             :skeletal_primary_index,
              :to => :index_proxy
 
     delegate :table_name,
@@ -367,7 +369,7 @@ class InventoryCollection
     def initialize(model_class: nil, manager_ref: nil, association: nil, parent: nil, strategy: nil,
                    custom_save_block: nil, delete_method: nil, dependency_attributes: nil,
                    attributes_blacklist: nil, attributes_whitelist: nil, complete: nil, update_only: nil,
-                   check_changed: nil, arel: nil, builder_params: {},
+                   check_changed: nil, arel: nil, builder_params: {}, create_only: nil,
                    inventory_object_attributes: nil, name: nil, saver_strategy: nil,
                    parent_inventory_collections: nil, manager_uuids: [], all_manager_uuids: nil, targeted_arel: nil,
                    targeted: nil, manager_ref_allowed_nil: nil, secondary_refs: {}, use_ar_object: nil,
@@ -387,6 +389,7 @@ def initialize(model_class: nil, manager_ref: nil, association: nil, parent: nil
       @internal_attributes    = %i(__feedback_edge_set_parent __parent_inventory_collections)
       @complete               = complete.nil? ? true : complete
       @update_only            = update_only.nil? ? false : update_only
+      @create_only            = create_only.nil? ? false : create_only
       @builder_params         = builder_params
       @name                   = name || association || model_class.to_s.demodulize.tableize
       @saver_strategy         = process_saver_strategy(saver_strategy)
@@ -399,7 +402,6 @@ def initialize(model_class: nil, manager_ref: nil, association: nil, parent: nil
       @manager_uuids                = Set.new.merge(manager_uuids)
       @all_manager_uuids            = all_manager_uuids
       @parent_inventory_collections = parent_inventory_collections
-      @skeletal_manager_uuids       = Set.new.merge(manager_uuids)
       @targeted_arel                = targeted_arel
       @targeted                     = !!targeted
 
@@ -489,6 +491,10 @@ def create_allowed?
       !update_only?
     end
 
+    def create_only?
+      create_only
+    end
+
     def saved?
       saved
     end
@@ -497,6 +503,10 @@ def saveable?
       dependencies.all?(&:saved?)
     end
 
+    def parallel_safe?
+      @parallel_safe_cache ||= %i(concurrent_safe concurrent_safe_batch).include?(saver_strategy)
+    end
+
     def data_collection_finalized?
       data_collection_finalized
     end
@@ -517,17 +527,19 @@ def object_index_with_keys(keys, record)
     def noop?
       # If this InventoryCollection doesn't do anything. it can easily happen for targeted/batched strategies.
       if targeted?
-        if parent_inventory_collections.nil? && manager_uuids.blank? && skeletal_manager_uuids.blank? &&
-           all_manager_uuids.nil? && parent_inventory_collections.blank? && custom_save_block.nil?
+        if parent_inventory_collections.nil? && manager_uuids.blank? &&
+           all_manager_uuids.nil? && parent_inventory_collections.blank? && custom_save_block.nil? &&
+           skeletal_primary_index.blank?
           # It's a noop Parent targeted InventoryCollection
           true
-        elsif !parent_inventory_collections.nil? && parent_inventory_collections.all? { |x| x.manager_uuids.blank? }
+        elsif !parent_inventory_collections.nil? && parent_inventory_collections.all? { |x| x.manager_uuids.blank? } &&
+              skeletal_primary_index.blank?
           # It's a noop Child targeted InventoryCollection
           true
         else
           false
         end
-      elsif data.blank? && !delete_allowed?
+      elsif data.blank? && !delete_allowed? && skeletal_primary_index.blank?
         # If we have no data to save and delete is not allowed, we can just skip
         true
       else
@@ -738,12 +750,12 @@ def db_collection_for_comparison
           targeted_arel.call(self)
         elsif manager_ref.count > 1
           # TODO(lsmola) optimize with ApplicationRecordIterator
-          hashes = extract_references(manager_uuids + skeletal_manager_uuids)
+          hashes = extract_references(manager_uuids)
           full_collection_for_comparison.where(build_multi_selection_condition(hashes))
         else
           ManagerRefresh::ApplicationRecordIterator.new(
             :inventory_collection => self,
-            :manager_uuids_set    => (manager_uuids + skeletal_manager_uuids).to_a.flatten.compact
+            :manager_uuids_set    => manager_uuids.to_a.flatten.compact
           )
         end
       else

app/models/manager_refresh/inventory_collection/data_storage.rb

@@ -10,10 +10,11 @@ class DataStorage
 
       delegate :each, :size, :to => :data
 
-      delegate :find,
-               :primary_index,
+      delegate :primary_index,
                :build_primary_index_for,
                :build_secondary_indexes_for,
+               :named_ref,
+               :skeletal_primary_index,
                :to => :index_proxy
 
       delegate :builder_params,
@@ -31,7 +32,7 @@ def initialize(inventory_collection, secondary_refs)
       end
 
       def <<(inventory_object)
-        unless primary_index.find(inventory_object.manager_uuid)
+        if inventory_object.manager_uuid.present? && !primary_index.find(inventory_object.manager_uuid)
           data << inventory_object
 
           # TODO(lsmola) Maybe we do not need the secondary indexes here?
@@ -53,27 +54,40 @@ def find_or_build(manager_uuid)
       end
 
       def find_or_build_by(manager_uuid_hash)
-        if !manager_uuid_hash.keys.all? { |x| manager_ref.include?(x) } || manager_uuid_hash.keys.size != manager_ref.size
-          raise "Allowed find_or_build_by keys are #{manager_ref}"
+        find_in_data(manager_uuid_hash) || build(manager_uuid_hash)
+      end
+
+      def find_in_data(hash)
+        hash = enrich_data(hash)
+
+        if manager_ref.any? { |x| !hash.key?(x) }
+          raise "Needed find_or_build_by keys are: #{manager_ref}, data provided: #{hash}"
         end
 
-        # Not using find by since if could take record from db, then any changes would be ignored, since such record will
-        # not be stored to DB, maybe we should rethink this?
-        primary_index.find(manager_uuid_hash) || build(manager_uuid_hash)
+        uuid = ::ManagerRefresh::InventoryCollection::Reference.build_stringified_reference(hash, named_ref)
+        primary_index.find(uuid)
       end
 
       def build(hash)
-        hash             = builder_params.merge(hash)
-        inventory_object = new_inventory_object(hash)
-
-        uuid = inventory_object.manager_uuid
-        # Each InventoryObject must be able to build an UUID, return nil if it can't
-        return nil if uuid.blank?
-        # Return existing InventoryObject if we have it
-        return primary_index.find(uuid) if primary_index.find(uuid)
+        # We will take existing skeletal record, so we don't duplicate references for saving. We can have duplicated
+        # reference from local_db index, (if we are using .find in parser, that causes N+1 db queries), but that is ok,
+        # since that one is not being saved.
+        uuid = ::ManagerRefresh::InventoryCollection::Reference.build_stringified_reference(hash, named_ref)
+        inventory_object = skeletal_primary_index.delete(uuid)
+
+        # We want to update the skeletal record with actual data
+        inventory_object&.assign_attributes(hash)
+
+        # Build the InventoryObject
+        inventory_object ||= new_inventory_object(enrich_data(hash))
         # Store new InventoryObject and return it
         push(inventory_object)
-        inventory_object
+
+        return inventory_object unless inventory_object.nil?
+
+        # TODO(lsmola) prepare for changing behavior, build will return nil if it can't build or the record is already
+        # there. Maybe we should even make build a private method.
+        find_in_data(enrich_data(hash))
       end
 
       def to_a
@@ -123,6 +137,13 @@ def to_raw_data
           end
         end
       end
+
+      private
+
+      def enrich_data(hash)
+        # This is 25% faster than builder_params.merge(hash)
+        {}.merge!(builder_params).merge!(hash)
+      end
     end
   end
 end

app/models/manager_refresh/inventory_collection/index/proxy.rb

@@ -4,6 +4,8 @@ module Index
       class Proxy
         include Vmdb::Logging
 
+        attr_reader :skeletal_primary_index
+
         def initialize(inventory_collection, secondary_refs = {})
           @inventory_collection = inventory_collection
 
@@ -28,6 +30,13 @@ def initialize(inventory_collection, secondary_refs = {})
               @data_indexes[index_name]
             )
           end
+
+          @skeletal_primary_index = ManagerRefresh::InventoryCollection::Index::Type::Skeletal.new(
+            inventory_collection,
+            :skeletal_primary_index_ref,
+            named_ref,
+            primary_index
+          )
         end
 
         def build_primary_index_for(inventory_object)
@@ -66,9 +75,9 @@ def find(reference, ref: primary_index_ref)
           when :local_db_find_references, :local_db_cache_all
             local_db_index_find(reference)
           when :local_db_find_missing_references
-            data_index_find(reference) || local_db_index_find(reference)
+            find_in_data_or_skeletal_index(reference) || local_db_index_find(reference)
           else
-            data_index_find(reference)
+            find_in_data_or_skeletal_index(reference)
           end
         end
 
@@ -95,19 +104,34 @@ def lazy_find(manager_uuid, ref: primary_index_ref, key: nil, default: nil)
                                                     :ref => ref, :key => key, :default => default)
         end
 
-        def named_ref(ref)
+        def named_ref(ref = primary_index_ref)
           all_refs[ref]
         end
 
         private
 
         delegate :association_to_foreign_key_mapping,
                  :build_stringified_reference,
+                 :parallel_safe?,
                  :strategy,
                  :to => :inventory_collection
 
         attr_reader :all_refs, :data_indexes, :inventory_collection, :primary_ref, :local_db_indexes, :secondary_refs
 
+        def find_in_data_or_skeletal_index(reference)
+          if parallel_safe?
+            # With parallel safe strategies, we create skeletal nodes that we can look for
+            data_index_find(reference) || skeletal_index_find(reference)
+          else
+            data_index_find(reference)
+          end
+        end
+
+        def skeletal_index_find(reference)
+          # Find in skeletal index, but we are able to create skeletal index only for primary indexes
+          skeletal_primary_index.find(reference.stringified_reference) if reference.primary?
+        end
+
         def data_index_find(reference)
           data_index(reference.ref).find(reference.stringified_reference)
         end

app/models/manager_refresh/inventory_collection/index/type/skeletal.rb

@@ -0,0 +1,64 @@
+module ManagerRefresh
+  class InventoryCollection
+    module Index
+      module Type
+        class Skeletal < ManagerRefresh::InventoryCollection::Index::Type::Base
+          def initialize(inventory_collection, index_name, attribute_names, primary_index)
+            super
+
+            @primary_index = primary_index
+          end
+
+          delegate :builder_params,
+                   :new_inventory_object,
+                   :named_ref,
+                   :to => :inventory_collection
+
+          delegate :blank?,
+                   :each,
+                   :to => :index
+
+          # Find value based on index_value
+          #
+          # @param index_value [String] a index_value of the InventoryObject we search for
+          # @return [InventoryObject|nil] Returns found value or nil
+          def find(index_value)
+            index[index_value]
+          end
+
+          # Deletes and returns the value on the index_value
+          #
+          # @param index_value [String] a index_value of the InventoryObject we search for
+          # @return [InventoryObject|nil] Returns found value or nil
+          def delete(index_value)
+            index.delete(index_value)
+          end
+
+          # Builds index record with skeletal InventoryObject and returns it, or returns nil if it's already present
+          # in primary_index or skeletal_primary_index
+          # @param attributes [Hash] Skeletal data of the index, must contain unique index keys and everything else
+          #        needed for creating the record in the Database
+          # @return [InventoryObject|nil] Returns built value or nil
+          def build(attributes)
+            attributes = {}.merge!(builder_params).merge!(attributes)
+
+            # If the primary index is already filled, we don't want populate skeletal index
+            uuid = ::ManagerRefresh::InventoryCollection::Reference.build_stringified_reference(attributes, named_ref)
+            return if primary_index.find(uuid)
+
+            # Return if skeletal index already exists
+            return if index[uuid]
+
+            # We want to populate a new skeletal index
+            inventory_object                     = new_inventory_object(attributes)
+            index[inventory_object.manager_uuid] = inventory_object
+          end
+
+          private
+
+          attr_reader :primary_index
+        end
+      end
+    end
+  end
+end

app/models/manager_refresh/inventory_collection/reference.rb

@@ -3,17 +3,22 @@ class InventoryCollection
     class Reference
       include Vmdb::Logging
 
-      attr_reader :full_reference, :ref, :stringified_reference
+      attr_reader :full_reference, :keys, :ref, :stringified_reference
 
       delegate :[], :to => :full_reference
 
       def initialize(data, ref, keys)
         @full_reference = build_full_reference(data, keys)
         @ref            = ref
+        @keys           = keys
 
         @stringified_reference = self.class.build_stringified_reference(full_reference, keys)
       end
 
+      def primary?
+        ref == :manager_ref
+      end
+
       def to_hash
       end