-Update stat file importing to include additional output when reading…

… statfile

-Update ahf_trees to use the raw finder_id values from tree data files
-Perform correct halo object initialization in manager.py
-If unset, make halo_number equal to the catalog index rather than finder_id
-Update crosslink to utilize catalog_index rather than finder_id
-Update property importer to expect correct number if outputs from stat file reader
-Update object cache to use catalog_index rather than finder_id

tangos/input_handlers/ahf_trees.py

@@ -59,10 +59,10 @@ def _load_mtree_file_standard(self):
         while i < len(data):
             for j in range(data[i][2]):
                 idx = i+1+j
-                if data[idx][1]+1 in self._fid: # check if this halo was loaded in case a minimum number of particles different to AHF was used to load halos into DB
+                if data[idx][1] in self._fid: # check if this halo was loaded in case a minimum number of particles different to AHF was used to load halos into DB
                 # keep in mind finder id and AHF id have an offset of 1
-                    results['id_desc'] = np.append(results['id_desc'],data[i][0]+1)
-                    results['id_this'] = np.append(results['id_this'],data[idx][1]+1)
+                    results['id_desc'] = np.append(results['id_desc'],data[i][0])
+                    results['id_this'] = np.append(results['id_this'],data[idx][1])
                     results['f_share'] = np.append(results['f_share'], float(data[idx][0] * data[idx][0]) / (data[i][1] * data[idx][2]) )
             i += data[i][2] + 1
 

tangos/input_handlers/halo_stat_files/__init__.py

@@ -102,7 +102,7 @@ def iter_rows(self, *args):
             values = [raw_values[0], raw_values[1]]
             for arg in args:
                 if arg in self._column_translations:
-                    values.append(self._column_translations[arg](raw_args, raw_values[1:]))
+                    values.append(self._column_translations[arg](raw_args, raw_values[2:]))
                 else:
                     values.append(raw_values[2:][raw_args.index(arg)])
             yield values

tangos/scripts/manager.py

@@ -109,7 +109,7 @@ def _db_import_export(target_session, from_session, *sims):
 
             logger.info("Transferring objects for %s", ts_ext)
             for h_ext in ts_ext.objects:
-                h = Halo(ts, h_ext.halo_number, h_ext.finder_id, h_ext.NDM,
+                h = Halo(ts, h_ext.halo_number, h_ext.finder_id, h_ext.catalog_index, h_ext.NDM,
                          h_ext.NStar, h_ext.NGas, h_ext.object_typecode)
                 h.external_id = h_ext.id
                 halos_this_ts.append(h)

tangos/tools/add_simulation.py

@@ -111,7 +111,7 @@ def add_objects_to_timestep(self, ts, create_class=core.halo.Halo):
                                                                  enumerator(ts.extension, object_typetag=create_class.tag,
                                                                             min_halo_particles=self.min_halo_particles)):
             if database_number is None:
-                database_number = finder_id
+                database_number = catalog_id
 
             if (NDM+Nstar+Ngas >= self.min_halo_particles or NDM==0) \
                     and (self.max_num_objects is None or database_number<=self.max_num_objects ):

tangos/tools/crosslink.py

@@ -56,7 +56,7 @@ def _generate_timestep_pairs(self):
         raise NotImplementedError("No implementation found for generating the timestep pairs")
 
     def get_halo_entry(self, ts, halo_number):
-        h = ts.halos.filter_by(finder_id=halo_number).first()
+        h = ts.halos.filter_by(halo_number=halo_number).first()
         return h
 
     def need_crosslink_ts(self, ts1, ts2, object_typecode=0):
@@ -85,13 +85,13 @@ def need_crosslink_ts(self, ts1, ts2, object_typecode=0):
             return False
         return True
 
-    def create_db_objects_from_catalog(self, cat, finder_id_to_halos_1, finder_id_to_halos_2, same_d_id):
+    def create_db_objects_from_catalog(self, cat, catalog_index_to_halos_1, catalog_index_to_halos_2, same_d_id):
         items = []
         missing_db_object = 0
         for i, possibilities in enumerate(cat):
-            h1 = finder_id_to_halos_1.get(i, None)
+            h1 = catalog_index_to_halos_1.get(i, None)
             for cat_i, weight in possibilities:
-                h2 = finder_id_to_halos_2.get(cat_i, None)
+                h2 = catalog_index_to_halos_2.get(cat_i, None)
 
                 if h1 is not None and h2 is not None:
                     items.append(core.halo_data.HaloLink(h1, h2, same_d_id, weight))
@@ -103,9 +103,9 @@ def create_db_objects_from_catalog(self, cat, finder_id_to_halos_1, finder_id_to
                         missing_db_object)
         return items
 
-    def make_finder_id_to_halo_map(self, ts, object_typecode):
+    def make_catalog_index_to_halo_map(self, ts, object_typecode):
         halos = ts.objects.filter_by(object_typecode=object_typecode).all()
-        halos_map = {h.finder_id: h for h in halos}
+        halos_map = {h.catalog_index: h for h in halos}
         return halos_map
 
     def crosslink_ts(self, ts1, ts2, halo_min=0, halo_max=None, dmonly=False, threshold=config.default_linking_threshold, object_typecode=0):
@@ -114,8 +114,8 @@ def crosslink_ts(self, ts1, ts2, halo_min=0, halo_max=None, dmonly=False, thresh
         :type ts1 tangos.core.TimeStep
         :type ts2 tangos.core.TimeStep"""
         logger.info("Gathering halo information for %r and %r", ts1, ts2)
-        halos1 = self.make_finder_id_to_halo_map(ts1, object_typecode)
-        halos2 = self.make_finder_id_to_halo_map(ts2, object_typecode)
+        halos1 = self.make_catalog_index_to_halo_map(ts1, object_typecode)
+        halos2 = self.make_catalog_index_to_halo_map(ts2, object_typecode)
 
         with parallel_tasks.ExclusiveLock("create_db_objects_from_catalog"):
             same_d_id = core.dictionary.get_or_create_dictionary_item(self.session, "ptcls_in_common")

tangos/tools/property_importer.py

@@ -89,7 +89,7 @@ def _import_properties_for_timestep(self, ts, property_names, object_typetag):
         for values in self.handler.iterate_object_properties_for_timestep(ts.extension, object_typetag, property_names):
             db_object = self._object_cache.resolve(values[0], object_typetag)
             if db_object is not None:
-                for db_name, value in zip(property_db_names, values[1:]):
+                for db_name, value in zip(property_db_names, values[2:]):
                     rows_to_store+=self._create_properties(db_name, db_object, value)
 
 

tangos/util/timestep_object_cache.py

@@ -18,15 +18,15 @@ def _initialise_cache(self):
             typetag = obj.object_typetag_from_code(obj.object_typecode)
             if typetag not in self._map:
                 self._map[typetag] = {}
-            self._map[typetag][obj.finder_id] = obj
+            self._map[typetag][obj.catalog_index] = obj
 
     def _ensure_cache(self):
         if not hasattr(self, "_map"):
             self._initialise_cache()
 
-    def resolve(self, finder_id, typetag):
+    def resolve(self, catalog_index, typetag):
         self._ensure_cache()
         try:
-            return self._map[typetag][finder_id]
+            return self._map[typetag][catalog_index]
         except KeyError:
             return None