From ea7c17537990d72385878880426bf82b09fdb4bb Mon Sep 17 00:00:00 2001 From: mtremmel Date: Fri, 26 Feb 2021 15:28:34 -0500 Subject: [PATCH] -Update stat file importing to include additional output when reading statfile -Update ahf_trees to use the raw finder_id values from tree data files -Perform correct halo object initialization in manager.py -If unset, make halo_number equal to the catalog index rather than finder_id -Update crosslink to utilize catalog_index rather than finder_id -Update property importer to expect correct number if outputs from stat file reader -Update object cache to use catalog_index rather than finder_id --- tangos/input_handlers/ahf_trees.py | 6 +++--- .../input_handlers/halo_stat_files/__init__.py | 2 +- tangos/scripts/manager.py | 2 +- tangos/tools/add_simulation.py | 2 +- tangos/tools/crosslink.py | 16 ++++++++-------- tangos/tools/property_importer.py | 2 +- tangos/util/timestep_object_cache.py | 6 +++--- 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/tangos/input_handlers/ahf_trees.py b/tangos/input_handlers/ahf_trees.py index 33596d11..1fb497b9 100644 --- a/tangos/input_handlers/ahf_trees.py +++ b/tangos/input_handlers/ahf_trees.py @@ -59,10 +59,10 @@ def _load_mtree_file_standard(self): while i < len(data): for j in range(data[i][2]): idx = i+1+j - if data[idx][1]+1 in self._fid: # check if this halo was loaded in case a minimum number of particles different to AHF was used to load halos into DB + if data[idx][1] in self._fid: # check if this halo was loaded in case a minimum number of particles different to AHF was used to load halos into DB # keep in mind finder id and AHF id have an offset of 1 - results['id_desc'] = np.append(results['id_desc'],data[i][0]+1) - results['id_this'] = np.append(results['id_this'],data[idx][1]+1) + results['id_desc'] = np.append(results['id_desc'],data[i][0]) + results['id_this'] = np.append(results['id_this'],data[idx][1]) results['f_share'] = np.append(results['f_share'], float(data[idx][0] * data[idx][0]) / (data[i][1] * data[idx][2]) ) i += data[i][2] + 1 diff --git a/tangos/input_handlers/halo_stat_files/__init__.py b/tangos/input_handlers/halo_stat_files/__init__.py index 1345a058..4d85f28c 100644 --- a/tangos/input_handlers/halo_stat_files/__init__.py +++ b/tangos/input_handlers/halo_stat_files/__init__.py @@ -102,7 +102,7 @@ def iter_rows(self, *args): values = [raw_values[0], raw_values[1]] for arg in args: if arg in self._column_translations: - values.append(self._column_translations[arg](raw_args, raw_values[1:])) + values.append(self._column_translations[arg](raw_args, raw_values[2:])) else: values.append(raw_values[2:][raw_args.index(arg)]) yield values diff --git a/tangos/scripts/manager.py b/tangos/scripts/manager.py index 659f760b..9dbff76e 100755 --- a/tangos/scripts/manager.py +++ b/tangos/scripts/manager.py @@ -109,7 +109,7 @@ def _db_import_export(target_session, from_session, *sims): logger.info("Transferring objects for %s", ts_ext) for h_ext in ts_ext.objects: - h = Halo(ts, h_ext.halo_number, h_ext.finder_id, h_ext.NDM, + h = Halo(ts, h_ext.halo_number, h_ext.finder_id, h_ext.catalog_index, h_ext.NDM, h_ext.NStar, h_ext.NGas, h_ext.object_typecode) h.external_id = h_ext.id halos_this_ts.append(h) diff --git a/tangos/tools/add_simulation.py b/tangos/tools/add_simulation.py index 75ac837f..d9328743 100644 --- a/tangos/tools/add_simulation.py +++ b/tangos/tools/add_simulation.py @@ -111,7 +111,7 @@ def add_objects_to_timestep(self, ts, create_class=core.halo.Halo): enumerator(ts.extension, object_typetag=create_class.tag, min_halo_particles=self.min_halo_particles)): if database_number is None: - database_number = finder_id + database_number = catalog_id if (NDM+Nstar+Ngas >= self.min_halo_particles or NDM==0) \ and (self.max_num_objects is None or database_number<=self.max_num_objects ): diff --git a/tangos/tools/crosslink.py b/tangos/tools/crosslink.py index 7cffb85d..515f51b4 100644 --- a/tangos/tools/crosslink.py +++ b/tangos/tools/crosslink.py @@ -56,7 +56,7 @@ def _generate_timestep_pairs(self): raise NotImplementedError("No implementation found for generating the timestep pairs") def get_halo_entry(self, ts, halo_number): - h = ts.halos.filter_by(finder_id=halo_number).first() + h = ts.halos.filter_by(halo_number=halo_number).first() return h def need_crosslink_ts(self, ts1, ts2, object_typecode=0): @@ -85,13 +85,13 @@ def need_crosslink_ts(self, ts1, ts2, object_typecode=0): return False return True - def create_db_objects_from_catalog(self, cat, finder_id_to_halos_1, finder_id_to_halos_2, same_d_id): + def create_db_objects_from_catalog(self, cat, catalog_index_to_halos_1, catalog_index_to_halos_2, same_d_id): items = [] missing_db_object = 0 for i, possibilities in enumerate(cat): - h1 = finder_id_to_halos_1.get(i, None) + h1 = catalog_index_to_halos_1.get(i, None) for cat_i, weight in possibilities: - h2 = finder_id_to_halos_2.get(cat_i, None) + h2 = catalog_index_to_halos_2.get(cat_i, None) if h1 is not None and h2 is not None: items.append(core.halo_data.HaloLink(h1, h2, same_d_id, weight)) @@ -103,9 +103,9 @@ def create_db_objects_from_catalog(self, cat, finder_id_to_halos_1, finder_id_to missing_db_object) return items - def make_finder_id_to_halo_map(self, ts, object_typecode): + def make_catalog_index_to_halo_map(self, ts, object_typecode): halos = ts.objects.filter_by(object_typecode=object_typecode).all() - halos_map = {h.finder_id: h for h in halos} + halos_map = {h.catalog_index: h for h in halos} return halos_map def crosslink_ts(self, ts1, ts2, halo_min=0, halo_max=None, dmonly=False, threshold=config.default_linking_threshold, object_typecode=0): @@ -114,8 +114,8 @@ def crosslink_ts(self, ts1, ts2, halo_min=0, halo_max=None, dmonly=False, thresh :type ts1 tangos.core.TimeStep :type ts2 tangos.core.TimeStep""" logger.info("Gathering halo information for %r and %r", ts1, ts2) - halos1 = self.make_finder_id_to_halo_map(ts1, object_typecode) - halos2 = self.make_finder_id_to_halo_map(ts2, object_typecode) + halos1 = self.make_catalog_index_to_halo_map(ts1, object_typecode) + halos2 = self.make_catalog_index_to_halo_map(ts2, object_typecode) with parallel_tasks.ExclusiveLock("create_db_objects_from_catalog"): same_d_id = core.dictionary.get_or_create_dictionary_item(self.session, "ptcls_in_common") diff --git a/tangos/tools/property_importer.py b/tangos/tools/property_importer.py index fb5e35cc..cdb30b60 100644 --- a/tangos/tools/property_importer.py +++ b/tangos/tools/property_importer.py @@ -89,7 +89,7 @@ def _import_properties_for_timestep(self, ts, property_names, object_typetag): for values in self.handler.iterate_object_properties_for_timestep(ts.extension, object_typetag, property_names): db_object = self._object_cache.resolve(values[0], object_typetag) if db_object is not None: - for db_name, value in zip(property_db_names, values[1:]): + for db_name, value in zip(property_db_names, values[2:]): rows_to_store+=self._create_properties(db_name, db_object, value) diff --git a/tangos/util/timestep_object_cache.py b/tangos/util/timestep_object_cache.py index f15ab02b..91f2d3bc 100644 --- a/tangos/util/timestep_object_cache.py +++ b/tangos/util/timestep_object_cache.py @@ -18,15 +18,15 @@ def _initialise_cache(self): typetag = obj.object_typetag_from_code(obj.object_typecode) if typetag not in self._map: self._map[typetag] = {} - self._map[typetag][obj.finder_id] = obj + self._map[typetag][obj.catalog_index] = obj def _ensure_cache(self): if not hasattr(self, "_map"): self._initialise_cache() - def resolve(self, finder_id, typetag): + def resolve(self, catalog_index, typetag): self._ensure_cache() try: - return self._map[typetag][finder_id] + return self._map[typetag][catalog_index] except KeyError: return None \ No newline at end of file