more units (#96)

* more units

src/scida/configfiles/simulations.yaml

@@ -4,15 +4,15 @@ defs:
     dataset_type:
       series: ArepoSimulation
       dataset: ArepoSnapshot
-    unitfile: units/gadget.yaml
+    unitfile: units/gadget_cosmological.yaml
   tng_props: &tng_props
     code: arepo
     suite: TNG
     tags: ["simulation"]
   illustris_params: &illustris_params
     dataset_type:
       dataset: ArepoSnapshot
-    unitfile: units/gadget.yaml
+    unitfile: units/gadget_cosmological.yaml
 
 data:
   TNG50:
@@ -84,17 +84,24 @@ data:
             content: tngclust
             match: substring
 
+  # Some sims do not have config or header attributes for identification in hdf5 files
 
+  Auriga:
+    dataset_type:
+      series: ArepoSimulation
+      dataset: ArepoSnapshot
+    unitfile: units/auriga.yaml
+    identifiers:
+      name_contains: "AURIGA"  # can only rely on path
 
-
-  # Illustris does not have config or header attributes for identification in hdf5 files
   Illustris:
       <<: *illustris_params
       aliases:
         - Illustris-1
         - Illustris100
       identifiers:
         name_contains: "Illustris-1"  # can only rely on path name in lack of metadata...
+
   Illustris-2:
       <<: *illustris_params
       identifiers:

src/scida/configfiles/units/auriga.yaml

@@ -0,0 +1,37 @@
+metadata_unitsystem: cgs  # this means we have a cgs conversion factor available
+units:
+fields:
+  _all:
+    Coordinates: Mpc
+    Masses: 10^10 Msun
+    Velocities: km / s
+    ParticleIDs: none
+    Softenings: Mpc  # guess
+  PartType0:
+    GFM_Metallicity: none
+    GFM_Metals: none
+    DivergenceOfMagneticField: unknown
+    HighResGasMass: 10^10 Msun  # guess
+  PartType4:
+    AccretedFlag: none
+    BirthPos: Mpc
+    BirthVel: km / s
+    GFM_StellarFormationTime: Gyr
+    GFM_Metallicity: none
+    GFM_Metals: none
+    GFM_StellarPhotometrics: # TODO: mag
+    GravForce: km / s^2 / Mpc
+    GravPotential: (km / s)^2
+  PartType5:
+    BH_HaloVvir: km / s  # guess
+    BH_HsmlDM: Mpc  # guess
+    BH_Mdot_Quasar: Msun / yr  # guess
+    BH_Mdot_Radio: Msun / yr  # guess
+    BH_RadioLuminosity: unknown
+    BH_RhoDM: unknown
+    BH_TimeStep: unknown
+    BH_VelDM: km / s  # guess
+    BH_XrayLuminosity: unknown
+  Group:
+    GroupRadioLuminosity: unknown
+    GroupXrayLuminosity: unknown

src/scida/configfiles/units/eagle.yaml

@@ -1,6 +1,7 @@
 ---
 # we expect a cgs conversion factor within each data set, thus we only specify the cgs units.
 fields:
+  HashTable: "no_units"
   PartType0:
     AExpMaximumTemperature:
       cgsunits:

src/scida/configfiles/units/gadget.yaml → src/scida/configfiles/units/gadget_cosmological.yaml

@@ -55,6 +55,8 @@ fields:
     Temperature: K
     TimebinHydro:
     Volume: code_length^3
+  PartType1_mostbound:
+    Acceleration: unknown  # TODO: From MTNG
   PartType1:
     SubfindDMDensity: code_mass / code_length^3
   PartType3:
@@ -108,6 +110,7 @@ fields:
     GroupGasMetallicity:
     GroupLen: none
     GroupLenType: none
+    GroupLenPrevMostBnd: none
     GroupMass: code_mass
     GroupMassType: code_mass
     GroupNsubs: none
@@ -138,21 +141,25 @@ fields:
     SubhaloBfldHalo: (h / a^2) * code_pressure^(1/2)
     SubhaloCM: code_length
     SubhaloFlag: none
+    SubhaloGasMassSFR: code_mass
     SubhaloGasMetalFractions:
     SubhaloGasMetalFractionsHalfRad:
     SubhaloGasMetalFractionsMaxRad:
     SubhaloGasMetalFractionsSfr:
     SubhaloGasMetalFractionsSfrWeighted:
     SubhaloGasMetallicity:
+    SubhaloGasMetallicityWeighted:  # MTNG alias?
     SubhaloGasMetallicityHalfRad:
     SubhaloGasMetallicityMaxRad:
     SubhaloGasMetallicitySfr:
     SubhaloGasMetallicitySfrWeighted:
     SubhaloGrNr: none
+    SubhaloGroupNr: none  # MTNG alias
     SubhaloHalfmassRad: code_length
     SubhaloHalfmassRadType: code_length
     SubhaloIDMostbound: none
     SubhaloLen: none
+    SubhaloLenPrevMostBnd: none
     SubhaloLenType: none
     SubhaloMass: code_mass
     SubhaloMassInHalfRad: code_mass
@@ -165,11 +172,16 @@ fields:
     SubhaloOffsetType: none
     SubhaloOrigHaloID: none
     SubhaloParent: none
+    SubhaloParentRank: none
     SubhaloPos: code_length
+    SubhaloRankInGr: none
     SubhaloSFR: Msun / yr
     SubhaloSFRinHalfRad: Msun / yr
+    SubhaloSfrInHalfRad: Msun / yr  # MTNG alias
     SubhaloSFRinMaxRad: Msun / yr
+    SubhaloSfrInMaxRad: Msun / yr  # MTNG alias
     SubhaloSFRinRad: Msun / yr
+    SubhaloSfrInRad: Msun / yr  # MTNG alias
     SubhaloSpin: kpc / h * km / s
     SubhaloStarMetalFractions:
     SubhaloStarMetalFractionsHalfRad:
@@ -180,6 +192,7 @@ fields:
     SubhaloStellarPhotometrics: "1.0" # for now lets not add unit "mag" (TODO?)
     SubhaloStellarPhotometricsMassInRad: code_mass
     SubhaloStellarPhotometricsRad: code_length
+    SubhaloTidalTensor: unknown  # MTNG field, units not documented
     SubhaloVel: km / s
     SubhaloVelDisp: km / s
     SubhaloVmax: km / s

src/scida/configfiles/units/gizmo.yaml

@@ -14,30 +14,51 @@ units:
   code_divdamp: code_magnetic * code_velocity
 fields:
   _all:
+    AGS-Softening: unknown
     Coordinates: code_length
     Velocities: a^0.5 * code_velocity
     ParticleIDs:
     ParticleChildIDsNumber:
     ParticleIDGenerationNumber:
     Masses: code_mass
     Potential: (km / s)^2  # TODO: not in GIZMO manual, taken from https://bitbucket.org/awetzel/gizmo_analysis/src/master/gizmo_io.py
+    HaloID: none
+    ID_Generations: none
   PartType0:
-    InternalEnergy: code_energy
+    ArtificialViscosity:
+    DelayTime: unknown
     Density: code_density
-    SmoothingLength: code_length
+    Dust_Masses: code_mass  # guess
+    Dust_Metallicity:
     ElectronAbundance:
+    FractionH2:
+    GrackleHI: unknown
+    GrackleHII: unknown
+    GrackleHM: unknown
+    GrackleHeI: unknown
+    GrackleHeII: unknown
+    GrackleHeIII: unknown
+    InternalEnergy: code_energy
+    Metallicity:
     NeutralHydrogenAbundance:
+    PhotonEnergy: code_energy
+    Sigma: unknown
+    SmoothingLength: code_length
     StarFormationRate: Msun / yr
-    Metallicity:
-    ArtificialViscosity:
+    StellarFormationTime: # guess
     MagneticField: code_magnetic
     MagneticFieldDivergence: code_divdamp
-    PhotonEnergy: code_energy
-    StellarFormationTime:
+    NWindLaunches: none
   PartType4:
+    Dust_Masses: code_mass  # guess
+    Dust_Metallicity:
     Metallicity:
-    StellarFormationTime:
+    StellarFormationTime: # guess
   PartType5:
+    BH_AccretionLength: code_length  # guess
+    BH_NProgs: none
     BH_Mass: code_mass
     BH_Mdot: code_mass / code_time
     BH_Mass_AlphaDisk: code_mass
+    BH_StellarFormationTime: # guess
+    StellarFormationTime: # guess

src/scida/customs/arepo/dataset.py

@@ -37,7 +37,7 @@ def __init__(self, path, chunksize="auto", catalog=None, **kwargs) -> None:
         self.iscatalog = kwargs.pop("iscatalog", False)
         self.header = {}
         self.config = {}
-        self._defaultunitfiles: List[str] = ["units/gadget.yaml"]
+        self._defaultunitfiles: List[str] = ["units/gadget_cosmological.yaml"]
         self.parameters = {}
         self._grouplengths = {}
         self._subhalolengths = {}

src/scida/customs/gizmo/dataset.py

@@ -1,5 +1,5 @@
 import os
-from typing import Union
+from typing import List, Union
 
 from scida import GadgetStyleSnapshot
 from scida.convenience import load
@@ -16,6 +16,7 @@ def __init__(self, path, chunksize="auto", catalog=None, **kwargs) -> None:
         self.header = {}
         self.config = {}
         self.parameters = {}
+        self._defaultunitfiles: List[str] = ["units/gizmo.yaml"]
         self._grouplengths = {}
         prfx = kwargs.pop("fileprefix", None)
         if prfx is None:

src/scida/interfaces/mixins/units.py

@@ -283,7 +283,10 @@ def __init__(self, *args, **kwargs):
 
         if unitfile != "":
             unithints = get_config_fromfile(unitfile)
-            unitdefs.update(unithints.get("units", {}))
+            newdefs = unithints.get("units", {})
+            if newdefs is None:
+                newdefs = {}
+            unitdefs.update(**newdefs)
 
         units = kwargs.pop("units")
         if isinstance(units, bool):
@@ -305,7 +308,7 @@ def __init__(self, *args, **kwargs):
 
         fieldudefs.append(unithints.get("fields", {}))
 
-        fwu = combine_configs(fieldudefs, mode="overwrite_keys")  # merge udefs
+        fwu = combine_configs(fieldudefs, mode="overwrite_values")  # merge udefs
         mode_metadata = unithints.get("metadata_unitsystem", "cgs")
 
         def add_units(container: FieldContainer, basepath: str):

src/scida/misc.py

@@ -158,7 +158,7 @@ def check_config_for_dataset(metadata, path: Optional[str] = None, unique: bool
                 # we only check the last three path elements
                 dirnames = [p.name, p.parents[0].name, p.parents[1].name]
                 substring = idtfrs["name_contains"]
-                if not any([substring in d for d in dirnames]):
+                if not any([substring.lower() in d.lower() for d in dirnames]):
                     possible_candidate = False
             if len(allkeys) == 0:
                 possible_candidate = False

tests/customs/test_all.py

@@ -0,0 +1,13 @@
+import logging
+
+from scida import load
+from tests.testdata_properties import require_testdata_path
+
+
+@require_testdata_path("interface")
+def test_allunitsdiscovered(testdatapath, caplog):
+    load(testdatapath)
+    caplog.set_level(logging.DEBUG)
+    assert (
+        "Cannot determine units from neither unit file nor metadata" not in caplog.text
+    )

tests/customs/test_auriga.py

@@ -0,0 +1,9 @@
+from scida import ArepoSnapshot
+from scida.convenience import load
+from tests.testdata_properties import require_testdata_path
+
+
+@require_testdata_path("interface", only=["AURIGA66_snapshot"])
+def test_series(testdatapath):
+    obj = load(testdatapath)
+    assert isinstance(obj, ArepoSnapshot)

tests/test_config.py

@@ -9,7 +9,7 @@ def test_load_defaultconf():
 
 
 def test_load_multiple_confs():
-    conf = get_config_fromfiles(["config.yaml", "units/gadget.yaml"])
+    conf = get_config_fromfiles(["config.yaml", "units/gadget_cosmological.yaml"])
     assert conf is not None
     assert "units" in conf
     assert "cache_path" in conf

tests/test_issues.py

@@ -1,9 +1,11 @@
 # tests for gh issues. to be cleaned up and moved to the right place eventually
 import pathlib
 
+import h5py
+import numpy as np
 import pytest
 
-from scida import ArepoSimulation, load
+from scida import ArepoSimulation, ArepoSnapshot, load
 from scida.convenience import find_path
 from tests.helpers import write_gadget_testfile
 from tests.testdata_properties import require_testdata_path
@@ -31,6 +33,7 @@ def test_issue_63(testdatapath, tmp_path):
 
     # check that units work for TNG100-3
     ds = load(testdatapath, units=True)
+    assert isinstance(ds, ArepoSnapshot)
     gas = ds.data["PartType0"]
     coords = gas["Coordinates"]
     assert coords.to_base_units().units == ds.ureg("centimeter")
@@ -44,8 +47,10 @@ def test_issue_63(testdatapath, tmp_path):
     # assert "unknown" units when no units known
     p = pathlib.Path(tmp_path) / "test.hdf5"
     write_gadget_testfile(p)
+    with h5py.File(p, "r+") as hf:
+        hf["PartType0"]["ExtraField"] = np.zeros((1,), dtype=float)
     ds = load(p, units=True)
-    assert ds.data["PartType0"]["Coordinates"].units == "unknown"
+    assert ds.data["PartType0"]["ExtraField"].units == "unknown"
 
 
 @require_testdata_path("series", only=["TNGvariation_simulation"])