Enable "lazy_tree" for all Datamodels (#358)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

CHANGES.rst

@@ -5,6 +5,8 @@
 
 - replace usages of ``copy_arrays`` with ``memmap`` [#360]
 
+- Enable asdf "lazy_tree" mode for all roman datamodels files [#358]
+
 0.20.0 (2024-05-15)
 ===================
 

pyproject.toml

@@ -13,14 +13,14 @@ classifiers = [
     "Programming Language :: Python :: 3",
 ]
 dependencies = [
-    "asdf >=3.1.0",
+    "asdf >=3.3.0",
     "asdf-astropy >=0.5.0",
     "gwcs >=0.19.0",
     "numpy >=1.22",
     "astropy >=5.3.0",
     "rad >= 0.20.0",
     # "rad @ git+https://github.com/spacetelescope/rad.git",
-    "asdf-standard >=1.0.3",
+    "asdf-standard >=1.1.0",
 ]
 dynamic = [
     "version",

src/roman_datamodels/datamodels/_core.py

@@ -19,6 +19,7 @@
 import asdf
 import numpy as np
 from asdf.exceptions import ValidationError
+from asdf.lazy_nodes import AsdfDictNode, AsdfListNode
 from astropy.time import Time
 
 from roman_datamodels import stnode, validate
@@ -315,10 +316,10 @@ def items(self):
         """
 
         def recurse(tree, path=[]):
-            if isinstance(tree, (stnode.DNode, dict)):
+            if isinstance(tree, (stnode.DNode, dict, AsdfDictNode)):
                 for key, val in tree.items():
                     yield from recurse(val, path + [key])
-            elif isinstance(tree, (stnode.LNode, list, tuple)):
+            elif isinstance(tree, (stnode.LNode, list, tuple, AsdfListNode)):
                 for i, val in enumerate(tree):
                     yield from recurse(val, path + [i])
             elif tree is not None:

src/roman_datamodels/datamodels/_datamodels.py

@@ -62,7 +62,7 @@ def append_individual_image_meta(self, meta):
         """
 
         # Convert input to a dictionary, if necessary
-        if not isinstance(meta, dict):
+        if not isinstance(meta, (dict, asdf.lazy_nodes.AsdfDictNode)):
             meta_dict = meta.to_flat_dict()
         else:
             meta_dict = meta
@@ -79,7 +79,7 @@ def append_individual_image_meta(self, meta):
 
             # Keys that are themselves Dnodes (subdirectories)
             # neccessitate a new table
-            if isinstance(value, (dict, asdf.tags.core.ndarray.NDArrayType, QTable)):
+            if isinstance(value, (dict, asdf.tags.core.ndarray.NDArrayType, QTable, asdf.lazy_nodes.AsdfDictNode)):
                 continue
 
             if isinstance(value, stnode.DNode):
@@ -94,7 +94,11 @@ def append_individual_image_meta(self, meta):
                         continue
 
                     subtable_cols.append(subkey)
-                    subtable_vals.append([str(subvalue)] if isinstance(subvalue, (list, dict)) else [subvalue])
+                    subtable_vals.append(
+                        [str(subvalue)]
+                        if isinstance(subvalue, (list, dict, asdf.lazy_nodes.AsdfDictNode, asdf.lazy_nodes.AsdfListNode))
+                        else [subvalue]
+                    )
 
                 # Skip this Table if it would be empty
                 if subtable_vals:
@@ -109,7 +113,7 @@ def append_individual_image_meta(self, meta):
             else:
                 # Store Basic keyword
                 basic_cols.append(key)
-                basic_vals.append([str(value)] if isinstance(value, list) else [value])
+                basic_vals.append([str(value)] if isinstance(value, (list, asdf.lazy_nodes.AsdfListNode)) else [value])
 
         # Make Basic Table if needed
         if self.meta.individual_image_meta.basic.colnames == ["dummy"]:

src/roman_datamodels/datamodels/_utils.py

@@ -14,7 +14,7 @@
 __all__ = ["rdm_open"]
 
 
-def _open_path_like(init, **kwargs):
+def _open_path_like(init, lazy_tree=True, **kwargs):
     """
     Attempt to open init as if it was a path-like object.
 
@@ -31,6 +31,9 @@ def _open_path_like(init, **kwargs):
     -------
     `asdf.AsdfFile`
     """
+    # asdf defaults to lazy_tree=False, this overwrites it to
+    # lazy_tree=True for roman_datamodels
+    kwargs["lazy_tree"] = lazy_tree
 
     try:
         asdf_file = asdf.open(init, **kwargs)

src/roman_datamodels/stnode/_converters.py

@@ -20,6 +20,8 @@ class _RomanConverter(Converter):
     Base class for the roman_datamodels converters.
     """
 
+    lazy = True
+
     def __init_subclass__(cls, **kwargs) -> None:
         """
         Automatically create the converter objects.
@@ -50,7 +52,7 @@ def select_tag(self, obj, tags, ctx):
         return obj.tag
 
     def to_yaml_tree(self, obj, tag, ctx):
-        return obj._data
+        return dict(obj._data)
 
     def from_yaml_tree(self, node, tag, ctx):
         return OBJECT_NODE_CLASSES_BY_TAG[tag](node)

src/roman_datamodels/stnode/_node.py

@@ -10,6 +10,7 @@
 from collections.abc import MutableMapping
 
 import asdf
+import asdf.lazy_nodes
 import asdf.schema as asdfschema
 import asdf.yamlutil as yamlutil
 import numpy as np
@@ -165,7 +166,7 @@ def __init__(self, node=None, parent=None, name=None):
         # Handle if we are passed different data types
         if node is None:
             self.__dict__["_data"] = {}
-        elif isinstance(node, dict):
+        elif isinstance(node, (dict, asdf.lazy_nodes.AsdfDictNode)):
             self.__dict__["_data"] = node
         else:
             raise ValueError("Initializer only accepts dicts")
@@ -209,10 +210,10 @@ def __getattr__(self, key):
             value = self._convert_to_scalar(key, self._data[key])
 
             # Return objects as node classes, if applicable
-            if isinstance(value, dict):
+            if isinstance(value, (dict, asdf.lazy_nodes.AsdfDictNode)):
                 return DNode(value, parent=self, name=key)
 
-            elif isinstance(value, list):
+            elif isinstance(value, (list, asdf.lazy_nodes.AsdfListNode)):
                 return LNode(value)
 
             else:
@@ -323,7 +324,7 @@ def __setitem__(self, key, value):
             value = self._convert_to_scalar(key, value)
 
         # If the value is a dictionary, loop over its keys and convert them to tagged scalars
-        if isinstance(value, dict):
+        if isinstance(value, (dict, asdf.lazy_nodes.AsdfDictNode)):
             for sub_key, sub_value in value.items():
                 if self._tag and "/tvac" in self._tag:
                     value[sub_key] = self._convert_to_scalar("tvac_" + sub_key, sub_value)
@@ -365,7 +366,7 @@ class LNode(UserList):
     def __init__(self, node=None):
         if node is None:
             self.data = []
-        elif isinstance(node, list):
+        elif isinstance(node, (list, asdf.lazy_nodes.AsdfListNode)):
             self.data = node
         elif isinstance(node, self.__class__):
             self.data = node.data
@@ -374,9 +375,9 @@ def __init__(self, node=None):
 
     def __getitem__(self, index):
         value = self.data[index]
-        if isinstance(value, dict):
+        if isinstance(value, (dict, asdf.lazy_nodes.AsdfDictNode)):
             return DNode(value)
-        elif isinstance(value, list):
+        elif isinstance(value, (list, asdf.lazy_nodes.AsdfListNode)):
             return LNode(value)
         else:
             return value