RCAL-941: Update romancal to use new L1/L2 schema.

changes/1473.general.rst

@@ -0,0 +1 @@
+Updates to support L1/L2 schema changes.

docs/roman/datamodels/metadata.rst

@@ -20,26 +20,31 @@ string will raise an exception.
         >>> from roman_datamodels.maker_utils import mk_datamodel
 
         >>> model = mk_datamodel(rdm.ImageModel)
-        >>> model.meta.target.ra = "foo"  # doctest: +IGNORE_EXCEPTION_DETAIL
+        >>> model.meta.pointing.target_ra = "foo"  # doctest: +IGNORE_EXCEPTION_DETAIL
         Traceback (most recent call last):
           File "<stdin>", line 1, in <module>
-          File "/Users/ddavis/miniconda3/envs/rcal_dev/lib/python3.9/site-packages/roman_datamodels/stnode.py", line 183, in __setattr__
-            if schema is None or _validate(key, value, schema, self.ctx):
-          File "/Users/ddavis/miniconda3/envs/rcal_dev/lib/python3.9/site-packages/roman_datamodels/stnode.py", line 97, in _validate
-            return _value_change(attr, tagged_tree, schema, False, strict_validation, ctx)
-          File "/Users/ddavis/miniconda3/envs/rcal_dev/lib/python3.9/site-packages/roman_datamodels/stnode.py", line 68, in _value_change
-            raise jsonschema.ValidationError(errmsg)
-        jsonschema.exceptions.ValidationError: While validating ra the following error occurred:
+          File "/path/to/python/lib/python3.12/site-packages/roman_datamodels/stnode/_node.py", line 251, in __setattr__
+            _validate(key, value, schema, self.ctx)
+          File "/path/to/python/lib/python3.12/site-packages/roman_datamodels/stnode/_node.py", line 78, in _validate
+            return _value_change(attr, tagged_tree, schema, False, will_strict_validate(), ctx)
+                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+          File "/path/to/python/lib/python3.12/site-packages/roman_datamodels/stnode/_node.py", line 47, in _value_change
+            raise ValidationError(errmsg)
+        asdf._jsonschema.exceptions.ValidationError: While validating target_ra the following error occurred:
         'foo' is not of type 'number'
 
         Failed validating 'type' in schema:
             {'$schema': 'http://stsci.edu/schemas/asdf-schema/0.1.0/asdf-schema',
-             'archive_catalog': {'datatype': 'float',
-                                 'destination': ['ScienceCommon.ra']},
-             'sdf': {'source': {'origin': 'PSS:fixed_target.ra_computed'},
-                     'special_processing': 'VALUE_REQUIRED'},
-             'title': 'Target RA at mid time of exposure',
-             'type': 'number'}
+            'archive_catalog': {'datatype': 'float',
+                                'destination': ['WFIExposure.target_ra',
+                                                'GuideWindow.target_ra']},
+            'description': 'Right ascension in units of degrees at the location '
+                            'of\n'
+                            'the target aperture.\n',
+            'sdf': {'source': {'origin': 'TBD'},
+                    'special_processing': 'VALUE_REQUIRED'},
+            'title': 'Right Ascension of the Target Aperture (deg)',
+            'type': 'number'}
 
         On instance:
             'foo'

docs/roman/datamodels/models.rst

@@ -142,16 +142,17 @@ You can examine the contents of your model from within python using
 .. code-block:: python
 
     >>> print("\n".join("{: >20}\t{}".format(k, v) for k, v in image_model.items()), "\n") # doctest: +ELLIPSIS
-    meta.calibration_software_version   9.9.0
-    meta.sdf_software_version   7.7.7
-           meta.filename        r0019106003005004023_03203_0034_WFI01_cal.asdf
-          meta.file_date        2020-01-01T00:00:00.000
-         meta.model_type        ImageModel
-             meta.origin        STSCI
-    meta.prd_software_version   8.8.8
-          meta.telescope        ROMAN
-      meta.aperture.name        WFI_06_FULL
-    meta.aperture.position_angle        30.0
+    meta.calibration_software_name	RomanCAL
+    meta.calibration_software_version	9.9.0
+       meta.product_type	l2
+           meta.filename	r0019106003005004023_03203_0034_WFI01_cal.asdf
+          meta.file_date	2020-01-01T00:00:00.000
+         meta.model_type	ImageModel
+             meta.origin	STSCI/SOC
+        meta.prd_version	8.8.8
+    meta.sdf_software_version	7.7.7
+          meta.telescope	ROMAN
+    meta.coordinates.reference_frame	ICRS
     ...
 
 or you can print specifics

pyproject.toml

@@ -24,9 +24,11 @@ dependencies = [
     "pyparsing >=2.4.7",
     "requests >=2.26",
     # "rad>=0.21.0,<0.22.0",
-    "rad @ git+https://github.com/spacetelescope/rad.git",
+    # "rad @ git+https://github.com/spacetelescope/rad.git",
     # "roman_datamodels>=0.21.0,<0.22.0",
-    "roman_datamodels @ git+https://github.com/spacetelescope/roman_datamodels.git",
+    # "roman_datamodels @ git+https://github.com/spacetelescope/roman_datamodels.git",
+    "rad @ git+https://github.com/PaulHuwe/rad.git@RAD-184_IMMandL1L2Updates",
+    "roman_datamodels @ git+https://github.com/PaulHuwe/roman_datamodels.git@RCAL-934_L1L2Update",
     "scipy >=1.11",
 #    "stcal>=1.8.0,<1.9.0",
     "stcal @ git+https://github.com/spacetelescope/stcal.git@main",

romancal/dark_current/dark_current_step.py

@@ -42,14 +42,8 @@ def process(self, input):
         with rdm.open(self.dark_name) as dark_model:
             # Temporary patch to utilize stcal dark step until MA table support
             # is fully implemented
-            if "ngroups" not in dark_model.meta.exposure:
-                dark_model.meta.exposure["ngroups"] = dark_model.data.shape[0]
-            if "nframes" not in dark_model.meta.exposure:
-                dark_model.meta.exposure["nframes"] = input_model.meta.exposure.nframes
-            if "groupgap" not in dark_model.meta.exposure:
-                dark_model.meta.exposure["groupgap"] = (
-                    input_model.meta.exposure.groupgap
-                )
+            if "nresultants" not in dark_model.meta.exposure:
+                dark_model.meta.exposure["nresultants"] = dark_model.data.shape[0]
 
             # Do the dark correction
             out_model = input_model

romancal/datamodels/library.py

@@ -75,4 +75,4 @@ def _mapping_to_group_id(mapping):
     """
     Combine a number of file metadata values into a ``group_id`` string
     """
-    return "{obs_id}".format_map(mapping)
+    return "{observation_id}".format_map(mapping)

romancal/datamodels/tests/test_library.py

@@ -26,7 +26,7 @@ def example_asn_path(tmp_path):
     fns = []
     for i in range(_N_MODELS):
         m = dm.ImageModel(mk_level2_image(shape=(2, 2)))
-        m.meta.observation.program = "0001"
+        m.meta.observation.program = 1
         m.meta.observation.observation = _OBSERVATION_NUMBERS[i]
         m.meta.observation.visit = 1
         m.meta.observation.visit_file_group = 1

romancal/dq_init/dq_init_step.py

@@ -44,8 +44,8 @@ def process(self, input):
         output_model = RampModel.from_science_raw(input_model)
 
         # guide window range information
-        x_start = input_model.meta.guidestar.gw_window_xstart
-        x_end = input_model.meta.guidestar.gw_window_xsize + x_start
+        x_start = input_model.meta.guide_star.window_xstart
+        x_end = input_model.meta.guide_star.window_xsize + x_start
         # set pixeldq array to GW_AFFECTED_DATA (2**4) for the given range
         output_model.pixeldq[int(x_start) : int(x_end), :] = pixel.GW_AFFECTED_DATA
         self.log.info(

romancal/dq_init/tests/test_dq_init.py

@@ -10,17 +10,17 @@
 RNG = np.random.default_rng(83)
 
 # Set parameters for multiple runs of data
-args = "xstart, ystart, xsize, ysize, ngroups, instrument, exp_type"
+args = "xstart, ystart, xsize, ysize, nresultants, instrument, exp_type"
 test_data = [(1, 1, 2048, 2048, 2, "WFI", "WFI_IMAGE")]
 ids = ["RampModel"]
 
 
 @pytest.mark.parametrize(args, test_data, ids=ids)
-def test_dq_im(xstart, ystart, xsize, ysize, ngroups, instrument, exp_type):
+def test_dq_im(xstart, ystart, xsize, ysize, nresultants, instrument, exp_type):
     """Check that PIXELDQ is initialized with the information from the reference file.
     test that a flagged value in the reference file flags the PIXELDQ array"""
 
-    csize = (ngroups, ysize, xsize)
+    csize = (nresultants, ysize, xsize)
 
     # create raw input data for step
     dm_ramp = maker_utils.mk_ramp(shape=csize)
@@ -75,10 +75,10 @@ def test_groupdq():
 
     # size of integration
     instrument = "WFI"
-    ngroups = 5
+    nresultants = 5
     xsize = 1032
     ysize = 1024
-    csize = (ngroups, ysize, xsize)
+    csize = (nresultants, ysize, xsize)
 
     # create raw input data for step
     dm_ramp = maker_utils.mk_ramp(shape=csize)
@@ -95,7 +95,7 @@ def test_groupdq():
     groupdq = outfile.groupdq
 
     np.testing.assert_array_equal(
-        np.full((ngroups, ysize, xsize), 0, dtype=int),
+        np.full((nresultants, ysize, xsize), 0, dtype=int),
         groupdq,
         err_msg="groupdq not initialized to zero",
     )
@@ -106,13 +106,13 @@ def test_err():
 
     # size of integration
     instrument = "WFI"
-    ngroups = 5
+    nresultants = 5
     xsize = 1032
     ysize = 1024
-    csize = (ngroups, ysize, xsize)
+    csize = (nresultants, ysize, xsize)
 
     # create raw input data for step
-    dm_ramp = maker_utils.mk_ramp(shape=(ngroups, ysize, xsize))
+    dm_ramp = maker_utils.mk_ramp(shape=(nresultants, ysize, xsize))
     dm_ramp.meta.instrument.name = instrument
 
     # create a MaskModel elements for the dq input mask
@@ -138,13 +138,13 @@ def test_dq_add1_groupdq():
 
     # size of integration
     instrument = "WFI"
-    ngroups = 5
+    nresultants = 5
     xsize = 1032
     ysize = 1024
-    csize = (ngroups, ysize, xsize)
+    csize = (nresultants, ysize, xsize)
 
     # create raw input data for step
-    dm_ramp = maker_utils.mk_ramp(shape=(ngroups, ysize, xsize))
+    dm_ramp = maker_utils.mk_ramp(shape=(nresultants, ysize, xsize))
     dm_ramp.meta.instrument.name = instrument
 
     # create a MaskModel elements for the dq input mask
@@ -195,8 +195,8 @@ def test_dqinit_step_interface(instrument, exptype):
     wfi_sci_raw.meta.instrument.name = instrument
     wfi_sci_raw.meta.instrument.detector = "WFI01"
     wfi_sci_raw.meta.instrument.optical_element = "F158"
-    wfi_sci_raw.meta["guidestar"]["gw_window_xstart"] = 1012
-    wfi_sci_raw.meta["guidestar"]["gw_window_xsize"] = 16
+    wfi_sci_raw.meta["guide_star"]["window_xstart"] = 1012
+    wfi_sci_raw.meta["guide_star"]["window_xsize"] = 16
     wfi_sci_raw.meta.exposure.type = exptype
     wfi_sci_raw.data = np.ones(shape, dtype=np.uint16)
     wfi_sci_raw[extra_key] = extra_value
@@ -245,8 +245,8 @@ def test_dqinit_refpix(instrument, exptype):
     wfi_sci_raw.meta.instrument.name = instrument
     wfi_sci_raw.meta.instrument.detector = "WFI01"
     wfi_sci_raw.meta.instrument.optical_element = "F158"
-    wfi_sci_raw.meta["guidestar"]["gw_window_xstart"] = 1012
-    wfi_sci_raw.meta["guidestar"]["gw_window_xsize"] = 16
+    wfi_sci_raw.meta["guide_star"]["window_xstart"] = 1012
+    wfi_sci_raw.meta["guide_star"]["window_xsize"] = 16
     wfi_sci_raw.meta.exposure.type = exptype
     wfi_sci_raw.data = np.ones(shape, dtype=np.uint16)
     wfi_sci_raw_model = ScienceRawModel(wfi_sci_raw)
@@ -295,8 +295,8 @@ def test_dqinit_resultantdq(instrument, exptype):
     wfi_sci_raw.meta.instrument.name = instrument
     wfi_sci_raw.meta.instrument.detector = "WFI01"
     wfi_sci_raw.meta.instrument.optical_element = "F158"
-    wfi_sci_raw.meta["guidestar"]["gw_window_xstart"] = 1012
-    wfi_sci_raw.meta["guidestar"]["gw_window_xsize"] = 16
+    wfi_sci_raw.meta["guide_star"]["window_xstart"] = 1012
+    wfi_sci_raw.meta["guide_star"]["window_xsize"] = 16
     wfi_sci_raw.meta.exposure.type = exptype
     wfi_sci_raw.resultantdq[1, 12, 12] = pixel["DROPOUT"]
     wfi_sci_raw.data = np.ones(shape, dtype=np.uint16)
@@ -344,8 +344,8 @@ def test_dqinit_getbestref(instrument, exptype):
     wfi_sci_raw.meta.instrument.name = instrument
     wfi_sci_raw.meta.instrument.detector = "WFI01"
     wfi_sci_raw.meta.instrument.optical_element = "F158"
-    wfi_sci_raw.meta["guidestar"]["gw_window_xstart"] = 1012
-    wfi_sci_raw.meta["guidestar"]["gw_window_xsize"] = 16
+    wfi_sci_raw.meta["guide_star"]["window_xstart"] = 1012
+    wfi_sci_raw.meta["guide_star"]["window_xsize"] = 16
     wfi_sci_raw.meta.exposure.type = exptype
     wfi_sci_raw.data = np.ones(shape, dtype=np.uint16)
     wfi_sci_raw_model = ScienceRawModel(wfi_sci_raw)

romancal/jump/jump_step.py

@@ -52,7 +52,6 @@
             input_model = rdd.open(input)
 
         # Extract the needed info from the Roman Data Model
-        meta = input_model.meta
         r_data = input_model.data
         r_gdq = input_model.groupdq
         r_pdq = input_model.pixeldq
@@ -64,7 +63,9 @@
             result.meta.cal_step.jump = "SKIPPED"
             return result
 
-        frames_per_group = meta.exposure.nframes
+        # FIXME: since frames_per_group => meta.exposure.nframes has been removed,
+        # we need to fix stcal.jump.jump to remove it from there too
+        frames_per_group = 1
 
         # Modify the arrays for input into the 'common' jump (4D)
         data = np.copy(r_data[np.newaxis, :])
@@ -74,11 +75,11 @@
 
         tstart = time.time()
 
-        # Check for an input model with NGROUPS<=2
-        ngroups = data.shape[1]
+        # Check for an input model with nresultants<=2
+        nresultants = data.shape[1]
 
-        if ngroups <= 2:
-            self.log.warning("Cannot apply jump detection as NGROUPS<=2;")
+        if nresultants <= 2:
+            self.log.warning("Cannot apply jump detection as nresultants<=2;")
             self.log.warning("Jump step will be skipped")
 
             result = input_model