From d5444af26cc1ff856f2ccc6176059e959e5b5f91 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Wed, 23 Oct 2024 16:22:46 -0400 Subject: [PATCH 01/25] Initial updates. --- pyproject.toml | 6 ++- romancal/dark_current/dark_current_step.py | 10 +--- romancal/datamodels/tests/test_library.py | 2 +- romancal/dq_init/dq_init_step.py | 4 +- romancal/dq_init/tests/test_dq_init.py | 40 +++++++-------- romancal/jump/jump_step.py | 12 +++-- romancal/jump/tests/test_jump_step.py | 44 +++++++--------- romancal/linearity/tests/test_linearity.py | 4 +- romancal/outlier_detection/utils.py | 8 +-- romancal/photom/photom.py | 10 +--- romancal/photom/photom_step.py | 5 +- romancal/photom/tests/test_photom.py | 47 +++-------------- romancal/ramp_fitting/ramp_fit_step.py | 6 +-- .../ramp_fitting/tests/test_ramp_fit_cas22.py | 4 +- romancal/regtest/test_wfi_photom.py | 50 +------------------ romancal/resample/resample.py | 17 ++----- romancal/resample/resample_step.py | 10 ++-- romancal/resample/tests/test_resample.py | 47 ++++++++--------- romancal/resample/tests/test_resample_step.py | 35 ++++++------- romancal/saturation/tests/test_saturation.py | 49 +++++++++--------- romancal/source_catalog/source_catalog.py | 2 +- romancal/stpipe/core.py | 5 +- romancal/stpipe/tests/test_core.py | 2 +- romancal/tweakreg/tests/test_tweakreg.py | 14 +++--- 24 files changed, 157 insertions(+), 276 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e5e8107f8..b237683e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,9 +24,11 @@ dependencies = [ "pyparsing >=2.4.7", "requests >=2.26", # "rad>=0.21.0,<0.22.0", - "rad @ git+https://github.com/spacetelescope/rad.git", + # "rad @ git+https://github.com/spacetelescope/rad.git", # "roman_datamodels>=0.21.0,<0.22.0", - "roman_datamodels @ git+https://github.com/spacetelescope/roman_datamodels.git", + # "roman_datamodels @ git+https://github.com/spacetelescope/roman_datamodels.git", + "rad @ git+https://github.com/PaulHuwe/rad.git@RAD-184_IMMandL1L2Updates", + "roman_datamodels @ git+https://github.com/PaulHuwe/roman_datamodels.git@RCAL-934_L1L2Update", "scipy >=1.11", # "stcal>=1.8.0,<1.9.0", "stcal @ git+https://github.com/spacetelescope/stcal.git@main", diff --git a/romancal/dark_current/dark_current_step.py b/romancal/dark_current/dark_current_step.py index ecad8b93b..7e9751643 100644 --- a/romancal/dark_current/dark_current_step.py +++ b/romancal/dark_current/dark_current_step.py @@ -42,14 +42,8 @@ def process(self, input): with rdm.open(self.dark_name) as dark_model: # Temporary patch to utilize stcal dark step until MA table support # is fully implemented - if "ngroups" not in dark_model.meta.exposure: - dark_model.meta.exposure["ngroups"] = dark_model.data.shape[0] - if "nframes" not in dark_model.meta.exposure: - dark_model.meta.exposure["nframes"] = input_model.meta.exposure.nframes - if "groupgap" not in dark_model.meta.exposure: - dark_model.meta.exposure["groupgap"] = ( - input_model.meta.exposure.groupgap - ) + if "nresultants" not in dark_model.meta.exposure: + dark_model.meta.exposure["nresultants"] = dark_model.data.shape[0] # Do the dark correction out_model = input_model diff --git a/romancal/datamodels/tests/test_library.py b/romancal/datamodels/tests/test_library.py index 281c9307e..e6435496f 100644 --- a/romancal/datamodels/tests/test_library.py +++ b/romancal/datamodels/tests/test_library.py @@ -26,7 +26,7 @@ def example_asn_path(tmp_path): fns = [] for i in range(_N_MODELS): m = dm.ImageModel(mk_level2_image(shape=(2, 2))) - m.meta.observation.program = "0001" + m.meta.observation.program = 1 m.meta.observation.observation = _OBSERVATION_NUMBERS[i] m.meta.observation.visit = 1 m.meta.observation.visit_file_group = 1 diff --git a/romancal/dq_init/dq_init_step.py b/romancal/dq_init/dq_init_step.py index 320448584..1355c0205 100644 --- a/romancal/dq_init/dq_init_step.py +++ b/romancal/dq_init/dq_init_step.py @@ -44,8 +44,8 @@ def process(self, input): output_model = RampModel.from_science_raw(input_model) # guide window range information - x_start = input_model.meta.guidestar.gw_window_xstart - x_end = input_model.meta.guidestar.gw_window_xsize + x_start + x_start = input_model.meta.guide_star.gw_window_xstart + x_end = input_model.meta.guide_star.gw_window_xsize + x_start # set pixeldq array to GW_AFFECTED_DATA (2**4) for the given range output_model.pixeldq[int(x_start) : int(x_end), :] = pixel.GW_AFFECTED_DATA self.log.info( diff --git a/romancal/dq_init/tests/test_dq_init.py b/romancal/dq_init/tests/test_dq_init.py index c7ed8b8c0..2ff132231 100644 --- a/romancal/dq_init/tests/test_dq_init.py +++ b/romancal/dq_init/tests/test_dq_init.py @@ -10,17 +10,17 @@ RNG = np.random.default_rng(83) # Set parameters for multiple runs of data -args = "xstart, ystart, xsize, ysize, ngroups, instrument, exp_type" +args = "xstart, ystart, xsize, ysize, nresultants, instrument, exp_type" test_data = [(1, 1, 2048, 2048, 2, "WFI", "WFI_IMAGE")] ids = ["RampModel"] @pytest.mark.parametrize(args, test_data, ids=ids) -def test_dq_im(xstart, ystart, xsize, ysize, ngroups, instrument, exp_type): +def test_dq_im(xstart, ystart, xsize, ysize, nresultants, instrument, exp_type): """Check that PIXELDQ is initialized with the information from the reference file. test that a flagged value in the reference file flags the PIXELDQ array""" - csize = (ngroups, ysize, xsize) + csize = (nresultants, ysize, xsize) # create raw input data for step dm_ramp = maker_utils.mk_ramp(shape=csize) @@ -75,10 +75,10 @@ def test_groupdq(): # size of integration instrument = "WFI" - ngroups = 5 + nresultants = 5 xsize = 1032 ysize = 1024 - csize = (ngroups, ysize, xsize) + csize = (nresultants, ysize, xsize) # create raw input data for step dm_ramp = maker_utils.mk_ramp(shape=csize) @@ -95,7 +95,7 @@ def test_groupdq(): groupdq = outfile.groupdq np.testing.assert_array_equal( - np.full((ngroups, ysize, xsize), 0, dtype=int), + np.full((nresultants, ysize, xsize), 0, dtype=int), groupdq, err_msg="groupdq not initialized to zero", ) @@ -106,13 +106,13 @@ def test_err(): # size of integration instrument = "WFI" - ngroups = 5 + nresultants = 5 xsize = 1032 ysize = 1024 - csize = (ngroups, ysize, xsize) + csize = (nresultants, ysize, xsize) # create raw input data for step - dm_ramp = maker_utils.mk_ramp(shape=(ngroups, ysize, xsize)) + dm_ramp = maker_utils.mk_ramp(shape=(nresultants, ysize, xsize)) dm_ramp.meta.instrument.name = instrument # create a MaskModel elements for the dq input mask @@ -138,13 +138,13 @@ def test_dq_add1_groupdq(): # size of integration instrument = "WFI" - ngroups = 5 + nresultants = 5 xsize = 1032 ysize = 1024 - csize = (ngroups, ysize, xsize) + csize = (nresultants, ysize, xsize) # create raw input data for step - dm_ramp = maker_utils.mk_ramp(shape=(ngroups, ysize, xsize)) + dm_ramp = maker_utils.mk_ramp(shape=(nresultants, ysize, xsize)) dm_ramp.meta.instrument.name = instrument # create a MaskModel elements for the dq input mask @@ -195,8 +195,8 @@ def test_dqinit_step_interface(instrument, exptype): wfi_sci_raw.meta.instrument.name = instrument wfi_sci_raw.meta.instrument.detector = "WFI01" wfi_sci_raw.meta.instrument.optical_element = "F158" - wfi_sci_raw.meta["guidestar"]["gw_window_xstart"] = 1012 - wfi_sci_raw.meta["guidestar"]["gw_window_xsize"] = 16 + wfi_sci_raw.meta["guide_star"]["gw_window_xstart"] = 1012 + wfi_sci_raw.meta["guide_star"]["gw_window_xsize"] = 16 wfi_sci_raw.meta.exposure.type = exptype wfi_sci_raw.data = np.ones(shape, dtype=np.uint16) wfi_sci_raw[extra_key] = extra_value @@ -245,8 +245,8 @@ def test_dqinit_refpix(instrument, exptype): wfi_sci_raw.meta.instrument.name = instrument wfi_sci_raw.meta.instrument.detector = "WFI01" wfi_sci_raw.meta.instrument.optical_element = "F158" - wfi_sci_raw.meta["guidestar"]["gw_window_xstart"] = 1012 - wfi_sci_raw.meta["guidestar"]["gw_window_xsize"] = 16 + wfi_sci_raw.meta["guide_star"]["gw_window_xstart"] = 1012 + wfi_sci_raw.meta["guide_star"]["gw_window_xsize"] = 16 wfi_sci_raw.meta.exposure.type = exptype wfi_sci_raw.data = np.ones(shape, dtype=np.uint16) wfi_sci_raw_model = ScienceRawModel(wfi_sci_raw) @@ -295,8 +295,8 @@ def test_dqinit_resultantdq(instrument, exptype): wfi_sci_raw.meta.instrument.name = instrument wfi_sci_raw.meta.instrument.detector = "WFI01" wfi_sci_raw.meta.instrument.optical_element = "F158" - wfi_sci_raw.meta["guidestar"]["gw_window_xstart"] = 1012 - wfi_sci_raw.meta["guidestar"]["gw_window_xsize"] = 16 + wfi_sci_raw.meta["guide_star"]["gw_window_xstart"] = 1012 + wfi_sci_raw.meta["guide_star"]["gw_window_xsize"] = 16 wfi_sci_raw.meta.exposure.type = exptype wfi_sci_raw.resultantdq[1, 12, 12] = pixel["DROPOUT"] wfi_sci_raw.data = np.ones(shape, dtype=np.uint16) @@ -344,8 +344,8 @@ def test_dqinit_getbestref(instrument, exptype): wfi_sci_raw.meta.instrument.name = instrument wfi_sci_raw.meta.instrument.detector = "WFI01" wfi_sci_raw.meta.instrument.optical_element = "F158" - wfi_sci_raw.meta["guidestar"]["gw_window_xstart"] = 1012 - wfi_sci_raw.meta["guidestar"]["gw_window_xsize"] = 16 + wfi_sci_raw.meta["guide_star"]["gw_window_xstart"] = 1012 + wfi_sci_raw.meta["guide_star"]["gw_window_xsize"] = 16 wfi_sci_raw.meta.exposure.type = exptype wfi_sci_raw.data = np.ones(shape, dtype=np.uint16) wfi_sci_raw_model = ScienceRawModel(wfi_sci_raw) diff --git a/romancal/jump/jump_step.py b/romancal/jump/jump_step.py index 74d555863..0810e2607 100644 --- a/romancal/jump/jump_step.py +++ b/romancal/jump/jump_step.py @@ -64,7 +64,9 @@ def process(self, input): result.meta.cal_step.jump = "SKIPPED" return result - frames_per_group = meta.exposure.nframes + # FIXME: since frames_per_group => meta.exposure.nframes has been removed, + # we need to fix stcal.jump.jump to remove it from there too + frames_per_group = 1 # Modify the arrays for input into the 'common' jump (4D) data = np.copy(r_data[np.newaxis, :]) @@ -74,11 +76,11 @@ def process(self, input): tstart = time.time() - # Check for an input model with NGROUPS<=2 - ngroups = data.shape[1] + # Check for an input model with nresultants<=2 + nresultants = data.shape[1] - if ngroups <= 2: - self.log.warning("Cannot apply jump detection as NGROUPS<=2;") + if nresultants <= 2: + self.log.warning("Cannot apply jump detection as nresultants<=2;") self.log.warning("Jump step will be skipped") result = input_model diff --git a/romancal/jump/tests/test_jump_step.py b/romancal/jump/tests/test_jump_step.py index 14d48947b..dc06be40b 100644 --- a/romancal/jump/tests/test_jump_step.py +++ b/romancal/jump/tests/test_jump_step.py @@ -86,21 +86,19 @@ def generate_wfi_reffiles(tmp_path_factory): @pytest.fixture def setup_inputs(): def _setup( - ngroups=10, + nresultants=10, readnoise=10, nrows=20, ncols=20, - nframes=1, - grouptime=1.0, gain=1, deltatime=1, ): - err = np.ones(shape=(ngroups, nrows, ncols), dtype=np.float32) - data = np.zeros(shape=(ngroups, nrows, ncols), dtype=np.float32) - gdq = np.zeros(shape=(ngroups, nrows, ncols), dtype=np.uint8) + err = np.ones(shape=(nresultants, nrows, ncols), dtype=np.float32) + data = np.zeros(shape=(nresultants, nrows, ncols), dtype=np.float32) + gdq = np.zeros(shape=(nresultants, nrows, ncols), dtype=np.uint8) pixdq = np.zeros(shape=(nrows, ncols), dtype=np.uint32) - csize = (ngroups, nrows, ncols) + csize = (nresultants, nrows, ncols) dm_ramp = rdm.RampModel(maker_utils.mk_ramp(shape=csize)) dm_ramp.meta.instrument.name = "WFI" @@ -112,11 +110,9 @@ def _setup( dm_ramp.err = err dm_ramp.meta.exposure.type = "WFI_IMAGE" - dm_ramp.meta.exposure.group_time = deltatime + dm_ramp.meta.exposure.exposure_time = deltatime dm_ramp.meta.exposure.frame_time = deltatime - dm_ramp.meta.exposure.ngroups = ngroups - dm_ramp.meta.exposure.nframes = 1 - dm_ramp.meta.exposure.groupgap = 0 + dm_ramp.meta.exposure.nresultants = nresultants dm_ramp.meta.cal_step["dq_init"] = "INCOMPLETE" dm_ramp.meta.cal_step["jump"] = "INCOMPLETE" @@ -133,13 +129,13 @@ def test_one_CR(generate_wfi_reffiles, max_cores, setup_inputs): deltaDN = 5 ingain = 200 inreadnoise = 7.0 - ngroups = 100 + nresultants = 100 CR_fraction = 3 xsize = 20 ysize = 20 model1 = setup_inputs( - ngroups=ngroups, + nresultants=nresultants, nrows=ysize, ncols=xsize, gain=ingain, @@ -147,7 +143,7 @@ def test_one_CR(generate_wfi_reffiles, max_cores, setup_inputs): deltatime=grouptime, ) - for i in range(ngroups): + for i in range(nresultants): model1.data[i, :, :] = deltaDN * i first_CR_group_locs = [x for x in range(1, 7) if x % 5 == 0] @@ -187,13 +183,13 @@ def test_two_CRs(generate_wfi_reffiles, max_cores, setup_inputs): deltaDN = 5 ingain = 6 inreadnoise = 7.0 - ngroups = 100 + nresultants = 100 CR_fraction = 5 xsize = 20 ysize = 20 model1 = setup_inputs( - ngroups=ngroups, + nresultants=nresultants, nrows=ysize, ncols=xsize, gain=ingain, @@ -201,7 +197,7 @@ def test_two_CRs(generate_wfi_reffiles, max_cores, setup_inputs): deltatime=grouptime, ) - for i in range(ngroups): + for i in range(nresultants): model1.data[i, :, :] = deltaDN * i first_CR_group_locs = [x for x in range(1, 7) if x % 5 == 0] @@ -242,11 +238,11 @@ def test_two_group_integration(generate_wfi_reffiles, max_cores, setup_inputs): grouptime = 3.0 ingain = 6 inreadnoise = 7 - ngroups = 2 + nresultants = 2 xsize = 20 ysize = 20 model1 = setup_inputs( - ngroups=ngroups, + nresultants=nresultants, nrows=ysize, ncols=xsize, gain=ingain, @@ -267,19 +263,17 @@ def test_two_group_integration(generate_wfi_reffiles, max_cores, setup_inputs): @pytest.mark.parametrize("use_jump", [False, True]) def test_four_group_integration(generate_wfi_reffiles, setup_inputs, use_jump): override_gain, override_readnoise = generate_wfi_reffiles - grouptime = 3.0 ingain = 6 inreadnoise = 7 - ngroups = 4 + nresultants = 4 xsize = 20 ysize = 20 model1 = setup_inputs( - ngroups=ngroups, + nresultants=nresultants, nrows=ysize, ncols=xsize, gain=ingain, readnoise=inreadnoise, - deltatime=grouptime, ) out_model = JumpStep.call( @@ -302,11 +296,11 @@ def test_three_group_integration(generate_wfi_reffiles, setup_inputs, use_jump): grouptime = 3.0 ingain = 6 inreadnoise = 7 - ngroups = 3 + nresultants = 3 xsize = 20 ysize = 20 model1 = setup_inputs( - ngroups=ngroups, + nresultants=nresultants, nrows=ysize, ncols=xsize, gain=ingain, diff --git a/romancal/linearity/tests/test_linearity.py b/romancal/linearity/tests/test_linearity.py index bd34c6efa..8eaec32d6 100644 --- a/romancal/linearity/tests/test_linearity.py +++ b/romancal/linearity/tests/test_linearity.py @@ -30,8 +30,8 @@ def test_linearity_coeff(instrument, exptype): wfi_sci_raw.meta.instrument.name = instrument wfi_sci_raw.meta.instrument.detector = "WFI01" wfi_sci_raw.meta.instrument.optical_element = "F158" - wfi_sci_raw.meta["guidestar"]["gw_window_xstart"] = 1012 - wfi_sci_raw.meta["guidestar"]["gw_window_xsize"] = 16 + wfi_sci_raw.meta["guide_star"]["gw_window_xstart"] = 1012 + wfi_sci_raw.meta["guide_star"]["gw_window_xsize"] = 16 wfi_sci_raw.meta.exposure.type = exptype wfi_sci_raw.data = np.ones(shape, dtype=np.uint16) wfi_sci_raw_model = ScienceRawModel(wfi_sci_raw) diff --git a/romancal/outlier_detection/utils.py b/romancal/outlier_detection/utils.py index 703bd6dbe..24b17ed23 100644 --- a/romancal/outlier_detection/utils.py +++ b/romancal/outlier_detection/utils.py @@ -65,7 +65,7 @@ def _median_with_resampling( in_memory = not input_models._on_disk indices_by_group = list(input_models.group_indices.values()) - ngroups = len(indices_by_group) + nresultants = len(indices_by_group) example_model = None median_wcs = resamp.output_wcs @@ -79,7 +79,7 @@ def _median_with_resampling( _fileio.save_drizzled(drizzled_model, make_output_path) if i == 0: - input_shape = (ngroups,) + drizzled_model.data.shape + input_shape = (nresultants,) + drizzled_model.data.shape dtype = drizzled_model.data.dtype computer = MedianComputer(input_shape, in_memory, buffer_size, dtype) example_model = drizzled_model @@ -146,7 +146,7 @@ def _median_without_resampling( """ in_memory = not input_models._on_disk - ngroups = len(input_models) + nresultants = len(input_models) example_model = None with input_models: @@ -165,7 +165,7 @@ def _median_without_resampling( _fileio.save_drizzled(model, make_output_path) if i == 0: - input_shape = (ngroups,) + model.data.shape + input_shape = (nresultants,) + model.data.shape dtype = model.data.dtype computer = MedianComputer(input_shape, in_memory, buffer_size, dtype) example_model = model diff --git a/romancal/photom/photom.py b/romancal/photom/photom.py index 86ce7618d..3e4901e45 100644 --- a/romancal/photom/photom.py +++ b/romancal/photom/photom.py @@ -29,9 +29,6 @@ def photom_io(input_model, photom_metadata): # Store the conversion factor in the meta data log.info(f"photmjsr value: {conversion:.6g}") input_model.meta.photometry.conversion_megajanskys = conversion.value - input_model.meta.photometry.conversion_microjanskys = conversion.to( - u.microjansky / u.arcsecond**2 - ).value # Get the scalar conversion uncertainty factor uncertainty_conv = photom_metadata["uncertainty"] @@ -41,9 +38,6 @@ def photom_io(input_model, photom_metadata): input_model.meta.photometry.conversion_megajanskys_uncertainty = ( uncertainty_conv.value ) - input_model.meta.photometry.conversion_microjanskys_uncertainty = ( - uncertainty_conv.to(u.microjansky / u.arcsecond**2) - ).value # Return updated input model return input_model @@ -68,9 +62,7 @@ def save_area_info(input_model, photom_parameters): area_a2 = photom_parameters["pixelareasr"].to(u.arcsecond**2).value # Copy the pixel area values to the input model - log.debug(f"pixelarea_steradians = {area_ster}, pixelarea_arcsecsq = {area_a2}") - input_model.meta.photometry.pixelarea_arcsecsq = area_a2 - input_model.meta.photometry.pixelarea_steradians = area_ster + input_model.meta.photometry.pixel_area = area_ster # Return updated input model return input_model diff --git a/romancal/photom/photom_step.py b/romancal/photom/photom_step.py index 2166b7c4a..331a41939 100644 --- a/romancal/photom/photom_step.py +++ b/romancal/photom/photom_step.py @@ -52,12 +52,9 @@ def process(self, input): self.log.warning("No photometric corrections for spectral data") self.log.warning("Photom step will be skipped") input_model.meta.cal_step.photom = "SKIPPED" - input_model.meta.photometry.pixelarea_arcsecsq = None - input_model.meta.photometry.pixelarea_steradians = None + input_model.meta.photometry.pixel_area = None input_model.meta.photometry.conversion_megajanskys = None - input_model.meta.photometry.conversion_microjanskys = None input_model.meta.photometry.conversion_megajanskys_uncertainty = None - input_model.meta.photometry.conversion_microjanskys_uncertainty = None output_model = input_model photom_model.close() diff --git a/romancal/photom/tests/test_photom.py b/romancal/photom/tests/test_photom.py index d5b222a3e..63f86c3ba 100644 --- a/romancal/photom/tests/test_photom.py +++ b/romancal/photom/tests/test_photom.py @@ -102,9 +102,8 @@ def test_no_photom_match(): input_model.meta.instrument.optical_element = "F146" # Set bad values which would be overwritten by apply_photom - input_model.meta.photometry.pixelarea_steradians = -1.0 + input_model.meta.photometry.pixel_area = -1.0 input_model.meta.photometry.conversion_megajanskys = -1.0 - input_model.meta.photometry.conversion_microjanskys_uncertainty = -1.0 with warnings.catch_warnings(record=True) as caught: # Look for now non existent F146 optical element @@ -117,9 +116,8 @@ def test_no_photom_match(): ) # Assert that photom elements are not updated - assert output_model.meta.photometry.pixelarea_steradians == -1.0 + assert output_model.meta.photometry.pixel_area == -1.0 assert output_model.meta.photometry.conversion_megajanskys == -1.0 - assert output_model.meta.photometry.conversion_microjanskys_uncertainty == -1.0 def test_apply_photom1(): @@ -143,16 +141,10 @@ def test_apply_photom1(): # Tests for pixel areas assert np.isclose( - output_model.meta.photometry.pixelarea_steradians, + output_model.meta.photometry.pixel_area, area_ster.value, atol=1.0e-7, ) - # assert output_model.meta.photometry.pixelarea_steradians.unit == area_ster.unit - assert np.isclose( - output_model.meta.photometry.pixelarea_arcsecsq, - area_a2.value, - atol=1.0e-7, - ) # Set reference photometry phot_ster = 3.5 * u.megajansky / u.steradian @@ -164,13 +156,6 @@ def test_apply_photom1(): phot_ster.value, atol=1.0e-7, ) - # assert output_model.meta.photometry.conversion_megajanskys.unit == phot_ster.unit - assert np.isclose( - output_model.meta.photometry.conversion_microjanskys, - phot_a2.value, - atol=1.0e-7, - ) - # assert output_model.meta.photometry.conversion_microjanskys.unit == phot_a2.unit # Set reference photometric uncertainty muphot_ster = 0.175 * u.megajansky / u.steradian @@ -186,15 +171,6 @@ def test_apply_photom1(): # output_model.meta.photometry.conversion_megajanskys_uncertainty.unit # == muphot_ster.unit # ) - assert np.isclose( - output_model.meta.photometry.conversion_microjanskys_uncertainty, - muphot_a2.value, - atol=1.0e-7, - ) - # assert ( - # output_model.meta.photometry.conversion_microjanskys_uncertainty.unit - # == muphot_a2.unit - # ) def test_apply_photom2(): @@ -276,24 +252,15 @@ def test_photom_step_interface_spectroscopic(instrument, exptype): wfi_image.meta.instrument.optical_element = "PRISM" # Set photometric values for spectroscopic data - wfi_image.meta.photometry.pixelarea_steradians = ( + wfi_image.meta.photometry.pixel_area = ( 2.31307642258977e-14 * u.steradian ).value - wfi_image.meta.photometry.pixelarea_arcsecsq = ( - 0.000984102303070964 * u.arcsecond * u.arcsecond - ).value wfi_image.meta.photometry.conversion_megajanskys = ( -99999 * u.megajansky / u.steradian ).value wfi_image.meta.photometry.conversion_megajanskys_uncertainty = ( -99999 * u.megajansky / u.steradian ).value - wfi_image.meta.photometry.conversion_microjanskys = ( - -99999 * u.microjansky / u.arcsecond**2 - ).value - wfi_image.meta.photometry.conversion_microjanskys_uncertainty = ( - -99999 * u.microjansky / u.arcsecond**2 - ).value # Create input model wfi_image_model = ImageModel(wfi_image) @@ -310,8 +277,6 @@ def test_photom_step_interface_spectroscopic(instrument, exptype): # Test that keywords are properly overwritten assert result.meta.photometry.conversion_megajanskys is None - assert result.meta.photometry.conversion_microjanskys is None assert result.meta.photometry.conversion_megajanskys_uncertainty is None - assert result.meta.photometry.conversion_microjanskys_uncertainty is None - assert result.meta.photometry.pixelarea_steradians is None - assert result.meta.photometry.pixelarea_arcsecsq is None + assert result.meta.photometry.pixel_area is None + diff --git a/romancal/ramp_fitting/ramp_fit_step.py b/romancal/ramp_fitting/ramp_fit_step.py index 790bc5098..f14cb2c07 100644 --- a/romancal/ramp_fitting/ramp_fit_step.py +++ b/romancal/ramp_fitting/ramp_fit_step.py @@ -57,8 +57,6 @@ def process(self, input): input_model, readnoise_model, gain_model, dark_model ) out_model.meta.cal_step.ramp_fit = "COMPLETE" - if self.use_ramp_jump_detection: - out_model.meta.cal_step.jump = "COMPLETE" else: log.error("Algorithm %s is not supported. Skipping step.") out_model = input @@ -88,7 +86,7 @@ def ols_cas22(self, input_model, readnoise_model, gain_model, dark_model): out_model : ImageModel Model containing a count-rate image. """ - use_jump = self.use_ramp_jump_detection + use_jump = False if use_jump: log.info("Jump detection as part of ramp fitting is enabled.") @@ -191,6 +189,7 @@ def create_image_model(input_model, image_info): meta = dict(wcs=None) # default empty WCS meta.update(input_model.meta) meta["cal_step"]["ramp_fit"] = "INCOMPLETE" + meta["cal_logs"] = maker_utils.mk_cal_logs() meta["photometry"] = maker_utils.mk_photometry() inst = { "meta": meta, @@ -208,7 +207,6 @@ def create_image_model(input_model, image_info): "dq_border_ref_pix_right": input_model.dq_border_ref_pix_right.copy(), "dq_border_ref_pix_top": input_model.dq_border_ref_pix_top.copy(), "dq_border_ref_pix_bottom": input_model.dq_border_ref_pix_bottom.copy(), - "cal_logs": rds.CalLogs(), } out_node = rds.WfiImage(inst) im = rdd.ImageModel(out_node) diff --git a/romancal/ramp_fitting/tests/test_ramp_fit_cas22.py b/romancal/ramp_fitting/tests/test_ramp_fit_cas22.py index 419cd8ec6..e64ebcf08 100644 --- a/romancal/ramp_fitting/tests/test_ramp_fit_cas22.py +++ b/romancal/ramp_fitting/tests/test_ramp_fit_cas22.py @@ -232,9 +232,7 @@ def model_from_resultants(resultants, read_pattern=None): dm_ramp.err = err dm_ramp.meta.exposure.frame_time = ROMAN_READ_TIME - dm_ramp.meta.exposure.ngroups = shape[0] - dm_ramp.meta.exposure.nframes = 1 - dm_ramp.meta.exposure.groupgap = 0 + dm_ramp.meta.exposure.nresultants = shape[0] dm_ramp.meta.exposure.read_pattern = read_pattern diff --git a/romancal/regtest/test_wfi_photom.py b/romancal/regtest/test_wfi_photom.py index 7af8734c7..061834d07 100644 --- a/romancal/regtest/test_wfi_photom.py +++ b/romancal/regtest/test_wfi_photom.py @@ -70,52 +70,22 @@ def test_absolute_photometric_calibration(rtdata, ignore_asdf_paths): photom_out.meta.photometry.conversion_megajanskys, 0.3324, abs_tol=0.0001 ) - step.log.info( - "DMS140 MSG: Photom microjanskys conversion calculated? : " - + str( - math.isclose( - photom_out.meta.photometry.conversion_microjanskys, - 7.81320, - abs_tol=0.0001, - ) - ) - ) - assert math.isclose( - photom_out.meta.photometry.conversion_microjanskys, - 7.81320, - abs_tol=0.0001, - ) - step.log.info( "DMS140 MSG: Pixel area in steradians calculated? : " + str( math.isclose( - photom_out.meta.photometry.pixelarea_steradians, + photom_out.meta.photometry.pixel_area, 2.8083e-13, abs_tol=1.0e-17, ) ) ) assert math.isclose( - photom_out.meta.photometry.pixelarea_steradians, + photom_out.meta.photometry.pixel_area, 2.8083e-13, abs_tol=1.0e-17, ) - step.log.info( - "DMS140 MSG: Pixel area in square arcseconds calculated? : " - + str( - math.isclose( - photom_out.meta.photometry.pixelarea_arcsecsq, - 0.011948, - abs_tol=1.0e-6, - ) - ) - ) - assert math.isclose( - photom_out.meta.photometry.pixelarea_arcsecsq, 0.011948, abs_tol=1.0e-6 - ) - step.log.info( "DMS140 MSG: Photom megajansky conversion uncertainty calculated? : " + str( @@ -132,22 +102,6 @@ def test_absolute_photometric_calibration(rtdata, ignore_asdf_paths): abs_tol=1.0e-6, ) - step.log.info( - "DMS140 MSG: Photom megajansky conversion uncertainty calculated? : " - + str( - math.isclose( - photom_out.meta.photometry.conversion_microjanskys_uncertainty, - 0.0, - abs_tol=1.0e-6, - ) - ) - ) - assert math.isclose( - photom_out.meta.photometry.conversion_microjanskys_uncertainty, - 0.0, - abs_tol=1.0e-6, - ) - rtdata.get_truth(f"truth/WFI/image/{output}") diff = compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) step.log.info( diff --git a/romancal/resample/resample.py b/romancal/resample/resample.py index d2a9b0b2e..964eca682 100644 --- a/romancal/resample/resample.py +++ b/romancal/resample/resample.py @@ -181,10 +181,10 @@ def __init__( l2_into_l3_meta(self.blank_output.meta, models[0].meta) self.blank_output.meta.wcs = self.output_wcs gwcs_into_l3(self.blank_output, self.output_wcs) - self.blank_output.cal_logs = stnode.CalLogs() - self.blank_output["individual_image_cal_logs"] = [ - model.cal_logs for model in models - ] + # self.blank_output.cal_logs = stnode.CalLogs() + # self.blank_output["individual_image_cal_logs"] = [ + # model.cal_logs for model in models + # ] for i, m in enumerate(models): self.input_models.shelve(m, i, modify=False) @@ -756,7 +756,6 @@ def l2_into_l3_meta(l3_meta, l2_meta): basic.segment: observation.segment basic.pass: observation.pass basic.program: observation.program - basic.survey: obervation.survey basic.optical_element: optical_element basic.instrument: instrument.name basic.telescope: telescope @@ -766,7 +765,6 @@ def l2_into_l3_meta(l3_meta, l2_meta): l3_meta.basic.segment = l2_meta.observation.segment l3_meta.basic["pass"] = l2_meta.observation["pass"] l3_meta.basic.program = l2_meta.observation.program - l3_meta.basic.survey = l2_meta.observation.survey l3_meta.basic.optical_element = l2_meta.instrument.optical_element l3_meta.basic.instrument = l2_meta.instrument.name l3_meta.coordinates = l2_meta.coordinates @@ -921,12 +919,7 @@ def populate_mosaic_basic( output_model.meta.basic.program = ( input_meta[0].observation.program if len({x.observation.program for x in input_meta}) == 1 - else "-1" - ) - output_model.meta.basic.survey = ( - input_meta[0].observation.survey - if len({x.observation.survey for x in input_meta}) == 1 - else "MULTIPLE" + else -1 ) # instrument data diff --git a/romancal/resample/resample_step.py b/romancal/resample/resample_step.py index 5315f150d..5bb1c283a 100644 --- a/romancal/resample/resample_step.py +++ b/romancal/resample/resample_step.py @@ -156,7 +156,7 @@ def _final_updates(self, model, input_models, kwargs): # calculate the actual value of pixel_scale_ratio based on pixel_scale # because source_catalog uses this value from the header. model.meta.resample.pixel_scale_ratio = ( - self.pixel_scale / np.sqrt(model.meta.photometry.pixelarea_arcsecsq) + self.pixel_scale / np.sqrt((model.meta.photometry.pixel_area * 4.254517e10)) if self.pixel_scale else self.pixel_scale_ratio ) @@ -240,12 +240,8 @@ def _load_custom_wcs(asdf_wcs_file, output_shape): def update_phot_keywords(self, model): """Update pixel scale keywords""" - if model.meta.photometry.pixelarea_steradians is not None: - model.meta.photometry.pixelarea_steradians *= ( - model.meta.resample.pixel_scale_ratio**2 - ) - if model.meta.photometry.pixelarea_arcsecsq is not None: - model.meta.photometry.pixelarea_arcsecsq *= ( + if model.meta.photometry.pixel_area is not None: + model.meta.photometry.pixel_area *= ( model.meta.resample.pixel_scale_ratio**2 ) diff --git a/romancal/resample/tests/test_resample.py b/romancal/resample/tests/test_resample.py index b20579f08..2d4d411cb 100644 --- a/romancal/resample/tests/test_resample.py +++ b/romancal/resample/tests/test_resample.py @@ -41,7 +41,6 @@ def create_mock_model( mock_model.meta.observation.segment = segment mock_model.meta.observation["pass"] = pass_ mock_model.meta.observation.program = program - mock_model.meta.observation.survey = survey mock_model.meta.instrument.optical_element = optical_element mock_model.meta.instrument.name = instrument_name mock_model.meta.wcsinfo.vparity = -1 @@ -82,7 +81,7 @@ def create_image(self): "effective_exposure_time": 3.04 * 6 * 8, }, "observation": { - "program": "00005", + "program": 5, "execution_plan": 1, "pass": 1, "observation": 1, @@ -754,12 +753,7 @@ def test_populate_mosaic_basic_single_exposure(exposure_1): assert output_model.meta.basic.program == ( input_meta[0].observation.program if len({x.observation.program for x in input_meta}) == 1 - else "-1" - ) - assert output_model.meta.basic.survey == ( - input_meta[0].observation.survey - if len({x.observation.survey for x in input_meta}) == 1 - else "MULTIPLE" + else -1 ) assert ( output_model.meta.basic.optical_element @@ -779,7 +773,7 @@ def test_populate_mosaic_basic_single_exposure(exposure_1): 1, 1, 1, - "12345", + 12345, "N/A", "F158", "WFI", @@ -790,7 +784,7 @@ def test_populate_mosaic_basic_single_exposure(exposure_1): 1, 1, 1, - "12345", + 12345, "N/A", "F158", "WFI", @@ -803,7 +797,7 @@ def test_populate_mosaic_basic_single_exposure(exposure_1): "visit": 1, "segment": 1, "pass": 1, - "program": "12345", + "program": 12345, "survey": "N/A", "optical_element": "F158", "instrument": "WFI", @@ -818,7 +812,7 @@ def test_populate_mosaic_basic_single_exposure(exposure_1): 1, 1, 1, - "12345", + 12345, "N/A", "F158", "WFI", @@ -829,7 +823,7 @@ def test_populate_mosaic_basic_single_exposure(exposure_1): 2, 1, 1, - "12345", + 12345, "N/A", "F158", "WFI", @@ -842,7 +836,7 @@ def test_populate_mosaic_basic_single_exposure(exposure_1): "visit": -1, "segment": 1, "pass": 1, - "program": "12345", + "program": 12345, "survey": "N/A", "optical_element": "F158", "instrument": "WFI", @@ -857,7 +851,7 @@ def test_populate_mosaic_basic_single_exposure(exposure_1): 1, 1, 1, - "12345", + 12345, "N/A", "F158", "WFI", @@ -868,7 +862,7 @@ def test_populate_mosaic_basic_single_exposure(exposure_1): 1, 2, 1, - "12345", + 12345, "N/A", "F158", "WFI", @@ -881,7 +875,7 @@ def test_populate_mosaic_basic_single_exposure(exposure_1): "visit": 1, "segment": -1, "pass": 1, - "program": "12345", + "program": 12345, "survey": "N/A", "optical_element": "F158", "instrument": "WFI", @@ -896,7 +890,7 @@ def test_populate_mosaic_basic_single_exposure(exposure_1): 1, 1, 1, - "12345", + 12345, "HLS", "F158", "WFI", @@ -907,7 +901,7 @@ def test_populate_mosaic_basic_single_exposure(exposure_1): 1, 1, 2, - "12345", + 12345, "EMS", "F158", "WFI", @@ -920,7 +914,7 @@ def test_populate_mosaic_basic_single_exposure(exposure_1): "visit": 1, "segment": 1, "pass": -1, - "program": "12345", + "program": 12345, "survey": "MULTIPLE", "optical_element": "F158", "instrument": "WFI", @@ -935,7 +929,7 @@ def test_populate_mosaic_basic_single_exposure(exposure_1): 1, 1, 1, - "12345", + 12345, "N/A", "F158", "WFI", @@ -946,7 +940,7 @@ def test_populate_mosaic_basic_single_exposure(exposure_1): 1, 1, 1, - "54321", + 54321, "N/A", "F158", "WFI", @@ -959,7 +953,7 @@ def test_populate_mosaic_basic_single_exposure(exposure_1): "visit": 1, "segment": 1, "pass": 1, - "program": "-1", + "program": -1, "survey": "N/A", "optical_element": "F158", "instrument": "WFI", @@ -974,7 +968,7 @@ def test_populate_mosaic_basic_single_exposure(exposure_1): 1, 1, 1, - "12345", + 12345, "HLS", "F158", "WFI", @@ -985,7 +979,7 @@ def test_populate_mosaic_basic_single_exposure(exposure_1): 1, 1, 1, - "12345", + 12345, "EMS", "F158", "WFI", @@ -998,7 +992,7 @@ def test_populate_mosaic_basic_single_exposure(exposure_1): "visit": 1, "segment": 1, "pass": 1, - "program": "12345", + "program": 12345, "survey": "MULTIPLE", "optical_element": "F158", "instrument": "WFI", @@ -1034,7 +1028,6 @@ def test_populate_mosaic_basic_different_observations( assert output_model.meta.basic.visit == expected_output["visit"] assert output_model.meta.basic.segment == expected_output["segment"] assert output_model.meta.basic.program == expected_output["program"] - assert output_model.meta.basic.survey == expected_output["survey"] assert output_model.meta.basic.optical_element == expected_output["optical_element"] assert output_model.meta.basic.instrument == expected_output["instrument"] diff --git a/romancal/resample/tests/test_resample_step.py b/romancal/resample/tests/test_resample_step.py index a221cc1f0..7ae775e3c 100644 --- a/romancal/resample/tests/test_resample_step.py +++ b/romancal/resample/tests/test_resample_step.py @@ -12,22 +12,21 @@ class MockModel: - def __init__(self, pixelarea_steradians, pixelarea_arcsecsq, pixel_scale_ratio): + def __init__(self, pixel_area, pixel_scale_ratio): self.meta = MockMeta( - pixelarea_steradians, pixelarea_arcsecsq, pixel_scale_ratio + pixel_area, pixel_scale_ratio ) class MockMeta: - def __init__(self, pixelarea_steradians, pixelarea_arcsecsq, pixel_scale_ratio): - self.photometry = MockPhotometry(pixelarea_steradians, pixelarea_arcsecsq) + def __init__(self, pixel_area, pixel_scale_ratio): + self.photometry = MockPhotometry(pixel_area) self.resample = MockResample(pixel_scale_ratio) class MockPhotometry: - def __init__(self, pixelarea_steradians, pixelarea_arcsecsq): - self.pixelarea_steradians = pixelarea_steradians - self.pixelarea_arcsecsq = pixelarea_arcsecsq + def __init__(self, pixel_area): + self.pixel_area = pixel_area class MockResample: @@ -238,32 +237,30 @@ def test_check_list_pars_exception(vals, name, min_vals): @pytest.mark.parametrize( - """pixelarea_steradians, pixelarea_arcsecsq, pixel_scale_ratio, + """pixel_area, pixel_scale_ratio, expected_steradians, expected_arcsecsq""", [ # Happy path tests - (1.0, 1.0, 2.0, 4.0, 4.0), - (2.0, 2.0, 0.5, 0.5, 0.5), - (0.0, 0.0, 2.0, 0.0, 0.0), - (1.0, 1.0, 0.0, 0.0, 0.0), - (None, 1.0, 2.0, None, 4.0), - (1.0, None, 2.0, 4.0, None), + (1.0, 2.0, 4.0, 4.0), + (2.0, 0.5, 0.5, 0.5), + (0.0, 2.0, 0.0, 0.0), + (1.0, 0.0, 0.0, 0.0), + (None, 2.0, None, 4.0), + (1.0, 2.0, 4.0, None), ], ) def test_update_phot_keywords( - pixelarea_steradians, - pixelarea_arcsecsq, + pixel_area, pixel_scale_ratio, expected_steradians, expected_arcsecsq, ): step = ResampleStep() - model = MockModel(pixelarea_steradians, pixelarea_arcsecsq, pixel_scale_ratio) + model = MockModel(pixel_area, pixel_scale_ratio) step.update_phot_keywords(model) - assert model.meta.photometry.pixelarea_steradians == expected_steradians - assert model.meta.photometry.pixelarea_arcsecsq == expected_arcsecsq + assert model.meta.photometry.pixel_area == expected_steradians @pytest.mark.parametrize( diff --git a/romancal/saturation/tests/test_saturation.py b/romancal/saturation/tests/test_saturation.py index f6bbd3df9..ca83e03d8 100644 --- a/romancal/saturation/tests/test_saturation.py +++ b/romancal/saturation/tests/test_saturation.py @@ -7,7 +7,7 @@ import numpy as np import pytest from roman_datamodels import maker_utils -from roman_datamodels.datamodels import ScienceRawModel +from roman_datamodels.datamodels import ScienceRawModel, RampModel from roman_datamodels.dqflags import group, pixel from romancal.saturation import SaturationStep @@ -19,11 +19,11 @@ def test_basic_saturation_flagging(setup_wfi_datamodels): threshold given by the reference file.""" # Create inputs, and data and saturation models - ngroups = 5 + nresultants = 5 nrows = 20 ncols = 20 satvalue = 60000 - ramp, satmap = setup_wfi_datamodels(ngroups, nrows, ncols) + ramp, satmap = setup_wfi_datamodels(nresultants, nrows, ncols) # Add ramp values up to the saturation limit ramp.data[0, 5, 5] = 0 @@ -48,11 +48,11 @@ def test_read_pattern_saturation_flagging(setup_wfi_datamodels): are allocated into resultants.""" # Create inputs, and data and saturation models - ngroups = 5 + nresultants = 5 nrows = 20 ncols = 20 satvalue = 60000 - ramp, satmap = setup_wfi_datamodels(ngroups, nrows, ncols) + ramp, satmap = setup_wfi_datamodels(nresultants, nrows, ncols) # Add ramp values up to the saturation limit ramp.data[0, 5, 5] = 0 @@ -91,12 +91,12 @@ def test_ad_floor_flagging(setup_wfi_datamodels): negative.""" # Create inputs, and data and saturation models - ngroups = 5 + nresultants = 5 nrows = 20 ncols = 20 satvalue = 60000 - ramp, satmap = setup_wfi_datamodels(ngroups, nrows, ncols) + ramp, satmap = setup_wfi_datamodels(nresultants, nrows, ncols) # Add ramp values up to the saturation limit ramp.data[0, 5, 5] = 0 # Signal at bottom rail - low saturation @@ -125,12 +125,12 @@ def test_ad_floor_and_saturation_flagging(setup_wfi_datamodels): """ # Create inputs, and data and saturation models - ngroups = 5 + nresultants = 5 nrows = 20 ncols = 20 satvalue = 60000 - ramp, satmap = setup_wfi_datamodels(ngroups, nrows, ncols) + ramp, satmap = setup_wfi_datamodels(nresultants, nrows, ncols) # Add ramp values up to the saturation limit ramp.data[0, 5, 5] = 0 # Signal at bottom rail - low saturation @@ -162,12 +162,12 @@ def test_signal_fluctuation_flagging(setup_wfi_datamodels): signal value drops back below saturation.""" # Create inputs, and data and saturation models - ngroups = 5 + nresultants = 5 nrows = 20 ncols = 20 satvalue = 60000 - ramp, satmap = setup_wfi_datamodels(ngroups, nrows, ncols) + ramp, satmap = setup_wfi_datamodels(nresultants, nrows, ncols) # Add ramp values up to the saturation limit ramp.data[0, 5, 5] = 10 @@ -191,12 +191,12 @@ def test_all_groups_saturated(setup_wfi_datamodels): """Check case where all groups are saturated.""" # Create inputs, and data and saturation models - ngroups = 5 + nresultants = 5 nrows = 20 ncols = 20 satvalue = 60000 - ramp, satmap = setup_wfi_datamodels(ngroups, nrows, ncols) + ramp, satmap = setup_wfi_datamodels(nresultants, nrows, ncols) # Add ramp values at or above saturation limit ramp.data[0, 5, 5] = 60000 @@ -219,13 +219,13 @@ def test_dq_propagation(setup_wfi_datamodels): """Check PIXELDQ propagation.""" # Create inputs, data, and saturation maps - ngroups = 5 + nresultants = 5 nrows = 20 ncols = 20 dqval1 = 5 dqval2 = 10 - ramp, satmap = setup_wfi_datamodels(ngroups, nrows, ncols) + ramp, satmap = setup_wfi_datamodels(nresultants, nrows, ncols) # Add DQ values to the data and reference file ramp.pixeldq[5, 5] = dqval1 @@ -243,12 +243,12 @@ def test_no_sat_check(setup_wfi_datamodels): added to the DQ mask and are not flagged as saturated.""" # Create inputs, and data and saturation models - ngroups = 5 + nresultants = 5 nrows = 20 ncols = 20 satvalue = 60000 - ramp, satmap = setup_wfi_datamodels(ngroups, nrows, ncols) + ramp, satmap = setup_wfi_datamodels(nresultants, nrows, ncols) # Add ramp values up to the saturation limit ramp.data[0, 5, 5] = 10 @@ -283,11 +283,11 @@ def test_nans_in_mask(setup_wfi_datamodels): pixel is set to NO_SAT_CHECK.""" # Create inputs, and data and saturation models - ngroups = 5 + nresultants = 5 nrows = 20 ncols = 20 - ramp, satmap = setup_wfi_datamodels(ngroups, nrows, ncols) + ramp, satmap = setup_wfi_datamodels(nresultants, nrows, ncols) # Add ramp values up to the saturation limit ramp.data[0, 5, 5] = 10 @@ -320,14 +320,15 @@ def test_saturation_getbestref(setup_wfi_datamodels): wfi_sci_raw.meta.instrument.name = "WFI" wfi_sci_raw.meta.instrument.detector = "WFI01" wfi_sci_raw.meta.instrument.optical_element = "F158" - wfi_sci_raw.meta["guidestar"]["gw_window_xstart"] = 1012 - wfi_sci_raw.meta["guidestar"]["gw_window_xsize"] = 16 + wfi_sci_raw.meta["guide_star"]["gw_window_xstart"] = 1012 + wfi_sci_raw.meta["guide_star"]["gw_window_xsize"] = 16 wfi_sci_raw.meta.exposure.type = "WFI_IMAGE" wfi_sci_raw.data = np.ones(shape, dtype=np.uint16) wfi_sci_raw_model = ScienceRawModel(wfi_sci_raw, dq=True) + input_model = RampModel.from_science_raw(wfi_sci_raw_model) # Run the pipeline - result = SaturationStep.call(wfi_sci_raw_model, override_saturation="N/A") + result = SaturationStep.call(input_model, override_saturation="N/A") assert result.meta.cal_step.saturation == "SKIPPED" @@ -335,9 +336,9 @@ def test_saturation_getbestref(setup_wfi_datamodels): def setup_wfi_datamodels(): """Set up fake WFI data to test.""" - def _models(ngroups, nrows, ncols): + def _models(nresultants, nrows, ncols): # Create ramp data - ramp_model = maker_utils.mk_ramp(shape=(ngroups, nrows, ncols)) + ramp_model = maker_utils.mk_ramp(shape=(nresultants, nrows, ncols)) # Create saturation reference data saturation_model = maker_utils.mk_saturation(shape=(nrows, ncols)) diff --git a/romancal/source_catalog/source_catalog.py b/romancal/source_catalog/source_catalog.py index 66dadaf82..59ceddbfa 100644 --- a/romancal/source_catalog/source_catalog.py +++ b/romancal/source_catalog/source_catalog.py @@ -157,7 +157,7 @@ def pixel_area(self): If the value is a negative placeholder (e.g., -999999 sr), the value is calculated from the WCS at the center of the image. """ - pixel_area = self.model.meta.photometry.pixelarea_steradians + pixel_area = self.model.meta.photometry.pixel_area if pixel_area < 0: pixel_area = (self._pixel_scale**2).to(u.sr).value return pixel_area diff --git a/romancal/stpipe/core.py b/romancal/stpipe/core.py index ca0f4f31d..651f522bb 100644 --- a/romancal/stpipe/core.py +++ b/romancal/stpipe/core.py @@ -75,8 +75,11 @@ def finalize_result(self, model, reference_files_used): model.meta.calibration_software_version = importlib.metadata.version("romancal") - if isinstance(model, (ImageModel, MosaicModel)): + # FIXME: should cal_logs be brought back into meta for a MosaicModel? + if isinstance(model, ImageModel): # convert to model.cal_logs type to avoid validation errors + model.meta.cal_logs = type(model.meta.cal_logs)(self.log_records) + if isinstance(model, MosaicModel): model.cal_logs = type(model.cal_logs)(self.log_records) if len(reference_files_used) > 0: diff --git a/romancal/stpipe/tests/test_core.py b/romancal/stpipe/tests/test_core.py index 8ecc57a35..0483280bf 100644 --- a/romancal/stpipe/tests/test_core.py +++ b/romancal/stpipe/tests/test_core.py @@ -109,7 +109,7 @@ def process(self): return ImageModel(mk_level2_image(shape=(20, 20))) result = LoggingStep().run() - assert any("Splines failed to reticulate" in l for l in result.cal_logs) + assert any("Splines failed to reticulate" in l for l in result.meta.cal_logs) def test_crds_meta(): diff --git a/romancal/tweakreg/tests/test_tweakreg.py b/romancal/tweakreg/tests/test_tweakreg.py index 63dad8e5a..2d34afabf 100644 --- a/romancal/tweakreg/tests/test_tweakreg.py +++ b/romancal/tweakreg/tests/test_tweakreg.py @@ -907,8 +907,8 @@ def test_tweakreg_handles_multiple_groups(tmp_path, base_image): add_tweakreg_catalog_attribute(tmp_path, img1, catalog_filename="img1") add_tweakreg_catalog_attribute(tmp_path, img2, catalog_filename="img2") - img1.meta.observation["program"] = "-program_id1" - img2.meta.observation["program"] = "-program_id2" + img1.meta.observation.program = 1 + img2.meta.observation.program = 2 img1.meta["filename"] = "file1.asdf" img2.meta["filename"] = "file2.asdf" @@ -918,10 +918,12 @@ def test_tweakreg_handles_multiple_groups(tmp_path, base_image): assert len(res.group_names) == 2 with res: for r, i in zip(res, [img1, img2]): - assert ( - r.meta.group_id.split("-")[1] - == i.meta.observation.program.split("-")[1] - ) + found = False + for log_entry in i.meta.cal_logs: + if f"'GROUP ID: {r.meta.group_id}'" in log_entry: + found = True + break + assert found, "The specified log entry was not found in meta.cal_logs." res.shelve(r, modify=False) From b4c910877185d160ed7248ef6ba79c3111ddce85 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 23 Oct 2024 20:25:13 +0000 Subject: [PATCH 02/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- romancal/photom/tests/test_photom.py | 5 +---- romancal/resample/resample.py | 2 +- romancal/resample/resample_step.py | 6 ++---- romancal/resample/tests/test_resample_step.py | 4 +--- romancal/saturation/tests/test_saturation.py | 2 +- 5 files changed, 6 insertions(+), 13 deletions(-) diff --git a/romancal/photom/tests/test_photom.py b/romancal/photom/tests/test_photom.py index 63f86c3ba..f43cc6aa2 100644 --- a/romancal/photom/tests/test_photom.py +++ b/romancal/photom/tests/test_photom.py @@ -252,9 +252,7 @@ def test_photom_step_interface_spectroscopic(instrument, exptype): wfi_image.meta.instrument.optical_element = "PRISM" # Set photometric values for spectroscopic data - wfi_image.meta.photometry.pixel_area = ( - 2.31307642258977e-14 * u.steradian - ).value + wfi_image.meta.photometry.pixel_area = (2.31307642258977e-14 * u.steradian).value wfi_image.meta.photometry.conversion_megajanskys = ( -99999 * u.megajansky / u.steradian ).value @@ -279,4 +277,3 @@ def test_photom_step_interface_spectroscopic(instrument, exptype): assert result.meta.photometry.conversion_megajanskys is None assert result.meta.photometry.conversion_megajanskys_uncertainty is None assert result.meta.photometry.pixel_area is None - diff --git a/romancal/resample/resample.py b/romancal/resample/resample.py index 964eca682..462f5b24c 100644 --- a/romancal/resample/resample.py +++ b/romancal/resample/resample.py @@ -7,7 +7,7 @@ from astropy import units as u from astropy.coordinates import SkyCoord from drizzle import cdrizzle, util -from roman_datamodels import datamodels, maker_utils, stnode +from roman_datamodels import datamodels, maker_utils from stcal.alignment.util import compute_scale from romancal.associations.asn_from_list import asn_from_list diff --git a/romancal/resample/resample_step.py b/romancal/resample/resample_step.py index 5bb1c283a..0ace7a4bb 100644 --- a/romancal/resample/resample_step.py +++ b/romancal/resample/resample_step.py @@ -156,7 +156,7 @@ def _final_updates(self, model, input_models, kwargs): # calculate the actual value of pixel_scale_ratio based on pixel_scale # because source_catalog uses this value from the header. model.meta.resample.pixel_scale_ratio = ( - self.pixel_scale / np.sqrt((model.meta.photometry.pixel_area * 4.254517e10)) + self.pixel_scale / np.sqrt(model.meta.photometry.pixel_area * 4.254517e10) if self.pixel_scale else self.pixel_scale_ratio ) @@ -241,9 +241,7 @@ def _load_custom_wcs(asdf_wcs_file, output_shape): def update_phot_keywords(self, model): """Update pixel scale keywords""" if model.meta.photometry.pixel_area is not None: - model.meta.photometry.pixel_area *= ( - model.meta.resample.pixel_scale_ratio**2 - ) + model.meta.photometry.pixel_area *= model.meta.resample.pixel_scale_ratio**2 def set_drizzle_defaults(self): """Set the default parameters for drizzle.""" diff --git a/romancal/resample/tests/test_resample_step.py b/romancal/resample/tests/test_resample_step.py index 7ae775e3c..dc16a8924 100644 --- a/romancal/resample/tests/test_resample_step.py +++ b/romancal/resample/tests/test_resample_step.py @@ -13,9 +13,7 @@ class MockModel: def __init__(self, pixel_area, pixel_scale_ratio): - self.meta = MockMeta( - pixel_area, pixel_scale_ratio - ) + self.meta = MockMeta(pixel_area, pixel_scale_ratio) class MockMeta: diff --git a/romancal/saturation/tests/test_saturation.py b/romancal/saturation/tests/test_saturation.py index ca83e03d8..b5f415f09 100644 --- a/romancal/saturation/tests/test_saturation.py +++ b/romancal/saturation/tests/test_saturation.py @@ -7,7 +7,7 @@ import numpy as np import pytest from roman_datamodels import maker_utils -from roman_datamodels.datamodels import ScienceRawModel, RampModel +from roman_datamodels.datamodels import RampModel, ScienceRawModel from roman_datamodels.dqflags import group, pixel from romancal.saturation import SaturationStep From 6f6d75b8c2a0f311dc5227193ced195353117e2f Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Wed, 23 Oct 2024 16:30:51 -0400 Subject: [PATCH 03/25] Fix style checks. --- romancal/jump/jump_step.py | 1 - romancal/photom/photom.py | 3 --- romancal/photom/tests/test_photom.py | 7 ------- 3 files changed, 11 deletions(-) diff --git a/romancal/jump/jump_step.py b/romancal/jump/jump_step.py index 0810e2607..03a27cf0e 100644 --- a/romancal/jump/jump_step.py +++ b/romancal/jump/jump_step.py @@ -52,7 +52,6 @@ def process(self, input): input_model = rdd.open(input) # Extract the needed info from the Roman Data Model - meta = input_model.meta r_data = input_model.data r_gdq = input_model.groupdq r_pdq = input_model.pixeldq diff --git a/romancal/photom/photom.py b/romancal/photom/photom.py index 3e4901e45..b06343afa 100644 --- a/romancal/photom/photom.py +++ b/romancal/photom/photom.py @@ -1,8 +1,6 @@ import logging import warnings -from astropy import units as u - log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) @@ -59,7 +57,6 @@ def save_area_info(input_model, photom_parameters): # Load the average pixel area values from the photom reference file header area_ster = photom_parameters["pixelareasr"].value - area_a2 = photom_parameters["pixelareasr"].to(u.arcsecond**2).value # Copy the pixel area values to the input model input_model.meta.photometry.pixel_area = area_ster diff --git a/romancal/photom/tests/test_photom.py b/romancal/photom/tests/test_photom.py index f43cc6aa2..b3ff879c5 100644 --- a/romancal/photom/tests/test_photom.py +++ b/romancal/photom/tests/test_photom.py @@ -137,7 +137,6 @@ def test_apply_photom1(): # Set reference photometry area_ster = 2.31307642258977e-14 * u.steradian - area_a2 = 0.000984102303070964 * u.arcsecond * u.arcsecond # Tests for pixel areas assert np.isclose( @@ -148,7 +147,6 @@ def test_apply_photom1(): # Set reference photometry phot_ster = 3.5 * u.megajansky / u.steradian - phot_a2 = phot_ster.to(u.microjansky / u.arcsecond**2) # Tests for photometry assert np.isclose( @@ -159,7 +157,6 @@ def test_apply_photom1(): # Set reference photometric uncertainty muphot_ster = 0.175 * u.megajansky / u.steradian - muphot_a2 = muphot_ster.to(u.microjansky / u.arcsecond**2) # Tests for photometric uncertainty assert np.isclose( @@ -167,10 +164,6 @@ def test_apply_photom1(): muphot_ster.value, atol=1.0e-7, ) - # assert ( - # output_model.meta.photometry.conversion_megajanskys_uncertainty.unit - # == muphot_ster.unit - # ) def test_apply_photom2(): From 13e840388a2233e82ecd9530d6ada36f79b365b4 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Thu, 24 Oct 2024 09:33:24 -0400 Subject: [PATCH 04/25] Set doctest to skip code in docs. --- docs/roman/datamodels/metadata.rst | 33 +++++++++++++++++------------- docs/roman/datamodels/models.rst | 4 ++-- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/docs/roman/datamodels/metadata.rst b/docs/roman/datamodels/metadata.rst index a8e57d6f1..24a42bd34 100644 --- a/docs/roman/datamodels/metadata.rst +++ b/docs/roman/datamodels/metadata.rst @@ -20,26 +20,31 @@ string will raise an exception. >>> from roman_datamodels.maker_utils import mk_datamodel >>> model = mk_datamodel(rdm.ImageModel) - >>> model.meta.target.ra = "foo" # doctest: +IGNORE_EXCEPTION_DETAIL + >>> model.meta.pointing.target_ra = "foo" # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): File "", line 1, in - File "/Users/ddavis/miniconda3/envs/rcal_dev/lib/python3.9/site-packages/roman_datamodels/stnode.py", line 183, in __setattr__ - if schema is None or _validate(key, value, schema, self.ctx): - File "/Users/ddavis/miniconda3/envs/rcal_dev/lib/python3.9/site-packages/roman_datamodels/stnode.py", line 97, in _validate - return _value_change(attr, tagged_tree, schema, False, strict_validation, ctx) - File "/Users/ddavis/miniconda3/envs/rcal_dev/lib/python3.9/site-packages/roman_datamodels/stnode.py", line 68, in _value_change - raise jsonschema.ValidationError(errmsg) - jsonschema.exceptions.ValidationError: While validating ra the following error occurred: + File "/path/to/python/lib/python3.12/site-packages/roman_datamodels/stnode/_node.py", line 251, in __setattr__ + _validate(key, value, schema, self.ctx) + File "/path/to/python/lib/python3.12/site-packages/roman_datamodels/stnode/_node.py", line 78, in _validate + return _value_change(attr, tagged_tree, schema, False, will_strict_validate(), ctx) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/path/to/python/lib/python3.12/site-packages/roman_datamodels/stnode/_node.py", line 47, in _value_change + raise ValidationError(errmsg) + asdf._jsonschema.exceptions.ValidationError: While validating target_ra the following error occurred: 'foo' is not of type 'number' Failed validating 'type' in schema: {'$schema': 'http://stsci.edu/schemas/asdf-schema/0.1.0/asdf-schema', - 'archive_catalog': {'datatype': 'float', - 'destination': ['ScienceCommon.ra']}, - 'sdf': {'source': {'origin': 'PSS:fixed_target.ra_computed'}, - 'special_processing': 'VALUE_REQUIRED'}, - 'title': 'Target RA at mid time of exposure', - 'type': 'number'} + 'archive_catalog': {'datatype': 'float', + 'destination': ['WFIExposure.target_ra', + 'GuideWindow.target_ra']}, + 'description': 'Right ascension in units of degrees at the location ' + 'of\n' + 'the target aperture.\n', + 'sdf': {'source': {'origin': 'TBD'}, + 'special_processing': 'VALUE_REQUIRED'}, + 'title': 'Right Ascension of the Target Aperture (deg)', + 'type': 'number'} On instance: 'foo' diff --git a/docs/roman/datamodels/models.rst b/docs/roman/datamodels/models.rst index 9c889031b..afab6f456 100644 --- a/docs/roman/datamodels/models.rst +++ b/docs/roman/datamodels/models.rst @@ -141,7 +141,7 @@ You can examine the contents of your model from within python using .. code-block:: python - >>> print("\n".join("{: >20}\t{}".format(k, v) for k, v in image_model.items()), "\n") # doctest: +ELLIPSIS + >>> print("\n".join("{: >20}\t{}".format(k, v) for k, v in image_model.items()), "\n") # doctest: +ELLIPSIS +SKIP meta.calibration_software_version 9.9.0 meta.sdf_software_version 7.7.7 meta.filename r0019106003005004023_03203_0034_WFI01_cal.asdf @@ -158,7 +158,7 @@ or you can print specifics .. code-block:: python - >>> print("\n".join("{: >20}\t{}".format(k, v) for k, v in image_model.meta.instrument.items())) + >>> print("\n".join("{: >20}\t{}".format(k, v) for k, v in image_model.meta.instrument.items())) # doctest: +SKIP name WFI detector WFI01 optical_element F158 From 0d4d07a487f7ba7a6f9df4576bccf337135ec54b Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Thu, 24 Oct 2024 10:52:42 -0400 Subject: [PATCH 05/25] Rename obs_id to observation_id. --- romancal/datamodels/library.py | 2 +- romancal/tweakreg/tests/test_tweakreg.py | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/romancal/datamodels/library.py b/romancal/datamodels/library.py index fa72195d7..1947e6859 100644 --- a/romancal/datamodels/library.py +++ b/romancal/datamodels/library.py @@ -75,4 +75,4 @@ def _mapping_to_group_id(mapping): """ Combine a number of file metadata values into a ``group_id`` string """ - return "{obs_id}".format_map(mapping) + return "{observation_id}".format_map(mapping) diff --git a/romancal/tweakreg/tests/test_tweakreg.py b/romancal/tweakreg/tests/test_tweakreg.py index 2d34afabf..956fa66c8 100644 --- a/romancal/tweakreg/tests/test_tweakreg.py +++ b/romancal/tweakreg/tests/test_tweakreg.py @@ -908,7 +908,9 @@ def test_tweakreg_handles_multiple_groups(tmp_path, base_image): add_tweakreg_catalog_attribute(tmp_path, img2, catalog_filename="img2") img1.meta.observation.program = 1 + img1.meta.observation["observation_id"] = "1" img2.meta.observation.program = 2 + img2.meta.observation["observation_id"] = "2" img1.meta["filename"] = "file1.asdf" img2.meta["filename"] = "file2.asdf" @@ -918,12 +920,7 @@ def test_tweakreg_handles_multiple_groups(tmp_path, base_image): assert len(res.group_names) == 2 with res: for r, i in zip(res, [img1, img2]): - found = False - for log_entry in i.meta.cal_logs: - if f"'GROUP ID: {r.meta.group_id}'" in log_entry: - found = True - break - assert found, "The specified log entry was not found in meta.cal_logs." + assert r.meta.group_id == i.meta.observation.observation_id res.shelve(r, modify=False) From 9299696ba4b0dc6209310efd1c7f0a564fba582f Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Thu, 24 Oct 2024 14:04:21 -0400 Subject: [PATCH 06/25] More obs_id fixes. --- .../outlier_detection/tests/test_outlier_detection.py | 2 +- romancal/resample/tests/test_resample.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/romancal/outlier_detection/tests/test_outlier_detection.py b/romancal/outlier_detection/tests/test_outlier_detection.py index ac9fbbb33..df60e3b6d 100644 --- a/romancal/outlier_detection/tests/test_outlier_detection.py +++ b/romancal/outlier_detection/tests/test_outlier_detection.py @@ -142,7 +142,7 @@ def test_find_outliers(tmp_path, base_image, on_disk): img.data[42, 72] = source_value img.err[:] = err_value img.meta.filename = str(tmp_path / f"img{i}_suffix.asdf") - img.meta.observation.obs_id = str(i) + img.meta.observation.observation_id = str(i) img.meta.background.level = 0 imgs.append(img) diff --git a/romancal/resample/tests/test_resample.py b/romancal/resample/tests/test_resample.py index 72b3130b9..4368a8359 100644 --- a/romancal/resample/tests/test_resample.py +++ b/romancal/resample/tests/test_resample.py @@ -251,8 +251,8 @@ def exposure_1(wfi_sca1, wfi_sca2, wfi_sca3): sca.meta.exposure["end_time"] = Time( "2020-02-01T00:02:30", format="isot", scale="utc" ) - sca.meta.observation["exposure"] = 1 - sca.meta.observation["obs_id"] = "1" + sca.meta.observation.exposure = 1 + sca.meta.observation.observation_id = "1" return [wfi_sca1, wfi_sca2, wfi_sca3] @@ -267,8 +267,8 @@ def exposure_2(wfi_sca4, wfi_sca5, wfi_sca6): sca.meta.exposure["end_time"] = Time( "2020-05-01T00:02:30", format="isot", scale="utc" ) - sca.meta.observation["exposure"] = 2 - sca.meta.observation["obs_id"] = "2" + sca.meta.observation.exposure = 2 + sca.meta.observation.observation_id = "2" return [wfi_sca4, wfi_sca5, wfi_sca6] From 4c83db06d13885cbe98a9837cd8de45cff000cef Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Thu, 24 Oct 2024 15:04:37 -0400 Subject: [PATCH 07/25] Fix tweakreg unit test. --- romancal/tweakreg/tests/test_tweakreg.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/romancal/tweakreg/tests/test_tweakreg.py b/romancal/tweakreg/tests/test_tweakreg.py index 956fa66c8..159d01020 100644 --- a/romancal/tweakreg/tests/test_tweakreg.py +++ b/romancal/tweakreg/tests/test_tweakreg.py @@ -498,6 +498,8 @@ def test_tweakreg_raises_attributeerror_on_missing_tweakreg_catalog(base_image): Test that TweakReg raises an AttributeError if meta.tweakreg_catalog is missing. """ img = base_image() + # remove attribute + del img.meta.source_detection["tweakreg_catalog_name"] with pytest.raises(AttributeError): trs.TweakRegStep.call([img]) From 1f0fa6bdacb7ba4829f19bcdc3c2533d6d513f6d Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Thu, 24 Oct 2024 15:24:11 -0400 Subject: [PATCH 08/25] Fix code block in models.rst. --- docs/roman/datamodels/models.rst | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/roman/datamodels/models.rst b/docs/roman/datamodels/models.rst index afab6f456..d46f30161 100644 --- a/docs/roman/datamodels/models.rst +++ b/docs/roman/datamodels/models.rst @@ -141,24 +141,25 @@ You can examine the contents of your model from within python using .. code-block:: python - >>> print("\n".join("{: >20}\t{}".format(k, v) for k, v in image_model.items()), "\n") # doctest: +ELLIPSIS +SKIP - meta.calibration_software_version 9.9.0 - meta.sdf_software_version 7.7.7 - meta.filename r0019106003005004023_03203_0034_WFI01_cal.asdf - meta.file_date 2020-01-01T00:00:00.000 - meta.model_type ImageModel - meta.origin STSCI - meta.prd_software_version 8.8.8 - meta.telescope ROMAN - meta.aperture.name WFI_06_FULL - meta.aperture.position_angle 30.0 + >>> print("\n".join("{: >20}\t{}".format(k, v) for k, v in image_model.items()), "\n") # doctest: +ELLIPSIS + meta.calibration_software_name RomanCAL + meta.calibration_software_version 9.9.0 + meta.product_type l2 + meta.filename r0019106003005004023_03203_0034_WFI01_cal.asdf + meta.file_date 2020-01-01T00:00:00.000 + meta.model_type ImageModel + meta.origin STSCI/SOC + meta.prd_version 8.8.8 + meta.sdf_software_version 7.7.7 + meta.telescope ROMAN + meta.coordinates.reference_frame ICRS ... or you can print specifics .. code-block:: python - >>> print("\n".join("{: >20}\t{}".format(k, v) for k, v in image_model.meta.instrument.items())) # doctest: +SKIP + >>> print("\n".join("{: >20}\t{}".format(k, v) for k, v in image_model.meta.instrument.items())) name WFI detector WFI01 optical_element F158 From f27ab1d49b81ca343b121cfe9900e2fda664d1a5 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Thu, 24 Oct 2024 15:41:29 -0400 Subject: [PATCH 09/25] Restore changes. --- romancal/ramp_fitting/ramp_fit_step.py | 2 +- romancal/resample/resample.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/romancal/ramp_fitting/ramp_fit_step.py b/romancal/ramp_fitting/ramp_fit_step.py index f14cb2c07..e534fdb30 100644 --- a/romancal/ramp_fitting/ramp_fit_step.py +++ b/romancal/ramp_fitting/ramp_fit_step.py @@ -86,7 +86,7 @@ def ols_cas22(self, input_model, readnoise_model, gain_model, dark_model): out_model : ImageModel Model containing a count-rate image. """ - use_jump = False + use_jump = self.use_ramp_jump_detection if use_jump: log.info("Jump detection as part of ramp fitting is enabled.") diff --git a/romancal/resample/resample.py b/romancal/resample/resample.py index 462f5b24c..f031f3a08 100644 --- a/romancal/resample/resample.py +++ b/romancal/resample/resample.py @@ -7,7 +7,7 @@ from astropy import units as u from astropy.coordinates import SkyCoord from drizzle import cdrizzle, util -from roman_datamodels import datamodels, maker_utils +from roman_datamodels import datamodels, maker_utils, stnode from stcal.alignment.util import compute_scale from romancal.associations.asn_from_list import asn_from_list @@ -181,10 +181,10 @@ def __init__( l2_into_l3_meta(self.blank_output.meta, models[0].meta) self.blank_output.meta.wcs = self.output_wcs gwcs_into_l3(self.blank_output, self.output_wcs) - # self.blank_output.cal_logs = stnode.CalLogs() - # self.blank_output["individual_image_cal_logs"] = [ - # model.cal_logs for model in models - # ] + self.blank_output.cal_logs = stnode.CalLogs() + self.blank_output["individual_image_cal_logs"] = [ + model.meta.cal_logs for model in models + ] for i, m in enumerate(models): self.input_models.shelve(m, i, modify=False) From 014b0eaec810d65469d3d2c73b626cc0a242906d Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Thu, 24 Oct 2024 16:14:58 -0400 Subject: [PATCH 10/25] Add sr to arcsec2conversion factor as a global variable. --- romancal/resample/resample_step.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/romancal/resample/resample_step.py b/romancal/resample/resample_step.py index 0ace7a4bb..de0154191 100644 --- a/romancal/resample/resample_step.py +++ b/romancal/resample/resample_step.py @@ -18,6 +18,9 @@ __all__ = ["ResampleStep"] +# conversion factor from steradian to squared arcsec +SR_TO_ARCSEC2 = 4.254517e10 + class ResampleStep(RomanStep): """ @@ -156,7 +159,7 @@ def _final_updates(self, model, input_models, kwargs): # calculate the actual value of pixel_scale_ratio based on pixel_scale # because source_catalog uses this value from the header. model.meta.resample.pixel_scale_ratio = ( - self.pixel_scale / np.sqrt(model.meta.photometry.pixel_area * 4.254517e10) + self.pixel_scale / np.sqrt(model.meta.photometry.pixel_area * SR_TO_ARCSEC2) if self.pixel_scale else self.pixel_scale_ratio ) From 180409df6430889a5feab8b4ea98fd9a71d27b61 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Fri, 25 Oct 2024 17:41:10 -0400 Subject: [PATCH 11/25] Updates to regtests and make_regtestdata.sh for the L1/L2 schema changes. --- romancal/dq_init/dq_init_step.py | 4 +- romancal/dq_init/tests/test_dq_init.py | 16 ++--- romancal/flatfield/flat_field.py | 10 +-- romancal/linearity/tests/test_linearity.py | 4 +- romancal/regtest/test_wfi_flat_field.py | 34 --------- .../regtest/test_wfi_grism_16resultants.py | 1 - romancal/regtest/test_wfi_grism_pipeline.py | 1 - .../regtest/test_wfi_image_16resultants.py | 1 - romancal/regtest/test_wfi_image_pipeline.py | 2 - romancal/saturation/tests/test_saturation.py | 4 +- romancal/scripts/make_regtestdata.sh | 72 ++++++++++--------- 11 files changed, 58 insertions(+), 91 deletions(-) diff --git a/romancal/dq_init/dq_init_step.py b/romancal/dq_init/dq_init_step.py index 1355c0205..1aecbe611 100644 --- a/romancal/dq_init/dq_init_step.py +++ b/romancal/dq_init/dq_init_step.py @@ -44,8 +44,8 @@ def process(self, input): output_model = RampModel.from_science_raw(input_model) # guide window range information - x_start = input_model.meta.guide_star.gw_window_xstart - x_end = input_model.meta.guide_star.gw_window_xsize + x_start + x_start = input_model.meta.guide_star.window_xstart + x_end = input_model.meta.guide_star.window_xsize + x_start # set pixeldq array to GW_AFFECTED_DATA (2**4) for the given range output_model.pixeldq[int(x_start) : int(x_end), :] = pixel.GW_AFFECTED_DATA self.log.info( diff --git a/romancal/dq_init/tests/test_dq_init.py b/romancal/dq_init/tests/test_dq_init.py index 2ff132231..2d8f7929c 100644 --- a/romancal/dq_init/tests/test_dq_init.py +++ b/romancal/dq_init/tests/test_dq_init.py @@ -195,8 +195,8 @@ def test_dqinit_step_interface(instrument, exptype): wfi_sci_raw.meta.instrument.name = instrument wfi_sci_raw.meta.instrument.detector = "WFI01" wfi_sci_raw.meta.instrument.optical_element = "F158" - wfi_sci_raw.meta["guide_star"]["gw_window_xstart"] = 1012 - wfi_sci_raw.meta["guide_star"]["gw_window_xsize"] = 16 + wfi_sci_raw.meta["guide_star"]["window_xstart"] = 1012 + wfi_sci_raw.meta["guide_star"]["window_xsize"] = 16 wfi_sci_raw.meta.exposure.type = exptype wfi_sci_raw.data = np.ones(shape, dtype=np.uint16) wfi_sci_raw[extra_key] = extra_value @@ -245,8 +245,8 @@ def test_dqinit_refpix(instrument, exptype): wfi_sci_raw.meta.instrument.name = instrument wfi_sci_raw.meta.instrument.detector = "WFI01" wfi_sci_raw.meta.instrument.optical_element = "F158" - wfi_sci_raw.meta["guide_star"]["gw_window_xstart"] = 1012 - wfi_sci_raw.meta["guide_star"]["gw_window_xsize"] = 16 + wfi_sci_raw.meta["guide_star"]["window_xstart"] = 1012 + wfi_sci_raw.meta["guide_star"]["window_xsize"] = 16 wfi_sci_raw.meta.exposure.type = exptype wfi_sci_raw.data = np.ones(shape, dtype=np.uint16) wfi_sci_raw_model = ScienceRawModel(wfi_sci_raw) @@ -295,8 +295,8 @@ def test_dqinit_resultantdq(instrument, exptype): wfi_sci_raw.meta.instrument.name = instrument wfi_sci_raw.meta.instrument.detector = "WFI01" wfi_sci_raw.meta.instrument.optical_element = "F158" - wfi_sci_raw.meta["guide_star"]["gw_window_xstart"] = 1012 - wfi_sci_raw.meta["guide_star"]["gw_window_xsize"] = 16 + wfi_sci_raw.meta["guide_star"]["window_xstart"] = 1012 + wfi_sci_raw.meta["guide_star"]["window_xsize"] = 16 wfi_sci_raw.meta.exposure.type = exptype wfi_sci_raw.resultantdq[1, 12, 12] = pixel["DROPOUT"] wfi_sci_raw.data = np.ones(shape, dtype=np.uint16) @@ -344,8 +344,8 @@ def test_dqinit_getbestref(instrument, exptype): wfi_sci_raw.meta.instrument.name = instrument wfi_sci_raw.meta.instrument.detector = "WFI01" wfi_sci_raw.meta.instrument.optical_element = "F158" - wfi_sci_raw.meta["guide_star"]["gw_window_xstart"] = 1012 - wfi_sci_raw.meta["guide_star"]["gw_window_xsize"] = 16 + wfi_sci_raw.meta["guide_star"]["window_xstart"] = 1012 + wfi_sci_raw.meta["guide_star"]["window_xsize"] = 16 wfi_sci_raw.meta.exposure.type = exptype wfi_sci_raw.data = np.ones(shape, dtype=np.uint16) wfi_sci_raw_model = ScienceRawModel(wfi_sci_raw) diff --git a/romancal/flatfield/flat_field.py b/romancal/flatfield/flat_field.py index b74475274..bc84ad9b9 100644 --- a/romancal/flatfield/flat_field.py +++ b/romancal/flatfield/flat_field.py @@ -6,6 +6,8 @@ import numpy as np from roman_datamodels.dqflags import pixel +from roman_datamodels import stnode + log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) @@ -51,18 +53,18 @@ def do_flat_field(output_model, flat_model): # Check to see if flat data array is smaller than science data log.warning("Flat data array is not the same shape as the science data") log.warning("Step will be skipped") - output_model.meta.cal_step.flat_field = "SKIPPED" + output_model.meta.cal_step["flat_field"] = "SKIPPED" elif output_model.meta.exposure.type != "WFI_IMAGE": # Check to see if attempt to flatten non-Image data log.info("Skipping flat field for spectral exposure.") - output_model.meta.cal_step.flat_field = "SKIPPED" + output_model.meta.cal_step["flat_field"] = "SKIPPED" elif flat_model is None: # Check to see if attempt to flatten non-Image data log.info("Skipping flat field - no flat reference file.") - output_model.meta.cal_step.flat_field = "SKIPPED" + output_model.meta.cal_step["flat_field"] = "SKIPPED" else: apply_flat_field(output_model, flat_model) - output_model.meta.cal_step.flat_field = "COMPLETE" + output_model.meta.cal_step["flat_field"] = "COMPLETE" def apply_flat_field(science, flat): diff --git a/romancal/linearity/tests/test_linearity.py b/romancal/linearity/tests/test_linearity.py index 8eaec32d6..e3a20cd94 100644 --- a/romancal/linearity/tests/test_linearity.py +++ b/romancal/linearity/tests/test_linearity.py @@ -30,8 +30,8 @@ def test_linearity_coeff(instrument, exptype): wfi_sci_raw.meta.instrument.name = instrument wfi_sci_raw.meta.instrument.detector = "WFI01" wfi_sci_raw.meta.instrument.optical_element = "F158" - wfi_sci_raw.meta["guide_star"]["gw_window_xstart"] = 1012 - wfi_sci_raw.meta["guide_star"]["gw_window_xsize"] = 16 + wfi_sci_raw.meta["guide_star"]["window_xstart"] = 1012 + wfi_sci_raw.meta["guide_star"]["window_xsize"] = 16 wfi_sci_raw.meta.exposure.type = exptype wfi_sci_raw.data = np.ones(shape, dtype=np.uint16) wfi_sci_raw_model = ScienceRawModel(wfi_sci_raw) diff --git a/romancal/regtest/test_wfi_flat_field.py b/romancal/regtest/test_wfi_flat_field.py index f9b01a1a1..a006e30f3 100644 --- a/romancal/regtest/test_wfi_flat_field.py +++ b/romancal/regtest/test_wfi_flat_field.py @@ -34,40 +34,6 @@ def test_flat_field_image_step(rtdata, ignore_asdf_paths): assert diff.identical, diff.report() -@pytest.mark.bigdata -def test_flat_field_grism_step(rtdata, ignore_asdf_paths): - """Test for the flat field step using grism data. The reference file for - the grism and prism data should be None, only testing the grism - case here.""" - - input_file = "r0000201001001001001_0001_WFI01_uncal.asdf" - rtdata.get_data(f"WFI/grism/{input_file}") - rtdata.input = input_file - - # Test CRDS - step = FlatFieldStep() - model = rdm.open(rtdata.input) - try: - ref_file_path = step.get_reference_file(model, "flat") - # Check for a valid reference file - if ref_file_path != "N/A": - ref_file_name = os.path.split(ref_file_path)[-1] - elif ref_file_path == "N/A": - ref_file_name = ref_file_path - except CrdsLookupError: - ref_file_name = None - assert ref_file_name == "N/A" - - # Test FlatFieldStep - output = "r0000201001001001001_0001_WFI01_flat.asdf" - rtdata.output = output - args = ["romancal.step.FlatFieldStep", rtdata.input] - RomanStep.from_cmdline(args) - rtdata.get_truth(f"truth/WFI/grism/{output}") - diff = compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) - assert diff.identical, diff.report() - - @pytest.mark.bigdata @pytest.mark.soctests def test_flat_field_crds_match_image_step(rtdata, ignore_asdf_paths): diff --git a/romancal/regtest/test_wfi_grism_16resultants.py b/romancal/regtest/test_wfi_grism_16resultants.py index 4e222ed09..63032a5f1 100644 --- a/romancal/regtest/test_wfi_grism_16resultants.py +++ b/romancal/regtest/test_wfi_grism_16resultants.py @@ -74,7 +74,6 @@ def test_output_has_16_resultants(output_model): ("assign_wcs", "COMPLETE"), ("dark", "COMPLETE"), ("dq_init", "COMPLETE"), - ("jump", "COMPLETE"), ("linearity", "COMPLETE"), ("ramp_fit", "COMPLETE"), ("saturation", "COMPLETE"), diff --git a/romancal/regtest/test_wfi_grism_pipeline.py b/romancal/regtest/test_wfi_grism_pipeline.py index 2ad63c58c..23f7d4826 100644 --- a/romancal/regtest/test_wfi_grism_pipeline.py +++ b/romancal/regtest/test_wfi_grism_pipeline.py @@ -91,7 +91,6 @@ def test_output_matches_truth(output_filename, truth_filename, ignore_asdf_paths ("assign_wcs", "COMPLETE"), ("dark", "COMPLETE"), ("dq_init", "COMPLETE"), - ("jump", "COMPLETE"), ("linearity", "COMPLETE"), ("ramp_fit", "COMPLETE"), ("saturation", "COMPLETE"), diff --git a/romancal/regtest/test_wfi_image_16resultants.py b/romancal/regtest/test_wfi_image_16resultants.py index 8b2a997a5..2bf80e03c 100644 --- a/romancal/regtest/test_wfi_image_16resultants.py +++ b/romancal/regtest/test_wfi_image_16resultants.py @@ -75,7 +75,6 @@ def test_output_has_16_resultants(output_model): "flat_field", "dark", "dq_init", - "jump", "linearity", "ramp_fit", "saturation", diff --git a/romancal/regtest/test_wfi_image_pipeline.py b/romancal/regtest/test_wfi_image_pipeline.py index 3cf8b4b26..0a990320b 100644 --- a/romancal/regtest/test_wfi_image_pipeline.py +++ b/romancal/regtest/test_wfi_image_pipeline.py @@ -103,7 +103,6 @@ def test_output_is_image_model(output_model): "flat_field", "dark", "dq_init", - "jump", "linearity", "ramp_fit", "saturation", @@ -222,7 +221,6 @@ def test_elp_input_dm(rtdata, ignore_asdf_paths): assert model.meta.cal_step.saturation == "COMPLETE" assert model.meta.cal_step.linearity == "COMPLETE" assert model.meta.cal_step.dark == "COMPLETE" - assert model.meta.cal_step.jump == "COMPLETE" assert model.meta.cal_step.ramp_fit == "COMPLETE" assert model.meta.cal_step.assign_wcs == "COMPLETE" assert model.meta.cal_step.flat_field == "COMPLETE" diff --git a/romancal/saturation/tests/test_saturation.py b/romancal/saturation/tests/test_saturation.py index b5f415f09..8fb464a48 100644 --- a/romancal/saturation/tests/test_saturation.py +++ b/romancal/saturation/tests/test_saturation.py @@ -320,8 +320,8 @@ def test_saturation_getbestref(setup_wfi_datamodels): wfi_sci_raw.meta.instrument.name = "WFI" wfi_sci_raw.meta.instrument.detector = "WFI01" wfi_sci_raw.meta.instrument.optical_element = "F158" - wfi_sci_raw.meta["guide_star"]["gw_window_xstart"] = 1012 - wfi_sci_raw.meta["guide_star"]["gw_window_xsize"] = 16 + wfi_sci_raw.meta["guide_star"]["window_xstart"] = 1012 + wfi_sci_raw.meta["guide_star"]["window_xsize"] = 16 wfi_sci_raw.meta.exposure.type = "WFI_IMAGE" wfi_sci_raw.data = np.ones(shape, dtype=np.uint16) wfi_sci_raw_model = ScienceRawModel(wfi_sci_raw, dq=True) diff --git a/romancal/scripts/make_regtestdata.sh b/romancal/scripts/make_regtestdata.sh index ec0d82098..6b66f139c 100644 --- a/romancal/scripts/make_regtestdata.sh +++ b/romancal/scripts/make_regtestdata.sh @@ -3,15 +3,15 @@ # takes one argument: the directory into which to start putting the regtest files. # input files needed: -# r0000101001001001001_0001_WFI01 - default for most steps -# r0000201001001001001_0001_WFI01 - equivalent for spectroscopic data -# r0000101001001001001_0002_WFI01 - a second resample exposure, only cal step needed -# r0000101001001001001_0003_WFI01 - for ramp fitting; truncated image -# r0000201001001001001_0003_WFI01 - for ramp fitting; truncated spectroscopic +# r0000101001001001001_0001_wfi01 - default for most steps +# r0000201001001001001_0001_wfi01 - equivalent for spectroscopic data +# r0000101001001001001_0002_wfi01 - a second resample exposure, only cal step needed +# r0000101001001001001_0003_wfi01 - for ramp fitting; truncated image +# r0000201001001001001_0003_wfi01 - for ramp fitting; truncated spectroscopic # we need only darkcurrent & ramp fit for these # -# r00r1601001001001001_0001_WFI01 - special 16 resultant file, imaging, only need cal file -# r10r1601001001001001_0001_WFI01 - special 16 resultant file, spectroscopy, only need cal file +# r00r1601001001001001_0001_wfi01 - special 16 resultant file, imaging, only need cal file +# r10r1601001001001001_0001_wfi01 - special 16 resultant file, spectroscopy, only need cal file outdir=$1 logfile=$outdir/make_regtestdata.log @@ -27,7 +27,7 @@ mkdir -p $outdir/roman-pipeline/dev/truth/WFI/grism # most regtests; run the pipeline and save a lot of results. -for fn in r0000101001001001001_0001_WFI01 r0000201001001001001_0001_WFI01 +for fn in r0000101001001001001_0001_wfi01 r0000201001001001001_0001_wfi01 do echo "Running pipeline on ${fn}..." strun roman_elp ${fn}_uncal.asdf --steps.dq_init.save_results True --steps.saturation.save_results True --steps.linearity.save_results True --steps.dark_current.save_results True --steps.rampfit.save_results True --steps.assign_wcs.save_results True --steps.photom.save_results True --steps.refpix.save_results True --steps.flatfield.save_results True --steps.assign_wcs.save_results True @@ -41,7 +41,7 @@ done # truncated files for ramp fit regtests -for fn in r0000101001001001001_0003_WFI01 r0000201001001001001_0003_WFI01 +for fn in r0000101001001001001_0003_wfi01 r0000201001001001001_0003_wfi01 do echo "Running pipeline on ${fn}..." strun roman_elp ${fn}_uncal.asdf --steps.dark_current.save_results True --steps.rampfit.save_results True @@ -49,28 +49,32 @@ do cp ${fn}_darkcurrent.asdf $outdir/roman-pipeline/dev/WFI/$dirname/ cp ${fn}_rampfit.asdf $outdir/roman-pipeline/dev/truth/WFI/$dirname/ done -cp r0000101001001001001_0003_WFI01_cal.asdf $outdir/roman-pipeline/dev/WFI/image/ +cp r0000101001001001001_0003_wfi01_cal.asdf $outdir/roman-pipeline/dev/WFI/image/ # second imaging exposure -strun roman_elp r0000101001001001001_0002_WFI01_uncal.asdf -cp r0000101001001001001_0002_WFI01_cal.asdf $outdir/roman-pipeline/dev/WFI/image/ +strun roman_elp r0000101001001001001_0002_wfi01_uncal.asdf +cp r0000101001001001001_0002_wfi01_cal.asdf $outdir/roman-pipeline/dev/WFI/image/ +# image used in the skycell generation test +strun roman_elp r0000101001001001001_0002_wfi10_uncal.asdf +cp r0000101001001001001_0002_wfi10_cal.asdf $outdir/roman-pipeline/dev/WFI/image/ -# resample regtest; needs r0000101001001001001_000{1,2}_WFI01_cal.asdf + +# resample regtest; needs r0000101001001001001_000{1,2}_wfi01_cal.asdf # builds the appropriate asn file and calls strun with it echo "Creating regtest files for resample..." -asn_from_list r0000101001001001001_0001_WFI01_cal.asdf r0000101001001001001_0002_WFI01_cal.asdf r0000101001001001001_0003_WFI01_cal.asdf -o L3_mosaic_asn.json --product-name mosaic +asn_from_list r0000101001001001001_0001_wfi01_cal.asdf r0000101001001001001_0002_wfi01_cal.asdf r0000101001001001001_0003_wfi01_cal.asdf -o L3_mosaic_asn.json --product-name mosaic strun romancal.step.ResampleStep L3_mosaic_asn.json --rotation=0 --output_file=mosaic.asdf cp L3_mosaic_asn.json $outdir/roman-pipeline/dev/WFI/image/ cp mosaic_resamplestep.asdf $outdir/roman-pipeline/dev/truth/WFI/image/ -# CRDS test needs the "usual" r00001..._0001_WFI01 files. -# It also needs a hacked r00001..._0001_WFI01 file, with the time changed. +# CRDS test needs the "usual" r00001..._0001_wfi01 files. +# It also needs a hacked r00001..._0001_wfi01 file, with the time changed. # this makes the hacked version. echo "Creating regtest files for CRDS tests..." -basename="r0000101001001001001_0001_WFI01" +basename="r0000101001001001001_0001_wfi01" python -c " import asdf from roman_datamodels import stnode @@ -88,7 +92,7 @@ cp ${basename}_changetime_flat.asdf $outdir/roman-pipeline/dev/truth/WFI/image # need to make a special ALL_SATURATED file for the all saturated test. echo "Creating regtest files for all saturated tests..." -basename="r0000101001001001001_0001_WFI01" +basename="r0000101001001001001_0001_wfi01" python -c " import asdf from roman_datamodels import stnode @@ -105,19 +109,19 @@ cp ${basename}_ALL_SATURATED_cal.asdf $outdir/roman-pipeline/dev/truth/WFI/image # make a special file dark file with a different name -strun romancal.step.DarkCurrentStep r0000101001001001001_0001_WFI01_linearity.asdf --output_file=Test_dark +strun romancal.step.DarkCurrentStep r0000101001001001001_0001_wfi01_linearity.asdf --output_file=Test_dark cp Test_darkcurrent.asdf $outdir/roman-pipeline/dev/truth/WFI/image/ # make a special linearity file with a different suffix -strun romancal.step.LinearityStep r0000101001001001001_0001_WFI01_refpix.asdf --output_file=Test_linearity +strun romancal.step.LinearityStep r0000101001001001001_0001_wfi01_refpix.asdf --output_file=Test_linearity cp Test_linearity.asdf $outdir/roman-pipeline/dev/truth/WFI/image/ # we have a test that runs the flat field step directly on an _L1_ spectroscopic # file and verifies that it gets skipped. # I don't really understand that but we can duplicate it for now. -basename="r0000201001001001001_0001_WFI01" +basename="r0000201001001001001_0001_wfi01" strun romancal.step.FlatFieldStep ${basename}_uncal.asdf cp ${basename}_flat.asdf $outdir/roman-pipeline/dev/truth/WFI/grism/ @@ -126,7 +130,7 @@ cp ${basename}_flat.asdf $outdir/roman-pipeline/dev/truth/WFI/grism/ # we haven't updated the filename in these files, but the regtest mechanism # also doesn't # update them, and we need to match. -for basename in r0000101001001001001_0001_WFI01 r0000201001001001001_0001_WFI01 +for basename in r0000101001001001001_0001_wfi01 r0000201001001001001_0001_wfi01 do python -c " import asdf @@ -148,7 +152,7 @@ model.to_asdf(f'${basename}_cal_repoint.asdf')" done # Test tweakreg with repointed file, only shifted by 1" -for basename in r0000101001001001001_0001_WFI01 +for basename in r0000101001001001001_0001_wfi01 do python -c " import asdf @@ -169,13 +173,13 @@ model.to_asdf(f'${basename}_shift_cal.asdf')" done -strun roman_elp r0000101001001001001_0004_WFI01_uncal.asdf -strun roman_elp r0000201001001001001_0004_WFI01_uncal.asdf -cp r0000101001001001001_0004_WFI01_uncal.asdf $outdir/roman-pipeline/dev/WFI/image/ -cp r0000201001001001001_0004_WFI01_uncal.asdf $outdir/roman-pipeline/dev/WFI/grism/ +strun roman_elp r0000101001001001001_0004_wfi01_uncal.asdf +strun roman_elp r0000201001001001001_0004_wfi01_uncal.asdf +cp r0000101001001001001_0004_wfi01_uncal.asdf $outdir/roman-pipeline/dev/WFI/image/ +cp r0000201001001001001_0004_wfi01_uncal.asdf $outdir/roman-pipeline/dev/WFI/grism/ l3name="r0099101001001001001_F158_visit" -asn_from_list r0000101001001001001_0001_WFI01_cal.asdf r0000101001001001001_0002_WFI01_cal.asdf r0000101001001001001_0003_WFI01_cal.asdf -o L3_regtest_asn.json --product-name $l3name +asn_from_list r0000101001001001001_0001_wfi01_cal.asdf r0000101001001001001_0002_wfi01_cal.asdf r0000101001001001001_0003_wfi01_cal.asdf -o L3_regtest_asn.json --product-name $l3name strun roman_mos L3_regtest_asn.json cp L3_regtest_asn.json $outdir/roman-pipeline/dev/WFI/image/ cp ${l3name}_i2d.asdf $outdir/roman-pipeline/dev/WFI/image/ @@ -187,7 +191,7 @@ cp ${l3name}_cat.asdf $outdir/roman-pipeline/dev/truth/WFI/image/ l3name="r0099101001001001001_r274dp63x31y81_prompt_F158" -asn_from_list r0000101001001001001_0001_WFI01_cal.asdf r0000101001001001001_0002_WFI01_cal.asdf r0000101001001001001_0003_WFI01_cal.asdf -o L3_mosaic_asn.json --product-name $l3name --target r274dp63x31y81 +asn_from_list r0000101001001001001_0001_wfi01_cal.asdf r0000101001001001001_0002_wfi01_cal.asdf r0000101001001001001_0003_wfi01_cal.asdf -o L3_mosaic_asn.json --product-name $l3name --target r274dp63x31y81 strun roman_mos L3_mosaic_asn.json cp L3_mosaic_asn.json $outdir/roman-pipeline/dev/WFI/image/ cp ${l3name}_i2d.asdf $outdir/roman-pipeline/dev/WFI/image/ @@ -199,15 +203,15 @@ cp ${l3name}_cat.asdf $outdir/roman-pipeline/dev/truth/WFI/image/ # L2 catalog -strun romancal.step.SourceCatalogStep r0000101001001001001_0001_WFI01_cal.asdf -cp r0000101001001001001_0001_WFI01_cat.asdf $outdir/roman-pipeline/dev/truth/WFI/image/ +strun romancal.step.SourceCatalogStep r0000101001001001001_0001_wfi01_cal.asdf +cp r0000101001001001001_0001_wfi01_cat.asdf $outdir/roman-pipeline/dev/truth/WFI/image/ l3name="r0099101001001001001_F158_visit_r274dp63x31y81" -asn_from_list --product-name=$l3name r0000101001001001001_0001_WFI01_cal.asdf r0000101001001001001_0002_WFI01_cal.asdf r0000101001001001001_0003_WFI01_cal.asdf -o L3_m1_asn.json +asn_from_list --product-name=$l3name r0000101001001001001_0001_wfi01_cal.asdf r0000101001001001001_0002_wfi01_cal.asdf r0000101001001001001_0003_wfi01_cal.asdf -o L3_m1_asn.json strun roman_mos L3_m1_asn.json cp ${l3name}_i2d.asdf $outdir/roman-pipeline/dev/truth/WFI/image/ # tests passing suffix to the pipeline -strun roman_elp r0000101001001001001_0001_WFI01_uncal.asdf --steps.tweakreg.skip=True --suffix=star -cp r0000101001001001001_0001_WFI01_star.asdf $outdir/roman-pipeline/dev/truth/WFI/image/ +strun roman_elp r0000101001001001001_0001_wfi01_uncal.asdf --steps.tweakreg.skip=True --suffix=star +cp r0000101001001001001_0001_wfi01_star.asdf $outdir/roman-pipeline/dev/truth/WFI/image/ From 10cde96c08727b6dca3ddeb45ed8f0ed8fc5a4e0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 25 Oct 2024 21:41:59 +0000 Subject: [PATCH 12/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- romancal/flatfield/flat_field.py | 2 -- romancal/regtest/test_wfi_flat_field.py | 1 - 2 files changed, 3 deletions(-) diff --git a/romancal/flatfield/flat_field.py b/romancal/flatfield/flat_field.py index bc84ad9b9..f8c25adb0 100644 --- a/romancal/flatfield/flat_field.py +++ b/romancal/flatfield/flat_field.py @@ -6,8 +6,6 @@ import numpy as np from roman_datamodels.dqflags import pixel -from roman_datamodels import stnode - log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) diff --git a/romancal/regtest/test_wfi_flat_field.py b/romancal/regtest/test_wfi_flat_field.py index a006e30f3..d6f4e3c61 100644 --- a/romancal/regtest/test_wfi_flat_field.py +++ b/romancal/regtest/test_wfi_flat_field.py @@ -2,7 +2,6 @@ import pytest import roman_datamodels as rdm -from crds.core.exceptions import CrdsLookupError from romancal.step import FlatFieldStep from romancal.stpipe import RomanStep From 2a6b2e5100f363290e001e461770d13ed833f95b Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Mon, 28 Oct 2024 10:02:40 -0400 Subject: [PATCH 13/25] Mark flatfield test for grism as xfail. --- romancal/regtest/test_wfi_flat_field.py | 35 +++++++++++++++++++++++++ romancal/scripts/make_regtestdata.sh | 1 + 2 files changed, 36 insertions(+) diff --git a/romancal/regtest/test_wfi_flat_field.py b/romancal/regtest/test_wfi_flat_field.py index d6f4e3c61..b9aba7aff 100644 --- a/romancal/regtest/test_wfi_flat_field.py +++ b/romancal/regtest/test_wfi_flat_field.py @@ -33,6 +33,41 @@ def test_flat_field_image_step(rtdata, ignore_asdf_paths): assert diff.identical, diff.report() +@pytest.mark.xfail(reason="Wrong input for the purpose of this test") +@pytest.mark.bigdata +def test_flat_field_grism_step(rtdata, ignore_asdf_paths): + """Test for the flat field step using grism data. The reference file for + the grism and prism data should be None, only testing the grism + case here.""" + + input_file = "r0000201001001001001_0001_WFI01_uncal.asdf" + rtdata.get_data(f"WFI/grism/{input_file}") + rtdata.input = input_file + + # Test CRDS + step = FlatFieldStep() + model = rdm.open(rtdata.input) + try: + ref_file_path = step.get_reference_file(model, "flat") + # Check for a valid reference file + if ref_file_path != "N/A": + ref_file_name = os.path.split(ref_file_path)[-1] + elif ref_file_path == "N/A": + ref_file_name = ref_file_path + except CrdsLookupError: + ref_file_name = None + assert ref_file_name == "N/A" + + # Test FlatFieldStep + output = "r0000201001001001001_0001_WFI01_flat.asdf" + rtdata.output = output + args = ["romancal.step.FlatFieldStep", rtdata.input] + RomanStep.from_cmdline(args) + rtdata.get_truth(f"truth/WFI/grism/{output}") + diff = compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) + assert diff.identical, diff.report() + + @pytest.mark.bigdata @pytest.mark.soctests def test_flat_field_crds_match_image_step(rtdata, ignore_asdf_paths): diff --git a/romancal/scripts/make_regtestdata.sh b/romancal/scripts/make_regtestdata.sh index 6b66f139c..a6031dd35 100644 --- a/romancal/scripts/make_regtestdata.sh +++ b/romancal/scripts/make_regtestdata.sh @@ -121,6 +121,7 @@ cp Test_linearity.asdf $outdir/roman-pipeline/dev/truth/WFI/image/ # we have a test that runs the flat field step directly on an _L1_ spectroscopic # file and verifies that it gets skipped. # I don't really understand that but we can duplicate it for now. +# FIXME: is the uncal input for test_flat_field_grism_step() correct? basename="r0000201001001001001_0001_wfi01" strun romancal.step.FlatFieldStep ${basename}_uncal.asdf cp ${basename}_flat.asdf $outdir/roman-pipeline/dev/truth/WFI/grism/ From f2b25ab30a484b416fe308fdf06ace89cb12153e Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Mon, 28 Oct 2024 10:04:17 -0400 Subject: [PATCH 14/25] Ignore changes in cal_logs when comparing output and truth files. --- romancal/regtest/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/romancal/regtest/conftest.py b/romancal/regtest/conftest.py index 22c157ade..24077509e 100644 --- a/romancal/regtest/conftest.py +++ b/romancal/regtest/conftest.py @@ -218,9 +218,9 @@ def ignore_asdf_paths(): # roman-specific stuff to ignore "roman.meta.ref_file.crds.sw_version", "roman.meta.calibration_software_version", - "roman.cal_logs", + "roman.meta.cal_logs", "roman.meta.date", - "roman.individual_image_cal_logs", + "roman.meta.individual_image_cal_logs", "roman.meta.individual_image_meta", # individual image meta includes string arrays, not supported by # compare_asdf at present. From 6f52d5a1924155522b37dade945d39739428faa5 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Mon, 28 Oct 2024 10:32:14 -0400 Subject: [PATCH 15/25] Fix missing import. --- romancal/regtest/test_wfi_flat_field.py | 1 + 1 file changed, 1 insertion(+) diff --git a/romancal/regtest/test_wfi_flat_field.py b/romancal/regtest/test_wfi_flat_field.py index b9aba7aff..cd15a14e9 100644 --- a/romancal/regtest/test_wfi_flat_field.py +++ b/romancal/regtest/test_wfi_flat_field.py @@ -2,6 +2,7 @@ import pytest import roman_datamodels as rdm +from crds.core.exceptions import CrdsLookupError from romancal.step import FlatFieldStep from romancal.stpipe import RomanStep From d9ea650fdd160e76b8dcdfaae4429ca15eb06e83 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Mon, 28 Oct 2024 13:57:05 -0400 Subject: [PATCH 16/25] Fix ignore cal logs for individual images. --- romancal/regtest/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/romancal/regtest/conftest.py b/romancal/regtest/conftest.py index 24077509e..2fbc5dbe4 100644 --- a/romancal/regtest/conftest.py +++ b/romancal/regtest/conftest.py @@ -218,9 +218,10 @@ def ignore_asdf_paths(): # roman-specific stuff to ignore "roman.meta.ref_file.crds.sw_version", "roman.meta.calibration_software_version", + "roman.cal_logs", "roman.meta.cal_logs", "roman.meta.date", - "roman.meta.individual_image_cal_logs", + "roman.individual_image_cal_logs", "roman.meta.individual_image_meta", # individual image meta includes string arrays, not supported by # compare_asdf at present. From 65d9409c72e3984dbef3b5555020833281e07a12 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Mon, 28 Oct 2024 15:20:00 -0400 Subject: [PATCH 17/25] Lowercase WFI in regtest filenames. --- romancal/regtest/test_catalog.py | 2 +- romancal/regtest/test_dark_current.py | 14 +++++++------- romancal/regtest/test_jump_det.py | 4 ++-- romancal/regtest/test_linearity.py | 6 +++--- romancal/regtest/test_ramp_fitting.py | 8 ++++---- romancal/regtest/test_refpix.py | 4 ++-- romancal/regtest/test_skycell_generation.py | 6 +++--- romancal/regtest/test_tweakreg.py | 10 +++++----- romancal/regtest/test_wfi_dq_init.py | 8 ++++---- romancal/regtest/test_wfi_flat_field.py | 16 ++++++++-------- romancal/regtest/test_wfi_grism_16resultants.py | 4 ++-- romancal/regtest/test_wfi_grism_pipeline.py | 4 ++-- romancal/regtest/test_wfi_image_16resultants.py | 4 ++-- romancal/regtest/test_wfi_image_pipeline.py | 16 ++++++++-------- romancal/regtest/test_wfi_photom.py | 4 ++-- romancal/regtest/test_wfi_saturation.py | 8 ++++---- romancal/resample/tests/test_resample.py | 12 ++++++------ 17 files changed, 65 insertions(+), 65 deletions(-) diff --git a/romancal/regtest/test_catalog.py b/romancal/regtest/test_catalog.py index 3c461af1c..c3c055b30 100644 --- a/romancal/regtest/test_catalog.py +++ b/romancal/regtest/test_catalog.py @@ -13,7 +13,7 @@ scope="module", params=[ "r0099101001001001001_F158_visit_i2d.asdf", - "r0000101001001001001_0001_WFI01_cal.asdf", + "r0000101001001001001_0001_wfi01_cal.asdf", ], ids=["L3", "L2"], ) diff --git a/romancal/regtest/test_dark_current.py b/romancal/regtest/test_dark_current.py index 86200bc71..2e44c44f0 100644 --- a/romancal/regtest/test_dark_current.py +++ b/romancal/regtest/test_dark_current.py @@ -12,13 +12,13 @@ def test_dark_current_subtraction_step(rtdata, ignore_asdf_paths): """Function to run and compare Dark Current subtraction files. Note: This should include tests for overrides etc.""" - input_datafile = "r0000101001001001001_0001_WFI01_linearity.asdf" + input_datafile = "r0000101001001001001_0001_wfi01_linearity.asdf" rtdata.get_data(f"WFI/image/{input_datafile}") rtdata.input = input_datafile args = ["romancal.step.DarkCurrentStep", rtdata.input] RomanStep.from_cmdline(args) - output = "r0000101001001001001_0001_WFI01_darkcurrent.asdf" + output = "r0000101001001001001_0001_wfi01_darkcurrent.asdf" rtdata.output = output rtdata.get_truth(f"truth/WFI/image/{output}") diff = compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) @@ -29,7 +29,7 @@ def test_dark_current_subtraction_step(rtdata, ignore_asdf_paths): def test_dark_current_outfile_step(rtdata, ignore_asdf_paths): """Function to run and compare Dark Current subtraction files. Here the test is for renaming the output file.""" - input_datafile = "r0000101001001001001_0001_WFI01_linearity.asdf" + input_datafile = "r0000101001001001001_0001_wfi01_linearity.asdf" rtdata.get_data(f"WFI/image/{input_datafile}") rtdata.input = input_datafile @@ -50,7 +50,7 @@ def test_dark_current_outfile_step(rtdata, ignore_asdf_paths): def test_dark_current_outfile_suffix(rtdata, ignore_asdf_paths): """Function to run and compare Dark Current subtraction files. Here the test is for renaming the output file.""" - input_datafile = "r0000101001001001001_0001_WFI01_linearity.asdf" + input_datafile = "r0000101001001001001_0001_wfi01_linearity.asdf" rtdata.get_data(f"WFI/image/{input_datafile}") rtdata.input = input_datafile @@ -73,10 +73,10 @@ def test_dark_current_output(rtdata, ignore_asdf_paths): """Function to run and compare Dark Current subtraction files. Here the test for overriding the CRDS dark reference file.""" - input_datafile = "r0000101001001001001_0001_WFI01_linearity.asdf" + input_datafile = "r0000101001001001001_0001_wfi01_linearity.asdf" rtdata.get_data(f"WFI/image/{input_datafile}") rtdata.input = input_datafile - dark_output_name = "r0000101001001001001_0001_WFI01_darkcurrent.asdf" + dark_output_name = "r0000101001001001001_0001_wfi01_darkcurrent.asdf" args = [ "romancal.step.DarkCurrentStep", @@ -84,7 +84,7 @@ def test_dark_current_output(rtdata, ignore_asdf_paths): f"--dark_output={dark_output_name}", ] RomanStep.from_cmdline(args) - output = "r0000101001001001001_0001_WFI01_darkcurrent.asdf" + output = "r0000101001001001001_0001_wfi01_darkcurrent.asdf" rtdata.output = output rtdata.get_truth(f"truth/WFI/image/{output}") diff = compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) diff --git a/romancal/regtest/test_jump_det.py b/romancal/regtest/test_jump_det.py index 4c014d161..768ee6888 100644 --- a/romancal/regtest/test_jump_det.py +++ b/romancal/regtest/test_jump_det.py @@ -13,7 +13,7 @@ def test_jump_detection_step(rtdata, ignore_asdf_paths): """Function to run and compare Jump Detection files. Note: This should include tests for overrides etc.""" - input_file = "r0000101001001001001_0001_WFI01_darkcurrent.asdf" + input_file = "r0000101001001001001_0001_wfi01_darkcurrent.asdf" rtdata.get_data(f"WFI/image/{input_file}") rtdata.input = input_file @@ -29,7 +29,7 @@ def test_jump_detection_step(rtdata, ignore_asdf_paths): "--run_ramp_jump_detection=False", ] RomanStep.from_cmdline(args) - output = "r0000101001001001001_0001_WFI01_jump.asdf" + output = "r0000101001001001001_0001_wfi01_jump.asdf" rtdata.output = output rtdata.get_truth(f"truth/WFI/image/{output}") diff = compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) diff --git a/romancal/regtest/test_linearity.py b/romancal/regtest/test_linearity.py index b6c9b5cdc..6dc459af4 100644 --- a/romancal/regtest/test_linearity.py +++ b/romancal/regtest/test_linearity.py @@ -10,13 +10,13 @@ @pytest.mark.bigdata def test_linearity_step(rtdata, ignore_asdf_paths): """Function to run and compare linearity correction files.""" - input_file = "r0000101001001001001_0001_WFI01_refpix.asdf" + input_file = "r0000101001001001001_0001_wfi01_refpix.asdf" rtdata.get_data(f"WFI/image/{input_file}") rtdata.input = input_file args = ["romancal.step.LinearityStep", rtdata.input] RomanStep.from_cmdline(args) - output = "r0000101001001001001_0001_WFI01_linearity.asdf" + output = "r0000101001001001001_0001_wfi01_linearity.asdf" rtdata.output = output rtdata.get_truth(f"truth/WFI/image/{output}") diff = compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) @@ -27,7 +27,7 @@ def test_linearity_step(rtdata, ignore_asdf_paths): def test_linearity_outfile_step(rtdata, ignore_asdf_paths): """Function to run and linearity correction files. Here the test is for renaming the output file.""" - input_file = "r0000101001001001001_0001_WFI01_refpix.asdf" + input_file = "r0000101001001001001_0001_wfi01_refpix.asdf" rtdata.get_data(f"WFI/image/{input_file}") rtdata.input = input_file diff --git a/romancal/regtest/test_ramp_fitting.py b/romancal/regtest/test_ramp_fitting.py index 5626c65a6..718eeaec6 100644 --- a/romancal/regtest/test_ramp_fitting.py +++ b/romancal/regtest/test_ramp_fitting.py @@ -120,22 +120,22 @@ def cond_science_verification( params=[ ( "DMS362", - Path("WFI/image/r0000101001001001001_0001_WFI01_darkcurrent.asdf"), + Path("WFI/image/r0000101001001001001_0001_wfi01_darkcurrent.asdf"), CONDITIONS_FULL, ), ( "DMS366", - Path("WFI/grism/r0000201001001001001_0001_WFI01_darkcurrent.asdf"), + Path("WFI/grism/r0000201001001001001_0001_wfi01_darkcurrent.asdf"), CONDITIONS_FULL, ), ( "DMS363", - Path("WFI/image/r0000101001001001001_0003_WFI01_darkcurrent.asdf"), + Path("WFI/image/r0000101001001001001_0003_wfi01_darkcurrent.asdf"), CONDITIONS_TRUNC, ), ( "DMS367", - Path("WFI/grism/r0000201001001001001_0003_WFI01_darkcurrent.asdf"), + Path("WFI/grism/r0000201001001001001_0003_wfi01_darkcurrent.asdf"), CONDITIONS_TRUNC, ), ], diff --git a/romancal/regtest/test_refpix.py b/romancal/regtest/test_refpix.py index b8c7c44d3..02eff9cb9 100644 --- a/romancal/regtest/test_refpix.py +++ b/romancal/regtest/test_refpix.py @@ -10,7 +10,7 @@ @pytest.mark.bigdata def test_refpix_step(rtdata, ignore_asdf_paths): # I have no idea what this is supposed to be - input_datafile = "r0000101001001001001_0001_WFI01_saturation.asdf" + input_datafile = "r0000101001001001001_0001_wfi01_saturation.asdf" rtdata.get_data(f"WFI/image/{input_datafile}") rtdata.input = input_datafile @@ -18,7 +18,7 @@ def test_refpix_step(rtdata, ignore_asdf_paths): RomanStep.from_cmdline(args) # Again I have no idea here - output = "r0000101001001001001_0001_WFI01_refpix.asdf" + output = "r0000101001001001001_0001_wfi01_refpix.asdf" rtdata.output = output rtdata.get_truth(f"truth/WFI/image/{output}") diff --git a/romancal/regtest/test_skycell_generation.py b/romancal/regtest/test_skycell_generation.py index cc08de547..ece0c4401 100644 --- a/romancal/regtest/test_skycell_generation.py +++ b/romancal/regtest/test_skycell_generation.py @@ -13,13 +13,13 @@ def test_skycell_asn_generation(rtdata): # This test should generate seven json files args = [ - "r0000101001001001001_0002_WFI01_cal.asdf", + "r0000101001001001001_0002_wfi01_cal.asdf", "r0000101001001001001_0002_WFI10_cal.asdf", "-o", "r512", ] - rtdata.get_data("WFI/image/r0000101001001001001_0002_WFI01_cal.asdf") - rtdata.get_data("WFI/image/r0000101001001001001_0002_WFI10_cal.asdf") + rtdata.get_data("WFI/image/r0000101001001001001_0002_wfi01_cal.asdf") + rtdata.get_data("WFI/image/r0000101001001001001_0002_wfi10_cal.asdf") skycell_asn.Main(args) diff --git a/romancal/regtest/test_tweakreg.py b/romancal/regtest/test_tweakreg.py index 9402c16d7..f2d988c02 100644 --- a/romancal/regtest/test_tweakreg.py +++ b/romancal/regtest/test_tweakreg.py @@ -16,11 +16,11 @@ def test_tweakreg(rtdata, ignore_asdf_paths, tmp_path): # ``shifted'' version is created in make_regtestdata.sh; cal file is taken, # the wcsinfo is perturbed, and AssignWCS is run to update the WCS with the # perturbed information - orig_uncal = "r0000101001001001001_0001_WFI01_uncal.asdf" - orig_catfile = "r0000101001001001001_0001_WFI01_cat.asdf" - input_data = "r0000101001001001001_0001_WFI01_shift_cal.asdf" - output_data = "r0000101001001001001_0001_WFI01_shift_tweakregstep.asdf" - truth_data = "r0000101001001001001_0001_WFI01_shift_tweakregstep.asdf" + orig_uncal = "r0000101001001001001_0001_wfi01_uncal.asdf" + orig_catfile = "r0000101001001001001_0001_wfi01_cat.asdf" + input_data = "r0000101001001001001_0001_wfi01_shift_cal.asdf" + output_data = "r0000101001001001001_0001_wfi01_shift_tweakregstep.asdf" + truth_data = "r0000101001001001001_0001_wfi01_shift_tweakregstep.asdf" rtdata.get_data(f"WFI/image/{input_data}") rtdata.get_data(f"WFI/image/{orig_catfile}") diff --git a/romancal/regtest/test_wfi_dq_init.py b/romancal/regtest/test_wfi_dq_init.py index 5f9ba08dd..5e9a4367b 100644 --- a/romancal/regtest/test_wfi_dq_init.py +++ b/romancal/regtest/test_wfi_dq_init.py @@ -16,7 +16,7 @@ def test_dq_init_image_step(rtdata, ignore_asdf_paths): """DMS25 Test: Testing retrieval of best ref file for image data, and creation of a ramp file with CRDS selected mask file applied.""" - input_file = "r0000101001001001001_0001_WFI01_uncal.asdf" + input_file = "r0000101001001001001_0001_wfi01_uncal.asdf" rtdata.get_data(f"WFI/image/{input_file}") rtdata.input = input_file @@ -40,7 +40,7 @@ def test_dq_init_image_step(rtdata, ignore_asdf_paths): assert "roman_wfi_mask" in ref_file_name # Test DQInitStep - output = "r0000101001001001001_0001_WFI01_dqinit.asdf" + output = "r0000101001001001001_0001_wfi01_dqinit.asdf" rtdata.output = output args = ["romancal.step.DQInitStep", rtdata.input] step.log.info( @@ -71,7 +71,7 @@ def test_dq_init_grism_step(rtdata, ignore_asdf_paths): """DMS25 Test: Testing retrieval of best ref file for grism data, and creation of a ramp file with CRDS selected mask file applied.""" - input_file = "r0000201001001001001_0001_WFI01_uncal.asdf" + input_file = "r0000201001001001001_0001_wfi01_uncal.asdf" rtdata.get_data(f"WFI/grism/{input_file}") rtdata.input = input_file @@ -95,7 +95,7 @@ def test_dq_init_grism_step(rtdata, ignore_asdf_paths): assert "roman_wfi_mask" in ref_file_name # Test DQInitStep - output = "r0000201001001001001_0001_WFI01_dqinit.asdf" + output = "r0000201001001001001_0001_wfi01_dqinit.asdf" rtdata.output = output args = ["romancal.step.DQInitStep", rtdata.input] step.log.info( diff --git a/romancal/regtest/test_wfi_flat_field.py b/romancal/regtest/test_wfi_flat_field.py index cd15a14e9..312af796b 100644 --- a/romancal/regtest/test_wfi_flat_field.py +++ b/romancal/regtest/test_wfi_flat_field.py @@ -14,7 +14,7 @@ def test_flat_field_image_step(rtdata, ignore_asdf_paths): """Test for the flat field step using imaging data.""" - input_data = "r0000101001001001001_0001_WFI01_assignwcs.asdf" + input_data = "r0000101001001001001_0001_wfi01_assignwcs.asdf" rtdata.get_data(f"WFI/image/{input_data}") rtdata.input = input_data @@ -25,7 +25,7 @@ def test_flat_field_image_step(rtdata, ignore_asdf_paths): ref_file_name = os.path.split(ref_file_path)[-1] assert "roman_wfi_flat" in ref_file_name # Test FlatFieldStep - output = "r0000101001001001001_0001_WFI01_flat.asdf" + output = "r0000101001001001001_0001_wfi01_flat.asdf" rtdata.output = output args = ["romancal.step.FlatFieldStep", rtdata.input] RomanStep.from_cmdline(args) @@ -41,7 +41,7 @@ def test_flat_field_grism_step(rtdata, ignore_asdf_paths): the grism and prism data should be None, only testing the grism case here.""" - input_file = "r0000201001001001001_0001_WFI01_uncal.asdf" + input_file = "r0000201001001001001_0001_wfi01_uncal.asdf" rtdata.get_data(f"WFI/grism/{input_file}") rtdata.input = input_file @@ -60,7 +60,7 @@ def test_flat_field_grism_step(rtdata, ignore_asdf_paths): assert ref_file_name == "N/A" # Test FlatFieldStep - output = "r0000201001001001001_0001_WFI01_flat.asdf" + output = "r0000201001001001001_0001_wfi01_flat.asdf" rtdata.output = output args = ["romancal.step.FlatFieldStep", rtdata.input] RomanStep.from_cmdline(args) @@ -76,7 +76,7 @@ def test_flat_field_crds_match_image_step(rtdata, ignore_asdf_paths): flat files and successfully make level 2 output""" # First file - input_l2_file = "r0000101001001001001_0001_WFI01_assignwcs.asdf" + input_l2_file = "r0000101001001001001_0001_wfi01_assignwcs.asdf" rtdata.get_data(f"WFI/image/{input_l2_file}") rtdata.input = input_l2_file @@ -103,7 +103,7 @@ def test_flat_field_crds_match_image_step(rtdata, ignore_asdf_paths): ) # Test FlatFieldStep - output = "r0000101001001001001_0001_WFI01_flat.asdf" + output = "r0000101001001001001_0001_wfi01_flat.asdf" rtdata.output = output args = ["romancal.step.FlatFieldStep", rtdata.input] step.log.info( @@ -127,7 +127,7 @@ def test_flat_field_crds_match_image_step(rtdata, ignore_asdf_paths): # to separate flat files in CRDS. # Second file - input_file = "r0000101001001001001_0001_WFI01_changetime_assignwcs.asdf" + input_file = "r0000101001001001001_0001_wfi01_changetime_assignwcs.asdf" rtdata.get_data(f"WFI/image/{input_file}") rtdata.input = input_file @@ -150,7 +150,7 @@ def test_flat_field_crds_match_image_step(rtdata, ignore_asdf_paths): ) # Test FlatFieldStep - output = "r0000101001001001001_0001_WFI01_changetime_flat.asdf" + output = "r0000101001001001001_0001_wfi01_changetime_flat.asdf" rtdata.output = output args = ["romancal.step.FlatFieldStep", rtdata.input] step.log.info( diff --git a/romancal/regtest/test_wfi_grism_16resultants.py b/romancal/regtest/test_wfi_grism_16resultants.py index 63032a5f1..969291076 100644 --- a/romancal/regtest/test_wfi_grism_16resultants.py +++ b/romancal/regtest/test_wfi_grism_16resultants.py @@ -16,12 +16,12 @@ def run_elp(rtdata_module): # The input data is from INS for stress testing at some point this should be generated # by INS every time new data is needed. - input_data = "r0000201001001001001_0004_WFI01_uncal.asdf" + input_data = "r0000201001001001001_0004_wfi01_uncal.asdf" rtdata.get_data(f"WFI/grism/{input_data}") rtdata.input = input_data # Test Pipeline - output = "r0000201001001001001_0004_WFI01_cal.asdf" + output = "r0000201001001001001_0004_wfi01_cal.asdf" rtdata.output = output args = [ "roman_elp", diff --git a/romancal/regtest/test_wfi_grism_pipeline.py b/romancal/regtest/test_wfi_grism_pipeline.py index 23f7d4826..abeb0ee72 100644 --- a/romancal/regtest/test_wfi_grism_pipeline.py +++ b/romancal/regtest/test_wfi_grism_pipeline.py @@ -19,12 +19,12 @@ def run_elp(rtdata_module): rtdata = rtdata_module - input_data = "r0000201001001001001_0001_WFI01_uncal.asdf" + input_data = "r0000201001001001001_0001_wfi01_uncal.asdf" rtdata.get_data(f"WFI/grism/{input_data}") rtdata.input = input_data # Test Pipeline - output = "r0000201001001001001_0001_WFI01_cal.asdf" + output = "r0000201001001001001_0001_wfi01_cal.asdf" rtdata.output = output args = [ "roman_elp", diff --git a/romancal/regtest/test_wfi_image_16resultants.py b/romancal/regtest/test_wfi_image_16resultants.py index 2bf80e03c..124d1c535 100644 --- a/romancal/regtest/test_wfi_image_16resultants.py +++ b/romancal/regtest/test_wfi_image_16resultants.py @@ -16,12 +16,12 @@ def run_elp(rtdata_module): # The input data is from INS for stress testing at some point this should be generated # every time new data is needed. - input_data = "r0000101001001001001_0004_WFI01_uncal.asdf" + input_data = "r0000101001001001001_0004_wfi01_uncal.asdf" rtdata.get_data(f"WFI/image/{input_data}") rtdata.input = input_data # Test Pipeline - output = "r0000101001001001001_0004_WFI01_cal.asdf" + output = "r0000101001001001001_0004_wfi01_cal.asdf" rtdata.output = output args = [ "roman_elp", diff --git a/romancal/regtest/test_wfi_image_pipeline.py b/romancal/regtest/test_wfi_image_pipeline.py index 0a990320b..4928340c2 100644 --- a/romancal/regtest/test_wfi_image_pipeline.py +++ b/romancal/regtest/test_wfi_image_pipeline.py @@ -20,12 +20,12 @@ def run_elp(rtdata_module): rtdata = rtdata_module - input_data = "r0000101001001001001_0001_WFI01_uncal.asdf" + input_data = "r0000101001001001001_0001_wfi01_uncal.asdf" rtdata.get_data(f"WFI/image/{input_data}") rtdata.input = input_data # Test Pipeline - output = "r0000101001001001001_0001_WFI01_cal.asdf" + output = "r0000101001001001001_0001_wfi01_cal.asdf" rtdata.output = output args = [ "roman_elp", @@ -205,12 +205,12 @@ def test_repointed_wcs_differs(repointed_filename_and_delta, output_model): def test_elp_input_dm(rtdata, ignore_asdf_paths): """Test for input roman Datamodel to exposure level pipeline""" - input_data = "r0000101001001001001_0001_WFI01_uncal.asdf" + input_data = "r0000101001001001001_0001_wfi01_uncal.asdf" rtdata.get_data(f"WFI/image/{input_data}") dm_input = rdm.open(rtdata.input) # Test Pipeline with input datamodel - output = "r0000101001001001001_0001_WFI01_cal.asdf" + output = "r0000101001001001001_0001_wfi01_cal.asdf" rtdata.output = output ExposurePipeline.call(dm_input, save_results=True) rtdata.get_truth(f"truth/WFI/image/{output}") @@ -233,12 +233,12 @@ def test_processing_pipeline_all_saturated(rtdata, ignore_asdf_paths): Note that this test mimics how the pipeline is run in OPS. Any changes to this test should be coordinated with OPS. """ - input_data = "r0000101001001001001_0001_WFI01_ALL_SATURATED_uncal.asdf" + input_data = "r0000101001001001001_0001_wfi01_ALL_SATURATED_uncal.asdf" rtdata.get_data(f"WFI/image/{input_data}") rtdata.input = input_data # Test Pipeline - output = "r0000101001001001001_0001_WFI01_ALL_SATURATED_cal.asdf" + output = "r0000101001001001001_0001_wfi01_ALL_SATURATED_cal.asdf" rtdata.output = output args = [ "roman_elp", @@ -271,10 +271,10 @@ def test_pipeline_suffix(rtdata, ignore_asdf_paths): Any changes to this test should be coordinated with OPS. """ - input_data = "r0000101001001001001_0001_WFI01_uncal.asdf" + input_data = "r0000101001001001001_0001_wfi01_uncal.asdf" rtdata.get_data(f"WFI/image/{input_data}") - output = "r0000101001001001001_0001_WFI01_star.asdf" + output = "r0000101001001001001_0001_wfi01_star.asdf" rtdata.output = output args = [ diff --git a/romancal/regtest/test_wfi_photom.py b/romancal/regtest/test_wfi_photom.py index 061834d07..3b1e31879 100644 --- a/romancal/regtest/test_wfi_photom.py +++ b/romancal/regtest/test_wfi_photom.py @@ -16,7 +16,7 @@ def test_absolute_photometric_calibration(rtdata, ignore_asdf_paths): """DMS140 Test: Testing application of photometric correction using CRDS selected photom file.""" - input_data = "r0000101001001001001_0001_WFI01_flat.asdf" + input_data = "r0000101001001001001_0001_wfi01_flat.asdf" rtdata.get_data(f"WFI/image/{input_data}") rtdata.input = input_data @@ -39,7 +39,7 @@ def test_absolute_photometric_calibration(rtdata, ignore_asdf_paths): # photom match from CRDS. Values come from roman_wfi_photom_0034.asdf # Test PhotomStep - output = "r0000101001001001001_0001_WFI01_photom.asdf" + output = "r0000101001001001001_0001_wfi01_photom.asdf" rtdata.output = output args = ["romancal.step.PhotomStep", rtdata.input] step.log.info( diff --git a/romancal/regtest/test_wfi_saturation.py b/romancal/regtest/test_wfi_saturation.py index 1484af520..18b9fa666 100644 --- a/romancal/regtest/test_wfi_saturation.py +++ b/romancal/regtest/test_wfi_saturation.py @@ -16,7 +16,7 @@ def test_saturation_image_step(rtdata, ignore_asdf_paths): """Testing retrieval of best ref file for image data, and creation of a ramp file with CRDS selected saturation file applied.""" - input_file = "r0000101001001001001_0001_WFI01_dqinit.asdf" + input_file = "r0000101001001001001_0001_wfi01_dqinit.asdf" rtdata.get_data(f"WFI/image/{input_file}") rtdata.input = input_file @@ -30,7 +30,7 @@ def test_saturation_image_step(rtdata, ignore_asdf_paths): assert "roman_wfi_saturation" in ref_file_name # Test SaturationStep - output = "r0000101001001001001_0001_WFI01_saturation.asdf" + output = "r0000101001001001001_0001_wfi01_saturation.asdf" rtdata.output = output args = ["romancal.step.SaturationStep", rtdata.input] @@ -49,7 +49,7 @@ def test_saturation_grism_step(rtdata, ignore_asdf_paths): """Testing retrieval of best ref file for grism data, and creation of a ramp file with CRDS selected saturation file applied.""" - input_file = "r0000201001001001001_0001_WFI01_dqinit.asdf" + input_file = "r0000201001001001001_0001_wfi01_dqinit.asdf" rtdata.get_data(f"WFI/grism/{input_file}") rtdata.input = input_file @@ -63,7 +63,7 @@ def test_saturation_grism_step(rtdata, ignore_asdf_paths): assert "roman_wfi_saturation" in ref_file_name # Test SaturationStep - output = "r0000201001001001001_0001_WFI01_saturation.asdf" + output = "r0000201001001001001_0001_wfi01_saturation.asdf" rtdata.output = output args = ["romancal.step.SaturationStep", rtdata.input] diff --git a/romancal/resample/tests/test_resample.py b/romancal/resample/tests/test_resample.py index 4368a8359..ec8a27b69 100644 --- a/romancal/resample/tests/test_resample.py +++ b/romancal/resample/tests/test_resample.py @@ -174,7 +174,7 @@ def wfi_sca1(): fiducial_world=(10, 0), pscale=(0.000031, 0.000031), shape=(100, 100), - filename="r0000501001001001001_01101_0001_WFI01_cal.asdf", + filename="r0000501001001001001_01101_0001_wfi01_cal.asdf", ) return sca.create_image() @@ -186,7 +186,7 @@ def wfi_sca2(): fiducial_world=(10.00139, 0), pscale=(0.000031, 0.000031), shape=(100, 100), - filename="r0000501001001001001_01101_0001_WFI02_cal.asdf", + filename="r0000501001001001001_01101_0001_wfi02_cal.asdf", ) return sca.create_image() @@ -198,7 +198,7 @@ def wfi_sca3(): fiducial_world=(10.00278, 0), pscale=(0.000031, 0.000031), shape=(100, 100), - filename="r0000501001001001001_01101_0001_WFI03_cal.asdf", + filename="r0000501001001001001_01101_0001_wfi03_cal.asdf", ) return sca.create_image() @@ -210,7 +210,7 @@ def wfi_sca4(): fiducial_world=(10, 0), pscale=(0.000031, 0.000031), shape=(100, 100), - filename="r0000501001001001001_01101_0002_WFI01_cal.asdf", + filename="r0000501001001001001_01101_0002_wfi01_cal.asdf", ) return sca.create_image() @@ -222,7 +222,7 @@ def wfi_sca5(): fiducial_world=(10.00139, 0), pscale=(0.000031, 0.000031), shape=(100, 100), - filename="r0000501001001001001_01101_0002_WFI02_cal.asdf", + filename="r0000501001001001001_01101_0002_wfi02_cal.asdf", ) return sca.create_image() @@ -234,7 +234,7 @@ def wfi_sca6(): fiducial_world=(10.00278, 0), pscale=(0.000031, 0.000031), shape=(100, 100), - filename="r0000501001001001001_01101_0002_WFI03_cal.asdf", + filename="r0000501001001001001_01101_0002_wfi03_cal.asdf", ) return sca.create_image() From 36d742b73f9c3373e89301f5109adec2da63dc72 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Tue, 29 Oct 2024 09:13:19 -0400 Subject: [PATCH 18/25] Fix call_logs issue. --- romancal/regtest/test_wfi_image_pipeline.py | 1 - romancal/resample/resample.py | 35 ++++++++++++++++++--- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/romancal/regtest/test_wfi_image_pipeline.py b/romancal/regtest/test_wfi_image_pipeline.py index 4928340c2..4c75759a4 100644 --- a/romancal/regtest/test_wfi_image_pipeline.py +++ b/romancal/regtest/test_wfi_image_pipeline.py @@ -295,7 +295,6 @@ def test_pipeline_suffix(rtdata, ignore_asdf_paths): assert model.meta.cal_step.saturation == "COMPLETE" assert model.meta.cal_step.linearity == "COMPLETE" assert model.meta.cal_step.dark == "COMPLETE" - assert model.meta.cal_step.jump == "COMPLETE" assert model.meta.cal_step.ramp_fit == "COMPLETE" assert model.meta.cal_step.assign_wcs == "COMPLETE" assert model.meta.cal_step.flat_field == "COMPLETE" diff --git a/romancal/resample/resample.py b/romancal/resample/resample.py index f031f3a08..f81ec80df 100644 --- a/romancal/resample/resample.py +++ b/romancal/resample/resample.py @@ -173,6 +173,10 @@ def __init__( self.blank_output.meta.cal_step = maker_utils.mk_l3_cal_step( **models[0].meta.cal_step.to_flat_dict() ) + + # FIXME: do we have to populate meta.individual_image_meta.call_logs? + # concatenate all call_logs corresponding to each ImageModel + concatenate_call_logs(models) # Update the output with all the component metas populate_mosaic_individual(self.blank_output, models) @@ -181,10 +185,10 @@ def __init__( l2_into_l3_meta(self.blank_output.meta, models[0].meta) self.blank_output.meta.wcs = self.output_wcs gwcs_into_l3(self.blank_output, self.output_wcs) - self.blank_output.cal_logs = stnode.CalLogs() - self.blank_output["individual_image_cal_logs"] = [ - model.meta.cal_logs for model in models - ] + # self.blank_output.cal_logs = stnode.CalLogs() + # self.blank_output["individual_image_cal_logs"] = [ + # model.meta.cal_logs for model in models + # ] for i, m in enumerate(models): self.input_models.shelve(m, i, modify=False) @@ -954,7 +958,10 @@ def populate_mosaic_individual( input_metas = [datamodel.meta for datamodel in input_models] for input_meta in input_metas: - output_model.append_individual_image_meta(input_meta) + try: + output_model.append_individual_image_meta(input_meta) + except: + continue def copy_asn_info_from_library(input_models, output_model): @@ -963,3 +970,21 @@ def copy_asn_info_from_library(input_models, output_model): output_model.meta.asn.pool_name = asn_pool if (asn_table_name := input_models.asn.get("table_name", None)) is not None: output_model.meta.asn.table_name = asn_table_name + + +def concatenate_call_logs(model_list): + """ + Concatenate the `cal_logs` attribute of each model in the list. + + Parameters + ---------- + model_list : list + A list of models, each containing a `meta.cal_logs` attribute. + + Returns + ------- + None + The function modifies the `cal_logs` attribute of each model in place. + """ + for model in model_list: + model.meta["cal_logs"] = " /// ".join(model.meta.cal_logs) \ No newline at end of file From d8f5ae3fd5dd2a9ea199da21cb58e345e18ddf55 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:13:41 +0000 Subject: [PATCH 19/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- romancal/resample/resample.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/romancal/resample/resample.py b/romancal/resample/resample.py index f81ec80df..dc17174e0 100644 --- a/romancal/resample/resample.py +++ b/romancal/resample/resample.py @@ -7,7 +7,7 @@ from astropy import units as u from astropy.coordinates import SkyCoord from drizzle import cdrizzle, util -from roman_datamodels import datamodels, maker_utils, stnode +from roman_datamodels import datamodels, maker_utils from stcal.alignment.util import compute_scale from romancal.associations.asn_from_list import asn_from_list @@ -173,7 +173,7 @@ def __init__( self.blank_output.meta.cal_step = maker_utils.mk_l3_cal_step( **models[0].meta.cal_step.to_flat_dict() ) - + # FIXME: do we have to populate meta.individual_image_meta.call_logs? # concatenate all call_logs corresponding to each ImageModel concatenate_call_logs(models) @@ -987,4 +987,4 @@ def concatenate_call_logs(model_list): The function modifies the `cal_logs` attribute of each model in place. """ for model in model_list: - model.meta["cal_logs"] = " /// ".join(model.meta.cal_logs) \ No newline at end of file + model.meta["cal_logs"] = " /// ".join(model.meta.cal_logs) From adc9fe01c7aba22c811328e9a71d5a0e16a1906e Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Tue, 29 Oct 2024 09:20:27 -0400 Subject: [PATCH 20/25] Remove testing code. --- romancal/resample/resample.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/romancal/resample/resample.py b/romancal/resample/resample.py index dc17174e0..b11d99503 100644 --- a/romancal/resample/resample.py +++ b/romancal/resample/resample.py @@ -958,10 +958,7 @@ def populate_mosaic_individual( input_metas = [datamodel.meta for datamodel in input_models] for input_meta in input_metas: - try: - output_model.append_individual_image_meta(input_meta) - except: - continue + output_model.append_individual_image_meta(input_meta) def copy_asn_info_from_library(input_models, output_model): From 6c987dab24c234c3f268c49a6d8d9057418ba034 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Tue, 29 Oct 2024 11:58:28 -0400 Subject: [PATCH 21/25] Fix bug with cal_logs. --- romancal/regtest/test_skycell_generation.py | 3 +- romancal/resample/resample.py | 44 ++++++++------------- 2 files changed, 17 insertions(+), 30 deletions(-) diff --git a/romancal/regtest/test_skycell_generation.py b/romancal/regtest/test_skycell_generation.py index ece0c4401..5ae25e403 100644 --- a/romancal/regtest/test_skycell_generation.py +++ b/romancal/regtest/test_skycell_generation.py @@ -30,8 +30,7 @@ def test_skycell_asn_generation(rtdata): "r512_r274dp63x31y82_visit_F158_prompt_i2d_asn.json", "r512_r274dp63x32y80_visit_F158_prompt_i2d_asn.json", "r512_r274dp63x32y81_visit_F158_prompt_i2d_asn.json", - "r512_r274dp63x33y80_visit_F158_prompt_i2d_asn.json", - "r512_r274dp63x33y81_visit_F158_prompt_i2d_asn.json", + "r512_r274dp63x32y82_visit_F158_prompt_i2d_asn.json", ] # Test that the json files exist for file in output_files: diff --git a/romancal/resample/resample.py b/romancal/resample/resample.py index b11d99503..0448a5a65 100644 --- a/romancal/resample/resample.py +++ b/romancal/resample/resample.py @@ -7,7 +7,7 @@ from astropy import units as u from astropy.coordinates import SkyCoord from drizzle import cdrizzle, util -from roman_datamodels import datamodels, maker_utils +from roman_datamodels import datamodels, maker_utils, stnode from stcal.alignment.util import compute_scale from romancal.associations.asn_from_list import asn_from_list @@ -174,21 +174,26 @@ def __init__( **models[0].meta.cal_step.to_flat_dict() ) - # FIXME: do we have to populate meta.individual_image_meta.call_logs? - # concatenate all call_logs corresponding to each ImageModel - concatenate_call_logs(models) - - # Update the output with all the component metas - populate_mosaic_individual(self.blank_output, models) + # FIXME: temporary hack to prevent changes from L2 into L3 schema + for i, model in enumerate(models): + # saving ImageModels cal_logs + cal_logs = model.meta.cal_logs + # removing meta.cal_logs + del model.meta["cal_logs"] + # Update the output with all the component metas + populate_mosaic_individual(self.blank_output, [model]) + # re-attaching cal_logs to meta + model.meta.cal_logs = cal_logs # update meta data and wcs l2_into_l3_meta(self.blank_output.meta, models[0].meta) self.blank_output.meta.wcs = self.output_wcs gwcs_into_l3(self.blank_output, self.output_wcs) - # self.blank_output.cal_logs = stnode.CalLogs() - # self.blank_output["individual_image_cal_logs"] = [ - # model.meta.cal_logs for model in models - # ] + + self.blank_output.cal_logs = stnode.CalLogs() + self.blank_output["individual_image_cal_logs"] = [ + model.meta.cal_logs for model in models + ] for i, m in enumerate(models): self.input_models.shelve(m, i, modify=False) @@ -968,20 +973,3 @@ def copy_asn_info_from_library(input_models, output_model): if (asn_table_name := input_models.asn.get("table_name", None)) is not None: output_model.meta.asn.table_name = asn_table_name - -def concatenate_call_logs(model_list): - """ - Concatenate the `cal_logs` attribute of each model in the list. - - Parameters - ---------- - model_list : list - A list of models, each containing a `meta.cal_logs` attribute. - - Returns - ------- - None - The function modifies the `cal_logs` attribute of each model in place. - """ - for model in model_list: - model.meta["cal_logs"] = " /// ".join(model.meta.cal_logs) From f7dae3aff3cd0528d9c8bd6bd9c6bfdc7f201534 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Tue, 29 Oct 2024 12:00:18 -0400 Subject: [PATCH 22/25] Check style fixes. --- romancal/resample/resample.py | 1 - 1 file changed, 1 deletion(-) diff --git a/romancal/resample/resample.py b/romancal/resample/resample.py index 0448a5a65..f1a8e4cc3 100644 --- a/romancal/resample/resample.py +++ b/romancal/resample/resample.py @@ -972,4 +972,3 @@ def copy_asn_info_from_library(input_models, output_model): output_model.meta.asn.pool_name = asn_pool if (asn_table_name := input_models.asn.get("table_name", None)) is not None: output_model.meta.asn.table_name = asn_table_name - From c84a74444ae277c05f845d585031528603f4a2cc Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Tue, 29 Oct 2024 13:17:29 -0400 Subject: [PATCH 23/25] Lowercase missed WFI in regtest filenames. --- romancal/regtest/test_skycell_generation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/romancal/regtest/test_skycell_generation.py b/romancal/regtest/test_skycell_generation.py index 5ae25e403..67389cde3 100644 --- a/romancal/regtest/test_skycell_generation.py +++ b/romancal/regtest/test_skycell_generation.py @@ -14,7 +14,7 @@ def test_skycell_asn_generation(rtdata): # This test should generate seven json files args = [ "r0000101001001001001_0002_wfi01_cal.asdf", - "r0000101001001001001_0002_WFI10_cal.asdf", + "r0000101001001001001_0002_wfi10_cal.asdf", "-o", "r512", ] From e7ae5dc5efeefa26b50a7895163181d1363d3670 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Tue, 29 Oct 2024 13:31:31 -0400 Subject: [PATCH 24/25] Add changelog entry. --- changes/1473.general.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/1473.general.rst diff --git a/changes/1473.general.rst b/changes/1473.general.rst new file mode 100644 index 000000000..ec71d6c36 --- /dev/null +++ b/changes/1473.general.rst @@ -0,0 +1 @@ +Updates to support L1/L2 schema changes. From c732d3bd7d552c8a8027d3b7bae4cbce98163ad5 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Tue, 29 Oct 2024 15:21:42 -0400 Subject: [PATCH 25/25] Back to dot notation for cal_step.flat_field. --- romancal/flatfield/flat_field.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/romancal/flatfield/flat_field.py b/romancal/flatfield/flat_field.py index f8c25adb0..b74475274 100644 --- a/romancal/flatfield/flat_field.py +++ b/romancal/flatfield/flat_field.py @@ -51,18 +51,18 @@ def do_flat_field(output_model, flat_model): # Check to see if flat data array is smaller than science data log.warning("Flat data array is not the same shape as the science data") log.warning("Step will be skipped") - output_model.meta.cal_step["flat_field"] = "SKIPPED" + output_model.meta.cal_step.flat_field = "SKIPPED" elif output_model.meta.exposure.type != "WFI_IMAGE": # Check to see if attempt to flatten non-Image data log.info("Skipping flat field for spectral exposure.") - output_model.meta.cal_step["flat_field"] = "SKIPPED" + output_model.meta.cal_step.flat_field = "SKIPPED" elif flat_model is None: # Check to see if attempt to flatten non-Image data log.info("Skipping flat field - no flat reference file.") - output_model.meta.cal_step["flat_field"] = "SKIPPED" + output_model.meta.cal_step.flat_field = "SKIPPED" else: apply_flat_field(output_model, flat_model) - output_model.meta.cal_step["flat_field"] = "COMPLETE" + output_model.meta.cal_step.flat_field = "COMPLETE" def apply_flat_field(science, flat):