From 25e40888f9ee4d2c0e79ec3e35a36ae3e043f85a Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Wed, 28 Jun 2023 15:18:41 -0400 Subject: [PATCH 01/82] Move methods that handle S_REGION into assign_wcs.utils. --- romancal/assign_wcs/assign_wcs_step.py | 71 ++++++----------------- romancal/assign_wcs/utils.py | 79 ++++++++++++++++++++++++-- 2 files changed, 89 insertions(+), 61 deletions(-) diff --git a/romancal/assign_wcs/assign_wcs_step.py b/romancal/assign_wcs/assign_wcs_step.py index ff9d24db7..aac0b005d 100644 --- a/romancal/assign_wcs/assign_wcs_step.py +++ b/romancal/assign_wcs/assign_wcs_step.py @@ -12,7 +12,7 @@ from ..stpipe import RomanStep from . import pointing -from .utils import wcs_bbox_from_shape +from .utils import wcs_bbox_from_shape, add_s_region log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) @@ -32,8 +32,10 @@ def process(self, input): for reftype in self.reference_file_types: log.info(f"reftype, {reftype}") reffile = self.get_reference_file(input_model, reftype) - reference_file_names[reftype] = reffile if reffile else "" - log.debug(f"reference files used in assign_wcs: {reference_file_names}") + reference_file_names[reftype] = reffile or "" + log.debug( + f"reference files used in assign_wcs: {reference_file_names}" + ) result = load_wcs(input_model, reference_file_names) if self.save_results: @@ -66,15 +68,16 @@ def load_wcs(input_model, reference_files=None): if reference_files is not None: for ref_type, ref_file in reference_files.items(): - if ref_file not in ["N/A", ""]: - reference_files[ref_type] = ref_file - else: - reference_files[ref_type] = None + reference_files[ref_type] = ( + ref_file if ref_file not in ["N/A", ""] else None + ) else: reference_files = {} # Frames - detector = cf.Frame2D(name="detector", axes_order=(0, 1), unit=(u.pix, u.pix)) + detector = cf.Frame2D( + name="detector", axes_order=(0, 1), unit=(u.pix, u.pix) + ) v2v3 = cf.Frame2D( name="v2v3", axes_order=(0, 1), @@ -87,7 +90,11 @@ def load_wcs(input_model, reference_files=None): distortion = wfi_distortion(output_model, reference_files) tel2sky = pointing.v23tosky(output_model) - pipeline = [Step(detector, distortion), Step(v2v3, tel2sky), Step(world, None)] + pipeline = [ + Step(detector, distortion), + Step(v2v3, tel2sky), + Step(world, None), + ] wcs = WCS(pipeline) if wcs.bounding_box is None: wcs.bounding_box = wcs_bbox_from_shape(output_model.data.shape) @@ -137,49 +144,3 @@ def wfi_distortion(model, reference_files): transform.bounding_box = bbox return transform - - -def add_s_region(model): - """ - Calculate the detector's footprint using ``WCS.footprint`` and save it in the - ``S_REGION`` keyword - - Parameters - ---------- - model : `~roman_datamodels.datamodels.ImageModel` - The data model for processing - - Returns - ------- - A formatted string representing the detector's footprint - """ - - bbox = model.meta.wcs.bounding_box - - if bbox is None: - bbox = wcs_bbox_from_shape(model.data.shape) - - # footprint is an array of shape (2, 4) - i.e. 4 values for RA and 4 values for - # Dec - as we are interested only in the footprint on the sky - footprint = model.meta.wcs.footprint(bbox, center=True, axis_type="spatial").T - # take only imaging footprint - footprint = footprint[:2, :] - - # Make sure RA values are all positive - negative_ind = footprint[0] < 0 - if negative_ind.any(): - footprint[0][negative_ind] = 360 + footprint[0][negative_ind] - - footprint = footprint.T - update_s_region_keyword(model, footprint) - - -def update_s_region_keyword(model, footprint): - s_region = "POLYGON ICRS " + " ".join([str(x) for x in footprint.ravel()]) + " " - log.info(f"S_REGION VALUES: {s_region}") - if "nan" in s_region: - # do not update s_region if there are NaNs. - log.info("There are NaNs in s_region, S_REGION not updated.") - else: - model.meta.wcsinfo.s_region = s_region - log.info(f"Update S_REGION to {model.meta.wcsinfo.s_region}") diff --git a/romancal/assign_wcs/utils.py b/romancal/assign_wcs/utils.py index 76b8ef847..8e3225ce6 100644 --- a/romancal/assign_wcs/utils.py +++ b/romancal/assign_wcs/utils.py @@ -1,5 +1,6 @@ import functools from typing import List, Tuple, Union +import logging import numpy as np from astropy.coordinates import SkyCoord @@ -9,6 +10,9 @@ from gwcs.wcstools import wcs_from_fiducial from roman_datamodels.datamodels import DataModel +log = logging.getLogger(__name__) +log.setLevel(logging.DEBUG) + _MAX_SIP_DEGREE = 6 @@ -148,7 +152,9 @@ def wcs_from_footprints( calc_rotation_matrix(roll_ref, v3yangle, vparity=vparity), (2, 2) ) - rotation = astmodels.AffineTransformation2D(pc, name="pc_rotation_matrix") + rotation = astmodels.AffineTransformation2D( + pc, name="pc_rotation_matrix" + ) transform.append(rotation) if sky_axes: @@ -181,7 +187,10 @@ def wcs_from_footprints( output_bounding_box = [] for axis in out_frame.axes_order: - axis_min, axis_max = domain_bounds[axis].min(), domain_bounds[axis].max() + axis_min, axis_max = ( + domain_bounds[axis].min(), + domain_bounds[axis].max(), + ) output_bounding_box.append((axis_min, axis_max)) output_bounding_box = tuple(output_bounding_box) @@ -199,7 +208,9 @@ def wcs_from_footprints( wnew.bounding_box = output_bounding_box if shape is None: - shape = [int(axs[1] - axs[0] + 0.5) for axs in output_bounding_box[::-1]] + shape = [ + int(axs[1] - axs[0] + 0.5) for axs in output_bounding_box[::-1] + ] wnew.pixel_shape = shape[::-1] wnew.array_shape = shape @@ -248,7 +259,9 @@ def compute_scale( spatial_idx = np.where(np.array(wcs.output_frame.axes_type) == "SPATIAL")[0] delta[spatial_idx[0]] = 1 - crpix_with_offsets = np.vstack((crpix, crpix + delta, crpix + np.roll(delta, 1))).T + crpix_with_offsets = np.vstack( + (crpix, crpix + delta, crpix + np.roll(delta, 1)) + ).T crval_with_offsets = wcs(*crpix_with_offsets, with_bounding_box=False) coords = SkyCoord( @@ -328,7 +341,9 @@ def compute_fiducial(wcslist, bounding_box=None): axes_types = wcslist[0].output_frame.axes_type spatial_axes = np.array(axes_types) == "SPATIAL" spectral_axes = np.array(axes_types) == "SPECTRAL" - footprints = np.hstack([w.footprint(bounding_box=bounding_box).T for w in wcslist]) + footprints = np.hstack( + [w.footprint(bounding_box=bounding_box).T for w in wcslist] + ) spatial_footprint = footprints[spatial_axes] spectral_footprint = footprints[spectral_axes] @@ -344,8 +359,60 @@ def compute_fiducial(wcslist, bounding_box=None): y_mid = (np.max(y) + np.min(y)) / 2.0 z_mid = (np.max(z) + np.min(z)) / 2.0 lon_fiducial = np.rad2deg(np.arctan2(y_mid, x_mid)) % 360.0 - lat_fiducial = np.rad2deg(np.arctan2(z_mid, np.sqrt(x_mid**2 + y_mid**2))) + lat_fiducial = np.rad2deg( + np.arctan2(z_mid, np.sqrt(x_mid**2 + y_mid**2)) + ) fiducial[spatial_axes] = lon_fiducial, lat_fiducial if spectral_footprint.any(): fiducial[spectral_axes] = spectral_footprint.min() return fiducial + + +def add_s_region(model): + """ + Calculate the detector's footprint using ``WCS.footprint`` and save it in the + ``S_REGION`` keyword + + Parameters + ---------- + model : `~roman_datamodels.datamodels.ImageModel` + The data model for processing + + Returns + ------- + A formatted string representing the detector's footprint + """ + + bbox = model.meta.wcs.bounding_box + + if bbox is None: + bbox = wcs_bbox_from_shape(model.data.shape) + + # footprint is an array of shape (2, 4) - i.e. 4 values for RA and 4 values for + # Dec - as we are interested only in the footprint on the sky + footprint = model.meta.wcs.footprint( + bbox, center=True, axis_type="spatial" + ).T + # take only imaging footprint + footprint = footprint[:2, :] + + # Make sure RA values are all positive + negative_ind = footprint[0] < 0 + if negative_ind.any(): + footprint[0][negative_ind] = 360 + footprint[0][negative_ind] + + footprint = footprint.T + update_s_region_keyword(model, footprint) + + +def update_s_region_keyword(model, footprint): + s_region = ( + "POLYGON ICRS " + " ".join([str(x) for x in footprint.ravel()]) + " " + ) + log.info(f"S_REGION VALUES: {s_region}") + if "nan" in s_region: + # do not update s_region if there are NaNs. + log.info("There are NaNs in s_region, S_REGION not updated.") + else: + model.meta.wcsinfo.s_region = s_region + log.info(f"Update S_REGION to {model.meta.wcsinfo.s_region}") From ff63981344d279fe68ae7feade68a29c2a472a6a Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Thu, 20 Jul 2023 15:57:05 -0400 Subject: [PATCH 02/82] Implement resample step for RomanCal. --- pyproject.toml | 3 +- romancal/resample/__init__.py | 3 + romancal/resample/resample_utils.py | 138 ++++++++++++++++++++++++++-- 3 files changed, 136 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 83f6f5272..d89a1e53b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,8 @@ dependencies = [ 'rad @ git+https://github.com/spacetelescope/rad.git@main', # 'roman_datamodels >=0.15.0', 'roman_datamodels @ git+https://github.com/spacetelescope/roman_datamodels.git@main', - 'stcal >=1.4.0', + # 'stcal >=1.4.0', + 'stcal @ git+https://github.com/mairanteodoro/stcal.git#egg=stcal-alignment', 'stpipe >=0.5.0', 'tweakwcs >=0.8.0', 'spherical-geometry >= 1.2.22', diff --git a/romancal/resample/__init__.py b/romancal/resample/__init__.py index e69de29bb..da34cce81 100644 --- a/romancal/resample/__init__.py +++ b/romancal/resample/__init__.py @@ -0,0 +1,3 @@ +from .resample_step import ResampleStep + +__all__ = ["ResampleStep"] diff --git a/romancal/resample/resample_utils.py b/romancal/resample/resample_utils.py index 46c4bcf70..0a2f67dd0 100644 --- a/romancal/resample/resample_utils.py +++ b/romancal/resample/resample_utils.py @@ -1,9 +1,17 @@ import logging from typing import Tuple +import warnings import numpy as np -from romancal.assign_wcs.utils import wcs_bbox_from_shape, wcs_from_footprints +from romancal.assign_wcs.utils import wcs_bbox_from_shape +from stcal.alignment.util import wcs_from_footprints +from astropy.nddata.bitmask import bitfield_to_boolean_mask, interpret_bit_flags +from astropy import units as u +from romancal.lib.dqflags import pixel +from astropy.modeling import Model +from astropy import wcs as fitswcs +import gwcs log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) @@ -15,8 +23,8 @@ def make_output_wcs( pscale=None, rotation=None, shape=None, - ref_pixel: Tuple[float, float] = None, - ref_coord: Tuple[float, float] = None, + crpix: Tuple[float, float] = None, + crval: Tuple[float, float] = None, ): """Generate output WCS here based on footprints of all input WCS objects Parameters @@ -45,12 +53,12 @@ def make_output_wcs( ``pixel_shape`` and ``array_shape`` properties of the returned WCS object. - ref_pixel : tuple of float, None, optional + crpix : tuple of float, None, optional Position of the reference pixel in the image array. If ``ref_pixel`` is not specified, it will be set to the center of the bounding box of the returned WCS object. - ref_coord : tuple of float, None, optional + crval : tuple of float, None, optional Right ascension and declination of the reference pixel. Automatically computed if not provided. @@ -75,8 +83,8 @@ def make_output_wcs( pscale=pscale, rotation=rotation, shape=shape, - ref_pixel=ref_pixel, - ref_coord=ref_coord, + crpix=crpix, + crval=crval, ) # Check that the output data shape has no zero length dimensions @@ -87,3 +95,119 @@ def make_output_wcs( ) return output_wcs + + +def build_driz_weight(model, weight_type=None, good_bits=None): + """Create a weight map for use by drizzle""" + dqmask = build_mask(model.dq, good_bits) + + if weight_type == "ivm": + if ( + hasattr(model, "var_rnoise") + and model.var_rnoise is not None + and model.var_rnoise.shape == model.data.shape + ): + with np.errstate(divide="ignore", invalid="ignore"): + inv_variance = model.var_rnoise**-1 + inv_variance[~np.isfinite(inv_variance)] = 1 * u.s**2 / u.electron**2 + else: + warnings.warn( + "var_rnoise array not available. Setting drizzle weight map to 1", + RuntimeWarning, + ) + inv_variance = 1.0 * u.s**2 / u.electron**2 + result = inv_variance * dqmask + elif weight_type == "exptime": + exptime = model.meta.exposure.exposure_time + result = exptime * dqmask + else: + result = np.ones(model.data.shape, dtype=model.data.dtype) * dqmask + + return result.astype(np.float32) + + +def build_mask(dqarr, bitvalue): + """Build a bit mask from an input DQ array and a bitvalue flag + + In the returned bit mask, 1 is good, 0 is bad + """ + bitvalue = interpret_bit_flags(bitvalue, flag_name_map=pixel) + + if bitvalue is None: + return np.ones(dqarr.shape, dtype=np.uint8) + return np.logical_not(np.bitwise_and(dqarr, ~bitvalue)).astype(np.uint8) + + +def calc_gwcs_pixmap(in_wcs, out_wcs, shape=None): + """Return a pixel grid map from input frame to output frame.""" + if shape: + bb = wcs_bbox_from_shape(shape) + log.debug("Bounding box from data shape: {}".format(bb)) + else: + bb = in_wcs.bounding_box + log.debug("Bounding box from WCS: {}".format(in_wcs.bounding_box)) + + grid = gwcs.wcstools.grid_from_bounding_box(bb) + pixmap = np.dstack(reproject(in_wcs, out_wcs)(grid[0], grid[1])) + + return pixmap + + +def reproject(wcs1, wcs2): + """ + Given two WCSs or transforms return a function which takes pixel + coordinates in the first WCS or transform and computes them in the second + one. It performs the forward transformation of ``wcs1`` followed by the + inverse of ``wcs2``. + + Parameters + ---------- + wcs1, wcs2 : `~astropy.wcs.WCS` or `~gwcs.wcs.WCS` or `~astropy.modeling.Model` + WCS objects. + + Returns + ------- + _reproject : func + Function to compute the transformations. It takes x, y + positions in ``wcs1`` and returns x, y positions in ``wcs2``. + """ + + if isinstance(wcs1, fitswcs.WCS): + forward_transform = wcs1.all_pix2world + elif isinstance(wcs1, gwcs.WCS): + forward_transform = wcs1.forward_transform + elif issubclass(wcs1, Model): + forward_transform = wcs1 + else: + raise TypeError( + "Expected input to be astropy.wcs.WCS or gwcs.WCS " + "object or astropy.modeling.Model subclass" + ) + + if isinstance(wcs2, fitswcs.WCS): + backward_transform = wcs2.all_world2pix + elif isinstance(wcs2, gwcs.WCS): + backward_transform = wcs2.backward_transform + elif issubclass(wcs2, Model): + backward_transform = wcs2.inverse + else: + raise TypeError( + "Expected input to be astropy.wcs.WCS or gwcs.WCS " + "object or astropy.modeling.Model subclass" + ) + + def _reproject(x, y): + sky = forward_transform(x, y) + flat_sky = [] + for axis in sky: + flat_sky.append(axis.flatten()) + # Filter out RuntimeWarnings due to computed NaNs in the WCS + warnings.simplefilter("ignore") + det = backward_transform(*tuple(flat_sky)) + warnings.resetwarnings() + det_reshaped = [] + for axis in det: + det_reshaped.append(axis.reshape(x.shape)) + return tuple(det_reshaped) + + return _reproject From 38f5c219a3eddcad106ab212f0eb3975b87e92e2 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Thu, 20 Jul 2023 15:58:47 -0400 Subject: [PATCH 03/82] Add resample files. --- romancal/resample/gwcs_drizzle.py | 460 ++++++++++++++++++++++++ romancal/resample/resample.py | 545 +++++++++++++++++++++++++++++ romancal/resample/resample_step.py | 389 ++++++++++++++++++++ 3 files changed, 1394 insertions(+) create mode 100644 romancal/resample/gwcs_drizzle.py create mode 100644 romancal/resample/resample.py create mode 100755 romancal/resample/resample_step.py diff --git a/romancal/resample/gwcs_drizzle.py b/romancal/resample/gwcs_drizzle.py new file mode 100644 index 000000000..69684c1ec --- /dev/null +++ b/romancal/resample/gwcs_drizzle.py @@ -0,0 +1,460 @@ +import numpy as np + +from drizzle import util +from drizzle import cdrizzle +from . import resample_utils + +import logging + +log = logging.getLogger(__name__) +log.setLevel(logging.DEBUG) + + +class GWCSDrizzle: + """ + Combine images using the drizzle algorithm + """ + + def __init__( + self, + product, + outwcs=None, + wt_scl=None, + pixfrac=1.0, + kernel="square", + fillval="INDEF", + ): + """ + Create a new Drizzle output object and set the drizzle parameters. + + Parameters + ---------- + + product : DataModel + A data model containing results from a previous run. The three + extensions SCI, WHT, and CTX contain the combined image, total counts + and image id bitmap, respectively. The WCS of the combined image is + also read from the SCI extension. + + outwcs : `gwcs.WCS` + The world coordinate system (WCS) of the resampled image. If not + provided, the WCS is taken from product. + + wt_scl : str, optional + How each input image should be scaled. The choices are `exptime`, + which scales each image by its exposure time, or `expsq`, which scales + each image by the exposure time squared. If not set, then each + input image is scaled by its own weight map. + + pixfrac : float, optional + The fraction of a pixel that the pixel flux is confined to. The + default value of 1 has the pixel flux evenly spread across the image. + A value of 0.5 confines it to half a pixel in the linear dimension, + so the flux is confined to a quarter of the pixel area when the square + kernel is used. + + kernel : str, optional + The name of the kernel used to combine the inputs. The choice of + kernel controls the distribution of flux over the kernel. The kernel + names are: "square", "gaussian", "point", "tophat", "turbo", "lanczos2", + and "lanczos3". The square kernel is the default. + + fillval : str, optional + The value a pixel is set to in the output if the input image does + not overlap it. The default value of INDEF does not set a value. + """ + + # Initialize the object fields + self._product = product + + self.outexptime = 0.0 + self.uniqid = 0 + + if wt_scl is None: + self.wt_scl = "" + else: + self.wt_scl = wt_scl + self.kernel = kernel + self.fillval = fillval + self.pixfrac = pixfrac + + self.sciext = "SCI" + self.whtext = "WHT" + self.conext = "CON" + + out_units = "cps" + + self.outexptime = getattr(product.meta.resample, "product_exposure_time", 0.0) + + self.outsci = product.data + if outwcs: + self.outwcs = outwcs + else: + self.outwcs = product.meta.wcs + + self.outwht = None + self.outcon = product.context + + if self.outcon.ndim == 2: + self.outcon = np.reshape( + self.outcon, (1, self.outcon.shape[0], self.outcon.shape[1]) + ) + elif self.outcon.ndim != 3: + raise ValueError( + "Drizzle context image has wrong dimensions: \ + {0}".format( + product + ) + ) + + # Check field values + if not self.outwcs: + raise ValueError("Either an existing file or wcs must be supplied") + + if out_units == "counts": + np.divide(self.outsci, self.outexptime, self.outsci) + elif out_units != "cps": + raise ValueError("Illegal value for out_units: %s" % out_units) + + # Since the context array is dynamic, it must be re-assigned + # back to the product's `con` attribute. + @property + def outcon(self): + """Return the context array""" + return self._product.context + + @outcon.setter + def outcon(self, value): + """Set new context array""" + self._product.context = value + + def add_image( + self, + insci, + inwcs, + inwht=None, + xmin=0, + xmax=0, + ymin=0, + ymax=0, + expin=1.0, + in_units="cps", + wt_scl=1.0, + ): + """ + Combine an input image with the output drizzled image. + + Instead of reading the parameters from a fits file, you can set + them by calling this lower level method. `Add_fits_file` calls + this method after doing its setup. + + Parameters + ---------- + + insci : array + A 2d numpy array containing the input image to be drizzled. + it is an error to not supply an image. + + inwcs : wcs + The world coordinate system of the input image. This is + used to convert the pixels to the output coordinate system. + + inwht : array, optional + A 2d numpy array containing the pixel by pixel weighting. + Must have the same dimensions as insci. If none is supplied, + the weighting is set to one. + + xmin : float, optional + This and the following three parameters set a bounding rectangle + on the output image. Only pixels on the output image inside this + rectangle will have their flux updated. Xmin sets the minimum value + of the x dimension. The x dimension is the dimension that varies + quickest on the image. If the value is zero or less, no minimum will + be set in the x dimension. All four parameters are zero based, + counting starts at zero. + + xmax : float, optional + Sets the maximum value of the x dimension on the bounding box + of the output image. If the value is zero or less, no maximum will + be set in the x dimension. + + ymin : float, optional + Sets the minimum value in the y dimension on the bounding box. The + y dimension varies less rapidly than the x and represents the line + index on the output image. If the value is zero or less, no minimum + will be set in the y dimension. + + ymax : float, optional + Sets the maximum value in the y dimension. If the value is zero or + less, no maximum will be set in the y dimension. + + expin : float, optional + The exposure time of the input image, a positive number. The + exposure time is used to scale the image if the units are counts and + to scale the image weighting if the drizzle was initialized with + wt_scl equal to "exptime" or "expsq." + + in_units : str, optional + The units of the input image. The units can either be "counts" + or "cps" (counts per second.) If the value is counts, before using + the input image it is scaled by dividing it by the exposure time. + + wt_scl : float, optional + If drizzle was initialized with wt_scl left blank, this value will + set a scaling factor for the pixel weighting. If drizzle was + initialized with wt_scl set to "exptime" or "expsq", the exposure time + will be used to set the weight scaling and the value of this parameter + will be ignored. + """ + if self.wt_scl == "exptime": + wt_scl = expin + elif self.wt_scl == "expsq": + wt_scl = expin * expin + + wt_scl = 1.0 # hard-coded for JWST count-rate data + self.increment_id() + + dodrizzle( + insci, + inwcs, + inwht, + self.outwcs, + self.outsci, + self.outwht, + self.outcon, + expin, + in_units, + wt_scl, + uniqid=self.uniqid, + xmin=xmin, + xmax=xmax, + ymin=ymin, + ymax=ymax, + pixfrac=self.pixfrac, + kernel=self.kernel, + fillval=self.fillval, + ) + + def increment_id(self): + """ + Increment the id count and add a plane to the context image if needed + + Drizzle tracks which input images contribute to the output image + by setting a bit in the corresponding pixel in the context image. + The uniqid indicates which bit. So it must be incremented each time + a new image is added. Each plane in the context image can hold 32 bits, + so after each 32 images, a new plane is added to the context. + """ + + # Compute what plane of the context image this input would + # correspond to: + planeid = int(self.uniqid / 32) + + # Add a new plane to the context image if planeid overflows + + if self.outcon.shape[0] == planeid: + plane = np.zeros_like(self.outcon[0]) + plane = plane.reshape((1, plane.shape[0], plane.shape[1])) + self.outcon = np.concatenate((self.outcon, plane)) + + # Increment the id + self.uniqid += 1 + + +def dodrizzle( + insci, + input_wcs, + inwht, + output_wcs, + outsci, + outwht, + outcon, + expin, + in_units, + wt_scl, + uniqid=1, + xmin=0, + xmax=0, + ymin=0, + ymax=0, + pixfrac=1.0, + kernel="square", + fillval="INDEF", +): + """ + Low level routine for performing 'drizzle' operation on one image. + + Parameters + ---------- + + insci : 2d array + A 2d numpy array containing the input image to be drizzled. + + input_wcs : gwcs.WCS object + The world coordinate system of the input image. + + inwht : 2d array + A 2d numpy array containing the pixel by pixel weighting. + Must have the same dimensions as insci. If none is supplied, + the weighting is set to one. + + output_wcs : gwcs.WCS object + The world coordinate system of the output image. + + outsci : 2d array + A 2d numpy array containing the output image produced by + drizzling. On the first call it should be set to zero. + Subsequent calls it will hold the intermediate results + + outwht : 2d array + A 2d numpy array containing the output counts. On the first + call it should be set to zero. On subsequent calls it will + hold the intermediate results. + + outcon : 2d or 3d array, optional + A 2d or 3d numpy array holding a bitmap of which image was an input + for each output pixel. Should be integer zero on first call. + Subsequent calls hold intermediate results. + + expin : float + The exposure time of the input image, a positive number. The + exposure time is used to scale the image if the units are counts. + + in_units : str + The units of the input image. The units can either be "counts" + or "cps" (counts per second.) + + wt_scl : float + A scaling factor applied to the pixel by pixel weighting. + + uniqid : int, optional + The id number of the input image. Should be one the first time + this function is called and incremented by one on each subsequent + call. + + xmin : float, optional + This and the following three parameters set a bounding rectangle + on the input image. Only pixels on the input image inside this + rectangle will have their flux added to the output image. Xmin + sets the minimum value of the x dimension. The x dimension is the + dimension that varies quickest on the image. If the value is zero, + no minimum will be set in the x dimension. All four parameters are + zero based, counting starts at zero. + + xmax : float, optional + Sets the maximum value of the x dimension on the bounding box + of the input image. If the value is zero, no maximum will + be set in the x dimension, the full x dimension of the output + image is the bounding box. + + ymin : float, optional + Sets the minimum value in the y dimension on the bounding box. The + y dimension varies less rapidly than the x and represents the line + index on the input image. If the value is zero, no minimum will be + set in the y dimension. + + ymax : float, optional + Sets the maximum value in the y dimension. If the value is zero, no + maximum will be set in the y dimension, the full x dimension + of the output image is the bounding box. + + pixfrac : float, optional + The fraction of a pixel that the pixel flux is confined to. The + default value of 1 has the pixel flux evenly spread across the image. + A value of 0.5 confines it to half a pixel in the linear dimension, + so the flux is confined to a quarter of the pixel area when the square + kernel is used. + + kernel: str, optional + The name of the kernel used to combine the input. The choice of + kernel controls the distribution of flux over the kernel. The kernel + names are: "square", "gaussian", "point", "tophat", "turbo", "lanczos2", + and "lanczos3". The square kernel is the default. + + fillval: str, optional + The value a pixel is set to in the output if the input image does + not overlap it. The default value of INDEF does not set a value. + + Returns + ------- + A tuple with three values: a version string, the number of pixels + on the input image that do not overlap the output image, and the + number of complete lines on the input image that do not overlap the + output input image. + + """ + + # Insure that the fillval parameter gets properly interpreted for use with tdriz + if util.is_blank(str(fillval)): + fillval = "INDEF" + else: + fillval = str(fillval) + + if in_units == "cps": + expscale = 1.0 + else: + expscale = expin + + if insci.dtype > np.float32: + insci = insci.astype(np.float32) + + # Add input weight image if it was not passed in + if inwht is None: + inwht = np.ones_like(insci) + + if xmax is None or xmax == xmin: + xmax = insci.shape[1] + if ymax is None or ymax == ymin: + ymax = insci.shape[0] + + # Compute what plane of the context image this input would + # correspond to: + planeid = int((uniqid - 1) / 32) + + # Check if the context image has this many planes + if outcon.ndim == 3: + nplanes = outcon.shape[0] + elif outcon.ndim == 2: + nplanes = 1 + else: + nplanes = 0 + + if nplanes <= planeid: + raise IndexError("Not enough planes in drizzle context image") + + # Alias context image to the requested plane if 3d + if outcon.ndim == 3: + outcon = outcon[planeid] + + # Compute the mapping between the input and output pixel coordinates + # for use in drizzle.cdrizzle.tdriz + pixmap = resample_utils.calc_gwcs_pixmap(input_wcs, output_wcs, insci.shape) + # inwht[np.isnan(pixmap[:,:,0])] = 0. + + log.debug(f"Pixmap shape: {pixmap[:,:,0].shape}") + log.debug(f"Input Sci shape: {insci.shape}") + log.debug(f"Output Sci shape: {outsci.shape}") + + # Call 'drizzle' to perform image combination + log.info(f"Drizzling {insci.shape} --> {outsci.shape}") + + _vers, nmiss, nskip = cdrizzle.tdriz( + insci.astype(np.float32), + inwht.astype(np.float32), + pixmap, + outsci.value, + outwht, + outcon, + uniqid=uniqid, + xmin=xmin, + xmax=xmax, + ymin=ymin, + ymax=ymax, + pixfrac=pixfrac, + kernel=kernel, + in_units=in_units, + expscale=expscale, + wtscale=wt_scl, + fillstr=fillval, + ) + return _vers, nmiss, nskip diff --git a/romancal/resample/resample.py b/romancal/resample/resample.py new file mode 100644 index 000000000..4b816a183 --- /dev/null +++ b/romancal/resample/resample.py @@ -0,0 +1,545 @@ +import logging + +import numpy as np +from drizzle import util +from drizzle import cdrizzle + +from roman_datamodels import datamodels + +from ..datamodels import ModelContainer +from roman_datamodels.maker_utils import mk_datamodel + +from . import gwcs_drizzle +from . import resample_utils +from ..lib.basic_utils import bytes2human + +log = logging.getLogger(__name__) +log.setLevel(logging.DEBUG) + +__all__ = ["OutputTooLargeError", "ResampleData"] + + +class OutputTooLargeError(RuntimeError): + """Raised when the output is too large for in-memory instantiation""" + + +class ResampleData: + """ + This is the controlling routine for the resampling process. + + Notes + ----- + This routine performs the following operations:: + + 1. Extracts parameter settings from input model, such as pixfrac, + weight type, exposure time (if relevant), and kernel, and merges + them with any user-provided values. + 2. Creates output WCS based on input images and define mapping function + between all input arrays and the output array. + 3. Updates output data model with output arrays from drizzle, including + a record of metadata from all input models. + """ + + def __init__( + self, + input_models, + output=None, + single=False, + blendheaders=True, + pixfrac=1.0, + kernel="square", + fillval="INDEF", + wht_type="ivm", + good_bits=0, + pscale_ratio=1.0, + pscale=None, + **kwargs, + ): + """ + Parameters + ---------- + input_models : list of objects + list of data models, one for each input image + + output : str + filename for output + + kwargs : dict + Other parameters. + + .. note:: + ``output_shape`` is in the ``x, y`` order. + + .. note:: + ``in_memory`` controls whether or not the resampled + array from ``resample_many_to_many()`` + should be kept in memory or written out to disk and + deleted from memory. Default value is `True` to keep + all products in memory. + """ + self.input_models = input_models + self.output_filename = output + self.pscale_ratio = pscale_ratio + self.single = single + self.blendheaders = blendheaders + self.pixfrac = pixfrac + self.kernel = kernel + self.fillval = fillval + self.weight_type = wht_type + self.good_bits = good_bits + self.in_memory = kwargs.get("in_memory", True) + + log.info(f"Driz parameter kernel: {self.kernel}") + log.info(f"Driz parameter pixfrac: {self.pixfrac}") + log.info(f"Driz parameter fillval: {self.fillval}") + log.info(f"Driz parameter weight_type: {self.weight_type}") + + output_wcs = kwargs.get("output_wcs", None) + output_shape = kwargs.get("output_shape", None) + crpix = kwargs.get("crpix", None) + crval = kwargs.get("crval", None) + rotation = kwargs.get("rotation", None) + + if pscale is not None: + log.info(f"Output pixel scale: {pscale} arcsec.") + pscale /= 3600.0 + else: + log.info(f"Output pixel scale ratio: {pscale_ratio}") + + if output_wcs: + # Use user-supplied reference WCS for the resampled image: + self.output_wcs = output_wcs + if output_shape is not None: + self.output_wcs.array_shape = output_shape[::-1] + + else: + # Define output WCS based on all inputs, including a reference WCS: + self.output_wcs = resample_utils.make_output_wcs( + self.input_models, + pscale_ratio=self.pscale_ratio, + pscale=pscale, + rotation=rotation, + shape=None if output_shape is None else output_shape[::-1], + crpix=crpix, + crval=crval, + ) + + log.debug("Output mosaic size: {}".format(self.output_wcs.array_shape)) + + # NOTE: should we enable memory allocation? + + # can_allocate, required_memory = datamodels.util.check_memory_allocation( + # self.output_wcs.array_shape, kwargs['allowed_memory'], datamodels.ImageModel + # ) + # if not can_allocate: + # raise OutputTooLargeError( + # f'Combined ImageModel size {self.output_wcs.array_shape} ' + # f'requires {bytes2human(required_memory)}. ' + # f'Model cannot be instantiated.' + # ) + + # NOTE: wait for William to fix bug in datamodels' init and then + # use datamodels.ImageModel(shape=(nx, ny)) instead of mk_datamodel() + + self.blank_output = mk_datamodel( + datamodels.MosaicModel, shape=tuple(self.output_wcs.array_shape) + ) + + # update meta data and wcs + self.blank_output.meta = dict(input_models[0].meta._data.items()) + self.blank_output.meta.wcs = self.output_wcs + + self.output_models = ModelContainer() + + def do_drizzle(self): + """Pick the correct drizzling mode based on self.single""" + if self.single: + return self.resample_many_to_many() + else: + return self.resample_many_to_one() + + def resample_many_to_many(self): + """Resample many inputs to many outputs where outputs have a common frame. + + Coadd only different detectors of the same exposure, i.e. map NRCA5 and + NRCB5 onto the same output image, as they image different areas of the + sky. + + Used for outlier detection + """ + for exposure in self.input_models.models_grouped: + output_model = self.blank_output + # Determine output file type from input exposure filenames + # Use this for defining the output filename + indx = exposure[0].meta.filename.rfind(".") + output_type = exposure[0].meta.filename[indx:] + output_root = "_".join( + exposure[0].meta.filename.replace(output_type, "").split("_")[:-1] + ) + output_model.meta.filename = f"{output_root}_outlier_i2d{output_type}" + + # Initialize the output with the wcs + driz = gwcs_drizzle.GWCSDrizzle( + output_model, + pixfrac=self.pixfrac, + kernel=self.kernel, + fillval=self.fillval, + ) + + log.info(f"{len(exposure)} exposures to drizzle together") + for img in exposure: + img = datamodels.open(img) + # TODO: should weight_type=None here? + inwht = resample_utils.build_driz_weight( + img, weight_type=self.weight_type, good_bits=self.good_bits + ) + + # apply sky subtraction + blevel = img.meta.background.level + if not img.meta.background.subtracted and blevel is not None: + data = img.data - blevel + else: + data = img.data + + driz.add_image(data, img.meta.wcs, inwht=inwht) + del data + img.close() + + if not self.in_memory: + # Write out model to disk, then return filename + output_name = output_model.meta.filename + output_model.save(output_name) + log.info(f"Exposure {output_name} saved to file") + self.output_models.append(output_name) + else: + self.output_models.append(output_model.copy()) + output_model.data *= 0.0 + output_model.wht *= 0.0 + + return self.output_models + + def resample_many_to_one(self): + """Resample and coadd many inputs to a single output. + + Used for stage 3 resampling + """ + output_model = self.blank_output.copy() + output_model.meta.filename = self.output_filename + output_model.meta["resample"] = {} + output_model.meta.resample["weight_type"] = self.weight_type + output_model.meta.resample["pointings"] = len(self.input_models.models_grouped) + + if self.blendheaders: + log.info("Skipping blendheaders for now.") + + # Initialize the output with the wcs + driz = gwcs_drizzle.GWCSDrizzle( + output_model, pixfrac=self.pixfrac, kernel=self.kernel, fillval=self.fillval + ) + + log.info("Resampling science data") + for img in self.input_models: + inwht = resample_utils.build_driz_weight( + img, weight_type=self.weight_type, good_bits=self.good_bits + ) + # apply sky subtraction + # NOTE: mocking a sky-subtracted image (remove this later on) + img.meta["background"] = {} + img.meta.background["level"] = 0 + img.meta.background["subtracted"] = True + blevel = img.meta.background.level + if not img.meta.background.subtracted and blevel is not None: + data = img.data - blevel + else: + data = img.data.copy() + + driz.add_image(data, img.meta.wcs, inwht=inwht) + del data, inwht + + # Resample variances array in self.input_models to output_model + self.resample_variance_array("var_rnoise", output_model) + self.resample_variance_array("var_poisson", output_model) + self.resample_variance_array("var_flat", output_model) + output_model.err = np.sqrt( + np.nansum( + [ + output_model.var_rnoise, + output_model.var_poisson, + output_model.var_flat, + ], + axis=0, + ) + ) + + self.update_exposure_times(output_model) + self.output_models.append(output_model) + + return self.output_models + + def resample_variance_array(self, name, output_model): + """Resample variance arrays from self.input_models to the output_model + + Resample the ``name`` variance array to the same name in output_model, + using a cumulative sum. + + This modifies output_model in-place. + """ + output_wcs = output_model.meta.wcs + inverse_variance_sum = np.full_like(output_model.data, np.nan) + + log.info(f"Resampling {name}") + for model in self.input_models: + variance = getattr(model, name) + if variance is None or variance.size == 0: + log.debug( + f"No data for '{name}' for model " + f"{repr(model.meta.filename)}. Skipping ..." + ) + continue + + elif variance.shape != model.data.shape: + log.warning( + f"Data shape mismatch for '{name}' for model " + f"{repr(model.meta.filename)}. Skipping ..." + ) + continue + + # Make input weight map of unity where there is science data + inwht = resample_utils.build_driz_weight( + model, weight_type=None, good_bits=self.good_bits + ) + + resampled_variance = np.zeros_like(output_model.data) + outwht = np.zeros_like(output_model.data) + outcon = np.zeros_like(output_model.con) + + # Resample the variance array. Fill "unpopulated" pixels with NaNs. + self.drizzle_arrays( + variance, + inwht, + model.meta.wcs, + output_wcs, + resampled_variance, + outwht, + outcon, + pixfrac=self.pixfrac, + kernel=self.kernel, + fillval=np.nan, + ) + + # Add the inverse of the resampled variance to a running sum. + # Update only pixels (in the running sum) with valid new values: + mask = resampled_variance > 0 + + inverse_variance_sum[mask] = np.nansum( + [inverse_variance_sum[mask], np.reciprocal(resampled_variance[mask])], + axis=0, + ) + + # We now have a sum of the inverse resampled variances. We need the + # inverse of that to get back to units of variance. + output_variance = np.reciprocal(inverse_variance_sum) + + setattr(output_model, name, output_variance) + + def update_exposure_times(self, output_model): + """Modify exposure time metadata in-place""" + total_exposure_time = 0.0 + exposure_times = {"start": [], "end": []} + for exposure in self.input_models.models_grouped: + total_exposure_time += exposure[0].meta.exposure.exposure_time + exposure_times["start"].append(exposure[0].meta.exposure.start_time) + exposure_times["end"].append(exposure[0].meta.exposure.end_time) + + # Update some basic exposure time values based on output_model + output_model.meta.exposure.exposure_time = total_exposure_time + output_model.meta.exposure.start_time = min(exposure_times["start"]) + output_model.meta.exposure.end_time = max(exposure_times["end"]) + output_model.meta.resample.product_exposure_time = total_exposure_time + + @staticmethod + def drizzle_arrays( + insci, + inwht, + input_wcs, + output_wcs, + outsci, + outwht, + outcon, + uniqid=1, + xmin=None, + xmax=None, + ymin=None, + ymax=None, + pixfrac=1.0, + kernel="square", + fillval="INDEF", + wtscale=1.0, + ): + """ + Low level routine for performing 'drizzle' operation on one image. + + The interface is compatible with STScI code. All images are Python + ndarrays, instead of filenames. File handling (input and output) is + performed by the calling routine. + + Parameters + ---------- + + insci : 2d array + A 2d numpy array containing the input image to be drizzled. + + inwht : 2d array + A 2d numpy array containing the pixel by pixel weighting. + Must have the same dimensions as insci. If none is supplied, + the weighting is set to one. + + input_wcs : gwcs.WCS object + The world coordinate system of the input image. + + output_wcs : gwcs.WCS object + The world coordinate system of the output image. + + outsci : 2d array + A 2d numpy array containing the output image produced by + drizzling. On the first call it should be set to zero. + Subsequent calls it will hold the intermediate results. This + is modified in-place. + + outwht : 2d array + A 2d numpy array containing the output counts. On the first + call it should be set to zero. On subsequent calls it will + hold the intermediate results. This is modified in-place. + + outcon : 2d or 3d array, optional + A 2d or 3d numpy array holding a bitmap of which image was an input + for each output pixel. Should be integer zero on first call. + Subsequent calls hold intermediate results. This is modified + in-place. + + uniqid : int, optional + The id number of the input image. Should be one the first time + this function is called and incremented by one on each subsequent + call. + + xmin : float, optional + This and the following three parameters set a bounding rectangle + on the input image. Only pixels on the input image inside this + rectangle will have their flux added to the output image. Xmin + sets the minimum value of the x dimension. The x dimension is the + dimension that varies quickest on the image. If the value is zero, + no minimum will be set in the x dimension. All four parameters are + zero based, counting starts at zero. + + xmax : float, optional + Sets the maximum value of the x dimension on the bounding box + of the input image. If the value is zero, no maximum will + be set in the x dimension, the full x dimension of the output + image is the bounding box. + + ymin : float, optional + Sets the minimum value in the y dimension on the bounding box. The + y dimension varies less rapidly than the x and represents the line + index on the input image. If the value is zero, no minimum will be + set in the y dimension. + + ymax : float, optional + Sets the maximum value in the y dimension. If the value is zero, no + maximum will be set in the y dimension, the full x dimension + of the output image is the bounding box. + + pixfrac : float, optional + The fraction of a pixel that the pixel flux is confined to. The + default value of 1 has the pixel flux evenly spread across the image. + A value of 0.5 confines it to half a pixel in the linear dimension, + so the flux is confined to a quarter of the pixel area when the square + kernel is used. + + kernel: str, optional + The name of the kernel used to combine the input. The choice of + kernel controls the distribution of flux over the kernel. The kernel + names are: "square", "gaussian", "point", "tophat", "turbo", "lanczos2", + and "lanczos3". The square kernel is the default. + + fillval: str, optional + The value a pixel is set to in the output if the input image does + not overlap it. The default value of INDEF does not set a value. + + Returns + ------- + A tuple with three values: a version string, the number of pixels + on the input image that do not overlap the output image, and the + number of complete lines on the input image that do not overlap the + output input image. + + """ + + # Insure that the fillval parameter gets properly interpreted for use with tdriz + if util.is_blank(str(fillval)): + fillval = "INDEF" + else: + fillval = str(fillval) + + if insci.dtype > np.float32: + insci = insci.astype(np.float32) + + # Add input weight image if it was not passed in + if inwht is None: + inwht = np.ones_like(insci) + + # Compute what plane of the context image this input would + # correspond to: + planeid = int((uniqid - 1) / 32) + + # Check if the context image has this many planes + if outcon.ndim == 3: + nplanes = outcon.shape[0] + elif outcon.ndim == 2: + nplanes = 1 + else: + nplanes = 0 + + if nplanes <= planeid: + raise IndexError("Not enough planes in drizzle context image") + + # Alias context image to the requested plane if 3d + if outcon.ndim == 3: + outcon = outcon[planeid] + + if xmin is xmax is ymin is ymax is None: + bb = input_wcs.bounding_box + ((x1, x2), (y1, y2)) = bb + xmin = int(min(x1, x2)) + ymin = int(min(y1, y2)) + xmax = int(max(x1, x2)) + ymax = int(max(y1, y2)) + + # Compute the mapping between the input and output pixel coordinates + # for use in drizzle.cdrizzle.tdriz + pixmap = resample_utils.calc_gwcs_pixmap(input_wcs, output_wcs, insci.shape) + + log.debug(f"Pixmap shape: {pixmap[:,:,0].shape}") + log.debug(f"Input Sci shape: {insci.shape}") + log.debug(f"Output Sci shape: {outsci.shape}") + + log.info(f"Drizzling {insci.shape} --> {outsci.shape}") + + _vers, _nmiss, _nskip = cdrizzle.tdriz( + insci, + inwht, + pixmap, + outsci, + outwht, + outcon, + uniqid=uniqid, + xmin=xmin, + xmax=xmax, + ymin=ymin, + ymax=ymax, + pixfrac=pixfrac, + kernel=kernel, + in_units="cps", + expscale=1.0, + wtscale=wtscale, + fillstr=fillval, + ) diff --git a/romancal/resample/resample_step.py b/romancal/resample/resample_step.py new file mode 100755 index 000000000..3fa3fc65d --- /dev/null +++ b/romancal/resample/resample_step.py @@ -0,0 +1,389 @@ +import logging +import re +from copy import deepcopy + +import numpy as np +import asdf +from stpipe.extern.configobj.validate import Validator +from stpipe.extern.configobj.configobj import ConfigObj + +from roman_datamodels import datamodels + +from ..datamodels import ModelContainer + +from . import resample +from ..stpipe import RomanStep +from ..assign_wcs import utils + +log = logging.getLogger(__name__) +log.setLevel(logging.DEBUG) + +__all__ = ["ResampleStep"] + + +# Force use of all DQ flagged data except for DO_NOT_USE and NON_SCIENCE +GOOD_BITS = "~DO_NOT_USE+NON_SCIENCE" + + +class ResampleStep(RomanStep): + """ + Resample input data onto a regular grid using the drizzle algorithm. + + .. note:: + When supplied via ``output_wcs``, a custom WCS overrides other custom + WCS parameters such as ``output_shape`` (now computed from by + ``output_wcs.bounding_box``), ``crpix`` + + Parameters + ----------- + input : ~jwst.datamodels.JwstDataModel or ~jwst.associations.Association + Single filename for either a single image or an association table. + """ + + class_alias = "resample" + + spec = """ + pixfrac = float(default=1.0) # change back to None when drizpar reference files are updated + kernel = string(default='square') # change back to None when drizpar reference files are updated + fillval = string(default='INDEF' ) # change back to None when drizpar reference files are updated + weight_type = option('ivm', 'exptime', None, default='ivm') # change back to None when drizpar ref update + output_shape = int_list(min=2, max=2, default=None) # [x, y] order + crpix = float_list(min=2, max=2, default=None) + crval = float_list(min=2, max=2, default=None) + rotation = float(default=None) + pixel_scale_ratio = float(default=1.0) # Ratio of input to output pixel scale + pixel_scale = float(default=None) # Absolute pixel scale in arcsec + output_wcs = string(default='') # Custom output WCS. + single = boolean(default=False) + blendheaders = boolean(default=True) + allowed_memory = float(default=None) # Fraction of memory to use for the combined image. + in_memory = boolean(default=True) + """ + + reference_file_types = [] + + def process(self, input): + if isinstance(input, datamodels.DataModel): + input_models = ModelContainer([input]) + output = input_models[0].meta.filename + self.blendheaders = False + elif isinstance(input, str): + input_models = ModelContainer(input) + output = input_models.meta.asn_table.products[0].name + + # if isinstance(input, ModelContainer): + # input_models = dm + # try: + # output = input_models.meta.asn_table.products[0].name + # except AttributeError: + # # coron data goes through this path by the time it gets to + # # resampling. + # # TODO: figure out why and make sure asn_table is carried along + # output = None + # else: + # input_models = ModelContainer([dm]) + # input_models.asn_pool_name = dm.meta.asn.pool_name + # input_models.asn_table_name = dm.meta.asn.table_name + # output = dm.meta.filename + # self.blendheaders = False + + # Check that input models are 2D images + if len(input_models[0].data.shape) != 2: + # resample can only handle 2D images, not 3D cubes, etc + raise RuntimeError(f"Input {input_models[0]} is not a 2D image.") + + # Get drizzle parameters reference file, if there is one + self.wht_type = self.weight_type + if "drizpars" in self.reference_file_types: + ref_filename = self.get_reference_file(input_models[0], "drizpars") + else: # no drizpars reference file found + ref_filename = "N/A" + + if ref_filename == "N/A": + self.log.info("No drizpars reference file found.") + kwargs = self._set_spec_defaults() + else: + self.log.info(f"Using drizpars reference file: {ref_filename}") + kwargs = self.get_drizpars(ref_filename, input_models) + + kwargs["allowed_memory"] = self.allowed_memory + + # Issue a warning about the use of exptime weighting + if self.wht_type == "exptime": + self.log.warning("Use of EXPTIME weighting will result in incorrect") + self.log.warning("propagated errors in the resampled product") + + # Custom output WCS parameters. + # Modify get_drizpars if any of these get into reference files: + kwargs["output_shape"] = self._check_list_pars( + self.output_shape, "output_shape", min_vals=[1, 1] + ) + kwargs["output_wcs"] = self._load_custom_wcs( + self.output_wcs, kwargs["output_shape"] + ) + kwargs["crpix"] = self._check_list_pars(self.crpix, "crpix") + kwargs["crval"] = self._check_list_pars(self.crval, "crval") + kwargs["rotation"] = self.rotation + kwargs["pscale"] = self.pixel_scale + kwargs["pscale_ratio"] = self.pixel_scale_ratio + kwargs["in_memory"] = self.in_memory + + # Call the resampling routine + resamp = resample.ResampleData(input_models, output=output, **kwargs) + result = resamp.do_drizzle() + + for model in result: + self._final_updates(model, input_models, kwargs) + if len(result) == 1: + result = result[0] + + input_models.close() + return result + + def _final_updates(self, model, input_models, kwargs): + model.meta.cal_step.resample = "COMPLETE" + self.update_fits_wcs(model) + utils.update_s_region_imaging(model) + model.meta.asn.pool_name = input_models.asn_pool_name + model.meta.asn.table_name = input_models.asn_table_name + + # if pixel_scale exists, it will override pixel_scale_ratio. + # calculate the actual value of pixel_scale_ratio based on pixel_scale + # because source_catalog uses this value from the header. + model.meta.resample.pixel_scale_ratio = ( + self.pixel_scale / np.sqrt(model.meta.photometry.pixelarea_arcsecsq) + if self.pixel_scale + else self.pixel_scale_ratio + ) + model.meta.resample.pixfrac = kwargs["pixfrac"] + self.update_phot_keywords(model) + + @staticmethod + def _check_list_pars(vals, name, min_vals=None): + if vals is None: + return None + if len(vals) != 2: + raise ValueError(f"List '{name}' must have exactly two elements.") + n = sum(x is None for x in vals) + if n == 2: + return None + elif n == 0: + if min_vals and sum(x >= y for x, y in zip(vals, min_vals)) != 2: + raise ValueError( + f"'{name}' values must be larger or equal to {list(min_vals)}" + ) + return list(vals) + else: + raise ValueError(f"Both '{name}' values must be either None or not None.") + + @staticmethod + def _load_custom_wcs(asdf_wcs_file, output_shape): + if not asdf_wcs_file: + return None + + with asdf.open(asdf_wcs_file) as af: + wcs = deepcopy(af.tree["wcs"]) + + if output_shape is not None or wcs is None: + wcs.array_shape = output_shape[::-1] + elif wcs.pixel_shape is not None: + wcs.array_shape = wcs.pixel_shape[::-1] + elif wcs.bounding_box is not None: + wcs.array_shape = tuple( + int(axs[1] - axs[0] + 0.5) + for axs in wcs.bounding_box.bounding_box(order="C") + ) + elif wcs.array_shape is None: + raise ValueError( + "Step argument 'output_shape' is required when custom WCS " + "does not have neither of 'array_shape', 'pixel_shape', or " + "'bounding_box' attributes set." + ) + + return wcs + + def update_phot_keywords(self, model): + """Update pixel scale keywords""" + if model.meta.photometry.pixelarea_steradians is not None: + model.meta.photometry.pixelarea_steradians *= ( + model.meta.resample.pixel_scale_ratio**2 + ) + if model.meta.photometry.pixelarea_arcsecsq is not None: + model.meta.photometry.pixelarea_arcsecsq *= ( + model.meta.resample.pixel_scale_ratio**2 + ) + + def get_drizpars(self, ref_filename, input_models): + """ + Extract drizzle parameters from reference file. + + This method extracts parameters from the drizpars reference file and + uses those to set defaults on the following ResampleStep configuration + parameters: + + pixfrac = float(default=None) + kernel = string(default=None) + fillval = string(default=None) + wht_type = option('ivm', 'exptime', None, default=None) + + Once the defaults are set from the reference file, if the user has + used a resample.cfg file or run ResampleStep using command line args, + then these will overwrite the defaults pulled from the reference file. + """ + with datamodels.DrizParsModel(ref_filename) as drpt: + drizpars_table = drpt.data + + num_groups = len(input_models.group_names) + filtname = input_models[0].meta.instrument.filter + row = None + filter_match = False + # look for row that applies to this set of input data models + for n, filt, num in zip( + range(0, len(drizpars_table)), + drizpars_table["filter"], + drizpars_table["numimages"], + ): + # only remember this row if no exact match has already been made for + # the filter. This allows the wild-card row to be anywhere in the + # table; since it may be placed at beginning or end of table. + + if str(filt) == "ANY" and not filter_match and num_groups >= num: + row = n + # always go for an exact match if present, though... + if filtname == filt and num_groups >= num: + row = n + filter_match = True + + # With presence of wild-card rows, code should never trigger this logic + if row is None: + self.log.error("No row found in %s matching input data.", ref_filename) + raise ValueError + + # Define the keys to pull from drizpars reffile table. + # All values should be None unless the user set them on the command + # line or in the call to the step + + drizpars = dict( + pixfrac=self.pixfrac, + kernel=self.kernel, + fillval=self.fillval, + wht_type=self.weight_type + # pscale_ratio=self.pixel_scale_ratio, # I think this can be removed JEM (??) + ) + + # For parameters that are set in drizpars table but not set by the + # user, use these. Otherwise, use values set by user. + reffile_drizpars = {k: v for k, v in drizpars.items() if v is None} + user_drizpars = {k: v for k, v in drizpars.items() if v is not None} + + # read in values from that row for each parameter + for k in reffile_drizpars: + if k in drizpars_table.names: + reffile_drizpars[k] = drizpars_table[k][row] + + # Convert the strings in the FITS binary table from np.bytes_ to str + for k, v in reffile_drizpars.items(): + if isinstance(v, np.bytes_): + reffile_drizpars[k] = v.decode("UTF-8") + + all_drizpars = {**reffile_drizpars, **user_drizpars} + + kwargs = dict( + good_bits=GOOD_BITS, + single=self.single, + blendheaders=self.blendheaders, + ) + + kwargs.update(all_drizpars) + + for k, v in kwargs.items(): + self.log.debug(" {}={}".format(k, v)) + + return kwargs + + def _set_spec_defaults(self): + """NIRSpec currently has no default drizpars reference file, so default + drizzle parameters are not set properly. This method sets them. + + Remove this class method when a drizpars reffile is delivered. + """ + configspec = self.load_spec_file() + config = ConfigObj(configspec=configspec) + if config.validate(Validator()): + kwargs = config.dict() + + if self.pixfrac is None: + self.pixfrac = 1.0 + if self.kernel is None: + self.kernel = "square" + if self.fillval is None: + self.fillval = "INDEF" + # Force definition of good bits + kwargs["good_bits"] = GOOD_BITS + kwargs["pixfrac"] = self.pixfrac + kwargs["kernel"] = str(self.kernel) + kwargs["fillval"] = str(self.fillval) + # self.weight_type has a default value of None + # The other instruments read this parameter from a reference file + if self.wht_type is None: + self.wht_type = "ivm" + + kwargs["wht_type"] = str(self.wht_type) + kwargs["pscale_ratio"] = self.pixel_scale_ratio + kwargs.pop("pixel_scale_ratio") + + for k, v in kwargs.items(): + if k in [ + "pixfrac", + "kernel", + "fillval", + "wht_type", + "pscale_ratio", + ]: + log.info(" using: %s=%s", k, repr(v)) + + return kwargs + + def update_fits_wcs(self, model): + """ + Update FITS WCS keywords of the resampled image. + """ + # Delete any SIP-related keywords first + pattern = r"^(cd[12]_[12]|[ab]p?_\d_\d|[ab]p?_order)$" + regex = re.compile(pattern) + + keys = list(model.meta.wcsinfo.instance.keys()) + for key in keys: + if regex.match(key): + del model.meta.wcsinfo.instance[key] + + # Write new PC-matrix-based WCS based on GWCS model + transform = model.meta.wcs.forward_transform + model.meta.wcsinfo.crpix1 = -transform[0].offset.value + 1 + model.meta.wcsinfo.crpix2 = -transform[1].offset.value + 1 + model.meta.wcsinfo.cdelt1 = transform[3].factor.value + model.meta.wcsinfo.cdelt2 = transform[4].factor.value + model.meta.wcsinfo.ra_ref = transform[6].lon.value + model.meta.wcsinfo.dec_ref = transform[6].lat.value + model.meta.wcsinfo.crval1 = model.meta.wcsinfo.ra_ref + model.meta.wcsinfo.crval2 = model.meta.wcsinfo.dec_ref + model.meta.wcsinfo.pc1_1 = transform[2].matrix.value[0][0] + model.meta.wcsinfo.pc1_2 = transform[2].matrix.value[0][1] + model.meta.wcsinfo.pc2_1 = transform[2].matrix.value[1][0] + model.meta.wcsinfo.pc2_2 = transform[2].matrix.value[1][1] + model.meta.wcsinfo.ctype1 = "RA---TAN" + model.meta.wcsinfo.ctype2 = "DEC--TAN" + + # Remove no longer relevant WCS keywords + rm_keys = [ + "v2_ref", + "v3_ref", + "ra_ref", + "dec_ref", + "roll_ref", + "v3yangle", + "vparity", + ] + for key in rm_keys: + if key in model.meta.wcsinfo.instance: + del model.meta.wcsinfo.instance[key] From 8aedb78d4e31086fb86feb5b6647d4942fefeec9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 1 Aug 2023 16:07:34 +0000 Subject: [PATCH 04/82] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- romancal/assign_wcs/assign_wcs_step.py | 6 ++---- romancal/assign_wcs/utils.py | 30 +++++++------------------- romancal/resample/gwcs_drizzle.py | 9 ++++---- romancal/resample/resample.py | 13 ++++------- romancal/resample/resample_step.py | 14 ++++++------ romancal/resample/resample_utils.py | 18 ++++++++-------- 6 files changed, 33 insertions(+), 57 deletions(-) diff --git a/romancal/assign_wcs/assign_wcs_step.py b/romancal/assign_wcs/assign_wcs_step.py index 35d9dea04..b33e5ba70 100644 --- a/romancal/assign_wcs/assign_wcs_step.py +++ b/romancal/assign_wcs/assign_wcs_step.py @@ -12,7 +12,7 @@ from ..stpipe import RomanStep from . import pointing -from .utils import wcs_bbox_from_shape, add_s_region +from .utils import add_s_region, wcs_bbox_from_shape log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) @@ -73,9 +73,7 @@ def load_wcs(input_model, reference_files=None): reference_files = {} # Frames - detector = cf.Frame2D( - name="detector", axes_order=(0, 1), unit=(u.pix, u.pix) - ) + detector = cf.Frame2D(name="detector", axes_order=(0, 1), unit=(u.pix, u.pix)) v2v3 = cf.Frame2D( name="v2v3", axes_order=(0, 1), diff --git a/romancal/assign_wcs/utils.py b/romancal/assign_wcs/utils.py index 8e3225ce6..7f957e5f7 100644 --- a/romancal/assign_wcs/utils.py +++ b/romancal/assign_wcs/utils.py @@ -1,6 +1,6 @@ import functools -from typing import List, Tuple, Union import logging +from typing import List, Tuple, Union import numpy as np from astropy.coordinates import SkyCoord @@ -152,9 +152,7 @@ def wcs_from_footprints( calc_rotation_matrix(roll_ref, v3yangle, vparity=vparity), (2, 2) ) - rotation = astmodels.AffineTransformation2D( - pc, name="pc_rotation_matrix" - ) + rotation = astmodels.AffineTransformation2D(pc, name="pc_rotation_matrix") transform.append(rotation) if sky_axes: @@ -208,9 +206,7 @@ def wcs_from_footprints( wnew.bounding_box = output_bounding_box if shape is None: - shape = [ - int(axs[1] - axs[0] + 0.5) for axs in output_bounding_box[::-1] - ] + shape = [int(axs[1] - axs[0] + 0.5) for axs in output_bounding_box[::-1]] wnew.pixel_shape = shape[::-1] wnew.array_shape = shape @@ -259,9 +255,7 @@ def compute_scale( spatial_idx = np.where(np.array(wcs.output_frame.axes_type) == "SPATIAL")[0] delta[spatial_idx[0]] = 1 - crpix_with_offsets = np.vstack( - (crpix, crpix + delta, crpix + np.roll(delta, 1)) - ).T + crpix_with_offsets = np.vstack((crpix, crpix + delta, crpix + np.roll(delta, 1))).T crval_with_offsets = wcs(*crpix_with_offsets, with_bounding_box=False) coords = SkyCoord( @@ -341,9 +335,7 @@ def compute_fiducial(wcslist, bounding_box=None): axes_types = wcslist[0].output_frame.axes_type spatial_axes = np.array(axes_types) == "SPATIAL" spectral_axes = np.array(axes_types) == "SPECTRAL" - footprints = np.hstack( - [w.footprint(bounding_box=bounding_box).T for w in wcslist] - ) + footprints = np.hstack([w.footprint(bounding_box=bounding_box).T for w in wcslist]) spatial_footprint = footprints[spatial_axes] spectral_footprint = footprints[spectral_axes] @@ -359,9 +351,7 @@ def compute_fiducial(wcslist, bounding_box=None): y_mid = (np.max(y) + np.min(y)) / 2.0 z_mid = (np.max(z) + np.min(z)) / 2.0 lon_fiducial = np.rad2deg(np.arctan2(y_mid, x_mid)) % 360.0 - lat_fiducial = np.rad2deg( - np.arctan2(z_mid, np.sqrt(x_mid**2 + y_mid**2)) - ) + lat_fiducial = np.rad2deg(np.arctan2(z_mid, np.sqrt(x_mid**2 + y_mid**2))) fiducial[spatial_axes] = lon_fiducial, lat_fiducial if spectral_footprint.any(): fiducial[spectral_axes] = spectral_footprint.min() @@ -390,9 +380,7 @@ def add_s_region(model): # footprint is an array of shape (2, 4) - i.e. 4 values for RA and 4 values for # Dec - as we are interested only in the footprint on the sky - footprint = model.meta.wcs.footprint( - bbox, center=True, axis_type="spatial" - ).T + footprint = model.meta.wcs.footprint(bbox, center=True, axis_type="spatial").T # take only imaging footprint footprint = footprint[:2, :] @@ -406,9 +394,7 @@ def add_s_region(model): def update_s_region_keyword(model, footprint): - s_region = ( - "POLYGON ICRS " + " ".join([str(x) for x in footprint.ravel()]) + " " - ) + s_region = "POLYGON ICRS " + " ".join([str(x) for x in footprint.ravel()]) + " " log.info(f"S_REGION VALUES: {s_region}") if "nan" in s_region: # do not update s_region if there are NaNs. diff --git a/romancal/resample/gwcs_drizzle.py b/romancal/resample/gwcs_drizzle.py index 69684c1ec..3a490d2cc 100644 --- a/romancal/resample/gwcs_drizzle.py +++ b/romancal/resample/gwcs_drizzle.py @@ -1,11 +1,10 @@ +import logging + import numpy as np +from drizzle import cdrizzle, util -from drizzle import util -from drizzle import cdrizzle from . import resample_utils -import logging - log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) @@ -102,7 +101,7 @@ def __init__( elif self.outcon.ndim != 3: raise ValueError( "Drizzle context image has wrong dimensions: \ - {0}".format( + {}".format( product ) ) diff --git a/romancal/resample/resample.py b/romancal/resample/resample.py index 4b816a183..ea3c0249a 100644 --- a/romancal/resample/resample.py +++ b/romancal/resample/resample.py @@ -1,17 +1,12 @@ import logging import numpy as np -from drizzle import util -from drizzle import cdrizzle - +from drizzle import cdrizzle, util from roman_datamodels import datamodels - -from ..datamodels import ModelContainer from roman_datamodels.maker_utils import mk_datamodel -from . import gwcs_drizzle -from . import resample_utils -from ..lib.basic_utils import bytes2human +from ..datamodels import ModelContainer +from . import gwcs_drizzle, resample_utils log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) @@ -124,7 +119,7 @@ def __init__( crval=crval, ) - log.debug("Output mosaic size: {}".format(self.output_wcs.array_shape)) + log.debug(f"Output mosaic size: {self.output_wcs.array_shape}") # NOTE: should we enable memory allocation? diff --git a/romancal/resample/resample_step.py b/romancal/resample/resample_step.py index 3fa3fc65d..4f8728439 100755 --- a/romancal/resample/resample_step.py +++ b/romancal/resample/resample_step.py @@ -2,18 +2,16 @@ import re from copy import deepcopy -import numpy as np import asdf -from stpipe.extern.configobj.validate import Validator -from stpipe.extern.configobj.configobj import ConfigObj - +import numpy as np from roman_datamodels import datamodels +from stpipe.extern.configobj.configobj import ConfigObj +from stpipe.extern.configobj.validate import Validator +from ..assign_wcs import utils from ..datamodels import ModelContainer - -from . import resample from ..stpipe import RomanStep -from ..assign_wcs import utils +from . import resample log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) @@ -297,7 +295,7 @@ def get_drizpars(self, ref_filename, input_models): kwargs.update(all_drizpars) for k, v in kwargs.items(): - self.log.debug(" {}={}".format(k, v)) + self.log.debug(f" {k}={v}") return kwargs diff --git a/romancal/resample/resample_utils.py b/romancal/resample/resample_utils.py index 0a2f67dd0..6f8a78c1d 100644 --- a/romancal/resample/resample_utils.py +++ b/romancal/resample/resample_utils.py @@ -1,17 +1,17 @@ import logging -from typing import Tuple import warnings +from typing import Tuple +import gwcs import numpy as np +from astropy import units as u +from astropy import wcs as fitswcs +from astropy.modeling import Model +from astropy.nddata.bitmask import interpret_bit_flags +from stcal.alignment.util import wcs_from_footprints from romancal.assign_wcs.utils import wcs_bbox_from_shape -from stcal.alignment.util import wcs_from_footprints -from astropy.nddata.bitmask import bitfield_to_boolean_mask, interpret_bit_flags -from astropy import units as u from romancal.lib.dqflags import pixel -from astropy.modeling import Model -from astropy import wcs as fitswcs -import gwcs log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) @@ -142,10 +142,10 @@ def calc_gwcs_pixmap(in_wcs, out_wcs, shape=None): """Return a pixel grid map from input frame to output frame.""" if shape: bb = wcs_bbox_from_shape(shape) - log.debug("Bounding box from data shape: {}".format(bb)) + log.debug(f"Bounding box from data shape: {bb}") else: bb = in_wcs.bounding_box - log.debug("Bounding box from WCS: {}".format(in_wcs.bounding_box)) + log.debug(f"Bounding box from WCS: {in_wcs.bounding_box}") grid = gwcs.wcstools.grid_from_bounding_box(bb) pixmap = np.dstack(reproject(in_wcs, out_wcs)(grid[0], grid[1])) From 635d88bec0b6f148d652f2b27167bec8ddb5ed0e Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Tue, 1 Aug 2023 14:58:46 -0400 Subject: [PATCH 05/82] Add drizzle to dependencies list. --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 03004ca86..7204815de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ dependencies = [ 'tweakwcs >=0.8.0', 'spherical-geometry >= 1.2.22', 'stsci.imagestats >= 1.6.3', + 'drizzle >= 1.13.7' ] dynamic = ['version'] From 3996e2a0ea542d824b62845052e419cfc08ea523 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Tue, 1 Aug 2023 15:19:56 -0400 Subject: [PATCH 06/82] Add resample step suffix to list. --- romancal/lib/suffix.py | 1 + 1 file changed, 1 insertion(+) diff --git a/romancal/lib/suffix.py b/romancal/lib/suffix.py index 995e2c5ec..9abaf691d 100644 --- a/romancal/lib/suffix.py +++ b/romancal/lib/suffix.py @@ -90,6 +90,7 @@ "outlierdetectionstep", "skymatchstep", "refpixstep", + "resamplestep", } From e9a7d37693afb9aaaff376bf972387bbbf6f4848 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Tue, 1 Aug 2023 15:20:15 -0400 Subject: [PATCH 07/82] Style check fixes. --- romancal/resample/resample.py | 14 +++++++++++--- romancal/resample/resample_step.py | 5 ++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/romancal/resample/resample.py b/romancal/resample/resample.py index ea3c0249a..abd92e7d3 100644 --- a/romancal/resample/resample.py +++ b/romancal/resample/resample.py @@ -124,7 +124,9 @@ def __init__( # NOTE: should we enable memory allocation? # can_allocate, required_memory = datamodels.util.check_memory_allocation( - # self.output_wcs.array_shape, kwargs['allowed_memory'], datamodels.ImageModel + # self.output_wcs.array_shape, + # kwargs['allowed_memory'], + # datamodels.ImageModel # ) # if not can_allocate: # raise OutputTooLargeError( @@ -229,7 +231,10 @@ def resample_many_to_one(self): # Initialize the output with the wcs driz = gwcs_drizzle.GWCSDrizzle( - output_model, pixfrac=self.pixfrac, kernel=self.kernel, fillval=self.fillval + output_model, + pixfrac=self.pixfrac, + kernel=self.kernel, + fillval=self.fillval, ) log.info("Resampling science data") @@ -327,7 +332,10 @@ def resample_variance_array(self, name, output_model): mask = resampled_variance > 0 inverse_variance_sum[mask] = np.nansum( - [inverse_variance_sum[mask], np.reciprocal(resampled_variance[mask])], + [ + inverse_variance_sum[mask], + np.reciprocal(resampled_variance[mask]), + ], axis=0, ) diff --git a/romancal/resample/resample_step.py b/romancal/resample/resample_step.py index 4f8728439..19b09d2c5 100755 --- a/romancal/resample/resample_step.py +++ b/romancal/resample/resample_step.py @@ -56,7 +56,7 @@ class ResampleStep(RomanStep): blendheaders = boolean(default=True) allowed_memory = float(default=None) # Fraction of memory to use for the combined image. in_memory = boolean(default=True) - """ + """ # noqa: E501 reference_file_types = [] @@ -265,8 +265,7 @@ def get_drizpars(self, ref_filename, input_models): pixfrac=self.pixfrac, kernel=self.kernel, fillval=self.fillval, - wht_type=self.weight_type - # pscale_ratio=self.pixel_scale_ratio, # I think this can be removed JEM (??) + wht_type=self.weight_type, ) # For parameters that are set in drizpars table but not set by the From d0b1167fae3c3eb6b993ea2d3d780f63c2632c05 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Tue, 1 Aug 2023 15:30:57 -0400 Subject: [PATCH 08/82] Use dev stcal-alignment branch. --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 1c805f3d0..d208f9714 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -9,5 +9,5 @@ numpy>=0.0.dev0 git+https://github.com/spacetelescope/roman_datamodels scipy>=0.0.dev0 git+https://github.com/spacetelescope/stpipe -git+https://github.com/spacetelescope/stcal +git+https://github.com/mairanteodoro/stcal.git#egg=stcal-alignment git+https://github.com/spacetelescope/tweakwcs From b2c6b4ae2271489116815c9e17b8d05c7bd84092 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Tue, 1 Aug 2023 16:04:08 -0400 Subject: [PATCH 09/82] Add sphinx-autobuild to docs dependency list. --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 7204815de..a7c15ee34 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ docs = [ 'sphinx-automodapi', 'sphinx-rtd-theme', 'stsci-rtd-theme', + 'sphinx-autobuild', 'tomli; python_version <="3.11"', ] test = [ From 1fd40b770e6b23e28164dcdb40d2115e72ffaeb4 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Tue, 1 Aug 2023 16:36:49 -0400 Subject: [PATCH 10/82] Fix requirements-dev entry for stcal-alignment. --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index d208f9714..874c8aa4d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -9,5 +9,5 @@ numpy>=0.0.dev0 git+https://github.com/spacetelescope/roman_datamodels scipy>=0.0.dev0 git+https://github.com/spacetelescope/stpipe -git+https://github.com/mairanteodoro/stcal.git#egg=stcal-alignment +git+https://github.com/mairanteodoro/stcal.git@stcal-alignment git+https://github.com/spacetelescope/tweakwcs From 4879bce4c2d34df38204dd5761566dd2728ac7b5 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Thu, 3 Aug 2023 16:24:01 -0400 Subject: [PATCH 11/82] Fix issue with invalid data format. --- romancal/resample/gwcs_drizzle.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/romancal/resample/gwcs_drizzle.py b/romancal/resample/gwcs_drizzle.py index 3a490d2cc..6b4368712 100644 --- a/romancal/resample/gwcs_drizzle.py +++ b/romancal/resample/gwcs_drizzle.py @@ -91,7 +91,9 @@ def __init__( else: self.outwcs = product.meta.wcs - self.outwht = None + self.outwht = np.zeros( + (self.outcon.shape[0], self.outcon.shape[1]), dtype=np.float32 + ) self.outcon = product.context if self.outcon.ndim == 2: @@ -437,13 +439,14 @@ def dodrizzle( # Call 'drizzle' to perform image combination log.info(f"Drizzling {insci.shape} --> {outsci.shape}") + breakpoint() _vers, nmiss, nskip = cdrizzle.tdriz( - insci.astype(np.float32), - inwht.astype(np.float32), + insci.astype(np.float32).value, + inwht.astype(np.float32).value, pixmap, - outsci.value, - outwht, - outcon, + outsci.astype(np.float32).value, + outwht.astype(np.float32), + outcon.astype(np.int32), uniqid=uniqid, xmin=xmin, xmax=xmax, @@ -456,4 +459,8 @@ def dodrizzle( wtscale=wt_scl, fillstr=fillval, ) + log.info( + f"Results from cdrizzle.tdriz(): \ + '_vers'={_vers}, 'nmiss'={nmiss}, 'nskip'={nskip}" + ) return _vers, nmiss, nskip From 3195b63baecbd1b8572182330791b93c0baac421 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Thu, 3 Aug 2023 16:24:52 -0400 Subject: [PATCH 12/82] Provide correct context image extension. --- romancal/resample/resample.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/romancal/resample/resample.py b/romancal/resample/resample.py index abd92e7d3..c48e64bc3 100644 --- a/romancal/resample/resample.py +++ b/romancal/resample/resample.py @@ -251,7 +251,7 @@ def resample_many_to_one(self): if not img.meta.background.subtracted and blevel is not None: data = img.data - blevel else: - data = img.data.copy() + data = img.data driz.add_image(data, img.meta.wcs, inwht=inwht) del data, inwht @@ -311,8 +311,8 @@ def resample_variance_array(self, name, output_model): resampled_variance = np.zeros_like(output_model.data) outwht = np.zeros_like(output_model.data) - outcon = np.zeros_like(output_model.con) - + outcon = np.zeros_like(output_model.context) + breakpoint() # Resample the variance array. Fill "unpopulated" pixels with NaNs. self.drizzle_arrays( variance, From e28f62718ddfd651c83ae6eeae2712cc792278c1 Mon Sep 17 00:00:00 2001 From: Nadia Dencheva Date: Thu, 3 Aug 2023 12:02:51 -0400 Subject: [PATCH 13/82] Use CRDS public server (#805) --- README.md | 2 +- docs/roman/pipeline_installation.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a3be94a7f..5083c5807 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ works with default CRDS setup with no modifications. To run the pipeline outside configured by setting two environment variables: export CRDS_PATH=$HOME/crds_cache - export CRDS_SERVER_URL=https://roman-crds-test.stsci.edu + export CRDS_SERVER_URL=https://roman-crds.stsci.edu ## Documentation diff --git a/docs/roman/pipeline_installation.rst b/docs/roman/pipeline_installation.rst index 3b7ec9bdf..43d10a207 100644 --- a/docs/roman/pipeline_installation.rst +++ b/docs/roman/pipeline_installation.rst @@ -118,4 +118,4 @@ configured by setting two environment variables: :: $ export CRDS_PATH=$HOME/crds_cache - $ export CRDS_SERVER_URL=https://roman-crds-test.stsci.edu + $ export CRDS_SERVER_URL=https://roman-crds.stsci.edu From 0f8ce5992aa4360b874af27c00026b9e1f6329fc Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Thu, 3 Aug 2023 16:27:31 -0400 Subject: [PATCH 14/82] Resolve conflict. --- romancal/datamodels/container.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/romancal/datamodels/container.py b/romancal/datamodels/container.py index ba4a9c999..404928f4b 100644 --- a/romancal/datamodels/container.py +++ b/romancal/datamodels/container.py @@ -39,7 +39,8 @@ class ModelContainer(Sequence): ---------- init : path to ASN file, list of either datamodels or path to ASDF files, or `None` If `None`, then an empty `ModelContainer` instance is initialized, to which - DataModels can later be added via the ``append()`` method. + datamodels can later be added via the ``insert()``, ``append()``, + or ``extend()`` method. iscopy : bool Presume this model is a copy. Members will not be closed From 95c7d162def7376f5fdc9c289d6d88c73e6b0832 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Thu, 10 Aug 2023 11:03:21 -0400 Subject: [PATCH 15/82] First functional implementation. --- romancal/resample/gwcs_drizzle.py | 26 ++++++----------- romancal/resample/resample.py | 40 +++++++++++++++----------- romancal/resample/resample_step.py | 45 +++++++++++++----------------- 3 files changed, 51 insertions(+), 60 deletions(-) diff --git a/romancal/resample/gwcs_drizzle.py b/romancal/resample/gwcs_drizzle.py index 6b4368712..a23e0eac7 100644 --- a/romancal/resample/gwcs_drizzle.py +++ b/romancal/resample/gwcs_drizzle.py @@ -69,10 +69,7 @@ def __init__( self.outexptime = 0.0 self.uniqid = 0 - if wt_scl is None: - self.wt_scl = "" - else: - self.wt_scl = wt_scl + self.wt_scl = "" if wt_scl is None else wt_scl self.kernel = kernel self.fillval = fillval self.pixfrac = pixfrac @@ -86,15 +83,9 @@ def __init__( self.outexptime = getattr(product.meta.resample, "product_exposure_time", 0.0) self.outsci = product.data - if outwcs: - self.outwcs = outwcs - else: - self.outwcs = product.meta.wcs - - self.outwht = np.zeros( - (self.outcon.shape[0], self.outcon.shape[1]), dtype=np.float32 - ) - self.outcon = product.context + self.outwcs = outwcs or product.meta.wcs + self.outwht = np.zeros(self.outsci.shape, dtype=np.float32) + self.outcon = np.zeros(self.outcon.shape, dtype=np.int32) if self.outcon.ndim == 2: self.outcon = np.reshape( @@ -115,7 +106,7 @@ def __init__( if out_units == "counts": np.divide(self.outsci, self.outexptime, self.outsci) elif out_units != "cps": - raise ValueError("Illegal value for out_units: %s" % out_units) + raise ValueError(f"Illegal value for out_units: {out_units}") # Since the context array is dynamic, it must be re-assigned # back to the product's `con` attribute. @@ -439,14 +430,13 @@ def dodrizzle( # Call 'drizzle' to perform image combination log.info(f"Drizzling {insci.shape} --> {outsci.shape}") - breakpoint() _vers, nmiss, nskip = cdrizzle.tdriz( insci.astype(np.float32).value, inwht.astype(np.float32).value, pixmap, - outsci.astype(np.float32).value, - outwht.astype(np.float32), - outcon.astype(np.int32), + outsci, + outwht, + outcon, uniqid=uniqid, xmin=xmin, xmax=xmax, diff --git a/romancal/resample/resample.py b/romancal/resample/resample.py index c48e64bc3..2963eb28a 100644 --- a/romancal/resample/resample.py +++ b/romancal/resample/resample.py @@ -4,6 +4,7 @@ from drizzle import cdrizzle, util from roman_datamodels import datamodels from roman_datamodels.maker_utils import mk_datamodel +from astropy import units as u from ..datamodels import ModelContainer from . import gwcs_drizzle, resample_utils @@ -260,15 +261,20 @@ def resample_many_to_one(self): self.resample_variance_array("var_rnoise", output_model) self.resample_variance_array("var_poisson", output_model) self.resample_variance_array("var_flat", output_model) - output_model.err = np.sqrt( - np.nansum( - [ - output_model.var_rnoise, - output_model.var_poisson, - output_model.var_flat, - ], - axis=0, - ) + + # TODO: fix unit here + output_model.err = u.Quantity( + np.sqrt( + np.nansum( + [ + output_model.var_rnoise, + output_model.var_poisson, + output_model.var_flat, + ], + axis=0, + ) + ), + unit=output_model.err.unit, ) self.update_exposure_times(output_model) @@ -285,7 +291,7 @@ def resample_variance_array(self, name, output_model): This modifies output_model in-place. """ output_wcs = output_model.meta.wcs - inverse_variance_sum = np.full_like(output_model.data, np.nan) + inverse_variance_sum = np.full_like(output_model.data.value, np.nan) log.info(f"Resampling {name}") for model in self.input_models: @@ -312,7 +318,6 @@ def resample_variance_array(self, name, output_model): resampled_variance = np.zeros_like(output_model.data) outwht = np.zeros_like(output_model.data) outcon = np.zeros_like(output_model.context) - breakpoint() # Resample the variance array. Fill "unpopulated" pixels with NaNs. self.drizzle_arrays( variance, @@ -341,7 +346,10 @@ def resample_variance_array(self, name, output_model): # We now have a sum of the inverse resampled variances. We need the # inverse of that to get back to units of variance. - output_variance = np.reciprocal(inverse_variance_sum) + # TODO: fix unit here + output_variance = u.Quantity( + np.reciprocal(inverse_variance_sum), unit=u.electron**2 / u.s**2 + ) setattr(output_model, name, output_variance) @@ -358,7 +366,7 @@ def update_exposure_times(self, output_model): output_model.meta.exposure.exposure_time = total_exposure_time output_model.meta.exposure.start_time = min(exposure_times["start"]) output_model.meta.exposure.end_time = max(exposure_times["end"]) - output_model.meta.resample.product_exposure_time = total_exposure_time + output_model.meta.resample["product_exposure_time"] = total_exposure_time @staticmethod def drizzle_arrays( @@ -528,11 +536,11 @@ def drizzle_arrays( log.info(f"Drizzling {insci.shape} --> {outsci.shape}") _vers, _nmiss, _nskip = cdrizzle.tdriz( - insci, + insci.astype(np.float32).value, inwht, pixmap, - outsci, - outwht, + outsci.value, + outwht.value, outcon, uniqid=uniqid, xmin=xmin, diff --git a/romancal/resample/resample_step.py b/romancal/resample/resample_step.py index 19b09d2c5..7dd106923 100755 --- a/romancal/resample/resample_step.py +++ b/romancal/resample/resample_step.py @@ -8,7 +8,7 @@ from stpipe.extern.configobj.configobj import ConfigObj from stpipe.extern.configobj.validate import Validator -from ..assign_wcs import utils +from stcal.alignment import util from ..datamodels import ModelContainer from ..stpipe import RomanStep from . import resample @@ -139,21 +139,26 @@ def process(self, input): return result def _final_updates(self, model, input_models, kwargs): - model.meta.cal_step.resample = "COMPLETE" - self.update_fits_wcs(model) - utils.update_s_region_imaging(model) - model.meta.asn.pool_name = input_models.asn_pool_name - model.meta.asn.table_name = input_models.asn_table_name + model.meta.cal_step["resample"] = "COMPLETE" + self.update_wcs(model) + util.update_s_region_imaging(model) + if ( + input_models.asn_pool_name is not None + and input_models.asn_table_name is not None + ): + # update ASN attributes + model.meta.asn.pool_name = input_models.asn_pool_name + model.meta.asn.table_name = input_models.asn_table_name # if pixel_scale exists, it will override pixel_scale_ratio. # calculate the actual value of pixel_scale_ratio based on pixel_scale # because source_catalog uses this value from the header. - model.meta.resample.pixel_scale_ratio = ( + model.meta.resample["pixel_scale_ratio"] = ( self.pixel_scale / np.sqrt(model.meta.photometry.pixelarea_arcsecsq) if self.pixel_scale else self.pixel_scale_ratio ) - model.meta.resample.pixfrac = kwargs["pixfrac"] + model.meta.resample["pixfrac"] = kwargs["pixfrac"] self.update_phot_keywords(model) @staticmethod @@ -341,35 +346,23 @@ def _set_spec_defaults(self): return kwargs - def update_fits_wcs(self, model): + def update_wcs(self, model): """ - Update FITS WCS keywords of the resampled image. + Update WCS keywords of the resampled image. """ # Delete any SIP-related keywords first pattern = r"^(cd[12]_[12]|[ab]p?_\d_\d|[ab]p?_order)$" regex = re.compile(pattern) - keys = list(model.meta.wcsinfo.instance.keys()) + keys = list(model._instance.meta.wcsinfo.keys()) for key in keys: if regex.match(key): - del model.meta.wcsinfo.instance[key] + del model._instance.meta.wcsinfo[key] # Write new PC-matrix-based WCS based on GWCS model transform = model.meta.wcs.forward_transform - model.meta.wcsinfo.crpix1 = -transform[0].offset.value + 1 - model.meta.wcsinfo.crpix2 = -transform[1].offset.value + 1 - model.meta.wcsinfo.cdelt1 = transform[3].factor.value - model.meta.wcsinfo.cdelt2 = transform[4].factor.value model.meta.wcsinfo.ra_ref = transform[6].lon.value model.meta.wcsinfo.dec_ref = transform[6].lat.value - model.meta.wcsinfo.crval1 = model.meta.wcsinfo.ra_ref - model.meta.wcsinfo.crval2 = model.meta.wcsinfo.dec_ref - model.meta.wcsinfo.pc1_1 = transform[2].matrix.value[0][0] - model.meta.wcsinfo.pc1_2 = transform[2].matrix.value[0][1] - model.meta.wcsinfo.pc2_1 = transform[2].matrix.value[1][0] - model.meta.wcsinfo.pc2_2 = transform[2].matrix.value[1][1] - model.meta.wcsinfo.ctype1 = "RA---TAN" - model.meta.wcsinfo.ctype2 = "DEC--TAN" # Remove no longer relevant WCS keywords rm_keys = [ @@ -382,5 +375,5 @@ def update_fits_wcs(self, model): "vparity", ] for key in rm_keys: - if key in model.meta.wcsinfo.instance: - del model.meta.wcsinfo.instance[key] + if key in model._instance.meta.wcsinfo: + del model._instance.meta.wcsinfo[key] From 220e4ea617f67d944467ac45d89957929687316e Mon Sep 17 00:00:00 2001 From: D Davis <49163225+ddavis-stsci@users.noreply.github.com> Date: Mon, 7 Aug 2023 11:37:34 -0400 Subject: [PATCH 16/82] Rcal 596 Add association processing to the pipeline code (#802) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- CHANGES.rst | 2 + requirements-dev.txt | 3 +- romancal/associations/load_as_asn.py | 207 ++++++++++++++++++ romancal/datamodels/filetype.py | 56 +++++ romancal/datamodels/tests/data/empty.asdf | 0 romancal/datamodels/tests/data/empty.json | 0 .../datamodels/tests/data/example_schema.json | 43 ++++ romancal/datamodels/tests/data/fake.asdf | 1 + romancal/datamodels/tests/data/fake.json | 1 + romancal/datamodels/tests/data/pluto.asdf | Bin 0 -> 805623 bytes romancal/datamodels/tests/test_filetype.py | 41 ++++ romancal/pipeline/exposure_pipeline.py | 154 ++++++++----- 12 files changed, 449 insertions(+), 59 deletions(-) create mode 100644 romancal/associations/load_as_asn.py create mode 100644 romancal/datamodels/filetype.py create mode 100644 romancal/datamodels/tests/data/empty.asdf create mode 100644 romancal/datamodels/tests/data/empty.json create mode 100755 romancal/datamodels/tests/data/example_schema.json create mode 100644 romancal/datamodels/tests/data/fake.asdf create mode 100644 romancal/datamodels/tests/data/fake.json create mode 100644 romancal/datamodels/tests/data/pluto.asdf create mode 100644 romancal/datamodels/tests/test_filetype.py diff --git a/CHANGES.rst b/CHANGES.rst index 99494f781..060e5daeb 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -45,6 +45,8 @@ refpix general ------- +- Update the pipeline code to process all the uncal files in an association [#802] + - `ModelContainer` supports slice and dice. [#710] - Add `ModelContainer` to `romancal.datamodels`. [#710] diff --git a/requirements-dev.txt b/requirements-dev.txt index 874c8aa4d..2c9922eb6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,7 +6,8 @@ git+https://github.com/astropy/asdf-astropy git+https://github.com/spacetelescope/crds git+https://github.com/spacetelescope/gwcs numpy>=0.0.dev0 -git+https://github.com/spacetelescope/roman_datamodels +git+https://github.com/spacetelescope/roman_datamodels@0.17.0 +git+https://github.com/spacetelescope/rad@0.17.0 scipy>=0.0.dev0 git+https://github.com/spacetelescope/stpipe git+https://github.com/mairanteodoro/stcal.git@stcal-alignment diff --git a/romancal/associations/load_as_asn.py b/romancal/associations/load_as_asn.py new file mode 100644 index 000000000..9e80e62ab --- /dev/null +++ b/romancal/associations/load_as_asn.py @@ -0,0 +1,207 @@ +"""Treat various objects as Associations""" + +from functools import partial +from os import path as os_path + +from ..associations import Association, AssociationRegistry, libpath, load_asn +from ..associations.asn_from_list import asn_from_list +from ..associations.lib.rules_elpp_base import DMS_ELPP_Base +from ..associations.lib.rules_level2 import Asn_Lv2Image + +__all__ = [ + "LoadAsAssociation", + "LoadAsLevel2Asn", +] + + +DEFAULT_NAME = "singleton" +DEFAULT_ASN_META = { + "program": DEFAULT_NAME, + "target": DEFAULT_NAME, + "asn_pool": DEFAULT_NAME, +} + + +class LoadAsAssociation(dict): + """Read in or create an association + + Parameters + ---------- + asn : dict or Association + An already existing association + + Notes + ----- + This class is normally not instantiated. + the `load` method should be used as the factory + method to read an association or create one from + a string or `Datamodel` object, or a list of such + objects. + """ + + @classmethod + def load( + cls, + obj, + meta=DEFAULT_ASN_META, + registry=AssociationRegistry, + rule=Association, + product_name_func=None, + ): + """Load object and return an association of it + + Parameters + ---------- + obj : Association, str, Datamodel, [str[,...]], [Datamodel[,...]] + The obj to return as an association + + registry : AssociationRegistry + The registry to use to load an association file with + + rule : Association + The rule to use if an association needs to be created + + product_name_func : func + A function, when given the argument of `obj`, or + if `obj` is a list, each item in `obj`, returns + a string that will be used as the product name in + the association. + + Attributes + ---------- + Along with the attributes belonging to an association, the + following are added: + + filename : str + The name of the association file, if such a file + where passed in. Otherwise a default value is given. + + Returns + ------- + association : Association + An association created using given obj + """ + try: + with open(obj) as fp: + pure_asn = load_asn(fp, registry=registry) + except Exception: + if not isinstance(obj, list): + obj = [obj] + asn = asn_from_list( + obj, + rule=rule, + meta=DEFAULT_ASN_META, + product_name_func=product_name_func, + ) + asn.filename = DEFAULT_NAME + else: + asn = rule() + asn.update(pure_asn) + asn.filename = obj + + return asn + + +class LoadAsLevel2Asn(LoadAsAssociation): + """Read in or create a Level2 association""" + + @classmethod + def load(cls, obj, basename=None): + """Open object and return a Level2 association of it + + Parameters + ---------- + obj : Association, str, Datamodel, [str[,...]], [Datamodel[,...]] + The obj to return as an association + + basename : str + If specified, use as the basename, with an index appended. + + Attributes + ---------- + Along with the attributes belonging to a Level2 association, the + following are added: + + filename : str + The name of the association file, if such a file + where passed in. Otherwise a default value is given. + + Returns + ------- + association : DMSLevel2bBase + An association created using given obj + """ + product_name_func = cls.model_product_name + if basename is not None: + product_name_func = partial(cls.name_with_index, basename) + + # if the input string is a FITS file create an asn and return + if isinstance(obj, str): + file_name, file_ext = os_path.splitext(obj) + + if file_ext == ".fits": + items = [(obj, "science")] + asn = asn_from_list( + items, + product_name=file_name, + rule=DMS_ELPP_Base, + with_exptype=True, + meta={"asn_pool": "singleton"}, + ) + return asn + + asn = super().load( + obj, + registry=AssociationRegistry( + definition_files=[libpath("rules_level2.py")], include_default=False + ), + rule=Asn_Lv2Image, + product_name_func=product_name_func, + ) + return asn + + @staticmethod + def model_product_name(model, *args, **kwargs): + """Product a model product name based on the model. + + Parameters + ---------- + model : DataModel + The model to get the name from + + Returns + ------- + product_name : str + The basename of filename from the model + """ + product_name, ext = os_path.splitext(model.meta.filename) + return product_name + + @staticmethod + def name_with_index(basename, obj, idx, *args, **kwargs): + """Produce a name with the basename and index appended + + Parameters + ---------- + basename : str + The base of the file name + + obj : object + The object being added to the association _(unused)_ + + idx : int + The current index of the added item. + + Returns + ------- + product_name : str + The concatenation of basename, '_', idx + + Notes + ----- + If the index is less than or equal to 1, no appending is done. + """ + basename, extension = os_path.splitext(os_path.basename(basename)) + if idx > 1: + basename = basename + "_" + str(idx) + return basename diff --git a/romancal/datamodels/filetype.py b/romancal/datamodels/filetype.py new file mode 100644 index 000000000..9cc8a4f62 --- /dev/null +++ b/romancal/datamodels/filetype.py @@ -0,0 +1,56 @@ +import io +import os +from pathlib import Path +from typing import Union + + +def check(init: Union[os.PathLike, Path, io.FileIO]) -> str: + """ + Determine the type of a file and return it as a string + + Parameters + ---------- + + init : str + file path or file object + + Returns + ------- + file_type: str + a string with the file type ("asdf" or "asn") + + """ + + supported = ("asdf", "json") + + if isinstance(init, (str, os.PathLike, Path)): + path, ext = os.path.splitext(init) + ext = ext.strip(".") + + if not ext: + raise ValueError(f"Input file path does not have an extension: {init}") + + if ext not in supported: # Could be the file is zipped; try splitting again + path, ext = os.path.splitext(path) + ext = ext.strip(".") + + if ext not in supported: + raise ValueError(f"Unrecognized file type for: {init}") + + if ext == "json": # Assume json input is an association + return "asn" + + return ext + elif hasattr(init, "read") and hasattr(init, "seek"): + magic = init.read(5) + init.seek(0, 0) + + if not magic or len(magic) < 5: + raise ValueError(f"Cannot get file type of {str(init)}") + + if magic == b"#ASDF": + return "asdf" + + return "asn" + else: + raise ValueError(f"Cannot get file type of {str(init)}") diff --git a/romancal/datamodels/tests/data/empty.asdf b/romancal/datamodels/tests/data/empty.asdf new file mode 100644 index 000000000..e69de29bb diff --git a/romancal/datamodels/tests/data/empty.json b/romancal/datamodels/tests/data/empty.json new file mode 100644 index 000000000..e69de29bb diff --git a/romancal/datamodels/tests/data/example_schema.json b/romancal/datamodels/tests/data/example_schema.json new file mode 100755 index 000000000..febac841c --- /dev/null +++ b/romancal/datamodels/tests/data/example_schema.json @@ -0,0 +1,43 @@ +{ + "date" : { + "title" : "[yyyy-mm-ddThh:mm:ss.ssssss] UTC date file created", + "type" : "string", + "sql_dtype" : "datetime2", + "fits_keyword" : "DATE", + "description" : "The UTC date and time when the HDU was created, in the form YYYY-MM-DDThh:mm:ss.ssssss, where YYYY shall be the four-digit calendar year number, MM the two-digit month number with January given by 01 and December by 12, and DD the two-digit day of the month. The literal T shall separate the date and time, hh shall be the two-digit hour in the day, mm the two-digit number of minutes after the hour, and ss.ssssss the number of seconds (two digits followed by a fraction accurate to microseconds) after the minute. Default values must not be given to any portion of the date/time string, and leading zeros must not be omitted.", + "calculation" : "Operating system time in the format of YYYY-MM-DDThh:mm:ss.ssssss", + "default_value" : "", + "example" : "2015-01-01T00:00:00.000001", + "units" : "", + "sw_source" : "calculation", + "source" : "Science Data Processing (SDP)", + "destination" : ["ScienceCommon.date","GuideStar.date"], + "level" : "1a", + "si" : "Multiple", + "section" : "Basic", + "mode" : "All", + "fits_hdu" : "PRIMARY", + "misc" : "" + }, + + "origin" : { + "title" : "institution responsible for creating FITS file", + "type" : "string", + "sql_dtype" : "nvarchar(20)", + "fits_keyword" : "ORIGIN", + "description" : "Identifies the organization or institution responsible for creating the FITS file.", + "calculation" : "", + "default_value" : "STSCI", + "example" : "STSCI", + "units" : "", + "sw_source" : "", + "source" : "Science Data Processing (SDP)", + "destination" : ["ScienceCommon.origin","GuideStar.origin"], + "level" : "1a", + "si" : "Multiple", + "section" : "Basic", + "mode" : "All", + "fits_hdu" : "PRIMARY", + "misc" : "" + } +} diff --git a/romancal/datamodels/tests/data/fake.asdf b/romancal/datamodels/tests/data/fake.asdf new file mode 100644 index 000000000..7cd6280d8 --- /dev/null +++ b/romancal/datamodels/tests/data/fake.asdf @@ -0,0 +1 @@ +not actually an ASDF file diff --git a/romancal/datamodels/tests/data/fake.json b/romancal/datamodels/tests/data/fake.json new file mode 100644 index 000000000..ed4e52fca --- /dev/null +++ b/romancal/datamodels/tests/data/fake.json @@ -0,0 +1 @@ +not actually a JSON file diff --git a/romancal/datamodels/tests/data/pluto.asdf b/romancal/datamodels/tests/data/pluto.asdf new file mode 100644 index 0000000000000000000000000000000000000000..c21b4cc3da1db72283f2cfa3cfe2aaf578e7f834 GIT binary patch literal 805623 zcmeIbf6Q*xb?3()QJhjoa@4wwQc~TB_=my|gY9%&i}yKkhNKzAaor};)J-bq#>R#g z`%<_}i^q{3rb)P{Hojn>kX)k6fZ-A@BS;{FC9t3ZPcp>`D2{zeQVgRR30x15)>weC z)U(cEzwGC8p8KwM-}7sq^L!s^-*eY{&e=cK{_eHDXYGCVIrRg-dhRdoIs44B&OGbn zQ|m35y7$uk=kL4t{d<1x{PXryH97aZe{$Z1`z|`~;!957bN;?h zop;gRPoDSwJ?}Vw|Ne{K|E_m^@`C;6U;0~Ter(^T>ViJ8@8VDG-CzCx9jBkXXV0hi zeyY6M3O;?$KPmILVBe?Tzvta&p8f7K&-(3?&%fZ3{rk$Ql-Hg2zwSTp({}a}lS19M z|N7BSowtAQ$M^2vTUWEZB)a9JAG>hxC70BtoVk6=nLmH&1sCqW;M2bla;|O)%{^Z8 zcdTzm75nI?_I~<;Pn>tj{xg5&_5ZBaak~j0z4YP>>SmRiuKKZ7@0t59{^Yx=&fUKB zl1eVMV+b+r?7pFA?7d|F#rrP0Eb5@M&Mdp=w@?193ohP&{>S&0SULN@e9u{Dob|pl z&i)4{f2z{;PhI-y3--V3f4+3@r}tm5|FRlid;Y2YvD9pJ&i;8FRR7Z_%e?nqcxj1* zvrCZu_Q|!DKYZb(`}duE(FGsdf9b{LsT!`I{`lUDFWzf8+5oV2Tzc`x%Cq)=1Hk^v zE-Ix@T)1!V{`b7Q{M&C`w*S0+7k~V`60#Rwuz&xB=h;2y@3o75{XOq_&*^*4{@Jrn z-}B!8;p8*VJoDs}R(^l=^B?@>lg?WC)0=+St*@Q=ReW~c^*rOXh;$MDp zsJk7l{w04|gkc z>@Ph3{34`UFEPUq--@*{Mg4HTag(vHw#)Vl1HynXAPkI!0rIb~mdK8Eonpoz{uOIu ziuzH!OSWb#TcdJ?Jz+o?5C-1B0N0l{<0CQSu>0blKo}4PI6uXB7~lVk*;kF9 z_%Ei<_&2^x{`1Xmd}9%k)t~!Uv>g*e&*yT#jkfjI{jv$dfG{8o2m`{v@E9N;V|*-p zE69%4Kiq#R?CoGjJJ0$4oOPwT zX)Fr^!hkR!3-! z{3_1P^uYCNIqODs)7TXTgaKhd7!U@8fqpO$GbGb-{A(}s{egbkAZ$pl!hkR!3KkD|7!U@80b!sU4BUFlsYOV&PP>h7alZ_$pYNvM zgn#K#7!U@80bxKGm<|SpUeCq-HH4MvU`2hUz7ht60bxKG5C%>R1B^F?y$KZ= zm)=)Av3?T1r8i+f7!U@80byW17zp1&v}4_`-SVwwJZ(O@(>PFH3IoD`Fdz&F1H!;= zFwpn@T-qN(INS{mb&YT$32C_!b6)0bxKG5C)nU z;P)eAwpHsZ{)^oo|9qMIL#Pj$eIT7kC&GX*APfit!oa*S!1eS0{yP^gLaO^djbFuA z+x`FYPyciglCU&yEUAChzruhpAPfit!oYeA@cY(Xub*%3ue!b->L2Mv7!U@80bxKG z5C-Oo0q+0a{2osnu`(RQKVNS0H(_b6SW>%$ZDBwd5C((+VPJO{2z$@lvDR4{zlyK6 z`?=pd`JMV@cl{zgNl(InFdz&F1H!<3Fwph;isW0bzH(U+l5jI0+^8?rm%@NBAPfit z!ocn^(DnO@VeP#gtsl5PUwyOtzLBn^D`7ww5C((+VPGB@X!4OAg<3hR?2801&Ko}4PrilUWfBNLHV_PBBeHt??Yh(Od7JR=k>;+**VPcw? zP`|0)gaKhd7!U@8fo?HCe#CfH%&uzP$A4+^t>%8J-PTwAFMSFF!hkR!3l`|p0kzB##N@}rMD zviRaP*DRiS`Zp{2mVf@w7k~faMce0}d-mC_JB}RLdibHAJLg2oW4x`-Wm3|2<`TNX`z)=fQf>CbXMSKC9sO1sg=bI+b%#s^Pd zT)sT1;!n4I>21X;FTcF7JvQwAW8eJ8#jUrTTHJWUk5_!QK`k9_ji7%lF7&7-dI;Zejk(kF&90n9`hPE>QD8jFdz&F13P2j zz!h&QLaKEYdvv=u{w)job5FmozH(U+l3mO7M`5kF9ql~hGh9FJm6v*+<;AV7^NRmc zxBtom2e#u=U;nFLHLg^B=g6+N_3=mVJgqn$pCZ4aOg=`wX8MoIk88KCzSM83esulL z<45|DeuM#GKo}4Pc7uTs%dum9eeA(ahw;z$-+F8jl3o9IfBWm($>b0#v!lseUw`$% zgIgi(raZ1U4`tiY?!WKe_iiVXz4s+~p7o1cYxyz0M1DnnME=xPX8w$SvA&c8m$qZA zZ;cD-K^PDQgaKhd7#JD@7MlsFE<5(v)*tb&`Mv7+>~8Mo^-nzh;gWy7rJQ@SU$eaW z@8MtnJm)(ZA0s~tIoHQ!6_@(^7hl}E`S9Ut{S^5P`A=Ium3lwpXPnPud-%TN(D+h$ zvT4GAFdz&F1H!qto0FlY@gr6SKIv-*DAS+&3Ua|&t&sk*I)P6;>rU* zQpKk@U&i&b6Y`BB%`|GR(t@#0_Hb;kC5CH2An2yL}v-Cm7*=|LC}2801&Ko}Sr z1Ki(<{+cr9*Z$;>-(6hsdp}%QOvvg^e!+N9s!YB^nf#6K|FQm*xgLu3&rPQO{_v)s zEWZ)(oy8N6KVIpd?X!LfX=rSzJlQj0Ko}4Pgn>=PK$ttUW37YOLmMvQ-_m&0;=X(D zUEFxX4V6D<{E7UB{DkLuKjTSd9H#izQ%_mEsw$5%`JJsFyZyGem-~X9QpIn`M?C(; z{TImJm>=6Ww9NWYe{t_w{|2sQ$4$k&+AVu83|>&!}>6uRo5f@+kF3!^POJ% z`#w*;MgAFDQXkjtZ?7@e_n1@u!k9213x=(|Mr$agGW7LwtCeDK8=_7x%7d2fGC zwwL>7alZ)Ghy3KK&;Mxg*I)m78DqPD?fNIKSEGLV=2y(0<&)pCz2vJJll_lL=~y}z z2801&V15|z*1MPAp{dqVU%p&c@9%N_6!{e6Rh(a>KWF*ee_?FQk$-Xjgs(mJ!Qu~o z|0nF5h*f-z@i(@Ye2x5v{HSZ0QnWpFdz)f6$6at(7#gV zd>Yqdk#873v2Xs>`78_jmY>Z>20qeU50*F9xSu`S!}yRF?;`&s|LK}vF+cKSu8(8? zH}R5pf6aB=s$IgJFdz&F19@X0d<(~p)>qB>q1eMVzlTgd!1&hj<^#j~h~z6%#|!tr zXZyLni}69O@8^6h`BB$0`7+-hjnQCzrTyje9^X?RPpZH0B@74ybHPBE3$kO~rr4t{ z3vWLI`g^WNGCoo64N>kPdQLT;HaC7n95mx|tNLWTkNdaAS6kk^p8D=uCf~AHWLd_d z{O-}(^=GVa+21yIS|zK$w_c9=lPudR3+XvB`og6p9u$5-3+W3T5T-zGow_8)8VPuY>3@S}6W zhcF-v2m|xLKx#aT^I0B0Xy!i;z3mqNjOW~R=+Gi0%Y*ZCj9-y&aeW;5Zrk`4^W%Ic z>qGwM)t}!3xAkEm?N+C{ZnMUa`dj@i3e9YDxDwCedMxh$ z$@lZ7sG~mX%=Jy2pQC;mPvUxY<0{5!tsku3UGpvSX})jC`f~n}d@@Ft)tBqr`qaJh zmdzFhgaKhd7#Ioz+>eU>ugRZQ_b;IT+_d#rZTpGwzB`V*^Yw2DSx+?AXL0>o-|;Q- zHTF08H{)TvpYbu`i};)SdcL;!(f#rj!hkR!4CILczHiu!KgAx;zIXFlxt9Hba!vc; zYJC>{Wqh^udGq^-dFmk-e17G@ts*3=6Ze~^zR7pE|EtG$$bUSU^Tk|`#dwyNANP~+ z_P-!L8IR4SA5{;Hb74Rj5C(*S7z2zKl>U1wr24v;K`jgVbMBAD@3n-z+U*#Zo9q9% zUOT)yXGg0u-yi6@9gHuMZ&CkTAC<~?JU+zp+<%tqv3Px|%>2A~m-h3LJ?nZ5DnDUL z7!U^fz(6zpw92m-zo5U5ueSQ#ddsQXsgL@0ohSJw+e4Z2eO=dw{DFLn{Hp8tRVqK~ z`+COj*dO8BOm^(LFO;8bt}q}B2m`{viDH2Bhu;1UTn|nESMs$j+l$oJZCjoCj{lr! zyTg5Kzqem2{d;(K(2gO`E|014t-0|nw#Vc5Ar@;#>xb|zqaC~K7v&|JD+~w&!hkT4 zf&s=)u0D8hD+%yKo}4Pgn>2~ARlPPXM(S?WApnv z=C?|JUp({l(^dQ+Tpji&ZSt4A`-|VR4(+LrR`>Yl%eI~=q&EE&?&W+e`7PHsaesxe z`A};4 zZ-o4e{Oi$2&RvwTR@+DPUH0?8i1ruhOe9APls@K$y$2qviADv13~y znaub|?2YWatrx%Yz=1{SN2~e5yf{LL$m{YCymzU0Zg zpYwO*JI1rW_0&_Z@4>n($Opap^E~lp`Z#*$Y1{EI*57b``)zM8=M7J()~|Wxb3czZ zV@>zURtp2dfH1JR7~uYo^gsMwF#R3lBitXH`@`oxf5`p5!n*)=tj9^L0mDuFOPAa7 z&n6#Otq=Q?{D*vmGWir`#xogT;(GnKcBapgf33{VGX6>Z@O@Z54=`TF_K*)Ue#ZK- zJmN#Pb8~H{`d)o63Ec2=Kftf>xb9# z`^?5~pLqQ7@}0zGH9yApL&=vYGyX$9MgHy0H?w}^Q_c9?&g~C-=i9NydE5i_`-SoQ z8EhZpU&MQyo8@25KdyZLQ|q~Q?w_4(kIqSV!hkTaNf_|nucLqEdLQ~%`tw&_e!1LV z{-W*u2k2jO*EQpBea|PR`fua2A=SE!8N#yQ`X|P7$ftO{$FJkktbFn*?tj~NJm`)i zN2>X6xs#)xc-O! zdT#!S`FXMXixVF1#5BWOv!}|9v zvp!t^&iO_1Ek0j(`Y?VLQr~S;{wHShq&MkJ7!U@=z(6y;6nm8B{tcf0;rcA@A0Ov# z?M0kO6284mWFu-_2)4#_arCOg-{K4=4L=om=7$1qRwmNhEh3nGL?p*Sv z{}1}8kIRDbnACc6ev|9DL+s6tmN(;RtWT<(8{hEOa}m#6zm>|ja^st+_|f(9J;H!6 zAPjVa0dM|^>#?}L$BS2SzLE1)Twlz1(A4{x?V-N;z9;!iDqmrIkovXtSRt*`ZIj={ zH8RY_KVQymKdPbm7WwQto$GJi*fQx+dK3nPft)eWeEwebr{qT--{Sf$uD9m;WbPlI zv#xeiH_qpgzof>Gn*3nr@x=WeQs4Jve6h**R`C%_wSMBi)ZbqWYsPn@zrL?&>T|We zUB|b?w|2&B-~FxprGH^S7#Ir!jJJe$*X&r=Y1RMmdnX>>;(BbZf8qINd?#;gFn+}E z;l$cor%&&E$#2W|HS1*evpnLa`95DUqU8h;rZ ztIhhZ>Mxe+y2gJooW;MU%=J}Xd%6BAa5X#Hz2w(ke%^V;zsO&>{*Tvt`S*RE@g{!X zhx@w{zr=gon|0mezf}9Pu6*yR@AfJG&9UX`YxT7-kT(Xnekk@1)_3$jeE*RChyID{ zi8-HW{@C_lD)*SI_yFhIa>q||{xjBKlRw30>$b&zXfH2S%2IxQOAIOKur|6F; zlaFyfBkt#vJKy4bT{Hd^YtV2S|I+2=`l40rb3QLF%`g(y!rReg#xvRf-h3eQ_hhgB zUE%md^Rf48%Qb1`H>%I6=FiJG;Nre}?_Jz@!wnVxCw|Gd7{8smK2B7Rx^Hz{N*~gP zFdz&N15JJtdyMAyfXSE0uRNLeGoH|_PhNdSzS1>6VSe6vE$XYu|5o+acD;Lyjv7nx zUzP>SCqE_rhL)T9T&+LfuO+{t?Cqyt^4XQ|JCgqn-F}s~Iv%7C=|dP028O~wbG~QQ z|8PHe@+FU7@jT}f$+w#I$qNhg*Zdw$ti8GqxPC1+{&_q->F=xK zM*5IGgaKioEe4wJ1I8YsUVn4*;lqn74;-lc3HcKFRH|(6LB90e{pDPa{gwsS)3j}0 z*Y`6%!T3epj_|MHGyeHEaIe*Cc zMo(W{A9uWZ>hJOi!hkR!49p7yoR9MM+p>5-9N!{;N|m`^7uP3q{SxC7W8<-FzQuS8 z+sFPQzw_qrSYMAHc;`7k#`>}TW6RBWVcbH0{*ZX(c!+bW+Zz96Sv1R^mpvJ~e^tKl zBn$`xxnh9+aX0Vt_sO>yUkKhR*Y77XUP68t>R%si9K^q-+}w|2b?h|Hujbd}Q)&0d zxz(7B|I++^V)K2-)$+#1J^4B3PdR@`KE>w;)9>xKy}h{U&?(jXi`*Y2*48@xG2S?~ zepG&5cvXL>KZJq#Vjz`o@%=0MANnUx=KPGee-YQGkBv?0kN({Azg*AbwTJs>HRl6& z#%hxv#Wl2h<6l#r+I*PTUd{({{*W@)YsK2C`!4>=vgBol79n|ktMfZ!SNf5DgaKh- z${67Dwl|;HHQ%B?^Y-U%`p>-Z$MsaP_cfewJv;XgC6*YUVgEGC+gTr8e2n!aA7p)4 z-`wQDb?j($tqo>{Clkvmd-WRT8JTEmq7Vcd6^=`F`LObevL0#v3_5nUdEWuH}5;6OVs* ztA2OYaJnoQk0PFkH{zA+2TOc!)!$C9^N&YEtv+Nx5CSURPC-v~f^W6Wz_-y&@ghh$NU9E>F9x2ZqpE+Peb&#(V284kr zVxalHTbKKw^v(G%myT()emnQz>v+J%;eV<`B z7}hoNEy92>APh_&1N3*EKcW9KzG9rf_JK4!wDpJdk1WrN|M2}g`fsj>3hfR1s~%6T z|DNuDLw?@`fJnTk$vMu`A^@+s1MbL!a$7y?iWBlLx02d#N0bxKGScies_4A(pBflcw zAiv;wix zAIoQZ$rpK^`H|0(|1#d@@fYJ-FMW4^$+hg)*vYVKk|JgR)#rH2{r)8&w0bxKG$P)v! z|NOp<`Lml3?XndcwxgIMn!E8qSYs%K%kYr=?w6W?J^@}ht3hfXs{89-=zr*+a+g_u`d|9b*xS{875}B_|KhW{ zzP)}ppGy8TG~S3G@^h{~Bpw*g^4{0w{ggRh+18%ue)&*gKp2=R2DrY0_Lh8!?d1GJ zm^-jzeQj-lkGH=P=ld9+;`|K#$oFGprFnnue2aJ{{>b-<$JG5Z7$57J-*EjI zpC1`t3%1RU)}LL+^J|RAA4w0wfH2@=z~)0jGJTbNtDMiPlgVN2haGEuuKFLopT_wx z+IQOjuH~W2ORYcaSL$_f-@W%PZoJ`!wfc>@w_bf z{E&)=+^)Chs(61e@=0%hjH?eGEWfW^C)0OZeiLky9W6gT?{U4S_q zgn_mg;PWBZt9$;5>xt+u$R~Kdt<3!Se9rk2v(vZV_V(hYL#I^Fw{6RBdq3+>f9C05 z=VJ}@&#%ZY#(th?i@((SiC6MV?r#zIPHn5=zbp&xFU_4w3pW^pr$Om~n^Yh9d z`h45@oVfSa^{39q8dw%>@jNs>ax0JHiR%Zrz981V;gkEXay=>en0(ph`ZD#s`aX9I zq{go-9v|;_$@vAIr~S0|599qF`TWc8v2Z;v{RzwW_?4~GxcTtm#gzvRRP%F;e=+~u zWcn}0zsP^7f61{1mIe3AAb!ZliMQPFk$ZhOo;m*G-l=V8{Fi0H^{)Ex>^%9C=Woad$Ya`#86^`pMoKjg3MPwJQJ zkJulK_Yf~#%PgegN^s^`rcrgnZlP_%`*m`g&{(n7<9l>Jrwf+Oe)%Y{8oQh0-6T`V-n)%H(5= zC(u6g`qcWgy`J@Nn_u7ev%ZvBKJ`n!#{RH#WPPoUEzuQ*LyPWXa2+k>%;mI zuWT=6wqNopUuvrl@>f1@aek5UFXDmxEVf=|&)UYnxZaELGwyHm!t>8BLbCFW?}ubE z*YA6BxYv$$Kj$CFC&%7T!|JFc9c(HFxE_o4hxUo_0>+2h+S}Co=?^KV=GXRmmhbs{ z@`I_z*Vz74nf{hC^Rx5j)6B2h80Pu5{i*xAUARp!PyLUhN-Pr9{`K$h0`k0nJ_`KuIHVsS{!0*e_o=|4|h3h49_cvYh70&l@{XXXhx!#EUmCxI`>r?fc7Qaw`t@an~7vs;g zf0VhNO7SfD`dodz>NP$#x&DCmgx}lXd|B7_n|#QN2l4ro{D<}RWZuvG*SWpaq0v0!B`mJ`zTxwO1{N- z4d>51dq+M(nfdX3BEH|v{h_%30QcuA<4arhw^6KRhP(LJl(~NZ^XGah@-eS{)EBSk zdCB4zeew(8YZYG{j~wr@^;l<5xPSdvc3$O*V+aG|V<5bXVn^!_zCS~I!gvhV8_`}{ zzVZ8EoKH)&UtCYY^-Y|A<9sLMN91#!%=$Av=IMj_k-X~Pq>r8TLI2PBud(&V`i)(m zw&k-uv=6k8l(~PseBSszPko?17z+bj-^qA5?Fac0?Ga`29ma<^zsBpyZx}zKJ!ScQ z%Pe2=&U``oNz)JcJD(T0zKHW*ee0X`<9OxsNZPU*s0kQS`;l*aKb!Hz*jlc$pPWyail0zD#c70r?lEBNKSEk(FKAD=K7;G8 zWrwl3?=^LsY(N7ME{L-PmXh4VSwf0BHk-(TT=NtF3L0)D@M^XDv|`%RJG zu|1wVw7xgJJoX3US;UiP|L(i@y+ufK@(XHFDQ|^C|^nE7ck^GA1iDysd_Yb%~6Zdmq`#iozK1BUd z=Jk?y9)A|kiob`!{_^^t-$Ud3Nb*tr`zYyHI-Uy#IN$iS#~v%+XRK4*#@HfRzsJ9( zOnb)oh5X0P{vkL2$^PPe9ODI?pGl1$#ON~ok<zbAKweG>#Q8Gv74A>O`8&^lF+hyLiY=DuBmOnz(*Cc^7cqW8`^otz`a|0DsXwo$)}QUCO#eiG%Xk#`zfGNw zBmd&}X~AU;OxqJIH*I$RV*mkt@<#%Q5+cH(E z?QMN${cm%8#b?)DPrkiF`M;KNzp1e_iTWgeU_62K zWxOnR{zU%6`jWqruTo#c6Zs{}v$c?RPn}Qmd4%I79!n?k|L?#0nIa^+pX&q2H^SJo zqg~H<565fRd&-V>ec*C-tm_+Vtbaad<0B+1pWibKy0BwiKiO;PK^PDQHUk5+|J;u? ztkt)pVZ!_VV{F}P48?!x-`+D`L^-a_5cT8y3g?G7AH;Yd`2zU~&r|00xyj_q-g+PM zE8{EIU-#CcHeVXj5O@!|v}2C?RGl`% zuc@Ea&vU>4?LGM!=To@;#d{vV`ryH>kZOI!9x|P;g}m&zE-vZBJ?fMEk^18NVkobC$FahL=|7}7 zj8C?K@v+eU`j|eRy!)H$NoDuhCz6E;VL%w@0|VakIr$soZNV4X(duXOXCbYVc|D*1 z$qzZ-$#@Ir>nPLTQ>Op#TjqQy*V~a#a{iODcfIF-d7k-`f03_w`l7zc?^r(he;=67 zDbJz#7USU@-#O_^_x>6ho(Kipq6@U0NfGJS-!3FB;NzQy=8pPx2C=W63{_FKA^u7!c=V1WLh z^xx(FvURe4)glc(0nV59XpzC8DE~ezSX|z*nahu`broW7XzGc4E{UBM;QNN z{HQITAivmjnfm1ZBcXo+zoN|d<2G%44b8X8b@Ju=pLH@_i*Jo<&o&M3YOnAn3``dT z77Gfg?#I}pH{)NNFJnAt)A^Ive(HVhx&Z;$M52?=SOx z$u8QYJT@EC>VNgWFt8aI2;9q#b)WC-uee@{e1q{Do}atS_S-%J-+JmP`#$2@_eo=I zoBla}#rfe)AD2V(E%r}XD`&?|?+dkmGkk>lS^X>w^n-y^{>Am>C0{81;P-!`uyx+n zM~v@Goo|s(@%>KD4|0DF`ftkme(}z@9-D7*|5n+do%>7rk$!{$VL%wz9R|4mi}New z3*6s}^JnBI$Q z|M;w3&v>B4%RW<(wTH73n-3peTzTL?wOs}6P@$6{*`06W{Z722XiTA5?C*27H!hkT4h5_z}&iy;R z^;_IOmHRggEt7w7Ke5=m8&*Q>&WG2b`-}aXcskSkz`i!MM&MUq(ExX3&8LlU-F*2<-sm|&zVPGB@ zpgm%I#`9<77q(u?Vp?H6);)LMUGk9dGo8i1il0<5q58UdKGI@Bf$xwH^(}Khe6IhB zYj0z~zGZad4L@GC`7ISc`#&H4K=J2)_DkjW6MwFfKli`?hZP4a*Lzhm^~3ybI&^3e zlI3sp3276?9Oq+$F6>yhKh{_iPx09zWskM>(it$kH*F3fMslM~@v+Bpq z>_>hN&YSO}|0?$-bGSvzM1{Q zc=1%*sd~@LuBkug*`Ivga_+CN7+$UCxW>zZe3bek|03V=^u_$B&%gfqudVPe z?nlA-xEL!__9-aJ${HSZ0>*KlK7wb#@#P(7jl&KHullf6*|FM32A98AA zf&P{4>sw~LgwHcm)92Lc%jX5YufX@8r`A5zSNud6m<|S-&vUT{vUbG3Wx@GA`ftWh zIR8gJK>on|DD(cfc7{9iZ}EPoUGpg~KkjeJcpK|WzQgu<`eD3~`SZM&Kkp}BnA+HG zu1||w*u-~ywr*?um-OqKd~Z6oVQPK7uD)E4rS}ztzjZj#--L@HFu?T-v3IfZIe$k! zMgL7c#{9hdJ$~WkZ}zl&m$6#&W%alDo7-=DdvVjDQ>yjJ9$#vEo_ve^kNQaEL(~uT z$@O5=7yFC(ksmW2!1a7XV6;sc&G=m01L^!JzPqkh{Fj>V2aejN-n#e1>_68ZmhY=9 zLNcB5dll02iRoE-liq{@9|L?J$J?(Vb-s)9Ta-P2Pyf&B84rrRo%O%PzyAKki)>G- zYP}Tqhi}WLy!*+w$p6;WH{S1p{E6+S%OdkV`PtZQm-+2C-{-5)G^#2~8;(5k@m_O@# z-L=;)N=~%RxA?t6k1w@7PrgO|$NG9Q`7Zet`-k~af9x-hzlPX=9c}zijc)~CYscDt z#Tv5l9RJeg=6vP!ZN!x9Jl9_lue5hlYMbh+u_O#k8v|TF+FZ}D>VN3Z>8~k!{+s!c zKagLo=E;m+2^mT->aNj{HANZRbsCQ$?(T`vFu~N_L2OF@z>`3aliFZe@nl@fH1II z3~)Uu{WtwF{k134|1*Eazj&VO&24?g_1C@i_3zv)3-YUy|5balKm5=`6(1{~cdI?y zjiZ@P+QzTE`x)ONpJM&^{v`L0;PuoG`PXejiIwm3{K+4`ySU=_ez^Lsvt944k0L)azF?dteBXzB$CJsoxE`PP zlW$qMwkK2h4rRp$*R7{7IYPPi%zjP(PyI1INd2?F>U_h$&HQ)6e*;Ii`sF^q*nB@Q zwB3%)_A?$P|2?i>U_6WSg-!l7ZhdP^sSeVEFdz&xG0^mvJNqMxn^^29#E(3GPybK; zMLxj&H{;p_{}}TT+t>6Bj$rMJf6L<0M;=*x@tSM4-&bV3jC{_MxxZI4|Ie2DzrU}H zE!nTGPh8*n@8REOeHjm=eyBg{m)D1q>SMD!@;A!f{oZ^I=dXtzgDjtXiq9M3SB8;% z66XiWx0?1VpStIyN7Y5R5C+D_0G}T{|H|{^U*3Ec`2l6}39k3%d>Hu&>%;nXEx*C_ zEazoGzR2}xbrdVPJN@k z5e8Nfe1|fx_hgpu)tBdaKlxcIpClh+{w&|) zTgGqW_+7cKf2)2s!H{6sihsV${TdjL(s*4RuXEQ&laH_BN&9DQ79Qq~ceQV`@XUBT zpC`$$=>I+c99v7n7UNOOk9>xFVCpj4$M#b{)E{N`5BaUfuXx_;&p*8BC(CyfzO#7Z z@yDzE%~IF*v;LI%zAW)6`!sL+6xIgX(fazXqemAZnat-y;bGo*Q2(lbr;GtUKYH;j z`a{m|k^i*i4=j)VpZT*sjsqJNbSs(H(zK>b* z^_BNQx!#TWc{2Gw_Ya@)SXABDy>AgCVp(u~opdJuD+~w&xnh9(F>*ib*t*qii2q{! z8vokApUC+-@`u#;PFy>yKlu#zOW^m3Jel`1f0pmbyr1nQU!%UrrzofLE8frkV0)}S zue|*7!f<5zw)Z`c-g(;g`<{G%lkaPiuh~8w7Aw1axvbvTWOmrlFAwT)F9qz7SO4jAC`fACFqtlJk`H0$&D=gW*Q(O-N1p4U@myvB8}s{Mnuu9flw7$3Z?tzD1eWGrzJvEBmLhd{5@{ z#T>@;oZGf{m2*J=zII$vMATb7uQ_D>dzKv<$Y`?mY*4ZcZm+(0zK2^5x7h{0&b*^vm{Ew~AiNEK}@7>t@V&8h| zDcfUfZ9J@Pe5-H%p!`2j{P~~#l6~)};zOQ3Jif*K&A5N8$EQ-yd;E*|_3+O4uI3-d zwWriS`$z8|j@v)6uhIY27n_cO=KeFW$1!}wzoyLi1lK!reNCL3>0W#*jn8;|XYBLj zTkPNGo;|;OKkW4F?@==Tl**@4ulM*D_wOfOJ(=<0O}8(7x8LIDAz4|RKh*n&^6`CR zQ27gE$HPE#Jxc6BntUogYk87CdEc|)dK%`(^+@#BWAg`zkYs?llwnNx4YM^ zbR}I01MM)t`9ePb(O*#JdMwYs(SI@iRmv;-S<)ZVUsD$UN#j3#>x=zE{!Bb^ewldj z@YVNu)`$2_V10p8~P*qKc46M9F|9!@3Zm!Li+c<`NXE@PkmD+UuJ)Deva`Y;wLwm zc=qD8Y(M4Z{Jrd3ihbkz($uf!bA_iAJn35Dr40tSe|C7c(2jM!v4t{S$G>I4?<>>a zr20epSMPnrxHjwRo65(?4>p}YdF`jZ$akrK_Sctg{ps@ki5IuZb@3Hn=lZO+cuK`z z*VnWDoL{FtJh{mqWdC-uf1EGm`#p*u3O_sHN9Tl(E-?^dS$3@RjV-?EI{r1|S@e%x z`zu?=a{Fy>FK#+?O0`~x{)+xOuI;+^aDOcBKR~`fK0(>zBU3xi_EDyOs6XnL{ZZz( zvcDDMKjbG}<7w>txPFoI@6;D%&hK;nzf0RN#k{y)>uZlawg}1E!|#`CeZ2hX6mhOP zPXo7no}>MyeWzT;A{O`Ed+*}L8*bRu{@z{tH|{6S`Nu~ed1Ud$Ypz+8etTE(Iku1c zkupBR^+x1VyhR;$zsh_5bf#!UK=Py^+XVE{yeuKXT=&^7TV8LzI&>^TlJnEcEhus9e2Yo z$WNx`FIivuf3v@rFHb6-dHU&!e=t5)@|BhM0b=XD8~u6Llkf2RJ(PJpWv{%{^NhDJ zzR7r1ZhU5H^&viqN3Z|F+?gFW(LOi%W~{Mg(cIr=dNzC$`&DfgPN#qYFTNgIqq;ru zUz+=8k-tpMpRvAd4`s&lxIY2?Vc+~L?wxi0<(=pEY4|<@<11XB$9N3&Mg2%;ho*Pgle zZ|?2m_ZJvnj7RS}{>}Zo=Dt3>xqn8ip=H7MvFBpv)ednMVPJd=n12pwojs1Nk)7xK zA^mgi{wcTmd3>ht^SS9m^=TLHBA@2-5yvyvcQv2C#%~9%jwk(u%nIlxqYvZ&zH8BZb#N81A1YsbPkitAe2>pVd|rx2e{DnJzbtKS=Suv0YU`W$ z{x$g(`PXjksIHUG6$XR>4+Gx&(6P0(y!d?;@f-0Q@tfoGo2iWte*c@}pK|kgEpMC4 z_YLp4`>aJsX5YlO^6o#+pXt2%FJ}y-@~t1d_Stgpf%lcMcl#|1zJES7|DyVeFK#kl z#3&}4?pzKuJ-H7eSXp7TVXFQJLX`YRR{3|VPHNO(0ro!lK9em_)^#Nnfx9m zpT8(`{S)_J7~6Mve2euTyS^%aKE76cslH5IFVFAyaJ_BU_Fnmk4~h?t#|Qa+F7N$C zeh(A+;~m0!h@>4EPnX}?qP zEAgxO@GJ5)@-2S9i}rzXZtJ;1tj&&Q&$xf9_?Gm8fOaF7m0PRn0N5j8Od-^AT{O;n4-}~XhzVT{$5#JJ@5}%qbpGtimvUpIuz93kN z`q;D={GQIxw)A^1U0Q@>Wiy@?zEvIU+0gYH_psK$S6E!No&uAds(w=llO z_Yt{%p4SguUzN8xzE^#{d41hn-x^!cCg0-wihQ3*d`f(3NIs?Yg#F@;d@FT-uIBzT z!w>!3IaaUoSX^`U zj}^cBJ3mydhtT|Dzy5F2`7HSr<1IWtHGSnao;aSlp1@nrmD~FH=KiL!g! zxsC&JNN!1KV$M=_G>tOcco8S1xBBUIAi)Y`O?{9CmKgglaR0m;jt{7;>|6>bk{T~08 z#SdQl?BdJ!zpspK*>73!du*xpT-WEtcgF4y@-4#Mqc{vn_9_Cw|RbiOY*v>kon`R5lQ)n(U~tNwRc zH1{WzuN%6LRGu)mDHxzVOnrZm`}NR2jP1Ww{pQreJRo0*D6RiFrjbT&0@5eTuH|6`r=qHsa+-)KTKL5El7a`T{k1eWU zBK|E4&WE(M@4A0l{HW{xB41!UXX<>5d@3GIHr{xie2ec3lmCz(we264pSu72*z2R3 z^M$d-tNplaPp&F$qPLremP_zENyZ4IlBZ8Sl#7ALiztH@!aOC*}ulK780X zMa3t$ekgZ*u|0eqiAT>mzC}EcUr|52zO9^Ef3_#Y%I!FYEp5iv;u$|-DY+`F>xZ!#eKCid+2fBaW@hUZiSCU%p&c?+Y;gXXDGtuyy!0{@S*O_4nd)Y)@FLZpShF2<=xHpId}v z_2K&dus6LO$EdH$++1I)zE|H91I_t?*ut8=;@{HNg01)m&X>@hxAhOYe_s5H@htMM z-20pO=KBA5wAff9zw+W?iRC)W6p-wANO--ylZT}#r^o$9+rIR+;+2v{OWjX_*XPDBIls&P;(mT(f3MK9S2rKtQ-svUF6v(SRbfCF z=o$mwdM@q{#P6}tK2zrNd~W_GH~&7h`Y=Aj^+?16=i|s{a%(^9$M_uEM?RTbdsrXN z@3K9VO&=k3ZHso3pSM3mnESNjZt~ML@~OgrFfbGbn(G~63tO+>i1!ELdU@`r6yI%a z;P+1GkLj=J4_6}QJ;pzIf7kVEn;-d>$4|IEs%`mw-|z9qs}COB3dzQE z;97PZ%0|WIn0`a5{~lYM=Kc_4&ws{s*Z<|$g@Mh+fcLz}{lGatNdDx-zsS#cKmA+Z z{zmz?<8$mU#;;;;Xtsv&&Z+S&#(UX5(?^&4IdqLr>YMR6wuf@yI(8hxcW{3Q){px` zj6MH3hK^L`X8UyYf4=?i?I+0h%lJMy{R_`?za`p7=1=}b|IoGnQGVU>Ezf^Xt$)cU zJv{SyBsY9%G+!oau~Fn<}6^+i~#VaK{}Vhff&A4Gq}{Q^Ay{M@t8ZryR@$X3bos{OdR{wMc% zLR){I`~5brbo*^@FTbmDO0_=On{S&M-(vf?e-@uda>pabqql!+Jo?QZK zkk;8N?GG`of2sRKb&wu}0mA^-H}L&$+FSA++V50;!2465w>cj||H%AP>!<5?o^NFQ zl6-{o#k@Xu{qp&M`&F@hbLU&l_iM(rquzd8W=}$r&l|U|q<`su(=ovJcf9owy!qRGeP^ETb*sr?!QxnO|%Nypa9>d5^YyXG(SSKj(-#v52)`bX|(!S&Ex z`|G*qM}2bsm+PIT#*fyxW>)zS?6MBRS)@mVL%wT z>CmA?NHrc-?HBhSqWx~`Uzi`)=X1Rb`5)tN^#8n{>sK@%x%2!X`-kr{#Aq@6a6MOB zd<=a*`4;0@f28N%d7l1{{+{tI@;AoEm_O^+ zw?4+spZa7xk^YVGN9up<_ON``k9>=GAb+QRnE%)^+c)+3MDi`~{pzXNW7St2LKv7Y z2Ex15cC7n3_Mpx0^9=1j7;lPeXZ1~;zoP$P{(S$D^G9AhY-s)TT^{)a`4szue2n_; zyS}OUvwn<^az2{t)2N@+^83D??VCE^;{Jgi-}>H5mlh$(kImPQsc+S{Q^0`7xA=U| z`MaV01^E@{dl;Wctsmo8T+hS!7~@}@4^1sUx9d5dNWMwFMg7p<=hi;fk9>>m6Njcnrzv64q+jv94R(f@RVC9+3Rf*nOz-g~3h5fXBD^e9w4L-~N;S z(DN6JuaGaat*_-5&v#O8n}6>2GrkjhHycA-PtX3#t$nN?>(BOyZ;k6KxPPFmuxB6~hZuv8w$?uQXxCs9)3&w|r#z(6DSU-<%F}^yr_!i@jq37zO*~IvlA72S` zgLX7~j}uZ|_Nu*&B|D#+KPA6neTR<6rTY8auc!XV zZ>ewcVe*;W+ROTR>zg>AnmeCo`=-vfxL>pPezm^uBcD37PgQxs-0?7A^KT*5*Tf#w z^cMfd=Uen|T;Cqo*4o4QNBW=K{QubXp?nd`tc;7c8nC!lH+P zTW>kF2&uj{W-O+^__wTlQ@Nh2c;@M+%Qaog>e=_`ou?JH54Xuf`y0knxW0&dfY%RQ z-`vZifA;tX_oJHH`+g&bQECjm;+*&tm-<4znnBTU5v61LvAT zdsN4H`B(L)`ZLBrD&OM#68R4OS>H0B-}(MzY<;cWe4lY@^Iv`IlltKP2h=C|2l-3i z^=+F!>&y1Az2swK<3F|iUVj=N3n{h}x&B{#Oa3fZY^q+uCNa?b9&PMlOkeSDS&(nh zU!~grzOU!|b6xL8-*^3o&Y$bu`8}f;4TdGg6LZG{^~w5r@w}<=E%IkRZ?HdKc>eiC zNVF9>%Hmt{X*uFjbrLSuVW7DmU~GY#d@FbV%6OOA<0}swSd@RS*7wJ)GkxT~9*z8< zZ<&0He3kkm|DIaBCzWq;eOK=7CEwZ&f3`7hX`Q{u?Qh~+@@u(aQuPrgkB5P|^DX*E z@&(2-=zsg>H@W4{{Yt4Xes5sv?;o(gz4cAR)71DDzhQjK*2>2FSJ1!kd7t;^#)rDD58F%rM13*d#r1N`uj~4Dogev@x4wz- zUBka&Y^?oIJbiHg6ZR+NseNxyd`o_N6aBW@JWbrU)u+F>u##| zt-R-Z#&@R1A5!&8zRdYm`eW*k`b{l=?CXhNZ~mElD>rF=<0fFBE#IQQ9@~Gi{M7X;jPKBYG5@K{>_4str+;QVkp6sX`XJx(){}F+T5kAY zeHedae{z3Q!$U|@uz%uP^3zkmsp=@48V2Uhx9ESo^*Q7lQ{xY*^)=rb&xd*XAs^v< zRciTTUr)Zp{qo6Aa_3v*&)$4G=i|Qj(xpX6Q}7YuTk>O5z^UpeoOZ#t=pDdsL zhx0-7r(D0o_!#q_y3F>m{d3n3@yqxf+dnnF#rm>6Vebb!PQfR9|JBbFAz6K!`{Bne zTo%p!1!ecA(9f#lJaPK!E0+}^Ss#a3x*bjC`ca+_&w_TGB1YQAx9G2O;~$J~dC&X& z-s05f%e?+ze9DXOF#cuZA&!rE<>hvs_?cOX**59eg#`?N2{;+ z);#_46tJi|HnB&(Wcx>lWco7u8j{K7{s3DcnauU1;hY`qeCdxDA(_nhy!oP#?0jzD zf88{`#q}&)|4o0y>&f48;}5CzA)h4Qq&~R+I{kZU`D0&id@=ssI=^?$_A_3TJ040q zwYcxzdlxs}aKqaD=Waf{rwGaHQ|^h+w_mvGszpe3-Qylpd@J|WFb364>;5K%& z>z+JzY%8StzSZ#J*n%|U!OGh+le(|<;K71?|2^TU7n7auA@ z>f6?Ikw2eTX-|n)%_sKfJG$skoNl$t-U=lep< z_*1w2lghWqKc?nyQ~lAp`Lj44GPZwBEuZ|ro=f8W$x8mURdY{6W^?@Tdghl}pWLpu zeY7lgee>bN=d5dg?tF{S6W;vrx1YP92&rywxAvv$Jk8#OWO>tG(w+t`X~*5vCFsJA zc0KJc?IrnKs?7XYU(TQIrVY9#ZwybVe;B{w`0`}V7w~!A##g*w6X#oap8E%IzJ=v^ z_2YTYXL7v-pZ8MxlJ677hk^recR=Snuhv#a+!Zw|2kQJTYNqwlNmqr{DaOP zpPS54H|2P+F;S(ueKF!p7xAxNuO5G&nD&Tr-}aFC)8Em5as0)t+fDn+{8rZQGagQi zaQ&E99_Ji+^xDtsnO|=0WqpW0&X=W z0Qd@=s-ji;&k1CB4Y*BejaS;vmn7qnmD-6T8KeHG`}&t{ zO0~^>T+euXlMmL2tYdV|cogGL9M2r@ls$ZmeV+9-dt;p6{DAcb$0PZ6YP`(WE{0_J zTUjBQto3B`;#=ecwEvX3{)TuQd%UzQpZ=VDlFxJ059KhH>}caByi09IlX*R5@-5nf zw(TAIe%8O#?~d!mOTDeUUr&EGbp2DyqkSfyVta+^Mbq>l@!w%RkH=+Hp7jpZtUV zvuiw$ogdrB@zgfH%<)BeZu%x)@}38&U*119KD_ejue^8;=NrBJb3gBmACGV6#h7cRE<2rj@+WEy3k3U{M<1DN9L37tH^*uG2e2nq1)cF&>Z@~Gf zAuy&gPW<^OtnIVoy76hxL-Bq|d|sa#{@d1<{)7IM&xgVO)<-{{#p~zBw`hOJFW7$C zqqhCm_x;2N`;YCT%=j_$>%0D4=g<0cys-V0bK?WFf8@_)`&QO-_8pHn-ui^V_xL)R{oXx=KKZi9mfM@zCX$NVsWDr=SF?#j{U{{q(9X7*?IiX z9;e2W`Ml5lFgRYP)*s|sTwhE4QKrAnO&_ce?J3(snd@Vi-_-h>^=JRF{gkOc<~OuH zy!tnvU%R#UVJ(;)*F7Ked5`h+)#t8DFRUpq%^jIooG`ai{gvY#;T({JQRszVl=K zJ-$MH(El*Mq3h3h8|&NTL+$vMx88%#b6oE?w7&Z8kGbbh{zbg;d7Sv8+&NxgJl}QwnLq1GJ{4=*aK`nlUDt2y{HP!LU$&q6A^#t{Jzn|r zk8BV5MJX58T>WFk@BYpYtzGZom6zLjufIKgjLo;GPxj{>M~+nAi#6QZ_)T5E%5(W{#CA{-n#LI8@B8@%=nt^NB`1y_m|ww ze$D-!a_fKgFYPPmJ87?Zp3kSbwRi0L*nFeSG1@$d;e-1V*0>4(8kdNFP1*7a$?{X` zR|n1Y2CMzU@h~>t$d5R_>5t>q7?!v`%IsylKR)wIZLep4SRcyt&#C24?Rxen->35Q zLw`7S`^lHQ=TDx`tvzk^$??GF5B6_ve2e~{^=+=NTD8}VA8`LGt}kGD)L+~FNWGus z6W`wa7VR0=BQd|!`fd94v=`h@fOw_M?@eia{p$Gbntyuxosy5xKaP!e+5^U~$)DIB z;*T=}Yk>))vVq4mY`JU+qpkY6&tRDI@lJ=^2yhvScYg!y&dUXEX` zr(pka{xHnV*s=CS@g2?ipT66ZsvkZtF#blq#s18#J*n-V+V#{o?IG=DjK+2NZ0_fl z#wWzTcGef;hm0?>e>fi5-&4aY?JMI^-ui9wFY1SUj{2m``25uM>*0azOV!uV{+R7y z{fQs;FJgT`}Zz@7s$A9Th$XBM0ceaQ2#p7SZ zH~lH`O__XS?)HrO=6nS8Lz&kPtuNL;HJ;D-iTKusx9Yvot*4vwZGHDI{T1W;!8Y2_ z>PP={$B}ow{_X0z{(S!GyZ)a2HN2V)-|c#9KF<)J%#Y=h&r!cT&-*2(?Po!Jkw4RZ zQs(?tTRx%tcjjx}dM?Iayzw^m@k0F4{t&N}`Ml2alsW&%@~5s()}QT5)i3!C^J95k z%j8?B{F?pE{JO61(D|`H`20frbG#4_%&%>Gm>=uM_)f{6R=(H6`sd!CzI7 zW!fM5gSqLK?W29Czl*hLV}PXP$n#+7E#F z^KE)EW%l>b?>kVxY!A;fejeIWADjFC@jPX&@8Wpm zdz^DXQ@%EpgF8k>u3V)=}pvwvtmIlkwPck)g0QR<5_pXWDSpR~u^ zFM;}_zUgoJ#$T#^BY&KGf72eYz3k8O{0blKn{RP`jC_Ih<9H=r+1|GJV}6vWPd@(< zZ)^``#=AJ*&GN`EsZZ9otv+=BYW#}&<#=KLdb08HkT#L;jNRtu`=N1m z{)qSEJs;OvYsR-|4{5*0Zr{-5lfQ62B1WH$4W6g{@4No&FZwIim+=7J-*biWs@67K3(BEy^cw~F|J}&(S?H||Q56u^bwg+B$sqsR_zvwT# z^4gwfe|!FqoZj!x;~AcF`mKobi{)v8-EX}Ywn=@+H+O26(QC7jC*wW_ld?Hue4X5 zKZ&(v`lCH(eq*<9=<>-oxju&aBVVQdyVfW3qd#N)C^Np$)_)J(KdI$WU+f?1hcflc z{My#X^H(gN{*L9de(Yb)&$Z=K)Fq9)J`U|eNW`3#lBmd#^4*SQ;pY>t9ljECspv?Jb&iAqYEZ?iI&QF_P z5wFCLCx=*x9lPcR%CCRE7S`6;vDSCo9iJ{b>)(X|e5 z^iRyMtvojD?0@2o>+_gDW%4`DFSGu+ z^{?vFj=#~~b9|706F;78?@xpzKGZ)S>RZR$KZ4)8r+=B7{a|}i{V&(w+W4EZ|Ml!M z^_j|FjrV`+si$7wb8=bkv+$dp>XZJK{?fC5>~H4h>1S%^sek$tPhaFm%+J#YW` zG{v{7jx}2q|CR;y#qq}eiA{+p8l2XpPGGBeOK)h^}**k;+y=N_8?W( z_oU~|pW5=Z`|f>j5mKF3jId=vd``_i@%h->&zI{vrrv(qpSCjlpX+I2^sK`Z^-urX zwtnny&KG(0W4xnneR98_{Ykug`byzTEpm_1l?0Q9tZo+9Tqt zZM^EHLx&b2nSIlKSmIYH9E|hF7JmrI@*)4{^C9t{o6Pa&`CqPoK{nP()zKQ)yndMO?|6}}x z`ktFi{PfMYh)>S)jYscU>O& zp_f1BtElg}@hi56?PYw>(;xXS{U`J1^Kk(Y)ohN_h{4xC< z^~vW?)|WE*A@gJWg8VU+PkHTUd+86@A71}W?L6DZ@$2c2{l)coj5iW*%%Aw?^TpKk zJGc50FO-QN;-l;Nz7U(TqxmE9DZLLU-qQu|33(7toL@G+6O!G-cq!NW*?dAs>+nlF z-*fj_i;(Pkz7Ix!O#936N&9MbkKf0q%<)cpPrg9=PMQ9n<+Zh+tS{To`8?LolQ}=f z`5N*c>YMeWKB*7Nyx)^~zb8}wod06{gnTMhf6R~Nvwb|DyUh9%kDk7`zJPf0{44DR z%V+&^*XPvh$MMJbKJ5qXi-#BQd^z40A(=nn`YEeVNOoRHI@k#VT%XSMQsgsc??bYD zUwHocMMx%_JqyWX^S9+Ysf!06IH%(KcN{sg_2jW*TlRN5&;3pV{;xjxmg;*=L9UP1 zm+@~|bUi=B{Ahm}-yk2Nza>9NUPGd;TcazV#p18}a#({)+WweRzKA zvh`OypH2TyzE6Cm#$SlPsq2^VJ&sS}k$fxl`G(_R>iQdc{7_%CSCq-0XfJ4AIlt%O zA>bqUlQRF6@5vcHLNa?6-i^1T$&44+-;|_-<6(e!x%HM)i;(Od+utfAlS6FCj&+^^ zv36|oo%pPs=X~B>M~^OE`tJTRFZ(SEK7Y_Y8kdXr`(`|f^FN-<`#FADKK&iXcioQg zZ&M!}|7E(XQtfTm>pQPLcyKEu^FN&L zGyf5iowxoE$>i_9`kC!yveIVhfb(wwD|R$25Kpu(o=kg2|4RNsKEUxt{zU#k+4JA2 z_JH=2GWh{zmgm_Q^0!pJM*Z{o-m|Z)FZ;vmf1ankH2oRY!utm-&#PbRdFD@<&%?wg z{VDNA{V@O3`gVOi>qGpq{@!{r>WBGvUH`WEG5*okA9Fl%Jh!dy*!QzPi5KdV_Lk$F z_Md#7_M&T<_Ry2NE}!`^J{IPB>}dX%^FfSvh_B_v*TP(|9j$K+?;)97`it^=H+8D} zE=EK%KEmfa<6~DIIIt+kWi?-CzVq_sN#%Fro~~s2FOEmf7twxkJo0&*_MYX@9`tR$ zXrEHcI`b*}|`tdyV$@$P`dCzXy9B;{+&$04pZ^^G%ep{LKr~f0L zAfAXf;*orx{lW6uwrA-3sXz8V`6c_G{muM_ZV$^N{@CAa59JcqEBmhzPr0?1?WfH8 z62E-@^`8I8U#LHx=lJ4yWO=mr)IY}~@!M8r{w$yP=XjyNOy&D!{a5BM!`d4=n!VNh z<*@wA>`6$b|FHJSjwUzpfBxdGqo*%Qj#Tj@`e)*w_+tEq6BgfBe1y@k0EvzFcq4{v)1Q zUTS@(c0K!#c&C4%|DZm{)*tbmzP|90#TT!+X8ZYa=;t{PKkND{9v{Rr`;X&+%;z`O#HFEls$c=o@f4)iEq}Q<&pnUUyKJ*|DODXtFBsvWd7gs56NWB zZ>I4xu162Lwc|Q|MgBv)6A#1-@x<}!^*_fW`3TD+o>{-XWtPwSay(FOTR)c1=etyW zGe63ye2#pE`L%7ocfY3(#xH2ka>JX~-rUYpfApv1>l{Brw}`-&|j9_z9`z4{3a4 z?)Us3*Z(KQwO{;?3gc1h_!Z~tQlH1j$Gq{&_%NSu+v0!d`{INg1@kD#c{8&HwN9ND?B>9<_Kg;uE=12R&`q6*VKQKS9e4XEUzenna`sMiP zdVH{a%ItsIBM@Xg?|Q z`HlKye$+49Lw!*8u4j2``g>(Qoc5D4zfaBm2jD&*k6=69z1)*cn-_+ z@WS%PmN_2izxn(?eD_^{uRl0m=wH|$>@UWHm>>Bi@zS>atRL+e>%-@9_8;?WTR+{O z#=pka7x7KJa6A)VeBX9xzQys&`ttdfe1Prec2K+e`FzUxYx+;>hyI27@cPg6Wj54!w{au3hxjJ`DI0$aY3^Lh+Qf``CWQ>v?}_`QG)kU-TEOAIA^tOM69o$NVUB zyfQzpJv>kS5szj4R-VUb-)JA1|E9~-@6htt`e%DRd=c+_{-D3>x_un)eBR)AW&7HW z$FAGM`ZC@^zDRyT{quh6hxUN_q3pGn*R%h5p5>Eovi@v8@kg1@|J1M79^!-kf$>n5 zPdRWeI~qTk+j=m=S4cI#j5W}FAB=b~ecSzk?-CDjZk9j$mw58}H@EZD2ixED@2{16 ztG>ZsIo|qiAN9@WBlZ{VlNayd^Az(Vf8+S3K2!C<`)Q9@ANCjTPc47y*R%cf*R-ef zPn@6S`cBHk3;ToRu|BjvL+hXZhV3EVIG%X_(Di434_$t0d8{vG+Be!4+5`3n`-}FG z>o3T+S)bJQ@p|S*nb))Z98Y}yCm*HE@yhZjvwkd}@jB|8a^NO*tog~b*ME8IwWu%Z zlQQiu^~e0iwwJ_9s?7HD`GVt_<1@9q)azM4`Y-Yy_7~${^bgFRWlo0{lWYsr_Dc8pR`|m zex!f$@K650@x$_m?mw2#_R}BGJ`%6A@64a=V|nDO%%3vX-&=kmO_6_*Z{2hE-Q~Pt z_${k<3$DNJt+sx?itkasl&MehW!gjD&;H6yW_|hm6f{>Kmj(4f{Po>F)|d8#^`*?` zA=aPy^ZAkak#F&NEL9(@5B(R*=koyTpIUzI*Ry@ZEBOoc#r|Ud5>K?>9PggY`msNW zN5&uM|HyY(KkBD#e=sZZwDwtl*QXTC;#Q)d5o<0qA$4BbC$AD=%teu&r9 z@kD#i^&HHfav2}6Z}3$~d}~>~*J$70sd?M7nrAF+3@ii zR6Mq=AMa;<`TWKDQl`HnzhVBgFU+s&cpu{z^v5iJ>iA>-QnvmvP8sI+*?;6C)DNEz z$QRh(L(6P0`;-2V{(^jm?PY!}FIA>~yzxi>&G<`d`MQ2*ewC^p_9yY;#lv!c-lKiz zc%r=^o~a+|m*+iu&GJfnw6Z_z)Hql8R-qm1y2mYC7JS~Mz9`e*(!Ua)?2p`K)}PPU zjK}hMVd(aG{S){0y8h>QWBb`2jz8v4fA8Uo`BSETY46xSynpI4`4*AYd&nPYuW0|f<~!uWPx&4U-W;pPrQF@{jff?55y({nEy1(1_7vt@oz0FPk{kcwx9N$e3kx~_vaq}A>N4}#`oDi zwtsAUNW3%t!}_<4hp@e#zF0o}37@yvKlE4RTO8l4AM?viru}9-a5wskpBHj#U*Gj% ze-lr{Pv7-n{i_?I``e3eJ@wS$Uw!qfYrmh!^&spI>W}*O_zCeqxo^Ct+C%cs zzU#;QsUPxN_6O|&`5E(Ld7jMth!@t!i>I-TT0A;Vfxt}MWWPEVb{4epq_S62; zKIJxEX%9I*7+>J^)JN`ko_hVLZ}tcKi}Q=Te`tMhywF~<{(K&$|KNTE#IGlte+bF= zhxy--rpTvK>m0bG9e2aOJbm&!{Ri>L{-u2+Ug&>`XYwbWr%e9t#cNnU=GXOjV*cKI zE}xIdhnQd6`cmIs{fH0p3EDIE598nDZ^TR6_H}(f_0RTFKa7V^-(A5qVM*3`E!1T{Y{zer@kl?A3X2LUi~-iJnawt7yFC&qrOx1$NN2*`Fqdf zH%ne8T4s#`lOX z>Yw^zf3g0wPn6mJyq_}hG`7t8QGe_o>f6)Dy7RI9r~Zi-@^6kGZ+?OGZ!6z*^ynfa z^N&{FkZOJ~?fv4%Zi9`dkgP8J9=P$JknDUpo>$&CxccD1t&r-r#68}gpT)VY%b)y> z{+%-OqkSjeVSI^l^ZERd#TT!+W>Jpw>id=CkDQ-neRB85w4WYdTGPH2zU}FQ_K5b= zlUd)sW!h`vi~f-HPpu#Cr+;LAsrEk=zvL6VpW}u2piF#>Jw8mou|Fj~S%1phe>Vx)= ze3^X7^GB)u?_E#4z3}|=i;&Epe&xZf?PT(+uUxjBYJV$!HKZQ+{mE}XcR>-7)gy3U zJDP0qkC03beQQUP%kjMxlF1%k=6;^zmCsA`m$}LG_k4b&{ZH*r@)i0AmPh+YdrkeZ z{+>)e!~SCZXwMlR^~z&?>A&dz7=QHY)7Jhof9jv@V}H;;dHA3{sSoO_YnkmKe_{Q@ z+7>&Sz5AOdKeC-nroSu4%gTP-yr21xEtBsNPaMDGAM8&)&r+tod7d)A4@P~_KC%A1 zJ~uhULhNXG;{Kc=mSV@+--51{T!0qg4=MP-*rtM@h`CgcN zwxgZD>CmA?NG8+1mHDpBx0e28g)j0v@8|fW{o(Un*ZyJX{K!YxKE{7&@2Ib?W%>u& zf7)x>cgAZdGe6pI=0|_y`G>Cc;pNBrvOmc;$w!Dko^Q*)SU=*A<)_MQpV=9EAL-$T z9@wH*XJUUwe@ebeK0|%a zO(tKYKFAk|H^##lkMiP8KV_#%HKc%DkWTr9P<7w)kZKGk=y(nfhe=DN~=R z<*|J0zmQ}zeVfVWUGhEhm9{eTXZh+gzt5y2=_nTr@cEDSnDGVrySDyn)9$DKDARw_ z-%=)j;d~$CE%evSFI8rJ*&fP6_c!?r@lAf4T3=pIzRB_^b3T{#<@#~npDG7#Wyf4> zkm@0SC=3V#n}7k%x6^)`U5@9==wFBSpDJ(ldu;Ugf7E-vpuRajTaTOYui3!(mo9Vt4&TSI`i3+O zdn3P~ei8W%4DiSLga2+2zf&%j$RayD%_64DkJ)ay{Vxm^k^k^{g7GWvFTwXKnIC1&_i6p}{A{TFqWV-A5C%>J z17U9*JJ$Ed9%NaNe|hl|`oF31ncVA3{d#&55Jw>ZDb^;_gql-tf1a(~AY8I!t4 zwp;ZTGk3{>9u+Pd0OYHdB47J{1Opfj%%0xR)L4{A0#tI*)(Lg7aDAWAu04 z`(pI37N0Q=VQ~wq56@Hf{P)z(bN-9_Gclgy>38h*v%fr<{GIP(#%MHra6bl~Pn8YN zAzA)fe|e9spmbu!~EEuSi7cwuGeDwm|t6&`eAvLS%1c-7$0SOxZeZY-?gmxRk!v- z_E`E92801&;KVS%`AF_(&+k!ieSX*ek@G{0Z_rD3f1MCSRn? z^;q10p5^g=mdEQUvwdv8#XaNw+Zb;nf98Jp1z);E3gGtVEVo-g^z*5%8S%5MkTulc?K=ljT)ntbH^ z#a%~FU(|Uv=WCfC`6ct`_2j3lf8CDouVE$rIuHhg0bwA24A39aUs}8+BN(0IG;-X@P(_c zT7+bJ5C6BL$ri5)sSlauc{1nUxW0+{2wd2XR)6Y~@lN&+`_J-Fk`9CcVL%uV2801& zAjSaaQ^_Yh`NbFZ6(L!^;s18*L-y(;{v&+H_}2f02Vp=M5C((+VL%uV2801&Ko}4P zgaKhd7!U@80bxKG5C((+VL%uV2801&Ko}4PgaKh79Rq*$^B?@>lg?Uw%ju+YWp4HR zk#PKn*Pi^&U%u?dhfiMoqm%#B|NOcCp#J+$|J>{66A!0;;8)N6#XUd&!TuG z^r3Tq;lDfiZ6Eo-uY7RN*=L@8@)>8Gaq<~^-uDmQYyY10v+w=c|Kj8`&ph+w{~tQ! BFOdKM literal 0 HcmV?d00001 diff --git a/romancal/datamodels/tests/test_filetype.py b/romancal/datamodels/tests/test_filetype.py new file mode 100644 index 000000000..f06085282 --- /dev/null +++ b/romancal/datamodels/tests/test_filetype.py @@ -0,0 +1,41 @@ +from pathlib import Path + +import pytest + +from romancal.datamodels import filetype + +DATA_DIRECTORY = Path(__file__).parent / "data" + + +def test_filetype(): + file_1 = filetype.check(DATA_DIRECTORY / "empty.json") + file_2 = filetype.check(DATA_DIRECTORY / "example_schema.json") + with open(DATA_DIRECTORY / "fake.json") as file_h: + file_3 = filetype.check(file_h) + file_4 = filetype.check(DATA_DIRECTORY / "empty.asdf") + file_5 = filetype.check(DATA_DIRECTORY / "pluto.asdf") + with open(DATA_DIRECTORY / "pluto.asdf", "rb") as file_h: + file_6 = filetype.check(file_h) + file_7 = filetype.check(DATA_DIRECTORY / "fake.asdf") + with open(DATA_DIRECTORY / "fake.json") as file_h: + file_8 = filetype.check(file_h) + file_9 = filetype.check(str(DATA_DIRECTORY / "pluto.asdf")) + + assert file_1 == "asn" + assert file_2 == "asn" + assert file_3 == "asn" + assert file_4 == "asdf" + assert file_5 == "asdf" + assert file_6 == "asdf" + assert file_7 == "asdf" + assert file_8 == "asn" + assert file_9 == "asdf" + + with pytest.raises(ValueError): + filetype.check(DATA_DIRECTORY / "empty.txt") + + with pytest.raises(ValueError): + filetype.check(2) + + with pytest.raises(ValueError): + filetype.check("test") diff --git a/romancal/pipeline/exposure_pipeline.py b/romancal/pipeline/exposure_pipeline.py index 25c3e2886..970e95dc9 100644 --- a/romancal/pipeline/exposure_pipeline.py +++ b/romancal/pipeline/exposure_pipeline.py @@ -3,10 +3,14 @@ from os.path import basename import numpy as np -from roman_datamodels import datamodels as rdd +from roman_datamodels import datamodels as rdm + +import romancal.datamodels.filetype as filetype # step imports from romancal.assign_wcs import AssignWcsStep +from romancal.associations.exceptions import AssociationNotValidError +from romancal.associations.load_as_asn import LoadAsLevel2Asn from romancal.dark_current import DarkCurrentStep from romancal.dq_init import dq_init_step from romancal.flatfield import FlatFieldStep @@ -71,66 +75,100 @@ def process(self, input): input_filename = None # open the input file - input = rdd.open(input) - - log.debug("Exposure Processing a WFI exposure") - - self.dq_init.suffix = "dq_init" - result = self.dq_init(input) - if input_filename: - result.meta.filename = input_filename - result = self.saturation(result) - - # Test for fully saturated data - if is_fully_saturated(result): - log.info("All pixels are saturated. Returning a zeroed-out image.") - - # Return zeroed-out image file (stopping pipeline) - return self.create_fully_saturated_zeroed_image(result) - - result = self.linearity(result) - result = self.dark_current(result) - result = self.jump(result) - result = self.rampfit(result) - - # Test for fully saturated data - if "groupdq" in result.keys(): + file_type = filetype.check(input) + asn = None + if file_type == "asdf": + try: + input = rdm.open(input) + except TypeError: + log.debug("Error opening file:") + return + + if file_type == "asn": + try: + asn = LoadAsLevel2Asn.load(input, basename=self.output_file) + except AssociationNotValidError: + log.debug("Error opening file:") + return + + # Build a list of observations to process + expos_file = [] + if file_type == "asdf": + expos_file = [input] + elif file_type == "asn": + for product in asn["products"]: + for member in product["members"]: + expos_file.append(member["expname"]) + + results = [] + for in_file in expos_file: + if isinstance(in_file, str): + input_filename = basename(in_file) + log.info(f"Input file name: {input_filename}") + else: + input_filename = None + + # Open the file + input = rdm.open(in_file) + log.info(f"Processing a WFI exposure {in_file}") + + self.dq_init.suffix = "dq_init" + result = self.dq_init(input) + if input_filename: + result.meta.filename = input_filename + result = self.saturation(result) + + # Test for fully saturated data if is_fully_saturated(result): - # Set all subsequent steps to skipped - for step_str in [ - "assign_wcs", - "flat_field", - "photom", - "source_detection", - ]: - result.meta.cal_step[step_str] = "SKIPPED" - - # Set suffix for proper output naming - self.suffix = "cal" + log.info("All pixels are saturated. Returning a zeroed-out image.") + + result = self.linearity(result) + result = self.dark_current(result) + result = self.jump(result) + result = self.rampfit(result) + + # Test for fully saturated data + if "groupdq" in result.keys(): + if is_fully_saturated(result): + # Set all subsequent steps to skipped + for step_str in [ + "assign_wcs", + "flat_field", + "photom", + "source_detection", + ]: + result.meta.cal_step[step_str] = "SKIPPED" + + # Set suffix for proper output naming + self.suffix = "cal" + results.append(result) # Return fully saturated image file (stopping pipeline) - return result - - result = self.assign_wcs(result) - if result.meta.exposure.type == "WFI_IMAGE": - result = self.flatfield(result) - else: - log.info("Flat Field step is being SKIPPED") - result.meta.cal_step.flat_field = "SKIPPED" - - result = self.photom(result) - - if result.meta.exposure.type == "WFI_IMAGE": - result = self.source_detection(result) - else: - log.info("Source Detection step is being SKIPPED") - result.meta.cal_step.source_detection = "SKIPPED" - - # setup output_file for saving - self.setup_output(result) - log.info("Roman exposure calibration pipeline ending...") - - return result + return results + + result = self.assign_wcs(result) + if result.meta.exposure.type == "WFI_IMAGE": + result = self.flatfield(result) + else: + log.info("Flat Field step is being SKIPPED") + result.meta.cal_step.flat_field = "SKIPPED" + + if result.meta.exposure.type == "WFI_IMAGE": + result = self.photom(result) + result = self.source_detection(result) + else: + log.info("Photom and source detection steps are being SKIPPED") + result.meta.cal_step.photom = "SKIPPED" + result.meta.cal_step.source_detection = "SKIPPED" + + # setup output_file for saving + self.setup_output(result) + log.info("Roman exposure calibration pipeline ending...") + + self.output_use_model = True + results.append(result) + + return results def setup_output(self, input): """Determine the proper file name suffix to use later""" From 7592b4325a5a498e04208ab093120171b8365d93 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 7 Aug 2023 21:05:27 -0400 Subject: [PATCH 17/82] [pre-commit.ci] pre-commit autoupdate (#806) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a21796b7a..b9dc295c3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,7 +32,7 @@ repos: args: ["--py38-plus"] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: 'v0.0.281' + rev: 'v0.0.282' hooks: - id: ruff args: ["--fix"] From 1b73fb5968b10d318e4c76ef1d34fa3651f0fac5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 10 Aug 2023 15:34:38 +0000 Subject: [PATCH 18/82] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- romancal/resample/resample.py | 2 +- romancal/resample/resample_step.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/romancal/resample/resample.py b/romancal/resample/resample.py index 2963eb28a..5407d592d 100644 --- a/romancal/resample/resample.py +++ b/romancal/resample/resample.py @@ -1,10 +1,10 @@ import logging import numpy as np +from astropy import units as u from drizzle import cdrizzle, util from roman_datamodels import datamodels from roman_datamodels.maker_utils import mk_datamodel -from astropy import units as u from ..datamodels import ModelContainer from . import gwcs_drizzle, resample_utils diff --git a/romancal/resample/resample_step.py b/romancal/resample/resample_step.py index 7dd106923..046b1a8e1 100755 --- a/romancal/resample/resample_step.py +++ b/romancal/resample/resample_step.py @@ -5,10 +5,10 @@ import asdf import numpy as np from roman_datamodels import datamodels +from stcal.alignment import util from stpipe.extern.configobj.configobj import ConfigObj from stpipe.extern.configobj.validate import Validator -from stcal.alignment import util from ..datamodels import ModelContainer from ..stpipe import RomanStep from . import resample From f538808f44cf1d83b430d0d929efe1ffe2862e47 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Thu, 10 Aug 2023 12:16:34 -0400 Subject: [PATCH 19/82] Temporarily pin RAD & RDM version. --- pyproject.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a7c15ee34..481f030a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,10 +22,10 @@ dependencies = [ 'photutils >=1.6.0', 'pyparsing >=2.4.7', 'requests >=2.22', - # 'rad >=0.15.0', - 'rad @ git+https://github.com/spacetelescope/rad.git@main', - # 'roman_datamodels >=0.15.0', - 'roman_datamodels @ git+https://github.com/spacetelescope/roman_datamodels.git@main', + 'rad == 0.17.0', + # 'rad @ git+https://github.com/spacetelescope/rad.git@main', + 'roman_datamodels == 0.17.0', + # 'roman_datamodels @ git+https://github.com/spacetelescope/roman_datamodels.git@main', # 'stcal >=1.4.0', 'stcal @ git+https://github.com/mairanteodoro/stcal.git#egg=stcal-alignment', 'stpipe >=0.5.0', From f5d448a0f2ebe7d4113069da1f19a7c38025a210 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Thu, 10 Aug 2023 12:42:27 -0400 Subject: [PATCH 20/82] Add entry to CHANGES.rst. --- CHANGES.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 060e5daeb..e1829977b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,10 @@ 0.11.1 (unreleased) =================== +resample +-------- +- Implement resampling step. [#787] + source_detection ---------------- - Skip the step if the data is not imaging mode. [#798] From 21ebac8f17953948e976ea5e84cfee3ea6f773c1 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Fri, 18 Aug 2023 16:59:22 -0400 Subject: [PATCH 21/82] Docstring update and refactoring to handle different types of input. --- romancal/resample/resample_step.py | 57 +++++++++++++++++++----------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/romancal/resample/resample_step.py b/romancal/resample/resample_step.py index 046b1a8e1..a0deb38cf 100755 --- a/romancal/resample/resample_step.py +++ b/romancal/resample/resample_step.py @@ -1,4 +1,5 @@ import logging +import os import re from copy import deepcopy @@ -34,9 +35,18 @@ class ResampleStep(RomanStep): Parameters ----------- - input : ~jwst.datamodels.JwstDataModel or ~jwst.associations.Association - Single filename for either a single image or an association table. - """ + input : str, ~roman_datamodels.datamodels.DataModel, or ~romancal.datamodels.ModelContainer + If a string is provided, it should correspond to either a single ASDF filename + or an association filename. Alternatively, a single DataModel instance can be + provided instead of an ASDF filename. + Multiple files can be processed via either an association file or wrapped by a + ModelContainer. + + Returns + ------- + result : ~roman_datamodels.datamodels.MosaicModel + A mosaic datamodel with the final output frame. + """ # noqa: E501 class_alias = "resample" @@ -66,24 +76,29 @@ def process(self, input): output = input_models[0].meta.filename self.blendheaders = False elif isinstance(input, str): - input_models = ModelContainer(input) - output = input_models.meta.asn_table.products[0].name - - # if isinstance(input, ModelContainer): - # input_models = dm - # try: - # output = input_models.meta.asn_table.products[0].name - # except AttributeError: - # # coron data goes through this path by the time it gets to - # # resampling. - # # TODO: figure out why and make sure asn_table is carried along - # output = None - # else: - # input_models = ModelContainer([dm]) - # input_models.asn_pool_name = dm.meta.asn.pool_name - # input_models.asn_table_name = dm.meta.asn.table_name - # output = dm.meta.filename - # self.blendheaders = False + # either a single asdf filename or an association filename + try: + # association filename + input_models = ModelContainer(input) + except Exception: + # single ASDF filename + input_models = ModelContainer([input]) + if hasattr(input_models, "asn_table") and len(input_models.asn_table): + output = input_models.asn_table["products"][0]["name"] + elif hasattr(input_models[0], "meta"): + output = input_models[0].meta.filename + elif isinstance(input, ModelContainer): + input_models = input + output = ( + f"{os.path.commonprefix([x.meta.filename for x in input_models])}.asdf" + ) + if len(output) == 0: + output = "resample_output.asdf" + else: + raise TypeError( + "Input must be an ASN filename, a ModelContainer, " + "a single ASDF filename, or a single Roman DataModel." + ) # Check that input models are 2D images if len(input_models[0].data.shape) != 2: From 7fef78dcbd2f329428a611533ff5aee5c943a093 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Tue, 12 Sep 2023 15:58:04 -0400 Subject: [PATCH 22/82] Small refactoring and unit tests. --- romancal/resample/resample.py | 47 +- romancal/resample/resample_step.py | 95 +-- romancal/resample/tests/__init__.py | 0 romancal/resample/tests/test_resample.py | 608 ++++++++++++++++++ romancal/resample/tests/test_resample_step.py | 281 ++++++++ 5 files changed, 971 insertions(+), 60 deletions(-) create mode 100644 romancal/resample/tests/__init__.py create mode 100644 romancal/resample/tests/test_resample.py create mode 100644 romancal/resample/tests/test_resample_step.py diff --git a/romancal/resample/resample.py b/romancal/resample/resample.py index 5407d592d..8e2f0c4df 100644 --- a/romancal/resample/resample.py +++ b/romancal/resample/resample.py @@ -31,7 +31,8 @@ class ResampleData: weight type, exposure time (if relevant), and kernel, and merges them with any user-provided values. 2. Creates output WCS based on input images and define mapping function - between all input arrays and the output array. + between all input arrays and the output array. Alternatively, a custom, + user-provided WCS object can be used instead. 3. Updates output data model with output arrays from drizzle, including a record of metadata from all input models. """ @@ -73,6 +74,16 @@ def __init__( deleted from memory. Default value is `True` to keep all products in memory. """ + if ( + (input_models is None) + or (len(input_models) == 0) + or (not any(input_models)) + ): + raise ValueError( + "No input has been provided. Input should be a list of datamodel(s) or " + "a ModelContainer." + ) + self.input_models = input_models self.output_filename = output self.pscale_ratio = pscale_ratio @@ -102,14 +113,14 @@ def __init__( else: log.info(f"Output pixel scale ratio: {pscale_ratio}") + # build the output WCS object if output_wcs: - # Use user-supplied reference WCS for the resampled image: + # use the provided WCS object self.output_wcs = output_wcs if output_shape is not None: self.output_wcs.array_shape = output_shape[::-1] - else: - # Define output WCS based on all inputs, including a reference WCS: + # determine output WCS based on all inputs, including a reference WCS self.output_wcs = resample_utils.make_output_wcs( self.input_models, pscale_ratio=self.pscale_ratio, @@ -172,9 +183,13 @@ def resample_many_to_many(self): indx = exposure[0].meta.filename.rfind(".") output_type = exposure[0].meta.filename[indx:] output_root = "_".join( - exposure[0].meta.filename.replace(output_type, "").split("_")[:-1] + exposure[0] + .meta.filename.replace(output_type, "") + .split("_")[:-1] + ) + output_model.meta.filename = ( + f"{output_root}_outlier_i2d{output_type}" ) - output_model.meta.filename = f"{output_root}_outlier_i2d{output_type}" # Initialize the output with the wcs driz = gwcs_drizzle.GWCSDrizzle( @@ -225,7 +240,9 @@ def resample_many_to_one(self): output_model.meta.filename = self.output_filename output_model.meta["resample"] = {} output_model.meta.resample["weight_type"] = self.weight_type - output_model.meta.resample["pointings"] = len(self.input_models.models_grouped) + output_model.meta.resample["pointings"] = len( + self.input_models.models_grouped + ) if self.blendheaders: log.info("Skipping blendheaders for now.") @@ -302,15 +319,14 @@ def resample_variance_array(self, name, output_model): f"{repr(model.meta.filename)}. Skipping ..." ) continue - elif variance.shape != model.data.shape: log.warning( f"Data shape mismatch for '{name}' for model " - f"{repr(model.meta.filename)}. Skipping ..." + f"{repr(model.meta.filename)}. Skipping..." ) continue - # Make input weight map of unity where there is science data + # create a unit weight map for all the input pixels with science data inwht = resample_utils.build_driz_weight( model, weight_type=None, good_bits=self.good_bits ) @@ -318,7 +334,8 @@ def resample_variance_array(self, name, output_model): resampled_variance = np.zeros_like(output_model.data) outwht = np.zeros_like(output_model.data) outcon = np.zeros_like(output_model.context) - # Resample the variance array. Fill "unpopulated" pixels with NaNs. + + # resample the variance array (fill "unpopulated" pixels with NaNs) self.drizzle_arrays( variance, inwht, @@ -366,7 +383,9 @@ def update_exposure_times(self, output_model): output_model.meta.exposure.exposure_time = total_exposure_time output_model.meta.exposure.start_time = min(exposure_times["start"]) output_model.meta.exposure.end_time = max(exposure_times["end"]) - output_model.meta.resample["product_exposure_time"] = total_exposure_time + output_model.meta.resample[ + "product_exposure_time" + ] = total_exposure_time @staticmethod def drizzle_arrays( @@ -527,7 +546,9 @@ def drizzle_arrays( # Compute the mapping between the input and output pixel coordinates # for use in drizzle.cdrizzle.tdriz - pixmap = resample_utils.calc_gwcs_pixmap(input_wcs, output_wcs, insci.shape) + pixmap = resample_utils.calc_gwcs_pixmap( + input_wcs, output_wcs, insci.shape + ) log.debug(f"Pixmap shape: {pixmap[:,:,0].shape}") log.debug(f"Input Sci shape: {insci.shape}") diff --git a/romancal/resample/resample_step.py b/romancal/resample/resample_step.py index a0deb38cf..77b61e936 100755 --- a/romancal/resample/resample_step.py +++ b/romancal/resample/resample_step.py @@ -68,6 +68,7 @@ class ResampleStep(RomanStep): in_memory = boolean(default=True) """ # noqa: E501 + # TODO: provide 'drizpars' file (then remove _set_spec_defaults?) reference_file_types = [] def process(self, input): @@ -83,15 +84,15 @@ def process(self, input): except Exception: # single ASDF filename input_models = ModelContainer([input]) - if hasattr(input_models, "asn_table") and len(input_models.asn_table): + if hasattr(input_models, "asn_table") and len( + input_models.asn_table + ): output = input_models.asn_table["products"][0]["name"] elif hasattr(input_models[0], "meta"): output = input_models[0].meta.filename elif isinstance(input, ModelContainer): input_models = input - output = ( - f"{os.path.commonprefix([x.meta.filename for x in input_models])}.asdf" - ) + output = f"{os.path.commonprefix([x.meta.filename for x in input_models])}.asdf" if len(output) == 0: output = "resample_output.asdf" else: @@ -109,21 +110,20 @@ def process(self, input): self.wht_type = self.weight_type if "drizpars" in self.reference_file_types: ref_filename = self.get_reference_file(input_models[0], "drizpars") - else: # no drizpars reference file found - ref_filename = "N/A" - - if ref_filename == "N/A": - self.log.info("No drizpars reference file found.") - kwargs = self._set_spec_defaults() - else: self.log.info(f"Using drizpars reference file: {ref_filename}") kwargs = self.get_drizpars(ref_filename, input_models) + else: + # no drizpars reference file found + self.log.info("No drizpars reference file found.") + kwargs = self._set_spec_defaults() kwargs["allowed_memory"] = self.allowed_memory # Issue a warning about the use of exptime weighting if self.wht_type == "exptime": - self.log.warning("Use of EXPTIME weighting will result in incorrect") + self.log.warning( + "Use of EXPTIME weighting will result in incorrect" + ) self.log.warning("propagated errors in the resampled product") # Custom output WCS parameters. @@ -155,7 +155,6 @@ def process(self, input): def _final_updates(self, model, input_models, kwargs): model.meta.cal_step["resample"] = "COMPLETE" - self.update_wcs(model) util.update_s_region_imaging(model) if ( input_models.asn_pool_name is not None @@ -178,6 +177,36 @@ def _final_updates(self, model, input_models, kwargs): @staticmethod def _check_list_pars(vals, name, min_vals=None): + """ + Check if a specific keyword parameter is properly formatted. + + Parameters + ---------- + vals : list or tuple + A list or tuple containing a pair of values currently assigned to the + keyword parameter `name`. Both values must be either `None` or not `None`. + name : str + The name of the keyword parameter. + min_vals : list or tuple, optional + A list or tuple containing a pair of minimum values to be assigned + to `name`, by default None. + + Returns + ------- + None or list + If either `vals` is set to `None` (or both of its elements), the + returned result will be `None`. Otherwise, the returned result will be + a list containing the current values assigned to `name`. + + Raises + ------ + ValueError + This error will be raised if any of the following conditions are met: + - the number of elements of `vals` is not 2; + - the currently assigned values of `vals` are smaller than the + minimum value provided; + - one element is `None` and the other is not `None`. + """ if vals is None: return None if len(vals) != 2: @@ -192,7 +221,9 @@ def _check_list_pars(vals, name, min_vals=None): ) return list(vals) else: - raise ValueError(f"Both '{name}' values must be either None or not None.") + raise ValueError( + f"Both '{name}' values must be either None or not None." + ) @staticmethod def _load_custom_wcs(asdf_wcs_file, output_shape): @@ -274,7 +305,9 @@ def get_drizpars(self, ref_filename, input_models): # With presence of wild-card rows, code should never trigger this logic if row is None: - self.log.error("No row found in %s matching input data.", ref_filename) + self.log.error( + "No row found in %s matching input data.", ref_filename + ) raise ValueError # Define the keys to pull from drizpars reffile table. @@ -360,35 +393,3 @@ def _set_spec_defaults(self): log.info(" using: %s=%s", k, repr(v)) return kwargs - - def update_wcs(self, model): - """ - Update WCS keywords of the resampled image. - """ - # Delete any SIP-related keywords first - pattern = r"^(cd[12]_[12]|[ab]p?_\d_\d|[ab]p?_order)$" - regex = re.compile(pattern) - - keys = list(model._instance.meta.wcsinfo.keys()) - for key in keys: - if regex.match(key): - del model._instance.meta.wcsinfo[key] - - # Write new PC-matrix-based WCS based on GWCS model - transform = model.meta.wcs.forward_transform - model.meta.wcsinfo.ra_ref = transform[6].lon.value - model.meta.wcsinfo.dec_ref = transform[6].lat.value - - # Remove no longer relevant WCS keywords - rm_keys = [ - "v2_ref", - "v3_ref", - "ra_ref", - "dec_ref", - "roll_ref", - "v3yangle", - "vparity", - ] - for key in rm_keys: - if key in model._instance.meta.wcsinfo: - del model._instance.meta.wcsinfo[key] diff --git a/romancal/resample/tests/__init__.py b/romancal/resample/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/romancal/resample/tests/test_resample.py b/romancal/resample/tests/test_resample.py new file mode 100644 index 000000000..4edab7ed6 --- /dev/null +++ b/romancal/resample/tests/test_resample.py @@ -0,0 +1,608 @@ +import pytest +import numpy as np +from astropy import units as u +from gwcs import WCS +from astropy.modeling import models +from astropy.time import Time +from astropy import coordinates as coord +from gwcs import coordinate_frames as cf +from roman_datamodels import datamodels, maker_utils +from romancal.resample.resample import ResampleData +from romancal.resample import gwcs_drizzle, resample_utils +from romancal.datamodels import ModelContainer + + +class WfiSca: + def __init__(self, fiducial_world, pscale, shape, filename): + self.fiducial_world = fiducial_world + self.pscale = pscale + self.shape = shape + self.filename = filename + + def create_image(self): + """ + Create a dummy L2 datamodel given the coordinates of the fiducial point, + a pixel scale, and the image shape and filename. + + Returns + ------- + datamodels.ImageModel + An L2 ImageModel datamodel. + """ + l2 = maker_utils.mk_level2_image( + shape=self.shape, + **{ + "meta": { + "wcsinfo": {"ra_ref": 10, "dec_ref": 0, "vparity": -1}, + "exposure": {"exposure_time": 152.04000000000002}, + "observation": { + "program": "00005", + "execution_plan": 1, + "pass": 1, + "observation": 1, + "segment": 1, + "visit": 1, + "visit_file_group": 1, + "visit_file_sequence": 1, + "visit_file_activity": "01", + "exposure": 1, + }, + }, + "data": u.Quantity( + np.random.poisson(2.5, size=self.shape).astype(np.float32), + u.electron / u.s, + dtype=np.float32, + ), + "var_rnoise": u.Quantity( + np.random.normal(1, 0.05, size=self.shape).astype( + np.float32 + ), + u.electron**2 / u.s**2, + dtype=np.float32, + ), + "var_poisson": u.Quantity( + np.random.poisson(1, size=self.shape).astype(np.float32), + u.electron**2 / u.s**2, + dtype=np.float32, + ), + "var_flat": u.Quantity( + np.random.uniform(0, 1, size=self.shape).astype(np.float32), + u.electron**2 / u.s**2, + dtype=np.float32, + ), + }, + ) + # data from WFISim simulation of SCA #01 + l2.meta.filename = self.filename + l2.meta["wcs"] = create_wcs_object_without_distortion( + fiducial_world=self.fiducial_world, + pscale=self.pscale, + shape=self.shape, + ) + return datamodels.ImageModel(l2) + + +def create_wcs_object_without_distortion(fiducial_world, pscale, shape): + """ + Create a simple WCS object without either distortion or rotation. + + Parameters + ---------- + fiducial_world : tuple + A pair of values corresponding to the fiducial's world coordinate. + pscale : tuple + A pair of values corresponding to the pixel scale in each axis. + shape : tuple + A pair of values specifying the dimensions of the WCS object. + + Returns + ------- + gwcs.WCS + A gwcs.WCS object. + """ + # components of the model + shift = models.Shift() & models.Shift() + + affine = models.AffineTransformation2D( + matrix=[[1, 0], [0, 1]], translation=[0, 0], name="pc_rotation_matrix" + ) + + scale = models.Scale(pscale[0]) & models.Scale(pscale[1]) + + tan = models.Pix2Sky_TAN() + celestial_rotation = models.RotateNative2Celestial( + fiducial_world[0], + fiducial_world[1], + 180, + ) + + det2sky = shift | affine | scale | tan | celestial_rotation + det2sky.name = "linear_transform" + + detector_frame = cf.Frame2D( + name="detector", axes_names=("x", "y"), unit=(u.pix, u.pix) + ) + sky_frame = cf.CelestialFrame( + reference_frame=coord.FK5(), name="fk5", unit=(u.deg, u.deg) + ) + + pipeline = [(detector_frame, det2sky), (sky_frame, None)] + + wcs_obj = WCS(pipeline) + + wcs_obj.bounding_box = ( + (-0.5, shape[-1] - 0.5), + (-0.5, shape[-2] - 0.5), + ) + + wcs_obj.pixel_shape = shape[::-1] + wcs_obj.array_shape = shape + + return wcs_obj + + +@pytest.fixture +def wfi_sca1(): + sca = WfiSca( + fiducial_world=(10, 0), + pscale=(0.000031, 0.000031), + shape=(100, 100), + filename="r0000501001001001001_01101_0001_WFI01_cal.asdf", + ) + + return sca.create_image() + + +@pytest.fixture +def wfi_sca2(): + sca = WfiSca( + fiducial_world=(10.00139, 0), + pscale=(0.000031, 0.000031), + shape=(100, 100), + filename="r0000501001001001001_01101_0001_WFI02_cal.asdf", + ) + + return sca.create_image() + + +@pytest.fixture +def wfi_sca3(): + sca = WfiSca( + fiducial_world=(10.00278, 0), + pscale=(0.000031, 0.000031), + shape=(100, 100), + filename="r0000501001001001001_01101_0001_WFI03_cal.asdf", + ) + + return sca.create_image() + + +@pytest.fixture +def wfi_sca4(): + sca = WfiSca( + fiducial_world=(10, 0), + pscale=(0.000031, 0.000031), + shape=(100, 100), + filename="r0000501001001001001_01101_0002_WFI01_cal.asdf", + ) + + return sca.create_image() + + +@pytest.fixture +def wfi_sca5(): + sca = WfiSca( + fiducial_world=(10.00139, 0), + pscale=(0.000031, 0.000031), + shape=(100, 100), + filename="r0000501001001001001_01101_0002_WFI02_cal.asdf", + ) + + return sca.create_image() + + +@pytest.fixture +def wfi_sca6(): + sca = WfiSca( + fiducial_world=(10.00278, 0), + pscale=(0.000031, 0.000031), + shape=(100, 100), + filename="r0000501001001001001_01101_0002_WFI03_cal.asdf", + ) + + return sca.create_image() + + +@pytest.fixture +def exposure_1(wfi_sca1, wfi_sca2, wfi_sca3): + """Returns a list with models corresponding to a dummy exposure 1.""" + # set the same exposure time for all SCAs + for sca in [wfi_sca1, wfi_sca2, wfi_sca3]: + sca.meta.exposure["start_time"] = Time( + "2020-02-01T00:00:00", format="isot", scale="utc" + ) + sca.meta.exposure["end_time"] = Time( + "2020-02-01T00:02:30", format="isot", scale="utc" + ) + sca.meta.observation["exposure"] = 1 + return [wfi_sca1, wfi_sca2, wfi_sca3] + + +@pytest.fixture +def exposure_2(wfi_sca4, wfi_sca5, wfi_sca6): + """Returns a list with models corresponding to a dummy exposure 2.""" + # set the same exposure time for all SCAs + for sca in [wfi_sca4, wfi_sca5, wfi_sca6]: + sca.meta.exposure["start_time"] = Time( + "2020-05-01T00:00:00", format="isot", scale="utc" + ) + sca.meta.exposure["end_time"] = Time( + "2020-05-01T00:02:30", format="isot", scale="utc" + ) + sca.meta.observation["exposure"] = 2 + return [wfi_sca4, wfi_sca5, wfi_sca6] + + +@pytest.fixture +def multiple_exposures(exposure_1, exposure_2): + """Returns a list with all the datamodels from exposure 1 and 2.""" + exposure_1.extend(exposure_2) + return exposure_1 + + +def test_resampledata_init(exposure_1): + """Test that ResampleData can set initial values.""" + input_models = exposure_1 + output = "output.asdf" + single = True + blendheaders = False + pixfrac = 0.8 + kernel = "turbo" + fillval = 0.0 + wht_type = "exp" + good_bits = 1 + pscale_ratio = 0.5 + pscale = 0.1 + kwargs = {"in_memory": False} + + resample_data = ResampleData( + input_models, + output=output, + single=single, + blendheaders=blendheaders, + pixfrac=pixfrac, + kernel=kernel, + fillval=fillval, + wht_type=wht_type, + good_bits=good_bits, + pscale_ratio=pscale_ratio, + pscale=pscale, + **kwargs, + ) + + # Assert + assert resample_data.input_models == input_models + assert resample_data.output_filename == output + assert resample_data.pscale_ratio == pscale_ratio + assert resample_data.single == single + assert resample_data.blendheaders == blendheaders + assert resample_data.pixfrac == pixfrac + assert resample_data.kernel == kernel + assert resample_data.fillval == fillval + assert resample_data.weight_type == wht_type + assert resample_data.good_bits == good_bits + assert resample_data.in_memory == kwargs["in_memory"] + + +def test_resampledata_init_default(exposure_1): + """Test instantiating ResampleData with default values.""" + input_models = exposure_1 + # Default parameter values + + resample_data = ResampleData(input_models) + + # Assert + assert resample_data.input_models == input_models + assert resample_data.output_filename is None + assert resample_data.pscale_ratio == 1.0 + assert not resample_data.single + assert resample_data.blendheaders + assert resample_data.pixfrac == 1.0 + assert resample_data.kernel == "square" + assert resample_data.fillval == "INDEF" + assert resample_data.weight_type == "ivm" + assert resample_data.good_bits == 0 + assert resample_data.in_memory + + +@pytest.mark.parametrize("input_models", [None, list(), [""], ModelContainer()]) +def test_resampledata_init_invalid_input(input_models): + """Test that ResampleData will raise an exception on invalid inputs.""" + with pytest.raises(Exception) as exec_info: + ResampleData(input_models) + + assert type(exec_info.value) == ValueError + + +def test_resampledata_do_drizzle_many_to_one_default_no_rotation_single_exposure( + exposure_1, +): + """Test that output WCS encompass the entire combined input WCS region + by checking that its extrema fall within the output WCS footprint. + + N.B.: since we are not providing the rotation parameter for the + resample_utils.make_output_wcs method, the output WCS will have + the same orientation (i.e. same PA) as the detector axes. + """ + + input_models = ModelContainer(exposure_1) + resample_data = ResampleData(input_models) + + output_models = resample_data.resample_many_to_one() + + output_min_value = np.min(output_models[0].meta.wcs.footprint()) + output_max_value = np.max(output_models[0].meta.wcs.footprint()) + + input_wcs_list = [sca.meta.wcs.footprint() for sca in input_models] + expected_min_value = np.min(np.stack(input_wcs_list)) + expected_max_value = np.max(np.stack(input_wcs_list)) + + # Assert + np.testing.assert_array_less(output_min_value, expected_min_value) + np.testing.assert_(output_max_value > expected_max_value) + + +def test_resampledata_do_drizzle_many_to_one_default_no_rotation_multiple_exposures( + multiple_exposures, +): + """Test that output WCS encompass the entire combined input WCS region + by checking that its extrema fall within the output WCS footprint. + + N.B.: since we are not providing the rotation parameter for the + resample_utils.make_output_wcs method, the output WCS will have + the same orientation (i.e. same PA) as the detector axes. + """ + + input_models = ModelContainer(multiple_exposures) + resample_data = ResampleData(input_models) + + output_models = resample_data.resample_many_to_one() + + output_min_value = np.min(output_models[0].meta.wcs.footprint()) + output_max_value = np.max(output_models[0].meta.wcs.footprint()) + + input_wcs_list = [sca.meta.wcs.footprint() for sca in multiple_exposures] + expected_min_value = np.min(np.stack(input_wcs_list)) + expected_max_value = np.max(np.stack(input_wcs_list)) + + # Assert + np.testing.assert_array_less(output_min_value, expected_min_value) + np.testing.assert_(output_max_value > expected_max_value) + + +def test_resampledata_do_drizzle_many_to_one_default_rotation_0(exposure_1): + """Test that output WCS encompass the entire combined input WCS region + by checking that the output WCS footprint vertices are close to the + expected vertices for the combined input WCS footprint. + + N.B.: in this case, rotation=0 will create a WCS that will be oriented North up. + """ + + input_models = ModelContainer(exposure_1) + resample_data = ResampleData(input_models, **{"rotation": 0}) + + output_models = resample_data.resample_many_to_one() + + output_min_value = np.min(output_models[0].meta.wcs.footprint()) + output_max_value = np.max(output_models[0].meta.wcs.footprint()) + + input_wcs_list = [sca.meta.wcs.footprint() for sca in exposure_1] + expected_min_value = np.min(np.stack(input_wcs_list)) + expected_max_value = np.max(np.stack(input_wcs_list)) + + # Assert + np.testing.assert_allclose(output_min_value, expected_min_value) + np.testing.assert_allclose(output_max_value, expected_max_value) + + +def test_resampledata_do_drizzle_many_to_one_default_rotation_0_multiple_exposures( + multiple_exposures, +): + """Test that output WCS encompass the entire combined input WCS region + by checking that the output WCS footprint vertices are close to the + expected vertices for the combined input WCS footprint. + + N.B.: in this case, rotation=0 will create a WCS that will be oriented North up. + """ + + input_models = ModelContainer(multiple_exposures) + resample_data = ResampleData(input_models, **{"rotation": 0}) + + output_models = resample_data.resample_many_to_one() + + output_min_value = np.min(output_models[0].meta.wcs.footprint()) + output_max_value = np.max(output_models[0].meta.wcs.footprint()) + + input_wcs_list = [sca.meta.wcs.footprint() for sca in multiple_exposures] + expected_min_value = np.min(np.stack(input_wcs_list)) + expected_max_value = np.max(np.stack(input_wcs_list)) + + # Assert + np.testing.assert_allclose(output_min_value, expected_min_value) + np.testing.assert_allclose(output_max_value, expected_max_value) + + +def test_resampledata_do_drizzle_many_to_one_single_input_model(wfi_sca1): + """Test that the output of resample from a single input file creates a WCS + footprint vertices that are close to the input WCS footprint's vertices.""" + + input_models = ModelContainer([wfi_sca1]) + resample_data = ResampleData( + input_models, output=wfi_sca1.meta.filename, **{"rotation": 0} + ) + + output_models = resample_data.resample_many_to_one() + + flat_1 = np.sort(wfi_sca1.meta.wcs.footprint().flatten()) + flat_2 = np.sort(output_models[0].meta.wcs.footprint().flatten()) + + # Assert + assert len(output_models) == 1 + assert output_models[0].meta.filename == resample_data.output_filename + np.testing.assert_allclose(flat_1, flat_2) + + +def test_update_exposure_times_different_sca_same_exposure(exposure_1): + """Test that update_exposure_times is properly updating the exposure parameters + for a set of different SCAs belonging to the same exposure.""" + input_models = ModelContainer(exposure_1) + resample_data = ResampleData(input_models) + + output_model = resample_data.blank_output.copy() + output_model.meta["resample"] = {} + + resample_data.update_exposure_times(output_model) + + assert ( + output_model.meta.resample.product_exposure_time + == exposure_1[0].meta.exposure.exposure_time + ) + assert ( + output_model.meta.exposure.exposure_time + == exposure_1[0].meta.exposure.exposure_time + ) + assert ( + output_model.meta.exposure.start_time + == exposure_1[0].meta.exposure.start_time + ) + assert ( + output_model.meta.exposure.end_time + == exposure_1[0].meta.exposure.end_time + ) + + +def test_update_exposure_times_same_sca_different_exposures( + exposure_1, exposure_2 +): + """Test that update_exposure_times is properly updating the exposure parameters + for a set of the same SCA but belonging to different exposures.""" + input_models = ModelContainer([exposure_1[0], exposure_2[0]]) + resample_data = ResampleData(input_models) + + output_model = resample_data.blank_output.copy() + output_model.meta["resample"] = {} + + resample_data.update_exposure_times(output_model) + + assert len(resample_data.input_models.models_grouped) == 2 + + assert output_model.meta.exposure.exposure_time == sum( + x.meta.exposure.exposure_time for x in input_models + ) + + assert output_model.meta.exposure.start_time == min( + x.meta.exposure.start_time for x in input_models + ) + + assert output_model.meta.exposure.end_time == max( + x.meta.exposure.end_time for x in input_models + ) + + assert output_model.meta.resample.product_exposure_time == sum( + x.meta.exposure.exposure_time for x in input_models + ) + + +@pytest.mark.parametrize( + "name", + ["var_rnoise", "var_poisson", "var_flat"], +) +def test_resample_variance_array(wfi_sca1, wfi_sca4, name): + """Test that the mean value for the variance array lies within 1% of the + expectation.""" + input_models = ModelContainer([wfi_sca1, wfi_sca4]) + resample_data = ResampleData(input_models, **{"rotation": 0}) + + output_model = resample_data.blank_output.copy() + output_model.meta["resample"] = {} + driz = gwcs_drizzle.GWCSDrizzle( + output_model, + pixfrac=resample_data.pixfrac, + kernel=resample_data.kernel, + fillval=resample_data.fillval, + ) + [driz.add_image(x.data, x.meta.wcs) for x in resample_data.input_models] + + resample_data.resample_variance_array(name, output_model) + + # combined variance is inversely proportional to the number of "measurements" + expected_combined_variance_value = np.nanmean( + [getattr(x, name) for x in input_models] + ) / len(input_models) + + np.isclose( + np.nanmean(getattr(output_model, name)).value, + expected_combined_variance_value, + atol=0.01, + ) + + +def test_custom_wcs_input_small_overlap_no_rotation(wfi_sca1, wfi_sca3): + """Test that resample can create a proper output in the edge case where the + desired output WCS does not encompass the entire input datamodel but, instead, have + just a small overlap.""" + input_models = ModelContainer([wfi_sca1]) + resample_data = ResampleData( + input_models, + **{"output_wcs": wfi_sca3.meta.wcs, "rotation": 0}, + ) + + output_models = resample_data.resample_many_to_one() + + # pixel scale in RA (N.B.: there's no shift in Dec.) + pixel_scale = np.abs( + wfi_sca3.meta.wcs(0, 0)[0] - wfi_sca3.meta.wcs(1, 0)[0] + ) + # overlap size in RA (N.B.: there's no shift in Dec.) + ra_overlap_size = np.ceil( + input_models[0].shape[0] + - np.abs(input_models[0].meta.wcs(0, 0)[0] - wfi_sca3.meta.wcs(0, 0)[0]) + / pixel_scale + ) + # determine the size of the region in the output that contains data + # (which should have come from the overlap with the input datamodel) + output_nonzero_region = np.nonzero(output_models[0].data) + ra_output_nonzero_size = np.ceil(len(set(output_nonzero_region[1]))) + + assert ra_output_nonzero_size == ra_overlap_size + + np.testing.assert_allclose( + output_models[0].meta.wcs(0, 0), wfi_sca3.meta.wcs(0, 0) + ) + + +def test_custom_wcs_input_entire_field_no_rotation(multiple_exposures): + """Test that resample can create a proper output that encompasses the entire + combined FOV of the input datamodels.""" + input_models = ModelContainer(multiple_exposures) + # create output WCS encompassing the entire exposure FOV + output_wcs = resample_utils.make_output_wcs( + input_models, + rotation=0, + ) + resample_data = ResampleData( + input_models, + **{"output_wcs": output_wcs}, + ) + + output_models = resample_data.resample_many_to_one() + + output_min_value = np.min(output_models[0].meta.wcs.footprint()) + output_max_value = np.max(output_models[0].meta.wcs.footprint()) + + input_wcs_list = [sca.meta.wcs.footprint() for sca in multiple_exposures] + expected_min_value = np.min(np.stack(input_wcs_list)) + expected_max_value = np.max(np.stack(input_wcs_list)) + + np.testing.assert_allclose(output_min_value, expected_min_value) + np.testing.assert_allclose(output_max_value, expected_max_value) diff --git a/romancal/resample/tests/test_resample_step.py b/romancal/resample/tests/test_resample_step.py new file mode 100644 index 000000000..dd771a95f --- /dev/null +++ b/romancal/resample/tests/test_resample_step.py @@ -0,0 +1,281 @@ +import pytest +import numpy as np +from astropy import units as u +from gwcs import WCS +from astropy.modeling import models +from astropy.time import Time +from astropy import coordinates as coord +from gwcs import coordinate_frames as cf +from roman_datamodels import datamodels, maker_utils +from romancal.resample import ResampleStep +from romancal.resample import gwcs_drizzle, resample_utils +from romancal.datamodels import ModelContainer +from asdf import AsdfFile + + +class MockModel: + def __init__( + self, pixelarea_steradians, pixelarea_arcsecsq, pixel_scale_ratio + ): + self.meta = MockMeta( + pixelarea_steradians, pixelarea_arcsecsq, pixel_scale_ratio + ) + + +class MockMeta: + def __init__( + self, pixelarea_steradians, pixelarea_arcsecsq, pixel_scale_ratio + ): + self.photometry = MockPhotometry( + pixelarea_steradians, pixelarea_arcsecsq + ) + self.resample = MockResample(pixel_scale_ratio) + + +class MockPhotometry: + def __init__(self, pixelarea_steradians, pixelarea_arcsecsq): + self.pixelarea_steradians = pixelarea_steradians + self.pixelarea_arcsecsq = pixelarea_arcsecsq + + +class MockResample: + def __init__(self, pixel_scale_ratio): + self.pixel_scale_ratio = pixel_scale_ratio + + +class Mosaic: + def __init__(self, fiducial_world, pscale, shape, filename, n_images): + self.fiducial_world = fiducial_world + self.pscale = pscale + self.shape = shape + self.filename = filename + self.n_images = n_images + + def create_mosaic(self): + """ + Create a dummy L3 datamodel given the coordinates of the fiducial point, + a pixel scale, and the image shape and filename. + + Returns + ------- + datamodels.MosaicModel + An L3 MosaicModel datamodel. + """ + l3 = maker_utils.mk_level3_mosaic( + shape=self.shape, + n_images=self.n_images, + ) + # data from WFISim simulation of SCA #01 + l3.meta.filename = self.filename + l3.meta["wcs"] = create_wcs_object_without_distortion( + fiducial_world=self.fiducial_world, + pscale=self.pscale, + shape=self.shape, + ) + l3.meta.wcs.forward_transform + return datamodels.MosaicModel(l3) + + +def create_wcs_object_without_distortion( + fiducial_world, pscale, shape, **kwargs +): + """ + Create a simple WCS object without either distortion or rotation. + + Parameters + ---------- + fiducial_world : tuple + A pair of values corresponding to the fiducial's world coordinate. + pscale : tuple + A pair of values corresponding to the pixel scale in each axis. + shape : tuple + A pair of values specifying the dimensions of the WCS object. + + Returns + ------- + gwcs.WCS + A gwcs.WCS object. + """ + # components of the model + shift = models.Shift() & models.Shift() + affine = models.AffineTransformation2D( + matrix=[[1, 0], [0, 1]], translation=[0, 0], name="pc_rotation_matrix" + ) + scale = models.Scale(pscale[0]) & models.Scale(pscale[1]) + tan = models.Pix2Sky_TAN() + celestial_rotation = models.RotateNative2Celestial( + fiducial_world[0], + fiducial_world[1], + 180, + ) + + # transforms between frames + # detector -> sky + det2sky = shift | affine | scale | tan | celestial_rotation + det2sky.name = "linear_transform" + + # frames + detector_frame = cf.Frame2D( + name="detector", + axes_order=(0, 1), + axes_names=("x", "y"), + unit=(u.pix, u.pix), + ) + sky_frame = cf.CelestialFrame( + reference_frame=coord.FK5(), name="fk5", unit=(u.deg, u.deg) + ) + + pipeline = [ + (detector_frame, det2sky), + (sky_frame, None), + ] + + wcs_obj = WCS(pipeline) + + wcs_obj.bounding_box = kwargs.get( + "bounding_box", + ( + (-0.5, shape[-1] - 0.5), + (-0.5, shape[-2] - 0.5), + ), + ) + + wcs_obj.pixel_shape = shape[::-1] + wcs_obj.array_shape = shape + + return wcs_obj + + +@pytest.fixture +def asdf_wcs_file(): + def _create_asdf_wcs_file(tmp_path, pixel_shape, bounding_box): + file_path = tmp_path / "wcs.asdf" + wcs_data = create_wcs_object_without_distortion( + (10, 0), + (0.000031, 0.000031), + (100, 100), + **{"pixel_shape": pixel_shape, "bounding_box": bounding_box}, + ) + wcs = {"wcs": wcs_data} + with AsdfFile(wcs) as af: + af.write_to(file_path) + return str(file_path) + + return _create_asdf_wcs_file + + +@pytest.mark.parametrize( + "vals, name, min_vals, expected", + [ + ([1, 2], "list1", None, [1, 2]), + ([None, None], "list2", None, None), + ([1, 2], "list4", [0, 0], [1, 2]), + ], +) +def test_check_list_pars_valid(vals, name, min_vals, expected): + step = ResampleStep() + + result = step._check_list_pars(vals, name, min_vals) + assert result == expected + + +@pytest.mark.parametrize( + "vals, name, min_vals", + [ + ([1, None], "list3", None), # One value is None + ([1, 2], "list5", [3, 3]), # Values do not meet minimum requirements + ([1, 2, 3], "list6", None), # Invalid number of elements + ], +) +def test_check_list_pars_exception(vals, name, min_vals): + step = ResampleStep() + with pytest.raises(ValueError): + step._check_list_pars(vals, name, min_vals) + + +def test_load_custom_wcs_no_file(): + step = ResampleStep() + result = step._load_custom_wcs(None, (512, 512)) + assert result is None + + +def test_load_custom_wcs_missing_output_shape(asdf_wcs_file): + with pytest.raises(ValueError): + step = ResampleStep() + step._load_custom_wcs(asdf_wcs_file, None) + + +def test_load_custom_wcs_invalid_file(tmp_path): + step = ResampleStep() + invalid_file = tmp_path / "invalid.asdf" + with open(invalid_file, "w") as f: + f.write("invalid asdf file") + + with pytest.raises(ValueError): + step._load_custom_wcs(str(invalid_file), (512, 512)) + + +@pytest.mark.parametrize( + "vals, name, min_vals, expected", + [ + ([1, 2], "test", [0, 0], [1, 2]), + ([None, None], "test", [0, 0], None), + ([0, 0], "test", [0, 0], [0, 0]), + ([1, 1], "test", [0, 0], [1, 1]), + ([0, 1], "test", [0, 0], [0, 1]), + ([1, 0], "test", [0, 0], [1, 0]), + ], +) +def test_check_list_pars(vals, name, min_vals, expected): + step = ResampleStep() + + result = step._check_list_pars(vals, name, min_vals) + assert result == expected + + +@pytest.mark.parametrize( + "vals, name, min_vals", + [ + ([None, 2], "test", [0, 0]), + ([1, None], "test", [0, 0]), + ([1], "test", [0, 0]), + ([1, 2, 3], "test", [0, 0]), + ([None, None, None], "test", [0, 0]), + ([1, 2], "test", [2, 2]), + ], +) +def test_check_list_pars_exception(vals, name, min_vals): + step = ResampleStep() + + with pytest.raises(ValueError): + step._check_list_pars(vals, name, min_vals) + + +@pytest.mark.parametrize( + "pixelarea_steradians, pixelarea_arcsecsq, pixel_scale_ratio, expected_steradians, expected_arcsecsq", + [ + # Happy path tests + (1.0, 1.0, 2.0, 4.0, 4.0), + (2.0, 2.0, 0.5, 0.5, 0.5), + (0.0, 0.0, 2.0, 0.0, 0.0), + (1.0, 1.0, 0.0, 0.0, 0.0), + (None, 1.0, 2.0, None, 4.0), + (1.0, None, 2.0, 4.0, None), + ], +) +def test_update_phot_keywords( + pixelarea_steradians, + pixelarea_arcsecsq, + pixel_scale_ratio, + expected_steradians, + expected_arcsecsq, +): + step = ResampleStep() + model = MockModel( + pixelarea_steradians, pixelarea_arcsecsq, pixel_scale_ratio + ) + + step.update_phot_keywords(model) + + assert model.meta.photometry.pixelarea_steradians == expected_steradians + assert model.meta.photometry.pixelarea_arcsecsq == expected_arcsecsq From 59f1d66dc254c3f8bfd1d6273a753e7072ca8203 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Wed, 23 Aug 2023 13:20:17 -0400 Subject: [PATCH 23/82] Fix `get_crds_parameters` for `ModelContainer` (#846) --- CHANGES.rst | 5 ++++- romancal/datamodels/container.py | 1 - romancal/datamodels/tests/test_datamodels.py | 10 ++-------- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 29991d53d..ff591545c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,7 +1,10 @@ 0.13.0 (unreleased) =================== -- +general +------- + +- Fix bug with ``ModelContainer.get_crds_parameters`` being a property not a method [#846] 0.12.0 (2023-08-18) =================== diff --git a/romancal/datamodels/container.py b/romancal/datamodels/container.py index 404928f4b..4d6652ffe 100644 --- a/romancal/datamodels/container.py +++ b/romancal/datamodels/container.py @@ -513,7 +513,6 @@ def crds_observatory(self): """ return "roman" - @property def get_crds_parameters(self): """ Get parameters used by CRDS to select references for this model. diff --git a/romancal/datamodels/tests/test_datamodels.py b/romancal/datamodels/tests/test_datamodels.py index 3d88853a5..4ce30148e 100644 --- a/romancal/datamodels/tests/test_datamodels.py +++ b/romancal/datamodels/tests/test_datamodels.py @@ -359,17 +359,11 @@ def test_get_crds_parameters(n, obj_type, tmp_path, request): n, obj_type, tmp_path ) - mc = ModelContainer(filepath_list) - - res = mc.get_crds_parameters - - assert isinstance(res, dict) + assert isinstance(ModelContainer(filepath_list).get_crds_parameters(), dict) def test_get_crds_parameters_empty(): - mc = ModelContainer() - - crds_param = mc.get_crds_parameters + crds_param = ModelContainer().get_crds_parameters() assert isinstance(crds_param, dict) assert len(crds_param) == 0 From 4455265eb415e62309dd40cc2aa3beab06058c34 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 28 Aug 2023 22:37:16 -0400 Subject: [PATCH 24/82] [pre-commit.ci] pre-commit autoupdate (#849) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1247b71de..7429c59e6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,7 +32,7 @@ repos: args: ["--py38-plus"] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: 'v0.0.285' + rev: 'v0.0.286' hooks: - id: ruff args: ["--fix"] From adaaaa3dcedca9a89fb849f1d2dba24e936cbbd2 Mon Sep 17 00:00:00 2001 From: Jonathan Eisenhamer Date: Wed, 30 Aug 2023 09:35:57 -0400 Subject: [PATCH 25/82] RCAL-511 Inititial implementation of the Uneven Ramp fitting (#779) Co-authored-by: Jonathan Eisenhamer Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- CHANGES.rst | 6 + romancal/ramp_fitting/ramp_fit_step.py | 337 ++++++++++------- .../ramp_fitting/tests/test_ramp_fit_cas22.py | 312 +++++++++++++++ ...{test_ramp_fit.py => test_ramp_fit_ols.py} | 357 +++++++++--------- 4 files changed, 715 insertions(+), 297 deletions(-) create mode 100644 romancal/ramp_fitting/tests/test_ramp_fit_cas22.py rename romancal/ramp_fitting/tests/{test_ramp_fit.py => test_ramp_fit_ols.py} (59%) diff --git a/CHANGES.rst b/CHANGES.rst index ff591545c..3a9baba42 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,6 +6,12 @@ general - Fix bug with ``ModelContainer.get_crds_parameters`` being a property not a method [#846] +ramp_fitting +------------ + +- Inititial implementation of the Uneven Ramp fitting [#779] + + 0.12.0 (2023-08-18) =================== diff --git a/romancal/ramp_fitting/ramp_fit_step.py b/romancal/ramp_fitting/ramp_fit_step.py index 984f4aed7..1d56d5e80 100644 --- a/romancal/ramp_fitting/ramp_fit_step.py +++ b/romancal/ramp_fitting/ramp_fit_step.py @@ -7,7 +7,7 @@ from roman_datamodels import datamodels as rdd from roman_datamodels import maker_utils from roman_datamodels import stnode as rds -from stcal.ramp_fitting import ramp_fit +from stcal.ramp_fitting import ols_cas22_fit, ramp_fit from romancal.lib import dqflags from romancal.stpipe import RomanStep @@ -15,68 +15,172 @@ log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) - __all__ = ["RampFitStep"] -def create_optional_results_model(input_model, opt_info): +class RampFitStep(RomanStep): + + """ + This step fits a straight line to the value of counts vs. time to + determine the mean count rate for each pixel. """ - Creates the optional output from the computed arrays from ramp_fit. - Parameters - ---------- - input_model : `~roman_datamodels.datamodels.RampModel` - The input data model. - opt_info : tuple - The ramp fitting arrays needed for the ``RampFitOutputModel``. + spec = """ + algorithm = option('ols','ols_cas22',default='ols') # Algorithm to use to fit. + save_opt = boolean(default=False) # Save optional output + opt_name = string(default='') + maximum_cores = option('none','quarter','half','all',default='none') # max number of processes to create + suffix = string(default='rampfit') # Default suffix of results + """ # noqa: E501 - Returns - ------- - opt_model : `~roman_datamodels.datamodels.RampFitOutputModel` - The optional ``RampFitOutputModel`` to be returned from the ramp fit step. - """ - ( - slope, - sigslope, - var_poisson, - var_rnoise, - yint, - sigyint, - pedestal, - weights, - crmag, - ) = opt_info - meta = {} - meta.update(input_model.meta) - crmag.shape = crmag.shape[1:] - crmag.dtype = np.float32 + weighting = "optimal" # Only weighting allowed for OLS - inst = { - "meta": meta, - "slope": u.Quantity(np.squeeze(slope), u.electron / u.s, dtype=slope.dtype), - "sigslope": u.Quantity( - np.squeeze(sigslope), u.electron / u.s, dtype=sigslope.dtype - ), - "var_poisson": u.Quantity( - np.squeeze(var_poisson), u.electron**2 / u.s**2, dtype=var_poisson.dtype - ), - "var_rnoise": u.Quantity( - np.squeeze(var_rnoise), u.electron**2 / u.s**2, dtype=var_rnoise.dtype - ), - "yint": u.Quantity(np.squeeze(yint), u.electron, dtype=yint.dtype), - "sigyint": u.Quantity(np.squeeze(sigyint), u.electron, dtype=sigyint.dtype), - "pedestal": u.Quantity(np.squeeze(pedestal), u.electron, dtype=pedestal.dtype), - "weights": np.squeeze(weights), - "crmag": u.Quantity(crmag, u.electron, dtype=pedestal.dtype), - } + reference_file_types = ["readnoise", "gain"] - out_node = rds.RampFitOutput(inst) - opt_model = rdd.RampFitOutputModel(out_node) - opt_model.meta.filename = input_model.meta.filename + def process(self, input): + with rdd.open(input, mode="rw") as input_model: + # Retrieve reference info + readnoise_filename = self.get_reference_file(input_model, "readnoise") + gain_filename = self.get_reference_file(input_model, "gain") + log.info("Using READNOISE reference file: %s", readnoise_filename) + readnoise_model = rdd.open(readnoise_filename, mode="rw") + log.info("Using GAIN reference file: %s", gain_filename) + gain_model = rdd.open(gain_filename, mode="rw") - return opt_model + # Do the fitting. + algorithm = self.algorithm.lower() + if algorithm == "ols": + out_model = self.ols(input_model, readnoise_model, gain_model) + out_model.meta.cal_step.ramp_fit = "COMPLETE" + elif algorithm == "ols_cas22": + out_model = self.ols_cas22(input_model, readnoise_model, gain_model) + out_model.meta.cal_step.ramp_fit = "COMPLETE" + else: + log.error("Algorithm %s is not supported. Skipping step.") + out_model = input + out_model.meta.cal_step.ramp_fit = "SKIPPED" + return out_model + + def ols(self, input_model, readnoise_model, gain_model): + """Perform Optimal Linear Fitting on evenly-spaced resultants + + The OLS algorithm used is the same used by JWST for it's ramp fitting. + + Parameters + ---------- + input_model : RampModel + Model containing ramps. + + readnoise_model : ReadnoiseRefModel + Model with the read noise reference information. + + gain_model : GainRefModel + Model with the gain reference information. + + Returns + ------- + out_model : ImageModel + Model containing a count-rate image. + """ + max_cores = self.maximum_cores + input_model.data = input_model.data[np.newaxis, :] + input_model.groupdq = input_model.groupdq[np.newaxis, :] + input_model.err = input_model.err[np.newaxis, :] + + log.info(f"Using algorithm = {self.algorithm}") + log.info(f"Using weighting = {self.weighting}") + + buffsize = ramp_fit.BUFSIZE + image_info, integ_info, opt_info, gls_opt_model = ramp_fit.ramp_fit( + input_model, + buffsize, + self.save_opt, + readnoise_model.data.value, + gain_model.data.value, + self.algorithm, + self.weighting, + max_cores, + dqflags.pixel, + ) + readnoise_model.close() + gain_model.close() + # Save the OLS optional fit product, if it exists + if opt_info is not None: + opt_model = create_optional_results_model(input_model, opt_info) + self.save_model(opt_model, "fitopt", output_file=self.opt_name) + + # All pixels saturated, therefore returning an image file with zero data + if image_info is None: + log.info("All pixels are saturated. Returning a zeroed-out image.") + + # Image info order is: data, dq, var_poisson, var_rnoise, err + image_info = ( + np.zeros(input_model.data.shape[2:], dtype=input_model.data.dtype), + input_model.pixeldq + | input_model.groupdq[0][0] + | dqflags.group["SATURATED"], + np.zeros(input_model.err.shape[2:], dtype=input_model.err.dtype), + np.zeros(input_model.err.shape[2:], dtype=input_model.err.dtype), + np.zeros(input_model.err.shape[2:], dtype=input_model.err.dtype), + ) + + out_model = create_image_model(input_model, image_info) + return out_model + + def ols_cas22(self, input_model, readnoise_model, gain_model): + """Peform Optimal Linear Fitting on arbitrarily space resulants + + Parameters + ---------- + input_model : RampModel + Model containing ramps. + + readnoise_model : ReadnoiseRefModel + Model with the read noise reference information. + + gain_model : GainRefModel + Model with the gain reference information. + + Returns + ------- + out_model : ImageModel + Model containing a count-rate image. + """ + resultants = input_model.data.value + dq = input_model.groupdq + read_noise = readnoise_model.data.value + gain = gain_model.data.value + read_pattern = input_model.meta.exposure.read_pattern + read_time = input_model.meta.exposure.frame_time + + # account for the gain + resultants *= gain + read_noise *= gain + + # Fit the ramps + ramppar, rampvar = ols_cas22_fit.fit_ramps_casertano( + resultants, dq, read_noise, read_time, read_pattern=read_pattern + ) + + # Break out the information and fix units + slopes = ramppar[..., 1] + var_rnoise = rampvar[..., 0] + var_poisson = rampvar[..., 1] + err = np.sqrt(var_poisson + var_rnoise) + + # Create the image model + image_info = (slopes, None, var_poisson, var_rnoise, err) + image_model = create_image_model(input_model, image_info) + + # That's all folks + return image_model + + +# ######### +# Utilities +# ######### def create_image_model(input_model, image_info): """ Creates an ImageModel from the computed arrays from ramp_fit. @@ -85,10 +189,9 @@ def create_image_model(input_model, image_info): ---------- input_model : `~roman_datamodels.datamodels.RampModel` Input ``RampModel`` for which the output ``ImageModel`` is created. + image_info : tuple The ramp fitting arrays needed for the ``ImageModel``. - refpix_info : tuple - The reference pixel arrays. Returns ------- @@ -105,6 +208,8 @@ def create_image_model(input_model, image_info): var_rnoise, u.electron**2 / u.s**2, dtype=var_rnoise.dtype ) err = u.Quantity(err, u.electron / u.s, dtype=err.dtype) + if dq is None: + dq = np.zeros(data.shape, dtype="u4") # Create output datamodel # ... and add all keys from input @@ -148,83 +253,59 @@ def create_image_model(input_model, image_info): return im -class RampFitStep(RomanStep): - - """ - This step fits a straight line to the value of counts vs. time to - determine the mean count rate for each pixel. +def create_optional_results_model(input_model, opt_info): """ + Creates the optional output from the computed arrays from ramp_fit. - spec = """ - opt_name = string(default='') - maximum_cores = option('none','quarter','half','all',default='none') # max number of processes to create - save_opt = boolean(default=False) # Save optional output - """ # noqa: E501 - algorithm = "ols" # Only algorithm allowed - - weighting = "optimal" # Only weighting allowed - - reference_file_types = ["readnoise", "gain"] - - def process(self, input): - with rdd.open(input, lazy_load=False) as input_model: - max_cores = self.maximum_cores - readnoise_filename = self.get_reference_file(input_model, "readnoise") - gain_filename = self.get_reference_file(input_model, "gain") - input_model.data = input_model.data[np.newaxis, :] - input_model.groupdq = input_model.groupdq[np.newaxis, :] - input_model.err = input_model.err[np.newaxis, :] - - log.info("Using READNOISE reference file: %s", readnoise_filename) - readnoise_model = rdd.open(readnoise_filename, mode="rw") - log.info("Using GAIN reference file: %s", gain_filename) - gain_model = rdd.open(gain_filename, mode="rw") - - log.info(f"Using algorithm = {self.algorithm}") - log.info(f"Using weighting = {self.weighting}") - - buffsize = ramp_fit.BUFSIZE - image_info, integ_info, opt_info, gls_opt_model = ramp_fit.ramp_fit( - input_model, - buffsize, - self.save_opt, - readnoise_model.data.value, - gain_model.data.value, - self.algorithm, - self.weighting, - max_cores, - dqflags.pixel, - ) - readnoise_model.close() - gain_model.close() - - # Save the OLS optional fit product, if it exists - if opt_info is not None: - opt_model = create_optional_results_model(input_model, opt_info) - self.save_model(opt_model, "fitopt", output_file=self.opt_name) - - # All pixels saturated, therefore returning an image file with zero data - if image_info is None: - log.info("All pixels are saturated. Returning a zeroed-out image.") + Parameters + ---------- + input_model : `~roman_datamodels.datamodels.RampModel` + The input data model. + opt_info : tuple + The ramp fitting arrays needed for the ``RampFitOutputModel``. - # Image info order is: data, dq, var_poisson, var_rnoise, err - image_info = ( - np.zeros(input_model.data.shape[2:], dtype=input_model.data.dtype), - input_model.pixeldq - | input_model.groupdq[0][0] - | dqflags.group["SATURATED"], - np.zeros(input_model.err.shape[2:], dtype=input_model.err.dtype), - np.zeros(input_model.err.shape[2:], dtype=input_model.err.dtype), - np.zeros(input_model.err.shape[2:], dtype=input_model.err.dtype), - ) + Returns + ------- + opt_model : `~roman_datamodels.datamodels.RampFitOutputModel` + The optional ``RampFitOutputModel`` to be returned from the ramp fit step. + """ + ( + slope, + sigslope, + var_poisson, + var_rnoise, + yint, + sigyint, + pedestal, + weights, + crmag, + ) = opt_info + meta = {} + meta.update(input_model.meta) + crmag.shape = crmag.shape[1:] + crmag.dtype = np.float32 - out_model = create_image_model(input_model, image_info) - out_model.meta.cal_step.ramp_fit = "COMPLETE" + inst = { + "meta": meta, + "slope": u.Quantity(np.squeeze(slope), u.electron / u.s, dtype=slope.dtype), + "sigslope": u.Quantity( + np.squeeze(sigslope), u.electron / u.s, dtype=sigslope.dtype + ), + "var_poisson": u.Quantity( + np.squeeze(var_poisson), u.electron**2 / u.s**2, dtype=var_poisson.dtype + ), + "var_rnoise": u.Quantity( + np.squeeze(var_rnoise), u.electron**2 / u.s**2, dtype=var_rnoise.dtype + ), + "yint": u.Quantity(np.squeeze(yint), u.electron, dtype=yint.dtype), + "sigyint": u.Quantity(np.squeeze(sigyint), u.electron, dtype=sigyint.dtype), + "pedestal": u.Quantity(np.squeeze(pedestal), u.electron, dtype=pedestal.dtype), + "weights": np.squeeze(weights), + "crmag": u.Quantity(crmag, u.electron, dtype=pedestal.dtype), + } - if self.save_results: - try: - self.suffix = "rampfit" - except AttributeError: - self["suffix"] = "rampfit" + out_node = rds.RampFitOutput(inst) + opt_model = rdd.RampFitOutputModel(out_node) + opt_model.meta.filename = input_model.meta.filename - return out_model + return opt_model diff --git a/romancal/ramp_fitting/tests/test_ramp_fit_cas22.py b/romancal/ramp_fitting/tests/test_ramp_fit_cas22.py new file mode 100644 index 000000000..b5cc87161 --- /dev/null +++ b/romancal/ramp_fitting/tests/test_ramp_fit_cas22.py @@ -0,0 +1,312 @@ +"""Ramp Fitting tests involving MultiAccum Tables""" +import os + +import numpy as np +import pytest +from astropy import units as u +from astropy.time import Time +from roman_datamodels import maker_utils +from roman_datamodels.datamodels import GainRefModel, RampModel, ReadnoiseRefModel + +from romancal.lib import dqflags +from romancal.ramp_fitting import RampFitStep + +# Currently Roman CRDS servers are not available publicly. +# Remove this test when one is. +pytestmark = pytest.mark.skipif( + os.environ.get("CI") == "true", + reason=( + "Roman CRDS servers are not currently available outside the internal network" + ), +) + +# Read Time in seconds +# For Roman, the read time of the detectors is a fixed value and is currently +# backed into code. Will need to refactor to consider the more general case. +# Used to deconstruct the MultiAccum tables into integration times. +ROMAN_READ_TIME = 3.04 + +DO_NOT_USE = dqflags.group["DO_NOT_USE"] +JUMP_DET = dqflags.group["JUMP_DET"] +SATURATED = dqflags.group["SATURATED"] + +dqflags = { + "DO_NOT_USE": 1, + "SATURATED": 2, + "JUMP_DET": 4, +} + +# Basic resultant +# +# The read pattern is `[[1], [2], [3], [4]]` +# The total expected counts is 7. +# The resultants were generated with +# `romanisim.l1.apportion_counts_to_resultants(counts, read_pattern)`. +SIMPLE_RESULTANTS = np.array( + [ + [[2.0, 2.0], [5.0, 1.0]], + [[4.0, 5.0], [6.0, 2.0]], + [[5.0, 6.0], [7.0, 6.0]], + [[7.0, 7.0], [7.0, 7.0]], + ], + dtype=np.float32, +) +SIMPLE_EXPECTED_DEFAULT = { + "data": np.array( + [[0.52631587, 0.52631587], [0.23026317, 0.7236843]], dtype=np.float32 + ), + "err": np.array( + [[0.24262409, 0.24262409], [0.16048454, 0.28450054]], dtype=np.float32 + ), + "var_poisson": np.array( + [[0.05886428, 0.05886428], [0.02575312, 0.08093839]], dtype=np.float32 + ), + "var_rnoise": np.array( + [[2.164128e-06, 2.164128e-06], [2.164128e-06, 2.164128e-06]], dtype=np.float32 + ), +} +SIMPLE_EXPECTED_GAIN = { + "data": np.array([[2.631579, 2.631579], [1.151316, 3.50926]], dtype=np.float32), + "err": np.array([[0.542564, 0.542564], [0.358915, 0.623119]], dtype=np.float32), + "var_poisson": np.array( + [[0.294321, 0.294321], [0.128766, 0.388223]], dtype=np.float32 + ), + "var_rnoise": np.array( + [[5.410319e-05, 5.410319e-05], [5.410319e-05, 5.476514e-05]], dtype=np.float32 + ), +} +SIMPLE_EXPECTED_RNOISE = { + "data": np.array( + [[0.52631587, 0.52631587], [0.23026317, 0.7236843]], dtype=np.float32 + ), + "err": np.array([[14.712976, 14.712976], [14.711851, 14.713726]], dtype=np.float32), + "var_poisson": np.array( + [[0.05886428, 0.05886428], [0.02575312, 0.08093839]], dtype=np.float32 + ), + "var_rnoise": np.array( + [[216.4128, 216.4128], [216.4128, 216.4128]], dtype=np.float32 + ), +} + + +# ##### +# Tests +# ##### +def test_bad_readpattern(): + """Ensure error is raised on bad readpattern""" + ramp_model, gain_model, readnoise_model = make_data( + SIMPLE_RESULTANTS, 1, 0.01, False + ) + bad_pattern = ramp_model.meta.exposure.read_pattern.data[1:] + ramp_model.meta.exposure.read_pattern = bad_pattern + + with pytest.raises(RuntimeError): + RampFitStep.call( + ramp_model, + algorithm="ols_cas22", + override_gain=gain_model, + override_readnoise=readnoise_model, + ) + + +@pytest.mark.parametrize( + "attribute", + ["data", "err", "var_poisson", "var_rnoise"], + ids=["data", "err", "var_poisson", "var_rnoise"], +) +def test_fits(fit_ramps, attribute): + """Check slopes""" + image_model, expected = fit_ramps + + value = getattr(image_model, attribute).value + np.testing.assert_allclose(value, expected[attribute], 1e-05) + + +# ######## +# Fixtures +# ######## +@pytest.fixture( + scope="module", + params=[ + pytest.param( + (SIMPLE_RESULTANTS, 1, 0.01, False, SIMPLE_EXPECTED_DEFAULT), id="default" + ), # No gain or noise + pytest.param( + (SIMPLE_RESULTANTS, 5, 0.01, False, SIMPLE_EXPECTED_GAIN), id="extragain" + ), # Increase gain + pytest.param( + (SIMPLE_RESULTANTS, 1, 100.0, False, SIMPLE_EXPECTED_RNOISE), + id="extranoise", + ), # Increase noise + ], +) +def fit_ramps(request): + """Test ramp fits""" + resultants, ingain, rnoise, randomize, expected = request.param + ramp_model, gain_model, readnoise_model = make_data( + resultants, ingain, rnoise, randomize + ) + out_model = RampFitStep.call( + ramp_model, + algorithm="ols_cas22", + override_gain=gain_model, + override_readnoise=readnoise_model, + ) + + return out_model, expected + + +# ######### +# Utilities +# ######### +def make_data(resultants, ingain, rnoise, randomize): + """Create test input data + + Parameters + ---------- + resultants : numpy.array.shape(3, xdim, ydim) + The resultant array. + + ingain : int + Gain to apply + + rnoise : float + Noise to apply + + randomize : bool, expected) + Randomize the gain and read noise across pixels. + + Returns + ------- + image, gain, readnoise : ImageModel, GainRefModel, ReadnoiseRefModel.array + Input image and related references + """ + ramp_model = model_from_resultants(resultants) + gain_model, readnoise_model = generate_wfi_reffiles( + ramp_model.shape[1:], ingain=ingain, rnoise=rnoise, randomize=randomize + ) + + return ramp_model, gain_model, readnoise_model + + +def model_from_resultants(resultants, read_pattern=None): + """Create a RampModel from resultants + + Parameters + ---------- + resultants : numpy.array.shape(reads, xdim, ydim) + The resultants to fit. + + read_pattern : [[int[,...]][,...]] + The read patter used to produce the resultants. + If None, presume a basic read pattern + """ + if read_pattern is None: + read_pattern = [[idx + 1] for idx in range(resultants.shape[0])] + + # Full WFI image has reference pixels all around. Add those on. + nrefpixs = 4 + full_wfi = np.ones( + ( + resultants.shape[0], + resultants.shape[1] + (nrefpixs * 2), + resultants.shape[2] + (nrefpixs * 2), + ), + dtype=np.float32, + ) + full_wfi[:, nrefpixs:-nrefpixs, nrefpixs:-nrefpixs] = resultants + shape = full_wfi.shape + + pixdq = np.zeros(shape=shape[1:], dtype=np.uint32) + err = np.zeros(shape=shape, dtype=np.float32) + gdq = np.zeros(shape=shape, dtype=np.uint8) + + dm_ramp = maker_utils.mk_ramp(shape=shape) + dm_ramp.data = u.Quantity(full_wfi, u.DN, dtype=np.float32) + dm_ramp.pixeldq = pixdq + dm_ramp.groupdq = gdq + dm_ramp.err = u.Quantity(err, u.DN, dtype=np.float32) + + dm_ramp.meta.exposure.frame_time = ROMAN_READ_TIME + dm_ramp.meta.exposure.ngroups = shape[0] + dm_ramp.meta.exposure.nframes = 1 + dm_ramp.meta.exposure.groupgap = 0 + + dm_ramp.meta.exposure.read_pattern = read_pattern + + ramp_model = RampModel(dm_ramp) + + return ramp_model + + +def generate_wfi_reffiles(shape, ingain=6, rnoise=0.01, randomize=True): + """Create GainRefModel and ReadnoiseRefModel + + Parameters + ---------- + shape : tuple + Shape of the arrays + + ingain : float + Maximum gain. + + rnoise : flota + Maximum noise + + randomize : bool + Randomize the gain and read noise data. + """ + # Create temporary gain reference file + gain_ref = maker_utils.mk_gain(shape=shape) + + gain_ref["meta"]["instrument"]["detector"] = "WFI01" + gain_ref["meta"]["instrument"]["name"] = "WFI" + gain_ref["meta"]["reftype"] = "GAIN" + gain_ref["meta"]["useafter"] = Time("2022-01-01T11:11:11.111") + + if randomize: + gain_ref["data"] = u.Quantity( + (np.random.random(shape) * 0.5).astype(np.float32) * ingain, + u.electron / u.DN, + dtype=np.float32, + ) + else: + gain_ref["data"] = u.Quantity( + np.ones(shape).astype(np.float32) * ingain, + u.electron / u.DN, + dtype=np.float32, + ) + gain_ref["dq"] = np.zeros(shape, dtype=np.uint16) + gain_ref["err"] = u.Quantity( + (np.random.random(shape) * 0.05).astype(np.float32), + u.electron / u.DN, + dtype=np.float32, + ) + + gain_ref_model = GainRefModel(gain_ref) + + # Create temporary readnoise reference file + rn_ref = maker_utils.mk_readnoise(shape=shape) + rn_ref["meta"]["instrument"]["detector"] = "WFI01" + rn_ref["meta"]["instrument"]["name"] = "WFI" + rn_ref["meta"]["reftype"] = "READNOISE" + rn_ref["meta"]["useafter"] = Time("2022-01-01T11:11:11.111") + + rn_ref["meta"]["exposure"]["type"] = "WFI_IMAGE" + rn_ref["meta"]["exposure"]["frame_time"] = 666 + + if randomize: + rn_ref["data"] = u.Quantity( + (np.random.random(shape) * rnoise).astype(np.float32), + u.DN, + dtype=np.float32, + ) + else: + rn_ref["data"] = u.Quantity( + np.ones(shape).astype(np.float32) * rnoise, u.DN, dtype=np.float32 + ) + + rn_ref_model = ReadnoiseRefModel(rn_ref) + + # return gainfile, readnoisefile + return gain_ref_model, rn_ref_model diff --git a/romancal/ramp_fitting/tests/test_ramp_fit.py b/romancal/ramp_fitting/tests/test_ramp_fit_ols.py similarity index 59% rename from romancal/ramp_fitting/tests/test_ramp_fit.py rename to romancal/ramp_fitting/tests/test_ramp_fit_ols.py index a19b99c28..f9f4e1ad5 100644 --- a/romancal/ramp_fitting/tests/test_ramp_fit.py +++ b/romancal/ramp_fitting/tests/test_ramp_fit_ols.py @@ -15,7 +15,14 @@ from romancal.lib import dqflags from romancal.ramp_fitting import RampFitStep -RNG = np.random.default_rng(619) +# Currently Roman CRDS servers are not available publicly. +# Remove this test when one is. +pytestmark = pytest.mark.skipif( + os.environ.get("CI") == "true", + reason=( + "Roman CRDS servers are not currently available outside the internal network" + ), +) MAXIMUM_CORES = ["none", "quarter", "half", "all"] @@ -30,206 +37,110 @@ } -def generate_ramp_model(shape, deltatime=1): - data = u.Quantity( - (RNG.uniform(size=shape) * 0.5).astype(np.float32), u.DN, dtype=np.float32 - ) - err = u.Quantity( - (RNG.uniform(size=shape) * 0.0001).astype(np.float32), u.DN, dtype=np.float32 - ) - pixdq = np.zeros(shape=shape[1:], dtype=np.uint32) - gdq = np.zeros(shape=shape, dtype=np.uint8) - - dm_ramp = maker_utils.mk_ramp(shape=shape) - dm_ramp.data = u.Quantity(data, u.DN, dtype=np.float32) - dm_ramp.pixeldq = pixdq - dm_ramp.groupdq = gdq - dm_ramp.err = u.Quantity(err, u.DN, dtype=np.float32) - - dm_ramp.meta.exposure.frame_time = deltatime - dm_ramp.meta.exposure.group_time = deltatime - dm_ramp.meta.exposure.ngroups = shape[0] - dm_ramp.meta.exposure.nframes = 1 - dm_ramp.meta.exposure.groupgap = 0 - - ramp_model = RampModel(dm_ramp) - - return ramp_model - - -def generate_wfi_reffiles(shape, ingain=6): - # Create temporary gain reference file - gain_ref = maker_utils.mk_gain(shape=shape) - - gain_ref["meta"]["instrument"]["detector"] = "WFI01" - gain_ref["meta"]["instrument"]["name"] = "WFI" - gain_ref["meta"]["reftype"] = "GAIN" - gain_ref["meta"]["useafter"] = Time("2022-01-01T11:11:11.111") - - gain_ref["data"] = u.Quantity( - (RNG.uniform(size=shape) * 0.5).astype(np.float32) * ingain, - u.electron / u.DN, - dtype=np.float32, - ) - gain_ref["dq"] = np.zeros(shape, dtype=np.uint16) - gain_ref["err"] = u.Quantity( - (RNG.uniform(size=shape) * 0.05).astype(np.float32), - u.electron / u.DN, - dtype=np.float32, - ) - - gain_ref_model = GainRefModel(gain_ref) - - # Create temporary readnoise reference file - rn_ref = maker_utils.mk_readnoise(shape=shape) - rn_ref["meta"]["instrument"]["detector"] = "WFI01" - rn_ref["meta"]["instrument"]["name"] = "WFI" - rn_ref["meta"]["reftype"] = "READNOISE" - rn_ref["meta"]["useafter"] = Time("2022-01-01T11:11:11.111") - - rn_ref["meta"]["exposure"]["type"] = "WFI_IMAGE" - rn_ref["meta"]["exposure"]["frame_time"] = 666 - - rn_ref["data"] = u.Quantity( - (RNG.uniform(size=shape) * 0.01).astype(np.float32), u.DN, dtype=np.float32 - ) - - rn_ref_model = ReadnoiseRefModel(rn_ref) - - # return gainfile, readnoisefile - return gain_ref_model, rn_ref_model - - -@pytest.mark.skipif( - os.environ.get("CI") == "true", - reason=( - "Roman CRDS servers are not currently available outside the internal network" - ), -) -@pytest.mark.parametrize("max_cores", MAXIMUM_CORES) -def test_one_group_small_buffer_fit_ols(max_cores): - ingain = 1.0 - deltatime = 1 - ngroups = 1 - xsize = 20 - ysize = 20 - shape = (ngroups, xsize, ysize) - - override_gain, override_readnoise = generate_wfi_reffiles(shape[1:], ingain) - - model1 = generate_ramp_model(shape, deltatime) - - model1.data[0, 15, 10] = 10.0 * model1.data.unit # add single CR - - out_model = RampFitStep.call( - model1, - override_gain=override_gain, - override_readnoise=override_readnoise, - maximum_cores=max_cores, - ) - - data = out_model.data.value - - # Index changes due to trimming of reference pixels - np.testing.assert_allclose(data[11, 6], 10, 1e-6) - - -@pytest.mark.skipif( - os.environ.get("CI") == "true", - reason=( - "Roman CRDS servers are not currently available outside the internal network" - ), -) -def test_multicore_ramp_fit_match(): - ingain = 1.0 - deltatime = 1 - ngroups = 4 - xsize = 20 - ysize = 20 - shape = (ngroups, xsize, ysize) - - override_gain, override_readnoise = generate_wfi_reffiles(shape[1:], ingain) - - model1 = generate_ramp_model(shape, deltatime) +def test_ols_multicore_ramp_fit_match(make_data): + """Test various core amount calculation""" + model, override_gain, override_readnoise = make_data # gain or read noise are also modified in place in an important way (!) # so we make copies here so that we can get agreement. out_model = RampFitStep.call( - model1.copy(), # model1 is modified in place now. - override_gain=override_gain.copy(), - override_readnoise=override_readnoise.copy(), + model.copy(), # model1 is modified in place now. + algorithm="ols", maximum_cores="none", + override_gain=override_gain, + override_readnoise=override_readnoise, ) all_out_model = RampFitStep.call( - model1.copy(), # model1 is modified in place now. - override_gain=override_gain.copy(), - override_readnoise=override_readnoise.copy(), + model.copy(), # model1 is modified in place now. + algorithm="ols", maximum_cores="all", + override_gain=override_gain, + override_readnoise=override_readnoise, ) # Original ramp parameters - np.testing.assert_allclose(out_model.data, all_out_model.data, 1e-6) - np.testing.assert_allclose(out_model.err, all_out_model.err, 1e-6) - np.testing.assert_allclose(out_model.amp33, all_out_model.amp33, 1e-6) + np.testing.assert_allclose(out_model.data, all_out_model.data, 1e-6, 1e-6) + np.testing.assert_allclose(out_model.err, all_out_model.err, 1e-6, 1e-6) + np.testing.assert_allclose(out_model.amp33, all_out_model.amp33, 1e-6, 1e-6) np.testing.assert_allclose( - out_model.border_ref_pix_left, all_out_model.border_ref_pix_left, 1e-6 + out_model.border_ref_pix_left, all_out_model.border_ref_pix_left, 1e-6, 1e-6 ) np.testing.assert_allclose( - out_model.border_ref_pix_right, all_out_model.border_ref_pix_right, 1e-6 + out_model.border_ref_pix_right, all_out_model.border_ref_pix_right, 1e-6, 1e-6 ) np.testing.assert_allclose( - out_model.border_ref_pix_top, all_out_model.border_ref_pix_top, 1e-6 + out_model.border_ref_pix_top, all_out_model.border_ref_pix_top, 1e-6, 1e-6 ) np.testing.assert_allclose( - out_model.border_ref_pix_bottom, all_out_model.border_ref_pix_bottom, 1e-6 + out_model.border_ref_pix_bottom, all_out_model.border_ref_pix_bottom, 1e-6, 1e-6 ) np.testing.assert_allclose( - out_model.dq_border_ref_pix_left, all_out_model.dq_border_ref_pix_left, 1e-6 + out_model.dq_border_ref_pix_left, + all_out_model.dq_border_ref_pix_left, + 1e-6, + 1e-6, ) np.testing.assert_allclose( - out_model.dq_border_ref_pix_right, all_out_model.dq_border_ref_pix_right, 1e-6 + out_model.dq_border_ref_pix_right, + all_out_model.dq_border_ref_pix_right, + 1e-6, + 1e-6, ) np.testing.assert_allclose( - out_model.dq_border_ref_pix_top, all_out_model.dq_border_ref_pix_top, 1e-6 + out_model.dq_border_ref_pix_top, all_out_model.dq_border_ref_pix_top, 1e-6, 1e-6 ) np.testing.assert_allclose( - out_model.dq_border_ref_pix_bottom, all_out_model.dq_border_ref_pix_bottom, 1e-6 + out_model.dq_border_ref_pix_bottom, + all_out_model.dq_border_ref_pix_bottom, + 1e-6, + 1e-6, ) # New rampfit parameters - np.testing.assert_allclose(out_model.var_poisson, all_out_model.var_poisson, 1e-6) - np.testing.assert_allclose(out_model.var_rnoise, all_out_model.var_rnoise, 1e-6) + np.testing.assert_allclose( + out_model.var_poisson, all_out_model.var_poisson, 1e-6, 1e-6 + ) + np.testing.assert_allclose( + out_model.var_rnoise, all_out_model.var_rnoise, 1e-6, 1e-6 + ) -@pytest.mark.skipif( - os.environ.get("CI") == "true", - reason=( - "Roman CRDS servers are not currently available outside the internal network" - ), -) +@pytest.mark.parametrize("make_data", [(1, 1, 1, 20, 20)], indirect=True) @pytest.mark.parametrize("max_cores", MAXIMUM_CORES) -def test_saturated_ramp_fit(max_cores): - ingain = 1.0 - deltatime = 1 - ngroups = 4 - xsize = 20 - ysize = 20 - shape = (ngroups, xsize, ysize) +def test_ols_one_group_small_buffer_fit(max_cores, make_data): + model, override_gain, override_readnoise = make_data + + model.data[0, 15, 10] = 10.0 * model.data.unit # add single CR + + out_model = RampFitStep.call( + model, + algorithm="ols", + maximum_cores=max_cores, + override_gain=override_gain, + override_readnoise=override_readnoise, + ) + + data = out_model.data.value + + # Index changes due to trimming of reference pixels + np.testing.assert_allclose(data[11, 6], -1.0e-5, 1e-5) + - # Create input model - override_gain, override_readnoise = generate_wfi_reffiles(shape[1:], ingain) - model1 = generate_ramp_model(shape, deltatime) +@pytest.mark.parametrize("max_cores", MAXIMUM_CORES) +def test_ols_saturated_ramp_fit(max_cores, make_data): + model, override_gain, override_readnoise = make_data # Set saturated flag - model1.groupdq = model1.groupdq | SATURATED + model.groupdq = model.groupdq | SATURATED # Run ramp fit step out_model = RampFitStep.call( - model1, + model, + algorithm="ols", + maximum_cores=max_cores, override_gain=override_gain, override_readnoise=override_readnoise, - maximum_cores=max_cores, ) # Test data and error arrays are zeroed out @@ -242,30 +153,30 @@ def test_saturated_ramp_fit(max_cores): assert np.all(np.bitwise_and(out_model.dq, SATURATED) == SATURATED) # Test that original ramp parameters preserved - np.testing.assert_allclose(out_model.amp33, model1.amp33, 1e-6) + np.testing.assert_allclose(out_model.amp33, model.amp33, 1e-6) np.testing.assert_allclose( - out_model.border_ref_pix_left, model1.border_ref_pix_left, 1e-6 + out_model.border_ref_pix_left, model.border_ref_pix_left, 1e-6 ) np.testing.assert_allclose( - out_model.border_ref_pix_right, model1.border_ref_pix_right, 1e-6 + out_model.border_ref_pix_right, model.border_ref_pix_right, 1e-6 ) np.testing.assert_allclose( - out_model.border_ref_pix_top, model1.border_ref_pix_top, 1e-6 + out_model.border_ref_pix_top, model.border_ref_pix_top, 1e-6 ) np.testing.assert_allclose( - out_model.border_ref_pix_bottom, model1.border_ref_pix_bottom, 1e-6 + out_model.border_ref_pix_bottom, model.border_ref_pix_bottom, 1e-6 ) np.testing.assert_allclose( - out_model.dq_border_ref_pix_left, model1.dq_border_ref_pix_left, 1e-6 + out_model.dq_border_ref_pix_left, model.dq_border_ref_pix_left, 1e-6 ) np.testing.assert_allclose( - out_model.dq_border_ref_pix_right, model1.dq_border_ref_pix_right, 1e-6 + out_model.dq_border_ref_pix_right, model.dq_border_ref_pix_right, 1e-6 ) np.testing.assert_allclose( - out_model.dq_border_ref_pix_top, model1.dq_border_ref_pix_top, 1e-6 + out_model.dq_border_ref_pix_top, model.dq_border_ref_pix_top, 1e-6 ) np.testing.assert_allclose( - out_model.dq_border_ref_pix_bottom, model1.dq_border_ref_pix_bottom, 1e-6 + out_model.dq_border_ref_pix_bottom, model.dq_border_ref_pix_bottom, 1e-6 ) # Test that an Image model was returned. @@ -273,3 +184,111 @@ def test_saturated_ramp_fit(max_cores): # Test that the ramp fit step was labeled complete assert out_model.meta.cal_step.ramp_fit == "COMPLETE" + + +# ######## +# fixtures +# ######## +@pytest.fixture +def make_data(request): + """Create test input data + + Parameters + ---------- + request.param : (ingain, deltatime, ngroups, xsize, ysize) + If specified, set the parameters of the created data. + If not specified, defaults are used. + + Returns + ------- + image, gain, readnoise : ImageModel, GainRefModel, ReadnoiseRefModel + Input image and related references + """ + if getattr(request, "param", None): + ingain, deltatime, ngroups, xsize, ysize = request.param + else: + ingain = 1 + deltatime = 1 + ngroups = 4 + xsize = 20 + ysize = 20 + shape = (ngroups, xsize, ysize) + + image = generate_ramp_model(shape, deltatime) + gain, readnoise = generate_wfi_reffiles(shape[1:], ingain) + + return image, gain, readnoise + + +# ######### +# Utilities +# ######### + + +def generate_ramp_model(shape, deltatime=1): + data = u.Quantity( + (np.random.random(shape) * 0.5).astype(np.float32), u.DN, dtype=np.float32 + ) + err = u.Quantity( + (np.random.random(shape) * 0.0001).astype(np.float32), u.DN, dtype=np.float32 + ) + pixdq = np.zeros(shape=shape[1:], dtype=np.uint32) + gdq = np.zeros(shape=shape, dtype=np.uint8) + + dm_ramp = maker_utils.mk_ramp(shape=shape) + dm_ramp.data = u.Quantity(data, u.DN, dtype=np.float32) + dm_ramp.pixeldq = pixdq + dm_ramp.groupdq = gdq + dm_ramp.err = u.Quantity(err, u.DN, dtype=np.float32) + + dm_ramp.meta.exposure.frame_time = deltatime + dm_ramp.meta.exposure.ngroups = shape[0] + dm_ramp.meta.exposure.nframes = 1 + dm_ramp.meta.exposure.groupgap = 0 + + ramp_model = RampModel(dm_ramp) + + return ramp_model + + +def generate_wfi_reffiles(shape, ingain=6): + # Create temporary gain reference file + gain_ref = maker_utils.mk_gain(shape=shape) + + gain_ref["meta"]["instrument"]["detector"] = "WFI01" + gain_ref["meta"]["instrument"]["name"] = "WFI" + gain_ref["meta"]["reftype"] = "GAIN" + gain_ref["meta"]["useafter"] = Time("2022-01-01T11:11:11.111") + + gain_ref["data"] = u.Quantity( + (np.random.random(shape) * 0.5).astype(np.float32) * ingain, + u.electron / u.DN, + dtype=np.float32, + ) + gain_ref["dq"] = np.zeros(shape, dtype=np.uint16) + gain_ref["err"] = u.Quantity( + (np.random.random(shape) * 0.05).astype(np.float32), + u.electron / u.DN, + dtype=np.float32, + ) + + gain_ref_model = GainRefModel(gain_ref) + + # Create temporary readnoise reference file + rn_ref = maker_utils.mk_readnoise(shape=shape) + rn_ref["meta"]["instrument"]["detector"] = "WFI01" + rn_ref["meta"]["instrument"]["name"] = "WFI" + rn_ref["meta"]["reftype"] = "READNOISE" + rn_ref["meta"]["useafter"] = Time("2022-01-01T11:11:11.111") + + rn_ref["meta"]["exposure"]["type"] = "WFI_IMAGE" + rn_ref["meta"]["exposure"]["frame_time"] = 666 + + rn_ref["data"] = u.Quantity( + (np.random.random(shape) * 0.01).astype(np.float32), u.DN, dtype=np.float32 + ) + + rn_ref_model = ReadnoiseRefModel(rn_ref) + + # return gainfile, readnoisefile + return gain_ref_model, rn_ref_model From 9fec5adb3207a22e220bd7c41c0f1e76d47331bf Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Thu, 31 Aug 2023 08:38:49 -0400 Subject: [PATCH 26/82] Fix RTD build (#853) --- .readthedocs.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index e1915f226..15010cb78 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -26,7 +26,6 @@ conda: # Optionally set the version of Python and requirements required to build your docs python: - system_packages: false install: - method: pip path: . From 8d727bb1805b453d6f3a4c5724a359026b4d7036 Mon Sep 17 00:00:00 2001 From: Jonathan Eisenhamer Date: Thu, 31 Aug 2023 15:25:22 -0400 Subject: [PATCH 27/82] fix opening mode for references to be read-only (#854) --- CHANGES.rst | 2 ++ romancal/ramp_fitting/ramp_fit_step.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 3a9baba42..4f454602d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -11,6 +11,8 @@ ramp_fitting - Inititial implementation of the Uneven Ramp fitting [#779] +- Fix opening mode for references to be read-only [#854] + 0.12.0 (2023-08-18) =================== diff --git a/romancal/ramp_fitting/ramp_fit_step.py b/romancal/ramp_fitting/ramp_fit_step.py index 1d56d5e80..4a62a623c 100644 --- a/romancal/ramp_fitting/ramp_fit_step.py +++ b/romancal/ramp_fitting/ramp_fit_step.py @@ -43,9 +43,9 @@ def process(self, input): readnoise_filename = self.get_reference_file(input_model, "readnoise") gain_filename = self.get_reference_file(input_model, "gain") log.info("Using READNOISE reference file: %s", readnoise_filename) - readnoise_model = rdd.open(readnoise_filename, mode="rw") + readnoise_model = rdd.open(readnoise_filename, mode="r") log.info("Using GAIN reference file: %s", gain_filename) - gain_model = rdd.open(gain_filename, mode="rw") + gain_model = rdd.open(gain_filename, mode="r") # Do the fitting. algorithm = self.algorithm.lower() From 8c186dc567c046b1ddec874ee62a398b18ba373f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 4 Sep 2023 21:58:02 -0400 Subject: [PATCH 28/82] [pre-commit.ci] pre-commit autoupdate (#855) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7429c59e6..bcfccac83 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,7 +32,7 @@ repos: args: ["--py38-plus"] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: 'v0.0.286' + rev: 'v0.0.287' hooks: - id: ruff args: ["--fix"] From db4aa8a9c0cac73b9b950ff48dcafaf757924fb6 Mon Sep 17 00:00:00 2001 From: "Brett M. Morris" Date: Mon, 11 Sep 2023 15:41:47 -0400 Subject: [PATCH 29/82] PSF fitting methods (#794) --- .github/workflows/roman_ci.yml | 45 ++- .github/workflows/roman_ci_cron.yaml | 2 +- CHANGES.rst | 2 + pyproject.toml | 5 +- requirements-dev.txt | 1 + romancal/lib/psf.py | 397 +++++++++++++++++++++++++++ romancal/lib/tests/test_psf.py | 179 ++++++++++++ tox.ini | 3 + 8 files changed, 621 insertions(+), 13 deletions(-) create mode 100644 romancal/lib/psf.py create mode 100644 romancal/lib/tests/test_psf.py diff --git a/.github/workflows/roman_ci.yml b/.github/workflows/roman_ci.yml index f01b7f25a..b3cb2486d 100644 --- a/.github/workflows/roman_ci.yml +++ b/.github/workflows/roman_ci.yml @@ -20,14 +20,21 @@ concurrency: cancel-in-progress: true jobs: - crds: - name: retrieve current CRDS context + data: + name: retrieve current CRDS context, and WebbPSF data runs-on: ubuntu-latest env: OBSERVATORY: roman CRDS_SERVER_URL: https://roman-crds-test.stsci.edu - CRDS_PATH: /tmp/crds_cache + CRDS_PATH: /tmp/data + outputs: + context: ${{ steps.context.outputs.pmap }} + path: ${{ steps.path.outputs.path }} + server: ${{ steps.server.outputs.url }} + hash: ${{ steps.data_hash.outputs.hash }} + webbpsf_path: ${{ steps.webbpsf_path.outputs.path }} steps: + # crds: - id: context run: > echo "pmap=$( @@ -40,10 +47,27 @@ jobs: run: echo "path=${{ env.CRDS_PATH }}" >> $GITHUB_OUTPUT - id: server run: echo "url=${{ env.CRDS_SERVER_URL }}" >> $GITHUB_OUTPUT - outputs: - context: ${{ steps.context.outputs.pmap }} - path: ${{ steps.path.outputs.path }} - server: ${{ steps.server.outputs.url }} + # webbpsf: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - id: data + run: | + echo "webbpsf_url=https://stsci.box.com/shared/static/n1fealx9q0m6sdnass6wnyfikvxtc0zz.gz" >> $GITHUB_OUTPUT + echo "path=/tmp/data" >> $GITHUB_OUTPUT + - run: | + mkdir -p ${{ steps.data.outputs.path }} + wget ${{ steps.data.outputs.webbpsf_url }} -O ${{ steps.data.outputs.path }}/minimal-webbpsf-data.tar.gz + cd ${{ steps.data.outputs.path }} + tar -xzvf minimal-webbpsf-data.tar.gz + - id: data_hash + run: echo "hash=${{ steps.data.outputs.hash }}" >> $GITHUB_OUTPUT + - uses: actions/cache@v3 + with: + path: ${{ steps.data.outputs.path }} + key: data-${{ steps.data_hash.outputs.hash }} + - id: webbpsf_path + run: echo "path=${{ steps.data.outputs.path }}/webbpsf-data" >> $GITHUB_OUTPUT check: uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1 with: @@ -53,7 +77,7 @@ jobs: - linux: build-dist test: uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@main - needs: [ crds ] + needs: [ data ] with: setenv: | CRDS_PATH: ${{ needs.crds.outputs.path }} @@ -65,8 +89,9 @@ jobs: DD_GIT_REPOSITORY_URL: ${{ github.repositoryUrl }} DD_GIT_COMMIT_SHA: ${{ github.sha }} DD_GIT_BRANCH: ${{ github.ref_name }} - cache-path: /tmp/crds_cache - cache-key: crds-${{ needs.crds_context.outputs.pmap }} + WEBBPSF_PATH: ${{ needs.data.outputs.webbpsf_path }} + cache-path: ${{ needs.data.outputs.path }} + cache-key: data-${{ needs.data.outputs.hash }} envs: | - linux: py39-oldestdeps-cov pytest-results-summary: true diff --git a/.github/workflows/roman_ci_cron.yaml b/.github/workflows/roman_ci_cron.yaml index c03cc66ad..7b2b2ef7f 100644 --- a/.github/workflows/roman_ci_cron.yaml +++ b/.github/workflows/roman_ci_cron.yaml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-latest env: OBSERVATORY: roman - CRDS_PATH: /tmp/crds_cache + CRDS_PATH: /tmp/data CRDS_SERVER_URL: https://roman-crds-test.stsci.edu steps: - id: context diff --git a/CHANGES.rst b/CHANGES.rst index 4f454602d..7c4d70cfa 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -100,6 +100,8 @@ general - Add ``dev`` install option. [#835] +- Add PSF photometry methods [#794] + 0.11.0 (2023-05-31) =================== diff --git a/pyproject.toml b/pyproject.toml index 79586e30b..c94e5cbc5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ dependencies = [ 'gwcs >=0.18.1', 'jsonschema >=4.8', 'numpy >=1.22', - 'photutils >=1.6.0', + 'photutils @ git+https://github.com/astropy/photutils.git', 'pyparsing >=2.4.7', 'requests >=2.22', 'rad >= 0.17.1', @@ -33,7 +33,8 @@ dependencies = [ 'tweakwcs >=0.8.0', 'spherical-geometry >= 1.2.22', 'stsci.imagestats >= 1.6.3', - 'drizzle >= 1.13.7' + 'drizzle >= 1.13.7', + 'webbpsf == 1.1.1', ] dynamic = ['version'] diff --git a/requirements-dev.txt b/requirements-dev.txt index b1142068d..a716df174 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -23,3 +23,4 @@ git+https://github.com/mairanteodoro/stcal.git@stcal-alignment git+https://github.com/spacetelescope/crds git+https://github.com/spacetelescope/tweakwcs git+https://github.com/spacetelescope/metrics_logger +git+https://github.com/astropy/photutils.git diff --git a/romancal/lib/psf.py b/romancal/lib/psf.py new file mode 100644 index 000000000..bd2ca74b3 --- /dev/null +++ b/romancal/lib/psf.py @@ -0,0 +1,397 @@ +""" +Utilities for fitting model PSFs to rate images. +""" + +import logging +import os + +import astropy.units as u +import numpy as np +import webbpsf +from astropy.modeling.fitting import LevMarLSQFitter +from astropy.nddata import CCDData, bitmask +from astropy.table import Table +from photutils.background import LocalBackground +from photutils.detection import DAOStarFinder +from photutils.psf import ( + GriddedPSFModel, + IterativePSFPhotometry, + PSFPhotometry, + SourceGrouper, +) +from roman_datamodels.datamodels import ImageModel +from webbpsf import conf, gridded_library, restart_logging + +from romancal.lib.dqflags import pixel as roman_dq_flag_map + +# set loggers to debug level by default: +log = logging.getLogger(__name__) +log.setLevel(logging.DEBUG) + +# Phase C central wavelengths [micron], released by Goddard (Jan 2023): +# https://roman.ipac.caltech.edu/sims/Param_db.html#wfi_filters +filter_central_wavelengths = { + "WFI_Filter_F062_Center": 0.620, + "WFI_Filter_F087_Center": 0.869, + "WFI_Filter_F106_Center": 1.060, + "WFI_Filter_F129_Center": 1.293, + "WFI_Filter_F146_Center": 1.464, + "WFI_Filter_F158_Center": 1.577, + "WFI_Filter_F184_Center": 1.842, + "WFI_Filter_F213_Center": 2.125, +} + +default_finder = DAOStarFinder( + # these defaults extracted from the + # romancal SourceDetectionStep + fwhm=2.0, + threshold=2.0, + sharplo=0.0, + sharphi=1.0, + roundlo=-1.0, + roundhi=1.0, + peakmax=1000.0, +) + + +def create_gridded_psf_model( + path_prefix, + filt, + detector, + oversample=12, + fov_pixels=12, + sqrt_n_psfs=4, + overwrite=False, + buffer_pixels=100, + instrument_options=None, + logging_level=None, +): + """ + Compute a gridded PSF model for one SCA via + `webbpsf.gridded_library.CreatePSFLibrary`. + + Parameters + ---------- + path_prefix : str or Path-like + Prefix to the output file path for the gridded PSF model + FITS file. A suffix denoting the detector name will + be appended. + filt : str + Filter name, starting with "F". For example: `"F184"`. + detector : str + Computed gridded PSF model for this SCA. + Examples include: `"SCA01"` or `"SCA18"`. + oversample : int, optional + Oversample factor, default is 12. See WebbPSF docs for details [1]_. + fov_pixels : int, optional + Field of view width [pixels]. Default is 12. + See WebbPSF docs for details [1]_. + sqrt_n_psfs : int, optional + Square root of the number of PSFs to calculate, distributed uniformly + across the detector. Default is 4. + overwrite : bool, optional + Overwrite output file if one already exists. Default is False. + buffer_pixels : int, optional + Calculate a grid of PSFs distributed uniformly across the detector + at least ``buffer_pixels`` away from the detector edges. Default is 100. + instrument_options : dict, optional + Instrument configuration options passed to WebbPSF. + For example, WebbPSF assumes Roman pointing jitter consistent with + mission specs by default, but this can be turned off with: + ``{'jitter': None, 'jitter_sigma': 0}``. + logging_level : str, optional + Set logging level by name if not `None`, otherwise inherit from + the romancal logger. + + Returns + ------- + gridmodel : `photutils.psf.GriddedPSFModel` + Gridded PSF model evaluated at several locations on one SCA. + model_psf_centroids : list of tuples + Pixel locations of the PSF models calculated for ``gridmodel``. + + References + ---------- + .. [1] `WebbPSF documentation for `webbpsf.JWInstrument.calc_psf` + `_ + + """ + if int(sqrt_n_psfs) != sqrt_n_psfs: + raise ValueError(f"`sqrt_n_psfs` must be an integer, got {sqrt_n_psfs}.") + n_psfs = int(sqrt_n_psfs) ** 2 + + # webbpsf appends "_sca??.fits" to the requested path: + expected_output_path = f"{path_prefix}_{detector.lower()}.fits" + + # Choose pixel boundaries for the grid of PSFs: + start_pix = 0 + stop_pix = 4096 + + # Choose locations on detector for each PSF: + pixel_range = np.linspace( + start_pix + buffer_pixels, stop_pix - buffer_pixels, int(sqrt_n_psfs) + ) + + # generate PSFs over a grid of detector positions [pix] + model_psf_centroids = [(int(x), int(y)) for y in pixel_range for x in pixel_range] + + if not os.path.exists(expected_output_path) or overwrite: + if logging_level is None: + # pass along logging level from __name__'s logger to WebbPSF: + logging_level = logging.getLevelName(log.level) + + # set the WebbPSF logging level (similar to webbpsf.utils.setup_logging): + conf.logging_level = logging_level + restart_logging(verbose=False) + + wfi = webbpsf.roman.WFI() + wfi.filter = filt + + if instrument_options is not None: + wfi.options.update(instrument_options) + + central_wavelength_meters = ( + filter_central_wavelengths[f"WFI_Filter_{filt}_Center"] * 1e-6 * u.m + ) + + # Initialize the PSF library + inst = gridded_library.CreatePSFLibrary( + instrument=wfi, + filter_name=filt, + detectors=detector.upper(), + num_psfs=n_psfs, + monochromatic=central_wavelength_meters, + oversample=oversample, + fov_pixels=fov_pixels, + add_distortion=False, + crop_psf=False, + save=True, + filename=path_prefix, + overwrite=overwrite, + verbose=False, + ) + + inst.location_list = model_psf_centroids + + # Create the PSF grid: + gridmodel = inst.create_grid() + + elif os.path.exists(expected_output_path): + logging.log( + logging.INFO, + f"Loading existing gridded PSF model from {expected_output_path}", + ) + psf_model = CCDData.read(expected_output_path, unit=u.electron / u.s, ext=0) + # the FITS file saved by webbpsf gets loaded with pixel + # axes flipped in both dimensions, so flip them back after loading: + psf_model.data = psf_model.data[::-1, ::-1] + psf_model.meta = dict(psf_model.meta) + psf_model.meta["oversampling"] = oversample + psf_model.meta["grid_xypos"] = np.array( + [list(tup)[::-1] for tup in model_psf_centroids] + ) + gridmodel = GriddedPSFModel(psf_model) + + return gridmodel, model_psf_centroids + + +def fit_psf_to_image_model( + image_model=None, + data=None, + error=None, + dq=None, + photometry_cls=PSFPhotometry, + psf_model=None, + grouper=None, + fitter=None, + localbkg_estimator=None, + finder=None, + x_init=None, + y_init=None, + progress_bar=False, + error_lower_limit=None, + fit_shape=(15, 15), + exclude_out_of_bounds=True, +): + """ + Fit PSF models to an ImageModel. + + Parameters + ---------- + image_model : `roman_datamodels.datamodels.ImageModel` + Image datamodel. If ``image_model`` is supplied, + ``data,error,dq`` should be `None`. + data : `astropy.units.Quantity` + Fit a PSF model to the rate image ``data``. + If ``data,error,dq`` are supplied, ``image_model`` should be `None`. + error : `astropy.units.Quantity` + Uncertainties on fluxes in ``data``. Should be `None` if + ``image_model`` is supplied. + dq : `numpy.ndarray` + Data quality bitmask for ``data``. Should be `None` if + ``image_model`` is supplied. + photometry_cls : {`photutils.psf.PSFPhotometry`, + `photutils.psf.IterativePSFPhotometry`} + Choose a photutils PSF photometry technique (default or iterative). + psf_model : `astropy.modeling.Fittable2DModel` + The 2D PSF model to fit to the rate image. Usually this model is an instance + of `photutils.psf.GriddedPSFModel`. + grouper : `photutils.psf.SourceGrouper` + Specifies rules for attempting joint fits of multiple PSFs when + there are nearby sources at small separations. + fitter : `astropy.modeling.fitting.Fitter`, optional + Modeling class which optimizes the PSF fit. + Default is `astropy.modeling.fitting.LevMarLSQFitter(calc_uncertainties=True)`. + localbkg_estimator : `photutils.background.LocalBackground`, optional + Specifies inner and outer radii for computing flux background near + a source. Default has ``inner_radius=10, outer_radius=30``. + finder : subclass of `photutils.detection.StarFinderBase`, optional + When ``photutils_cls`` is `photutils.psf.IterativePSFPhotometry`, the + ``finder`` is called to determine if sources remain in the rate image + after one PSF model is fit to the observations and removed. + Default was extracted from the `DAOStarFinder` call in the + Source Detection step. + x_init : `numpy.ndarray`, optional + Initial guesses for the ``x`` pixel coordinates of each source to fit. + y_init : `numpy.ndarray`, optional + Initial guesses for the ``y`` pixel coordinates of each source to fit. + progress_bar : bool, optional + Render a progress bar via photutils. Default is False. + error_lower_limit : `astropy.units.Quantity`, optional + Since some synthetic images may have bright sources with very + small statistical uncertainties, the ``error`` can be clipped at + ``error_lower_limit`` to prevent over-confident fits. + fit_shape : int, or tuple of length 2, optional + Rectangular shape around the center of a star that will + be used to define the PSF-fitting data. See docs for + `photutils.psf.PSFPhotometry` for details. Default is ``(16, 16)``. + exclude_out_of_bounds : bool, optional + If `True`, do not attempt to fit stars which have initial centroids + that fall outside the pixel limits of the SCA. Default is False. + + Returns + ------- + results_table : `astropy.table.QTable` + PSF photometry results. + photometry : instance of class ``photutils_cls`` + PSF photometry instance with configuration settings and results. + + """ + if grouper is None: + # minimum separation before sources are fit simultaneously: + grouper = SourceGrouper(min_separation=20) # [pix] + + if fitter is None: + fitter = LevMarLSQFitter(calc_uncertainties=True) + + # the iterative PSF method requires a finder: + psf_photometry_kwargs = {} + if photometry_cls is IterativePSFPhotometry or (x_init is None and y_init is None): + if finder is None: + finder = default_finder + psf_photometry_kwargs["finder"] = finder + + if localbkg_estimator is not None: + localbkg_estimator = LocalBackground( + inner_radius=10, # [pix] + outer_radius=30, # [pix] + ) + + photometry = photometry_cls( + grouper=grouper, + localbkg_estimator=localbkg_estimator, + psf_model=psf_model, + fitter=fitter, + fit_shape=fit_shape, + aperture_radius=fit_shape[0], + progress_bar=progress_bar, + **psf_photometry_kwargs, + ) + + if x_init is not None and y_init is not None: + guesses = Table(np.column_stack([x_init, y_init]), names=["x_init", "y_init"]) + else: + guesses = None + + if image_model is None: + if data is None and error is None: + raise ValueError( + "PSF fitting requires either an ImageModel, " + "or arrays for the data and error." + ) + + if dq is None: + if image_model is not None: + mask = dq_to_boolean_mask(image_model) + else: + mask = None + else: + mask = dq_to_boolean_mask(dq) + + if data is None and image_model is not None: + data = image_model.data + + if error is None and image_model is not None: + error = image_model.err + + if error_lower_limit is not None: + # option to enforce a lower limit on the flux uncertainties + error = np.clip(error, error_lower_limit, None) + + # we also mask non-finite values in the data and error arrays: + non_finite = ~np.isfinite(data) | ~np.isfinite(error) + + if exclude_out_of_bounds and guesses is not None: + # don't attempt to fit PSFs for objects with initial centroids + # outside the detector boundaries: + init_centroid_in_range = ( + (guesses["x_init"] > 0) + & (guesses["x_init"] < data.shape[0]) + & (guesses["y_init"] > 0) + & (guesses["y_init"] < data.shape[1]) + ) + guesses = guesses[init_centroid_in_range] + + # fit the model PSF to the data: + results_table = photometry( + data=data, error=error, init_params=guesses, mask=mask | non_finite + ) + + # results are stored on the PSFPhotometry instance: + return results_table, photometry + + +def dq_to_boolean_mask(image_model_or_dq, ignore_flags=0, flag_map_name="ROMAN_DQ"): + """ + Convert a DQ bitmask to a boolean mask. Useful for photutils methods. + + Parameters + ---------- + image_model_or_dq : `roman_datamodels.datamodels.ImageModel` or `numpy.ndarray` + ImageModel containing the DQ bitmask to convert to a boolean mask, + or the DQ bitmask itself. + ignore_flags : int, str, list, None (default = 0) + See docs for `astropy.nddata.bitmask.extend_bit_flag_map` + flag_map_name : str + Name for the bitmask flag map in the astropy bitmask registry + + Returns + ------- + mask : `numpy.ndarray` + Boolean mask + """ + + if isinstance(image_model_or_dq, ImageModel): + dq = image_model_or_dq.dq + else: + dq = image_model_or_dq + + # add the Roman DQ flags to the astropy bitmask registry: + dq_flag_map = dict(roman_dq_flag_map) + dq_flag_map.pop("GOOD") + + bitmask.extend_bit_flag_map(flag_map_name, **dq_flag_map) + + # convert the bitmask to a boolean mask: + mask = bitmask.bitfield_to_boolean_mask(dq, ignore_flags=ignore_flags) + return mask.astype(bool) diff --git a/romancal/lib/tests/test_psf.py b/romancal/lib/tests/test_psf.py new file mode 100644 index 000000000..23f18012a --- /dev/null +++ b/romancal/lib/tests/test_psf.py @@ -0,0 +1,179 @@ +""" + Unit tests for the Roman source detection step code +""" + +import os +import tempfile + +import numpy as np +import pytest +from astropy import units as u +from astropy.nddata import overlap_slices +from photutils.psf import PSFPhotometry +from roman_datamodels import maker_utils as testutil +from roman_datamodels.datamodels import ImageModel + +from romancal.lib.psf import create_gridded_psf_model, fit_psf_to_image_model + +n_sources = 10 +image_model_shape = (100, 100) +rng = np.random.default_rng(0) + + +@pytest.fixture +def setup_inputs(): + def _setup( + nrows=image_model_shape[0], ncols=image_model_shape[1], noise=1.0, seed=None + ): + """ + Return ImageModel of level 2 image. + """ + shape = (nrows, ncols) + wfi_image = testutil.mk_level2_image(shape=shape) + wfi_image.data = u.Quantity( + np.ones(shape, dtype=np.float32), u.electron / u.s, dtype=np.float32 + ) + wfi_image.meta.filename = "filename" + + # add noise to data + if noise is not None: + setup_rng = np.random.default_rng(seed or 19) + wfi_image.data = u.Quantity( + setup_rng.normal(scale=noise, size=shape), + u.electron / u.s, + dtype=np.float32, + ) + wfi_image.err = noise * np.ones(shape, dtype=np.float32) * u.electron / u.s + + # add dq array + wfi_image.dq = np.zeros(shape, dtype=np.uint32) + + # construct ImageModel + mod = ImageModel(wfi_image) + + return mod + + return _setup + + +def add_synthetic_sources( + image_model, + psf_model, + true_x, + true_y, + true_amp, + oversample, + xname="x_0", + yname="y_0", +): + fit_models = [] + + # ensure truths are arrays: + true_x, true_y, true_amp = ( + np.atleast_1d(truth) for truth in [true_x, true_y, true_amp] + ) + + for x, y, amp in zip(true_x, true_y, true_amp): + psf = psf_model.copy() + psf.parameters = [amp, x, y] + fit_models.append(psf) + + synth_image = image_model.data + synth_err = image_model.err + psf_shape = np.array(psf_model.data.shape[1:]) // oversample + + for fit_model in fit_models: + x0 = getattr(fit_model, xname).value + y0 = getattr(fit_model, yname).value + slc_lg, _ = overlap_slices(synth_image.shape, psf_shape, (y0, x0), mode="trim") + yy, xx = np.mgrid[slc_lg] + model_data = fit_model(xx, yy) * image_model.data.unit + model_err = np.sqrt(model_data.value) * model_data.unit + synth_image[slc_lg] += ( + np.random.normal( + model_data.to_value(image_model.data.unit), + model_err.to_value(image_model.data.unit), + size=model_data.shape, + ) + * image_model.data.unit + ) + synth_err[slc_lg] = np.sqrt(synth_err[slc_lg] ** 2 + model_err**2) + + +@pytest.mark.parametrize( + "dx, dy, true_amp", + zip( + rng.uniform(-1, 1, n_sources), + rng.uniform(-1, 1, n_sources), + np.geomspace(10, 10_000, n_sources), + ), +) +def test_psf_fit(setup_inputs, dx, dy, true_amp, seed=42): + # input parameters for PSF model: + filt = "F087" + detector = "SCA01" + oversample = 12 + fov_pixels = 15 + + dir_path = tempfile.gettempdir() + filename_prefix = f"psf_model_{filt}" + file_path = os.path.join(dir_path, filename_prefix) + + # compute gridded PSF model: + psf_model, centroids = create_gridded_psf_model( + file_path, + filt, + detector, + oversample=oversample, + fov_pixels=fov_pixels, + overwrite=False, + logging_level="ERROR", + ) + + # generate an ImageModel + image_model = setup_inputs(seed=seed) + init_data_stddev = np.std(image_model.data.value) + + # add synthetic sources to the ImageModel: + true_x = image_model_shape[0] / 2 + dx + true_y = image_model_shape[1] / 2 + dy + add_synthetic_sources( + image_model, psf_model, true_x, true_y, true_amp, oversample=oversample + ) + + if fov_pixels % 2 == 0: + fit_shape = (fov_pixels + 1, fov_pixels + 1) + else: + fit_shape = (fov_pixels, fov_pixels) + + # fit the PSF to the ImageModel: + results_table, photometry = fit_psf_to_image_model( + image_model=image_model, + photometry_cls=PSFPhotometry, + psf_model=psf_model, + x_init=true_x, + y_init=true_y, + fit_shape=fit_shape, + ) + + # difference between input and output, normalized by the + # uncertainty. Has units of sigma: + delta_x = np.abs(true_x - results_table["x_fit"]) / results_table["x_err"] + delta_y = np.abs(true_y - results_table["y_fit"]) / results_table["y_err"] + + sigma_threshold = 3.5 + assert np.all(delta_x < sigma_threshold) + assert np.all(delta_y < sigma_threshold) + + # now check that the uncertainties aren't way too large, which could cause + # the above test to pass even when the fits are bad. Use overly-simple approximation + # that astrometric uncertainty be proportional to the PSF's FWHM / SNR: + approx_snr = true_amp / init_data_stddev + approx_fwhm = 1 + approx_centroid_err = approx_fwhm / approx_snr + + # centroid err heuristic above is an underestimate, so we scale it up: + scale_factor_approx = 100 + + assert np.all(results_table["x_err"] < scale_factor_approx * approx_centroid_err) + assert np.all(results_table["y_err"] < scale_factor_approx * approx_centroid_err) diff --git a/tox.ini b/tox.ini index 37766beb7..935da7520 100644 --- a/tox.ini +++ b/tox.ini @@ -53,6 +53,9 @@ pass_env = TEST_BIGDATA CODECOV_* DD_* + WEBBPSF_PATH +set_env = + devdeps: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/scientific-python-nightly-wheels/simpl extras = test From b16b87f89f48a8a39bd860eb7820a0ad2664bb11 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 12 Sep 2023 10:13:08 -0400 Subject: [PATCH 30/82] [pre-commit.ci] pre-commit autoupdate (#860) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bcfccac83..890e87d8f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,7 +32,7 @@ repos: args: ["--py38-plus"] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: 'v0.0.287' + rev: 'v0.0.288' hooks: - id: ruff args: ["--fix"] @@ -43,7 +43,7 @@ repos: - id: isort - repo: https://github.com/psf/black - rev: 23.7.0 + rev: 23.9.1 hooks: - id: black From 21d814c63bf7ff2e8406f547b46dea786984e519 Mon Sep 17 00:00:00 2001 From: "Brett M. Morris" Date: Tue, 12 Sep 2023 12:22:35 -0400 Subject: [PATCH 31/82] PSF fitting random seed fix (#862) --- CHANGES.rst | 2 ++ romancal/lib/tests/test_psf.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 7c4d70cfa..729df6af9 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,6 +6,8 @@ general - Fix bug with ``ModelContainer.get_crds_parameters`` being a property not a method [#846] +- Fix random seed bug in PSF fitting methods [#862] + ramp_fitting ------------ diff --git a/romancal/lib/tests/test_psf.py b/romancal/lib/tests/test_psf.py index 23f18012a..29a412360 100644 --- a/romancal/lib/tests/test_psf.py +++ b/romancal/lib/tests/test_psf.py @@ -90,7 +90,7 @@ def add_synthetic_sources( model_data = fit_model(xx, yy) * image_model.data.unit model_err = np.sqrt(model_data.value) * model_data.unit synth_image[slc_lg] += ( - np.random.normal( + rng.normal( model_data.to_value(image_model.data.unit), model_err.to_value(image_model.data.unit), size=model_data.shape, From f78a127fa673190bb65d3eba6e0a6b852fbae773 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Tue, 12 Sep 2023 12:44:54 -0400 Subject: [PATCH 32/82] Remove style check from CI, this is repeating the work done by the `pre-commit.ci` bot on every PR --- .github/workflows/roman_ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/roman_ci.yml b/.github/workflows/roman_ci.yml index b3cb2486d..cd02ba3ef 100644 --- a/.github/workflows/roman_ci.yml +++ b/.github/workflows/roman_ci.yml @@ -72,7 +72,6 @@ jobs: uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1 with: envs: | - - linux: check-style - linux: check-dependencies - linux: build-dist test: From 361e91a3b0aa18e4644e14bddaf511337533f64c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 13 Sep 2023 13:32:18 +0000 Subject: [PATCH 33/82] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- romancal/resample/resample.py | 20 +++-------- romancal/resample/resample_step.py | 21 ++++------- romancal/resample/tests/test_resample.py | 35 +++++++------------ romancal/resample/tests/test_resample_step.py | 31 +++++----------- 4 files changed, 33 insertions(+), 74 deletions(-) diff --git a/romancal/resample/resample.py b/romancal/resample/resample.py index 8e2f0c4df..6ce8fd5fe 100644 --- a/romancal/resample/resample.py +++ b/romancal/resample/resample.py @@ -183,13 +183,9 @@ def resample_many_to_many(self): indx = exposure[0].meta.filename.rfind(".") output_type = exposure[0].meta.filename[indx:] output_root = "_".join( - exposure[0] - .meta.filename.replace(output_type, "") - .split("_")[:-1] - ) - output_model.meta.filename = ( - f"{output_root}_outlier_i2d{output_type}" + exposure[0].meta.filename.replace(output_type, "").split("_")[:-1] ) + output_model.meta.filename = f"{output_root}_outlier_i2d{output_type}" # Initialize the output with the wcs driz = gwcs_drizzle.GWCSDrizzle( @@ -240,9 +236,7 @@ def resample_many_to_one(self): output_model.meta.filename = self.output_filename output_model.meta["resample"] = {} output_model.meta.resample["weight_type"] = self.weight_type - output_model.meta.resample["pointings"] = len( - self.input_models.models_grouped - ) + output_model.meta.resample["pointings"] = len(self.input_models.models_grouped) if self.blendheaders: log.info("Skipping blendheaders for now.") @@ -383,9 +377,7 @@ def update_exposure_times(self, output_model): output_model.meta.exposure.exposure_time = total_exposure_time output_model.meta.exposure.start_time = min(exposure_times["start"]) output_model.meta.exposure.end_time = max(exposure_times["end"]) - output_model.meta.resample[ - "product_exposure_time" - ] = total_exposure_time + output_model.meta.resample["product_exposure_time"] = total_exposure_time @staticmethod def drizzle_arrays( @@ -546,9 +538,7 @@ def drizzle_arrays( # Compute the mapping between the input and output pixel coordinates # for use in drizzle.cdrizzle.tdriz - pixmap = resample_utils.calc_gwcs_pixmap( - input_wcs, output_wcs, insci.shape - ) + pixmap = resample_utils.calc_gwcs_pixmap(input_wcs, output_wcs, insci.shape) log.debug(f"Pixmap shape: {pixmap[:,:,0].shape}") log.debug(f"Input Sci shape: {insci.shape}") diff --git a/romancal/resample/resample_step.py b/romancal/resample/resample_step.py index 77b61e936..68063c3e4 100755 --- a/romancal/resample/resample_step.py +++ b/romancal/resample/resample_step.py @@ -1,6 +1,5 @@ import logging import os -import re from copy import deepcopy import asdf @@ -84,15 +83,15 @@ def process(self, input): except Exception: # single ASDF filename input_models = ModelContainer([input]) - if hasattr(input_models, "asn_table") and len( - input_models.asn_table - ): + if hasattr(input_models, "asn_table") and len(input_models.asn_table): output = input_models.asn_table["products"][0]["name"] elif hasattr(input_models[0], "meta"): output = input_models[0].meta.filename elif isinstance(input, ModelContainer): input_models = input - output = f"{os.path.commonprefix([x.meta.filename for x in input_models])}.asdf" + output = ( + f"{os.path.commonprefix([x.meta.filename for x in input_models])}.asdf" + ) if len(output) == 0: output = "resample_output.asdf" else: @@ -121,9 +120,7 @@ def process(self, input): # Issue a warning about the use of exptime weighting if self.wht_type == "exptime": - self.log.warning( - "Use of EXPTIME weighting will result in incorrect" - ) + self.log.warning("Use of EXPTIME weighting will result in incorrect") self.log.warning("propagated errors in the resampled product") # Custom output WCS parameters. @@ -221,9 +218,7 @@ def _check_list_pars(vals, name, min_vals=None): ) return list(vals) else: - raise ValueError( - f"Both '{name}' values must be either None or not None." - ) + raise ValueError(f"Both '{name}' values must be either None or not None.") @staticmethod def _load_custom_wcs(asdf_wcs_file, output_shape): @@ -305,9 +300,7 @@ def get_drizpars(self, ref_filename, input_models): # With presence of wild-card rows, code should never trigger this logic if row is None: - self.log.error( - "No row found in %s matching input data.", ref_filename - ) + self.log.error("No row found in %s matching input data.", ref_filename) raise ValueError # Define the keys to pull from drizpars reffile table. diff --git a/romancal/resample/tests/test_resample.py b/romancal/resample/tests/test_resample.py index 4edab7ed6..37607ec4a 100644 --- a/romancal/resample/tests/test_resample.py +++ b/romancal/resample/tests/test_resample.py @@ -1,15 +1,16 @@ -import pytest import numpy as np +import pytest +from astropy import coordinates as coord from astropy import units as u -from gwcs import WCS from astropy.modeling import models from astropy.time import Time -from astropy import coordinates as coord +from gwcs import WCS from gwcs import coordinate_frames as cf from roman_datamodels import datamodels, maker_utils -from romancal.resample.resample import ResampleData -from romancal.resample import gwcs_drizzle, resample_utils + from romancal.datamodels import ModelContainer +from romancal.resample import gwcs_drizzle, resample_utils +from romancal.resample.resample import ResampleData class WfiSca: @@ -54,9 +55,7 @@ def create_image(self): dtype=np.float32, ), "var_rnoise": u.Quantity( - np.random.normal(1, 0.05, size=self.shape).astype( - np.float32 - ), + np.random.normal(1, 0.05, size=self.shape).astype(np.float32), u.electron**2 / u.s**2, dtype=np.float32, ), @@ -472,18 +471,12 @@ def test_update_exposure_times_different_sca_same_exposure(exposure_1): == exposure_1[0].meta.exposure.exposure_time ) assert ( - output_model.meta.exposure.start_time - == exposure_1[0].meta.exposure.start_time - ) - assert ( - output_model.meta.exposure.end_time - == exposure_1[0].meta.exposure.end_time + output_model.meta.exposure.start_time == exposure_1[0].meta.exposure.start_time ) + assert output_model.meta.exposure.end_time == exposure_1[0].meta.exposure.end_time -def test_update_exposure_times_same_sca_different_exposures( - exposure_1, exposure_2 -): +def test_update_exposure_times_same_sca_different_exposures(exposure_1, exposure_2): """Test that update_exposure_times is properly updating the exposure parameters for a set of the same SCA but belonging to different exposures.""" input_models = ModelContainer([exposure_1[0], exposure_2[0]]) @@ -560,9 +553,7 @@ def test_custom_wcs_input_small_overlap_no_rotation(wfi_sca1, wfi_sca3): output_models = resample_data.resample_many_to_one() # pixel scale in RA (N.B.: there's no shift in Dec.) - pixel_scale = np.abs( - wfi_sca3.meta.wcs(0, 0)[0] - wfi_sca3.meta.wcs(1, 0)[0] - ) + pixel_scale = np.abs(wfi_sca3.meta.wcs(0, 0)[0] - wfi_sca3.meta.wcs(1, 0)[0]) # overlap size in RA (N.B.: there's no shift in Dec.) ra_overlap_size = np.ceil( input_models[0].shape[0] @@ -576,9 +567,7 @@ def test_custom_wcs_input_small_overlap_no_rotation(wfi_sca1, wfi_sca3): assert ra_output_nonzero_size == ra_overlap_size - np.testing.assert_allclose( - output_models[0].meta.wcs(0, 0), wfi_sca3.meta.wcs(0, 0) - ) + np.testing.assert_allclose(output_models[0].meta.wcs(0, 0), wfi_sca3.meta.wcs(0, 0)) def test_custom_wcs_input_entire_field_no_rotation(multiple_exposures): diff --git a/romancal/resample/tests/test_resample_step.py b/romancal/resample/tests/test_resample_step.py index dd771a95f..7dba0be6a 100644 --- a/romancal/resample/tests/test_resample_step.py +++ b/romancal/resample/tests/test_resample_step.py @@ -1,34 +1,25 @@ import pytest -import numpy as np +from asdf import AsdfFile +from astropy import coordinates as coord from astropy import units as u -from gwcs import WCS from astropy.modeling import models -from astropy.time import Time -from astropy import coordinates as coord +from gwcs import WCS from gwcs import coordinate_frames as cf from roman_datamodels import datamodels, maker_utils + from romancal.resample import ResampleStep -from romancal.resample import gwcs_drizzle, resample_utils -from romancal.datamodels import ModelContainer -from asdf import AsdfFile class MockModel: - def __init__( - self, pixelarea_steradians, pixelarea_arcsecsq, pixel_scale_ratio - ): + def __init__(self, pixelarea_steradians, pixelarea_arcsecsq, pixel_scale_ratio): self.meta = MockMeta( pixelarea_steradians, pixelarea_arcsecsq, pixel_scale_ratio ) class MockMeta: - def __init__( - self, pixelarea_steradians, pixelarea_arcsecsq, pixel_scale_ratio - ): - self.photometry = MockPhotometry( - pixelarea_steradians, pixelarea_arcsecsq - ) + def __init__(self, pixelarea_steradians, pixelarea_arcsecsq, pixel_scale_ratio): + self.photometry = MockPhotometry(pixelarea_steradians, pixelarea_arcsecsq) self.resample = MockResample(pixel_scale_ratio) @@ -76,9 +67,7 @@ def create_mosaic(self): return datamodels.MosaicModel(l3) -def create_wcs_object_without_distortion( - fiducial_world, pscale, shape, **kwargs -): +def create_wcs_object_without_distortion(fiducial_world, pscale, shape, **kwargs): """ Create a simple WCS object without either distortion or rotation. @@ -271,9 +260,7 @@ def test_update_phot_keywords( expected_arcsecsq, ): step = ResampleStep() - model = MockModel( - pixelarea_steradians, pixelarea_arcsecsq, pixel_scale_ratio - ) + model = MockModel(pixelarea_steradians, pixelarea_arcsecsq, pixel_scale_ratio) step.update_phot_keywords(model) From 2a22b5defd2ee48a5012897721a1c1ce802c2d5b Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Wed, 13 Sep 2023 10:25:36 -0400 Subject: [PATCH 34/82] Fix check styles. --- romancal/resample/tests/test_resample_step.py | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/romancal/resample/tests/test_resample_step.py b/romancal/resample/tests/test_resample_step.py index 7dba0be6a..95bf997e6 100644 --- a/romancal/resample/tests/test_resample_step.py +++ b/romancal/resample/tests/test_resample_step.py @@ -168,20 +168,6 @@ def test_check_list_pars_valid(vals, name, min_vals, expected): assert result == expected -@pytest.mark.parametrize( - "vals, name, min_vals", - [ - ([1, None], "list3", None), # One value is None - ([1, 2], "list5", [3, 3]), # Values do not meet minimum requirements - ([1, 2, 3], "list6", None), # Invalid number of elements - ], -) -def test_check_list_pars_exception(vals, name, min_vals): - step = ResampleStep() - with pytest.raises(ValueError): - step._check_list_pars(vals, name, min_vals) - - def test_load_custom_wcs_no_file(): step = ResampleStep() result = step._load_custom_wcs(None, (512, 512)) @@ -241,7 +227,8 @@ def test_check_list_pars_exception(vals, name, min_vals): @pytest.mark.parametrize( - "pixelarea_steradians, pixelarea_arcsecsq, pixel_scale_ratio, expected_steradians, expected_arcsecsq", + """pixelarea_steradians, pixelarea_arcsecsq, pixel_scale_ratio, + expected_steradians, expected_arcsecsq""", [ # Happy path tests (1.0, 1.0, 2.0, 4.0, 4.0), From 639412c41b0fc46620484a9fb33a54d27b14ff3b Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Wed, 13 Sep 2023 10:44:07 -0400 Subject: [PATCH 35/82] Small refactories. --- romancal/resample/resample_step.py | 20 +++++++++---------- romancal/resample/tests/test_resample_step.py | 14 +++++++++++-- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/romancal/resample/resample_step.py b/romancal/resample/resample_step.py index 68063c3e4..82e75956d 100755 --- a/romancal/resample/resample_step.py +++ b/romancal/resample/resample_step.py @@ -228,7 +228,7 @@ def _load_custom_wcs(asdf_wcs_file, output_shape): with asdf.open(asdf_wcs_file) as af: wcs = deepcopy(af.tree["wcs"]) - if output_shape is not None or wcs is None: + if output_shape is not None: wcs.array_shape = output_shape[::-1] elif wcs.pixel_shape is not None: wcs.array_shape = wcs.pixel_shape[::-1] @@ -283,7 +283,7 @@ def get_drizpars(self, ref_filename, input_models): filter_match = False # look for row that applies to this set of input data models for n, filt, num in zip( - range(0, len(drizpars_table)), + range(len(drizpars_table)), drizpars_table["filter"], drizpars_table["numimages"], ): @@ -329,16 +329,16 @@ def get_drizpars(self, ref_filename, input_models): if isinstance(v, np.bytes_): reffile_drizpars[k] = v.decode("UTF-8") - all_drizpars = {**reffile_drizpars, **user_drizpars} + all_drizpars = reffile_drizpars | user_drizpars - kwargs = dict( - good_bits=GOOD_BITS, - single=self.single, - blendheaders=self.blendheaders, + kwargs = ( + dict( + good_bits=GOOD_BITS, + single=self.single, + blendheaders=self.blendheaders, + ) + | all_drizpars ) - - kwargs.update(all_drizpars) - for k, v in kwargs.items(): self.log.debug(f" {k}={v}") diff --git a/romancal/resample/tests/test_resample_step.py b/romancal/resample/tests/test_resample_step.py index 95bf997e6..829d17408 100644 --- a/romancal/resample/tests/test_resample_step.py +++ b/romancal/resample/tests/test_resample_step.py @@ -137,13 +137,12 @@ def create_wcs_object_without_distortion(fiducial_world, pscale, shape, **kwargs @pytest.fixture def asdf_wcs_file(): - def _create_asdf_wcs_file(tmp_path, pixel_shape, bounding_box): + def _create_asdf_wcs_file(tmp_path): file_path = tmp_path / "wcs.asdf" wcs_data = create_wcs_object_without_distortion( (10, 0), (0.000031, 0.000031), (100, 100), - **{"pixel_shape": pixel_shape, "bounding_box": bounding_box}, ) wcs = {"wcs": wcs_data} with AsdfFile(wcs) as af: @@ -190,6 +189,17 @@ def test_load_custom_wcs_invalid_file(tmp_path): step._load_custom_wcs(str(invalid_file), (512, 512)) +def test_load_custom_wcs_asdf_without_wcs_attribute(tmp_path): + step = ResampleStep() + file_path = tmp_path / "asdf_file.asdf" + wcs = {} + with AsdfFile(wcs) as af: + af.write_to(file_path) + + with pytest.raises(KeyError): + step._load_custom_wcs(str(file_path), (100, 100)) + + @pytest.mark.parametrize( "vals, name, min_vals, expected", [ From ac9c90e76e74fd7276a9baef14d6375ac3e6e33b Mon Sep 17 00:00:00 2001 From: mairan Date: Wed, 13 Sep 2023 11:08:38 -0400 Subject: [PATCH 36/82] Update pyproject.toml Co-authored-by: Zach Burnett --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c94e5cbc5..6e8c00fca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ # 'roman_datamodels @ git+https://github.com/spacetelescope/roman_datamodels.git@main', 'scipy >=1.11', # 'stcal >=1.4.0', - 'stcal @ git+https://github.com/mairanteodoro/stcal.git#egg=stcal-alignment', + 'stcal @ git+https://github.com/mairanteodoro/stcal.git@stcal-alignment', 'stpipe >=0.5.0', 'tweakwcs >=0.8.0', 'spherical-geometry >= 1.2.22', From 37a61163f8fc35b33a3ad367f242e302e73b1cf3 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Wed, 20 Sep 2023 14:04:13 -0400 Subject: [PATCH 37/82] Updates to fix validation bug. --- romancal/regtest/test_resample.py | 164 +++++++++++++++++++++++++++++ romancal/resample/resample.py | 28 ++--- romancal/resample/resample_step.py | 10 +- romancal/step.py | 2 + 4 files changed, 188 insertions(+), 16 deletions(-) create mode 100644 romancal/regtest/test_resample.py diff --git a/romancal/regtest/test_resample.py b/romancal/regtest/test_resample.py new file mode 100644 index 000000000..e43c3186d --- /dev/null +++ b/romancal/regtest/test_resample.py @@ -0,0 +1,164 @@ +from io import StringIO + +import pytest +from metrics_logger.decorators import metrics_logger +from roman_datamodels import datamodels as rdm + +from romancal.stpipe import RomanStep +from romancal.resample.resample_step import ResampleStep + +from .regtestdata import compare_asdf +import json + + +def create_asn_file( + output_filename: str = "resample_output.asdf", + members_filename_list: list = None, +): + asn_dict = { + "asn_type": "None", + "asn_rule": "DMS_ELPP_Base", + "version_id": "null", + "code_version": "0.9.1.dev28+ge987cc9.d20230106", + "degraded_status": "No known degraded exposures in association.", + "program": "noprogram", + "constraints": "No constraints", + "asn_id": "a3001", + "target": "none", + "asn_pool": "test_pool_name", + "products": [ + { + "name": output_filename, + "members": [ + {"expname": x, "exptype": "science"} + for x in members_filename_list + ], + } + ], + } + asn_content = json.dumps(asn_dict) + asn_file_path = "sample_asn.json" + asn_file = StringIO() + asn_file.write(asn_content) + with open(asn_file_path, mode="w") as f: + print(asn_file.getvalue(), file=f) + + return asn_file_path + + +@metrics_logger( + "DMS342" +) # got DMS342 from here: https://jira.stsci.edu/browse/RSUBREQ-1051 +@pytest.mark.bigdata +def test_resample_single_file(rtdata, ignore_asdf_paths): + input_data = [ + "r0000501001001001001_01101_0001_WFI02_cal_proc.asdf", + "r0000501001001001001_01101_0002_WFI02_cal_proc.asdf", + ] + output_data = "resample_output_resamplestep.asdf" + # truth_data = "r0000401001001001001_01101_0001_WFI01_cal_twkreg_proc.asdf" + + [ + rtdata.get_data( + f"/Users/mteodoro/ROMAN/SYNTHETIC_IMAGES/IMAGES/23Q4_B11/L2/PROC/{data}" + ) + for data in input_data + ] + # rtdata.get_truth(f"truth/WFI/image/{truth_data}") + + rtdata.input = create_asn_file(members_filename_list=input_data) + rtdata.output = output_data + + # instantiate ResampleStep (for running and log access) + step = ResampleStep() + + args = [ + "romancal.step.ResampleStep", + rtdata.input, + "--rotation=0", + f"--output_file='{rtdata.output}'", + ] + RomanStep.from_cmdline(args) + resample_out = rdm.open(rtdata.output) + + # Can you add the metrics_logger decorator + # from metrics_logger.decorators import metrics_logger + # and indicate the this satisfies the requirement SOC-581 (RSOCREQ-28) + + # Also can you add SOC-582 (RSOCREQ-73) and this should test + # (a) Data quality and uncertainty information (DMS343) + # (b) Total exposure time (DMS344) + # (c) Metadata used in the mosaic generation process (DMS345) + + step.log.info( + "ResampleStep recorded as complete? :" + f' {resample_out.meta.cal_step.resample == "COMPLETE"}' + ) + assert resample_out.meta.cal_step.resample == "COMPLETE" + + step.log.info( + "ResampleStep created 'meta.resample'? :" + f' {hasattr(resample_out.meta, "resample")}' + ) + assert hasattr(resample_out.meta, "resample") + + step.log.info( + f"""DMS343 MSG: ResampleStep created new attribute data quality information? :\ + { + all( + hasattr(resample_out, x) for x in [ + "data", + "err", + "var_poisson", + "var_rnoise", + "var_flat", + ] + ) + }""" + ) + assert all( + hasattr(resample_out, x) + for x in ["data", "err", "var_poisson", "var_rnoise", "var_flat"] + ) + + step.log.info( + f"""DMS344 MSG: ResampleStep created new attribute with total exposure time? :\ + {"product_exposure_time" in resample_out.meta.resample}""" + ) + assert "product_exposure_time" in resample_out.meta.resample + + step.log.info( + f"""DMS345 MSG: ResampleStep included all metadata relevant to the creation of the mosaic? :\ + { + all( + hasattr(resample_out.meta.resample, x) + and bool(getattr(resample_out.meta.resample, x)) + for x in [ + "pixel_scale_ratio", + "pixfrac", + "pointings", + "product_exposure_time", + "weight_type", + "members", + ] + ) + }""" # noqa: E501 + ) + assert all( + hasattr(resample_out.meta.resample, x) + and bool(getattr(resample_out.meta.resample, x)) + for x in [ + "pixel_scale_ratio", + "pixfrac", + "pointings", + "product_exposure_time", + "weight_type", + "members", + ] + ) + + diff = compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) + step.log.info( + "Was the proper Resample data produced?" f" : {diff.identical}" + ) + assert diff.identical, diff.report() diff --git a/romancal/resample/resample.py b/romancal/resample/resample.py index 6ce8fd5fe..18f658dac 100644 --- a/romancal/resample/resample.py +++ b/romancal/resample/resample.py @@ -4,7 +4,7 @@ from astropy import units as u from drizzle import cdrizzle, util from roman_datamodels import datamodels -from roman_datamodels.maker_utils import mk_datamodel +from roman_datamodels.maker_utils import mk_datamodel, mk_resample from ..datamodels import ModelContainer from . import gwcs_drizzle, resample_utils @@ -234,9 +234,10 @@ def resample_many_to_one(self): """ output_model = self.blank_output.copy() output_model.meta.filename = self.output_filename - output_model.meta["resample"] = {} - output_model.meta.resample["weight_type"] = self.weight_type - output_model.meta.resample["pointings"] = len(self.input_models.models_grouped) + output_model.meta["resample"] = mk_resample() + output_model.meta.resample["members"] = [] + output_model.meta.resample.weight_type = self.weight_type + output_model.meta.resample.pointings = len(self.input_models.models_grouped) if self.blendheaders: log.info("Skipping blendheaders for now.") @@ -267,6 +268,7 @@ def resample_many_to_one(self): driz.add_image(data, img.meta.wcs, inwht=inwht) del data, inwht + output_model.meta.resample.members.append(img.meta.filename) # Resample variances array in self.input_models to output_model self.resample_variance_array("var_rnoise", output_model) @@ -289,6 +291,10 @@ def resample_many_to_one(self): ) self.update_exposure_times(output_model) + + # TODO: fix RAD to expect a context image datatype of int32 + output_model.context = output_model.context.astype(np.uint32) + self.output_models.append(output_model) return self.output_models @@ -377,7 +383,7 @@ def update_exposure_times(self, output_model): output_model.meta.exposure.exposure_time = total_exposure_time output_model.meta.exposure.start_time = min(exposure_times["start"]) output_model.meta.exposure.end_time = max(exposure_times["end"]) - output_model.meta.resample["product_exposure_time"] = total_exposure_time + output_model.meta.resample.product_exposure_time = total_exposure_time @staticmethod def drizzle_arrays( @@ -497,11 +503,7 @@ def drizzle_arrays( """ # Insure that the fillval parameter gets properly interpreted for use with tdriz - if util.is_blank(str(fillval)): - fillval = "INDEF" - else: - fillval = str(fillval) - + fillval = "INDEF" if util.is_blank(str(fillval)) else str(fillval) if insci.dtype > np.float32: insci = insci.astype(np.float32) @@ -514,10 +516,10 @@ def drizzle_arrays( planeid = int((uniqid - 1) / 32) # Check if the context image has this many planes - if outcon.ndim == 3: - nplanes = outcon.shape[0] - elif outcon.ndim == 2: + if outcon.ndim == 2: nplanes = 1 + elif outcon.ndim == 3: + nplanes = outcon.shape[0] else: nplanes = 0 diff --git a/romancal/resample/resample_step.py b/romancal/resample/resample_step.py index 82e75956d..2f2e4a3e3 100755 --- a/romancal/resample/resample_step.py +++ b/romancal/resample/resample_step.py @@ -67,12 +67,12 @@ class ResampleStep(RomanStep): in_memory = boolean(default=True) """ # noqa: E501 - # TODO: provide 'drizpars' file (then remove _set_spec_defaults?) reference_file_types = [] def process(self, input): if isinstance(input, datamodels.DataModel): input_models = ModelContainer([input]) + # set output filename from meta.filename found in the first datamodel output = input_models[0].meta.filename self.blendheaders = False elif isinstance(input, str): @@ -84,15 +84,19 @@ def process(self, input): # single ASDF filename input_models = ModelContainer([input]) if hasattr(input_models, "asn_table") and len(input_models.asn_table): + # set output filename from ASN table output = input_models.asn_table["products"][0]["name"] elif hasattr(input_models[0], "meta"): + # set output filename from meta.filename found in the first datamodel output = input_models[0].meta.filename elif isinstance(input, ModelContainer): input_models = input + # set output filename using the common prefix of all datamodels output = ( f"{os.path.commonprefix([x.meta.filename for x in input_models])}.asdf" ) if len(output) == 0: + # set default filename if no common prefix can be determined output = "resample_output.asdf" else: raise TypeError( @@ -164,12 +168,12 @@ def _final_updates(self, model, input_models, kwargs): # if pixel_scale exists, it will override pixel_scale_ratio. # calculate the actual value of pixel_scale_ratio based on pixel_scale # because source_catalog uses this value from the header. - model.meta.resample["pixel_scale_ratio"] = ( + model.meta.resample.pixel_scale_ratio = ( self.pixel_scale / np.sqrt(model.meta.photometry.pixelarea_arcsecsq) if self.pixel_scale else self.pixel_scale_ratio ) - model.meta.resample["pixfrac"] = kwargs["pixfrac"] + model.meta.resample.pixfrac = kwargs["pixfrac"] self.update_phot_keywords(model) @staticmethod diff --git a/romancal/step.py b/romancal/step.py index dc2ebd4e2..91bf7ff32 100644 --- a/romancal/step.py +++ b/romancal/step.py @@ -12,6 +12,7 @@ from .photom.photom_step import PhotomStep from .ramp_fitting.ramp_fit_step import RampFitStep from .refpix.refpix_step import RefPixStep +from .resample.resample_step import ResampleStep from .saturation.saturation_step import SaturationStep from .skymatch.skymatch_step import SkyMatchStep from .source_detection.source_detection_step import SourceDetectionStep @@ -32,4 +33,5 @@ "SourceDetectionStep", "SkyMatchStep", "TweakRegStep", + "ResampleStep", ] From 8352b09871b514dac1c343e72a4002a3a98c6d38 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 20 Sep 2023 18:05:10 +0000 Subject: [PATCH 38/82] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- romancal/regtest/test_resample.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/romancal/regtest/test_resample.py b/romancal/regtest/test_resample.py index e43c3186d..e14aeafa9 100644 --- a/romancal/regtest/test_resample.py +++ b/romancal/regtest/test_resample.py @@ -1,14 +1,14 @@ +import json from io import StringIO import pytest from metrics_logger.decorators import metrics_logger from roman_datamodels import datamodels as rdm -from romancal.stpipe import RomanStep from romancal.resample.resample_step import ResampleStep +from romancal.stpipe import RomanStep from .regtestdata import compare_asdf -import json def create_asn_file( @@ -30,8 +30,7 @@ def create_asn_file( { "name": output_filename, "members": [ - {"expname": x, "exptype": "science"} - for x in members_filename_list + {"expname": x, "exptype": "science"} for x in members_filename_list ], } ], @@ -131,7 +130,7 @@ def test_resample_single_file(rtdata, ignore_asdf_paths): f"""DMS345 MSG: ResampleStep included all metadata relevant to the creation of the mosaic? :\ { all( - hasattr(resample_out.meta.resample, x) + hasattr(resample_out.meta.resample, x) and bool(getattr(resample_out.meta.resample, x)) for x in [ "pixel_scale_ratio", @@ -158,7 +157,5 @@ def test_resample_single_file(rtdata, ignore_asdf_paths): ) diff = compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) - step.log.info( - "Was the proper Resample data produced?" f" : {diff.identical}" - ) + step.log.info("Was the proper Resample data produced?" f" : {diff.identical}") assert diff.identical, diff.report() From db2978b49e60221032af63200ed0d3360b472a46 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Wed, 20 Sep 2023 15:09:18 -0400 Subject: [PATCH 39/82] Fix numpy legacy issue. --- romancal/resample/tests/test_resample.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/romancal/resample/tests/test_resample.py b/romancal/resample/tests/test_resample.py index 37607ec4a..fc420053a 100644 --- a/romancal/resample/tests/test_resample.py +++ b/romancal/resample/tests/test_resample.py @@ -30,6 +30,7 @@ def create_image(self): datamodels.ImageModel An L2 ImageModel datamodel. """ + rng = np.random.default_rng() l2 = maker_utils.mk_level2_image( shape=self.shape, **{ @@ -50,22 +51,22 @@ def create_image(self): }, }, "data": u.Quantity( - np.random.poisson(2.5, size=self.shape).astype(np.float32), + rng.poisson(2.5, size=self.shape).astype(np.float32), u.electron / u.s, dtype=np.float32, ), "var_rnoise": u.Quantity( - np.random.normal(1, 0.05, size=self.shape).astype(np.float32), + rng.normal(1, 0.05, size=self.shape).astype(np.float32), u.electron**2 / u.s**2, dtype=np.float32, ), "var_poisson": u.Quantity( - np.random.poisson(1, size=self.shape).astype(np.float32), + rng.poisson(1, size=self.shape).astype(np.float32), u.electron**2 / u.s**2, dtype=np.float32, ), "var_flat": u.Quantity( - np.random.uniform(0, 1, size=self.shape).astype(np.float32), + rng.uniform(0, 1, size=self.shape).astype(np.float32), u.electron**2 / u.s**2, dtype=np.float32, ), @@ -458,7 +459,7 @@ def test_update_exposure_times_different_sca_same_exposure(exposure_1): resample_data = ResampleData(input_models) output_model = resample_data.blank_output.copy() - output_model.meta["resample"] = {} + output_model.meta["resample"] = maker_utils.mk_resample() resample_data.update_exposure_times(output_model) @@ -483,7 +484,7 @@ def test_update_exposure_times_same_sca_different_exposures(exposure_1, exposure resample_data = ResampleData(input_models) output_model = resample_data.blank_output.copy() - output_model.meta["resample"] = {} + output_model.meta["resample"] = maker_utils.mk_resample() resample_data.update_exposure_times(output_model) From 167a0d70187e1a3a3839de44f2a0d6b9253c52a4 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Wed, 20 Sep 2023 15:34:28 -0400 Subject: [PATCH 40/82] Add ResampleStep to integration test. --- romancal/stpipe/integration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/romancal/stpipe/integration.py b/romancal/stpipe/integration.py index 5b4017c9b..e74507734 100644 --- a/romancal/stpipe/integration.py +++ b/romancal/stpipe/integration.py @@ -36,4 +36,5 @@ def get_steps(): ("romancal.step.SourceDetectionStep", None, False), ("romancal.step.SkyMatchStep", "skymatch", False), ("romancal.step.TweakRegStep", "tweakreg", False), + ("romancal.step.ResampleStep", "resample", False), ] From 0a47e2a2c207d8aaa483507babe55911665f8c7f Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Tue, 26 Sep 2023 21:32:25 -0400 Subject: [PATCH 41/82] Add resample step documentation. --- docs/roman/package_index.rst | 1 + docs/roman/pipeline_steps.rst | 1 + docs/roman/resample/arguments.rst | 104 ++++++++++++++++++++++ docs/roman/resample/index.rst | 16 ++++ docs/roman/resample/main.rst | 118 +++++++++++++++++++++++++ docs/roman/resample/resample.rst | 6 ++ docs/roman/resample/resample_step.rst | 6 ++ docs/roman/resample/resample_utils.rst | 8 ++ 8 files changed, 260 insertions(+) create mode 100644 docs/roman/resample/arguments.rst create mode 100644 docs/roman/resample/index.rst create mode 100644 docs/roman/resample/main.rst create mode 100644 docs/roman/resample/resample.rst create mode 100644 docs/roman/resample/resample_step.rst create mode 100644 docs/roman/resample/resample_utils.rst diff --git a/docs/roman/package_index.rst b/docs/roman/package_index.rst index fc8965dfa..1851c9489 100644 --- a/docs/roman/package_index.rst +++ b/docs/roman/package_index.rst @@ -16,3 +16,4 @@ tweakreg/index.rst outlier_detection/index.rst skymatch/index.rst + resample/index.rst diff --git a/docs/roman/pipeline_steps.rst b/docs/roman/pipeline_steps.rst index c82689e0e..02ef4f98e 100644 --- a/docs/roman/pipeline_steps.rst +++ b/docs/roman/pipeline_steps.rst @@ -14,3 +14,4 @@ Pipeline Steps photom/index.rst source_detection/index.rst tweakreg/index.rst + resample/index.rst diff --git a/docs/roman/resample/arguments.rst b/docs/roman/resample/arguments.rst new file mode 100644 index 000000000..5a690cc9e --- /dev/null +++ b/docs/roman/resample/arguments.rst @@ -0,0 +1,104 @@ +.. _resample_step_args: + +Step Arguments +============== +The ``resample`` step has the following optional arguments that control +the behavior of the processing and the characteristics of the resampled +image. + +``--pixfrac`` (float, default=1.0) + The fraction by which input pixels are "shrunk" before being drizzled + onto the output image grid, given as a real number between 0 and 1. + +``--kernel`` (str, default='square') + The form of the kernel function used to distribute flux onto the output + image. Available kernels are `square`, `gaussian`, `point`, `tophat`, + `turbo`, `lanczos2`, and `lanczos3`. + +``--pixel_scale_ratio`` (float, default=1.0) + Ratio of input to output pixel scale. A value of 0.5 means the output + image would have 4 pixels sampling each input pixel. + Ignored when ``pixel_scale`` or ``output_wcs`` are provided. + +``--pixel_scale`` (float, default=None) + Absolute pixel scale in ``arcsec``. When provided, overrides + ``pixel_scale_ratio``. Ignored when ``output_wcs`` is provided. + +``--rotation`` (float, default=None) + Position angle of output image’s Y-axis relative to North. + A value of 0.0 would orient the final output image to be North up. + The default of `None` specifies that the images will not be rotated, + but will instead be resampled in the default orientation for the camera + with the x and y axes of the resampled image corresponding + approximately to the detector axes. Ignored when ``pixel_scale`` + or ``output_wcs`` are provided. + +``--crpix`` (tuple of float, default=None) + Position of the reference pixel in the image array in the ``x, y`` order. + If ``crpix`` is not specified, it will be set to the center of the bounding + box of the returned WCS object. When supplied from command line, it should + be a comma-separated list of floats. Ignored when ``output_wcs`` + is provided. + +``--crval`` (tuple of float, default=None) + Right ascension and declination of the reference pixel. Automatically + computed if not provided. When supplied from command line, it should be a + comma-separated list of floats. Ignored when ``output_wcs`` is provided. + +``--output_shape`` (tuple of int, default=None) + Shape of the image (data array) using "standard" ``nx`` first and ``ny`` + second (as opposite to the ``numpy.ndarray`` convention - ``ny`` first and + ``nx`` second). This value will be assigned to + ``pixel_shape`` and ``array_shape`` properties of the returned + WCS object. When supplied from command line, it should be a comma-separated + list of integers ``nx, ny``. + + .. note:: + Specifying ``output_shape`` *is required* when the WCS in + ``output_wcs`` does not have ``bounding_box`` property set. + +``--output_wcs`` (str, default='') + File name of a ``ASDF`` file with a GWCS stored under the ``"wcs"`` key + under the root of the file. The output image size is determined from the + bounding box of the WCS (if any). Argument ``output_shape`` overrides + computed image size and it is required when output WCS does not have + ``bounding_box`` property set. + + .. note:: + When ``output_wcs`` is specified, WCS-related arguments such as + ``pixel_scale_ratio``, ``pixel_scale``, ``rotation``, ``crpix``, + and ``crval`` will be ignored. + +``--fillval`` (str, default='INDEF') + The value to assign to output pixels that have zero weight or do not + receive any flux from any input pixels during drizzling. + +``--weight_type`` (str, default='ivm') + The weighting type for each input image. + If `weight_type=ivm` (the default), the scaling value + will be determined per-pixel using the inverse of the read noise + (VAR_RNOISE) array stored in each input image. If the VAR_RNOISE array does + not exist, the variance is set to 1 for all pixels (equal weighting). + If `weight_type=exptime`, the scaling value will be set equal to the + exposure time found in the image header. + +``--single`` (bool, default=False) + If set to `True`, resample each input image into a separate output. If + `False` (the default), each input is resampled additively (with weights) to + a common output + +``--blendheaders`` (bool, default=True) + Blend metadata from all input images into the resampled output image. + +``--allowed_memory`` (float, default=None) + Specifies the fractional amount of free memory to allow when creating the + resampled image. If ``None``, the environment variable + ``DMODEL_ALLOWED_MEMORY`` is used. If not defined, no check is made. If the + resampled image would be larger than specified, an ``OutputTooLargeError`` + exception will be generated. + + For example, if set to ``0.5``, only resampled images that use less than + half the available memory can be created. + +``--in_memory`` (bool, default=True) + If set to `False`, write output datamodel to disk. diff --git a/docs/roman/resample/index.rst b/docs/roman/resample/index.rst new file mode 100644 index 000000000..9f0adea95 --- /dev/null +++ b/docs/roman/resample/index.rst @@ -0,0 +1,16 @@ +.. _resample_step: + +========== +Resample +========== + +.. toctree:: + :maxdepth: 2 + + main.rst + arguments.rst + resample_step.rst + resample.rst + resample_utils.rst + +.. automodapi:: romancal.resample diff --git a/docs/roman/resample/main.rst b/docs/roman/resample/main.rst new file mode 100644 index 000000000..9b1f3d06b --- /dev/null +++ b/docs/roman/resample/main.rst @@ -0,0 +1,118 @@ +Description +=========== + +:Classes: `romancal.resample.ResampleStep` +:Alias: resample + +This routine will resample each input 2D image based on the WCS and +distortion information, and will combine multiple resampled images +into a single undistorted product. The distortion information should have +been incorporated into the image using the :ref:`assign_wcs ` +step. + +The ``resample`` step can take: + + * a single 2D input image (in the format of either a string with the full + path and filename of an ASDF file or a Roman + Datamodel/:py:class:`~romancal.datamodels.container.ModelContainer`); + * an association table (in JSON format). + +The parameters for the drizzle operation itself are set by +:py:func:`~romancal.resample.resample_step.ResampleStep.set_drizzle_defaults`. +The exact values used depends on the number of input images being combined +and the filter being used. Other information may be added as selection criteria +later, but during the :py:class:`~romancal.resample.resample_step.ResampleStep` +instantiation, only basic information is set. + +The output product is determined by using the WCS information of all inputs, +even if it is just a single image. The output WCS defines a +field-of-view that encompasses the undistorted footprints on the sky +of all the input images with the same orientation and plate scale +as the first listed input image. + +This step uses the interface to the C-based `cdriz` routine to do the +resampling via the +drizzle method (`Fruchter and Hook, PASP 2002`_). +The input-to-output pixel mapping is determined via a mapping function +derived from the WCS of each input image and the WCS of the defined +output product. The mapping function is created by +:py:func:`~romancal.resample.resample_utils.reproject` and passed on to +`cdriz` to drive the actual drizzling to create the output product. + +Context Image +------------- + +In addition to the resampled image data, resample step also creates a +"context image" stored in the ``con`` attribute in the output data model. +Each pixel in the context image is a bit field that encodes +information about which input image has contributed to the corresponding +pixel in the resampled data array. Context image uses 32 bit integers to encode +this information and hence it can keep track of 32 input images at most. + +For any given pixel, the first bit corresponds to the first input image, +the second bit corrsponds to the second input image, and so on. +If the number of input images is larger than 32, then it is necessary to +have multiple context images ("planes") to hold information about all input +images with the first plane encoding which of the first 32 images +(indexed from 0 through 32) contributed to the output data pixel, second plane +representing next 32 input images (indexed from 33 through 64), etc. +For this reason, the context image is a 3D array of type `numpy.int32` and shape +``(np, ny, nx)``, where ``nx`` and ``ny`` are the dimensions of the image's data +and ``np`` is the number of "planes", which is equal to +``(number of input images - 1) // 32 + 1``. If a bit at position ``k`` in a pixel +with coordinates ``(p, y, x)`` is 0 then input image number ``32 * p + k`` +(0-indexed) did not contribute to the output data pixel with array coordinates +``(y, x)``, and if that bit is 1 then input image number ``32 * p + k`` did +contribute to the pixel ``(y, x)`` in the resampled image. + +As an example, let's assume we have 8 input images. Then, when ``con`` pixel +values are displayed using binary representation (and decimal in parenthesis), +one could see values like this:: + + 00000001 (1) - only first input image contributed to this output pixel; + 00000010 (2) - 2nd input image contributed; + 00000100 (4) - 3rd input image contributed; + 10000000 (128) - 8th input image contributed; + 10000100 (132=128+4) - 3rd and 8th input images contributed; + 11001101 (205=1+4+8+64+128) - input images 1, 3, 4, 7, 8 have contributed + to this output pixel. + +In order to test if a specific input image contributed to an output pixel, +one needs to use bitwise operations. Using the example above, to test whether +input images number 4 and 5 have contributed to the output pixel whose +corresponding ``con`` value is 205 (11001101 in binary form) we can do +the following: + +>>> bool(205 & (1 << (5 - 1))) # (205 & 16) = 0 (== 0 => False): did NOT contribute +False +>>> bool(205 & (1 << (4 - 1))) # (205 & 8) = 8 (!= 0 => True): did contribute +True + +In general, to get a list of all input images that have contributed to an +output resampled pixel with image coordinates ``(x, y)``, and given a +context array ``con``, one can do something like this: + +.. doctest-skip:: + + >>> import numpy as np + >>> np.flatnonzero([v & (1 << k) for v in con[:, y, x] for k in range(32)]) + +For convenience, this functionality was implemented in the +:py:func:`~romancal.resample.resample_utils.decode_context` function. + + +References +---------- + +* `Fruchter and Hook, PASP 2002`_: full description of the drizzling algorithm. + +* `Casertano et al., AJ 2000`_ (Appendix A2): description of the inverse variance + map method. + +* `DrizzlePac Handbook`_: description of the drizzle parameters and other useful + drizzle-related resources. + + +.. _Fruchter and Hook, PASP 2002: https://doi.org/10.1086/338393 +.. _Casertano et al., AJ 2000: https://doi.org/10.1086/316851 +.. _DrizzlePac Handbook: http://drizzlepac.stsci.edu \ No newline at end of file diff --git a/docs/roman/resample/resample.rst b/docs/roman/resample/resample.rst new file mode 100644 index 000000000..665fd8d6d --- /dev/null +++ b/docs/roman/resample/resample.rst @@ -0,0 +1,6 @@ +.. resample_: + +Python Interface to Drizzle: ResampleData() +=========================================== + +.. automodapi:: romancal.resample.resample diff --git a/docs/roman/resample/resample_step.rst b/docs/roman/resample/resample_step.rst new file mode 100644 index 000000000..43c1dd6c6 --- /dev/null +++ b/docs/roman/resample/resample_step.rst @@ -0,0 +1,6 @@ +.. resample_step_: + +Python Step Interface: ResampleStep() +===================================== + +.. automodapi:: romancal.resample.resample_step diff --git a/docs/roman/resample/resample_utils.rst b/docs/roman/resample/resample_utils.rst new file mode 100644 index 000000000..14627b8a8 --- /dev/null +++ b/docs/roman/resample/resample_utils.rst @@ -0,0 +1,8 @@ +Resample Utilities +================== + +.. currentmodule:: romancal.resample.resample_utils + +.. automodule:: romancal.resample.resample_utils + :members: + From cade4f218b470dad31a0f4715edc4f18b97e899b Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Tue, 26 Sep 2023 21:32:58 -0400 Subject: [PATCH 42/82] Add gwcs and astropy to sphinx mapping. --- docs/conf.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index d51bdeae0..58228b539 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -41,7 +41,9 @@ def setup(app): sys.path.insert(0, os.path.abspath("exts/")) # -- General configuration ------------------------------------------------ -with open(Path(__file__).parent.parent / "pyproject.toml", "rb") as configuration_file: +with open( + Path(__file__).parent.parent / "pyproject.toml", "rb" +) as configuration_file: conf = tomllib.load(configuration_file) setup_cfg = conf["project"] @@ -67,6 +69,8 @@ def check_sphinx_version(expected_version): "numpy": ("https://numpy.org/devdocs", None), "scipy": ("http://scipy.github.io/devdocs", None), "matplotlib": ("http://matplotlib.org/", None), + "gwcs": ("https://gwcs.readthedocs.io/en/latest/", None), + "astropy": ("https://docs.astropy.org/en/stable/", None), } if sys.version_info[0] == 2: @@ -74,7 +78,9 @@ def check_sphinx_version(expected_version): intersphinx_mapping["pythonloc"] = ( "http://docs.python.org/", os.path.abspath( - os.path.join(os.path.dirname(__file__), "local/python2_local_links.inv") + os.path.join( + os.path.dirname(__file__), "local/python2_local_links.inv" + ) ), ) @@ -135,7 +141,9 @@ def check_sphinx_version(expected_version): # General information about the project project = setup_cfg["name"] -author = f'{setup_cfg["authors"][0]["name"]} <{setup_cfg["authors"][0]["email"]}>' +author = ( + f'{setup_cfg["authors"][0]["name"]} <{setup_cfg["authors"][0]["email"]}>' +) copyright = f"{datetime.datetime.now().year}, {author}" # The version info for the project you're documenting, acts as replacement for @@ -361,7 +369,9 @@ def check_sphinx_version(expected_version): # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [("index", "romancal", "Roman Pipeline Documentation", ["romancal"], 1)] +man_pages = [ + ("index", "romancal", "Roman Pipeline Documentation", ["romancal"], 1) +] # If true, show URL addresses after external links. man_show_urls = True From 9751af1a50ee5f20bea55bcb47bb4a4a0d31ef51 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Tue, 26 Sep 2023 21:33:42 -0400 Subject: [PATCH 43/82] Small refactoring and docstring improvements. --- romancal/resample/resample.py | 64 +++++++------ romancal/resample/resample_step.py | 133 ++++------------------------ romancal/resample/resample_utils.py | 124 ++++++++++++++++++++++---- 3 files changed, 161 insertions(+), 160 deletions(-) diff --git a/romancal/resample/resample.py b/romancal/resample/resample.py index 18f658dac..0bc3ffa76 100644 --- a/romancal/resample/resample.py +++ b/romancal/resample/resample.py @@ -25,7 +25,7 @@ class ResampleData: Notes ----- - This routine performs the following operations:: + This routine performs the following operations: 1. Extracts parameter settings from input model, such as pixfrac, weight type, exposure time (if relevant), and kernel, and merges @@ -161,7 +161,7 @@ def __init__( self.output_models = ModelContainer() def do_drizzle(self): - """Pick the correct drizzling mode based on self.single""" + """Pick the correct drizzling mode based on ``self.single``.""" if self.single: return self.resample_many_to_many() else: @@ -170,8 +170,8 @@ def do_drizzle(self): def resample_many_to_many(self): """Resample many inputs to many outputs where outputs have a common frame. - Coadd only different detectors of the same exposure, i.e. map NRCA5 and - NRCB5 onto the same output image, as they image different areas of the + Coadd only different detectors of the same exposure (e.g. map SCA 1 and + 10 onto the same output image), as they image different areas of the sky. Used for outlier detection @@ -183,9 +183,13 @@ def resample_many_to_many(self): indx = exposure[0].meta.filename.rfind(".") output_type = exposure[0].meta.filename[indx:] output_root = "_".join( - exposure[0].meta.filename.replace(output_type, "").split("_")[:-1] + exposure[0] + .meta.filename.replace(output_type, "") + .split("_")[:-1] + ) + output_model.meta.filename = ( + f"{output_root}_outlier_i2d{output_type}" ) - output_model.meta.filename = f"{output_root}_outlier_i2d{output_type}" # Initialize the output with the wcs driz = gwcs_drizzle.GWCSDrizzle( @@ -229,15 +233,16 @@ def resample_many_to_many(self): def resample_many_to_one(self): """Resample and coadd many inputs to a single output. - - Used for stage 3 resampling + Used for level 3 resampling """ output_model = self.blank_output.copy() output_model.meta.filename = self.output_filename output_model.meta["resample"] = mk_resample() output_model.meta.resample["members"] = [] output_model.meta.resample.weight_type = self.weight_type - output_model.meta.resample.pointings = len(self.input_models.models_grouped) + output_model.meta.resample.pointings = len( + self.input_models.models_grouped + ) if self.blendheaders: log.info("Skipping blendheaders for now.") @@ -300,12 +305,12 @@ def resample_many_to_one(self): return self.output_models def resample_variance_array(self, name, output_model): - """Resample variance arrays from self.input_models to the output_model + """Resample variance arrays from ``self.input_models`` to the ``output_model``. - Resample the ``name`` variance array to the same name in output_model, + Resample the ``name`` variance array to the same name in ``output_model``, using a cumulative sum. - This modifies output_model in-place. + This modifies ``output_model`` in-place. """ output_wcs = output_model.meta.wcs inverse_variance_sum = np.full_like(output_model.data.value, np.nan) @@ -371,7 +376,7 @@ def resample_variance_array(self, name, output_model): setattr(output_model, name, output_variance) def update_exposure_times(self, output_model): - """Modify exposure time metadata in-place""" + """Update exposure time metadata (in-place).""" total_exposure_time = 0.0 exposure_times = {"start": [], "end": []} for exposure in self.input_models.models_grouped: @@ -408,39 +413,39 @@ def drizzle_arrays( Low level routine for performing 'drizzle' operation on one image. The interface is compatible with STScI code. All images are Python - ndarrays, instead of filenames. File handling (input and output) is + `ndarrays`, instead of filenames. File handling (input and output) is performed by the calling routine. Parameters ---------- insci : 2d array - A 2d numpy array containing the input image to be drizzled. + A 2d `numpy` array containing the input image to be drizzled. inwht : 2d array - A 2d numpy array containing the pixel by pixel weighting. + A 2d `numpy` array containing the pixel by pixel weighting. Must have the same dimensions as insci. If none is supplied, the weighting is set to one. - input_wcs : gwcs.WCS object + input_wcs : `gwcs.wcs.WCS` object The world coordinate system of the input image. - output_wcs : gwcs.WCS object + output_wcs : `gwcs.wcs.WCS` object The world coordinate system of the output image. outsci : 2d array - A 2d numpy array containing the output image produced by + A 2d `numpy` array containing the output image produced by drizzling. On the first call it should be set to zero. Subsequent calls it will hold the intermediate results. This is modified in-place. outwht : 2d array - A 2d numpy array containing the output counts. On the first + A 2d `numpy` array containing the output counts. On the first call it should be set to zero. On subsequent calls it will hold the intermediate results. This is modified in-place. outcon : 2d or 3d array, optional - A 2d or 3d numpy array holding a bitmap of which image was an input + A 2d or 3d `numpy` array holding a bitmap of which image was an input for each output pixel. Should be integer zero on first call. Subsequent calls hold intermediate results. This is modified in-place. @@ -486,8 +491,8 @@ def drizzle_arrays( kernel: str, optional The name of the kernel used to combine the input. The choice of kernel controls the distribution of flux over the kernel. The kernel - names are: "square", "gaussian", "point", "tophat", "turbo", "lanczos2", - and "lanczos3". The square kernel is the default. + names are: `'square'`, `'gaussian'`, `'point'`, `'tophat'`, `'turbo'`, + `'lanczos2'`, and `'lanczos3'`. The `'square'` kernel is the default. fillval: str, optional The value a pixel is set to in the output if the input image does @@ -495,10 +500,11 @@ def drizzle_arrays( Returns ------- - A tuple with three values: a version string, the number of pixels - on the input image that do not overlap the output image, and the - number of complete lines on the input image that do not overlap the - output input image. + : tuple + A tuple with three values: a version string, the number of pixels + on the input image that do not overlap the output image, and the + number of complete lines on the input image that do not overlap the + output input image. """ @@ -540,7 +546,9 @@ def drizzle_arrays( # Compute the mapping between the input and output pixel coordinates # for use in drizzle.cdrizzle.tdriz - pixmap = resample_utils.calc_gwcs_pixmap(input_wcs, output_wcs, insci.shape) + pixmap = resample_utils.calc_gwcs_pixmap( + input_wcs, output_wcs, insci.shape + ) log.debug(f"Pixmap shape: {pixmap[:,:,0].shape}") log.debug(f"Input Sci shape: {insci.shape}") diff --git a/romancal/resample/resample_step.py b/romancal/resample/resample_step.py index 2f2e4a3e3..3f5e370ab 100755 --- a/romancal/resample/resample_step.py +++ b/romancal/resample/resample_step.py @@ -34,16 +34,16 @@ class ResampleStep(RomanStep): Parameters ----------- - input : str, ~roman_datamodels.datamodels.DataModel, or ~romancal.datamodels.ModelContainer + input : str, `roman_datamodels.datamodels.DataModel`, or `~romancal.datamodels.container.ModelContainer` If a string is provided, it should correspond to either a single ASDF filename or an association filename. Alternatively, a single DataModel instance can be - provided instead of an ASDF filename. - Multiple files can be processed via either an association file or wrapped by a - ModelContainer. + provided instead of an ASDF filename. Multiple files can be processed via + either an association file or wrapped by a + `~romancal.datamodels.container.ModelContainer`. Returns ------- - result : ~roman_datamodels.datamodels.MosaicModel + : `roman_datamodels.datamodels.MosaicModel` A mosaic datamodel with the final output frame. """ # noqa: E501 @@ -83,7 +83,9 @@ def process(self, input): except Exception: # single ASDF filename input_models = ModelContainer([input]) - if hasattr(input_models, "asn_table") and len(input_models.asn_table): + if hasattr(input_models, "asn_table") and len( + input_models.asn_table + ): # set output filename from ASN table output = input_models.asn_table["products"][0]["name"] elif hasattr(input_models[0], "meta"): @@ -92,9 +94,7 @@ def process(self, input): elif isinstance(input, ModelContainer): input_models = input # set output filename using the common prefix of all datamodels - output = ( - f"{os.path.commonprefix([x.meta.filename for x in input_models])}.asdf" - ) + output = f"{os.path.commonprefix([x.meta.filename for x in input_models])}.asdf" if len(output) == 0: # set default filename if no common prefix can be determined output = "resample_output.asdf" @@ -109,22 +109,16 @@ def process(self, input): # resample can only handle 2D images, not 3D cubes, etc raise RuntimeError(f"Input {input_models[0]} is not a 2D image.") - # Get drizzle parameters reference file, if there is one self.wht_type = self.weight_type - if "drizpars" in self.reference_file_types: - ref_filename = self.get_reference_file(input_models[0], "drizpars") - self.log.info(f"Using drizpars reference file: {ref_filename}") - kwargs = self.get_drizpars(ref_filename, input_models) - else: - # no drizpars reference file found - self.log.info("No drizpars reference file found.") - kwargs = self._set_spec_defaults() - + self.log.info("Setting drizzle's default parameters...") + kwargs = self.set_drizzle_defaults() kwargs["allowed_memory"] = self.allowed_memory # Issue a warning about the use of exptime weighting if self.wht_type == "exptime": - self.log.warning("Use of EXPTIME weighting will result in incorrect") + self.log.warning( + "Use of EXPTIME weighting will result in incorrect" + ) self.log.warning("propagated errors in the resampled product") # Custom output WCS parameters. @@ -222,7 +216,9 @@ def _check_list_pars(vals, name, min_vals=None): ) return list(vals) else: - raise ValueError(f"Both '{name}' values must be either None or not None.") + raise ValueError( + f"Both '{name}' values must be either None or not None." + ) @staticmethod def _load_custom_wcs(asdf_wcs_file, output_shape): @@ -261,99 +257,8 @@ def update_phot_keywords(self, model): model.meta.resample.pixel_scale_ratio**2 ) - def get_drizpars(self, ref_filename, input_models): - """ - Extract drizzle parameters from reference file. - - This method extracts parameters from the drizpars reference file and - uses those to set defaults on the following ResampleStep configuration - parameters: - - pixfrac = float(default=None) - kernel = string(default=None) - fillval = string(default=None) - wht_type = option('ivm', 'exptime', None, default=None) - - Once the defaults are set from the reference file, if the user has - used a resample.cfg file or run ResampleStep using command line args, - then these will overwrite the defaults pulled from the reference file. - """ - with datamodels.DrizParsModel(ref_filename) as drpt: - drizpars_table = drpt.data - - num_groups = len(input_models.group_names) - filtname = input_models[0].meta.instrument.filter - row = None - filter_match = False - # look for row that applies to this set of input data models - for n, filt, num in zip( - range(len(drizpars_table)), - drizpars_table["filter"], - drizpars_table["numimages"], - ): - # only remember this row if no exact match has already been made for - # the filter. This allows the wild-card row to be anywhere in the - # table; since it may be placed at beginning or end of table. - - if str(filt) == "ANY" and not filter_match and num_groups >= num: - row = n - # always go for an exact match if present, though... - if filtname == filt and num_groups >= num: - row = n - filter_match = True - - # With presence of wild-card rows, code should never trigger this logic - if row is None: - self.log.error("No row found in %s matching input data.", ref_filename) - raise ValueError - - # Define the keys to pull from drizpars reffile table. - # All values should be None unless the user set them on the command - # line or in the call to the step - - drizpars = dict( - pixfrac=self.pixfrac, - kernel=self.kernel, - fillval=self.fillval, - wht_type=self.weight_type, - ) - - # For parameters that are set in drizpars table but not set by the - # user, use these. Otherwise, use values set by user. - reffile_drizpars = {k: v for k, v in drizpars.items() if v is None} - user_drizpars = {k: v for k, v in drizpars.items() if v is not None} - - # read in values from that row for each parameter - for k in reffile_drizpars: - if k in drizpars_table.names: - reffile_drizpars[k] = drizpars_table[k][row] - - # Convert the strings in the FITS binary table from np.bytes_ to str - for k, v in reffile_drizpars.items(): - if isinstance(v, np.bytes_): - reffile_drizpars[k] = v.decode("UTF-8") - - all_drizpars = reffile_drizpars | user_drizpars - - kwargs = ( - dict( - good_bits=GOOD_BITS, - single=self.single, - blendheaders=self.blendheaders, - ) - | all_drizpars - ) - for k, v in kwargs.items(): - self.log.debug(f" {k}={v}") - - return kwargs - - def _set_spec_defaults(self): - """NIRSpec currently has no default drizpars reference file, so default - drizzle parameters are not set properly. This method sets them. - - Remove this class method when a drizpars reffile is delivered. - """ + def set_drizzle_defaults(self): + """Set the default parameters for drizzle.""" configspec = self.load_spec_file() config = ConfigObj(configspec=configspec) if config.validate(Validator()): diff --git a/romancal/resample/resample_utils.py b/romancal/resample/resample_utils.py index ff037ee0a..a05422450 100644 --- a/romancal/resample/resample_utils.py +++ b/romancal/resample/resample_utils.py @@ -26,18 +26,19 @@ def make_output_wcs( crpix: Tuple[float, float] = None, crval: Tuple[float, float] = None, ): - """Generate output WCS here based on footprints of all input WCS objects + """ + Generate output WCS here based on footprints of all input WCS objects + Parameters ---------- - input_models : list of `~roman_datamodels.datamodels.DataModel` - Each datamodel must have a ~gwcs.WCS object. + input_models : list of `roman_datamodels.datamodels.DataModel` + Each datamodel must have a `gwcs.wcs.WCS` object. pscale_ratio : float, optional Ratio of input to output pixel scale. Ignored when ``pscale`` is provided. pscale : float, None, optional - Absolute pixel scale in degrees. When provided, overrides - ``pscale_ratio``. + Absolute pixel scale in degrees. When provided, overrides ``pscale_ratio``. rotation : float, None, optional Position angle (in degrees) of output image's Y-axis relative to North. @@ -49,8 +50,8 @@ def make_output_wcs( shape : tuple of int, None, optional Shape of the image (data array) using ``numpy.ndarray`` convention - (``ny`` first and ``nx`` second). This value will be assigned to - ``pixel_shape`` and ``array_shape`` properties of the returned + (``ny`` first and ``nx`` second). This value will be assigned + to ``pixel_shape`` and ``array_shape`` properties of the returned WCS object. crpix : tuple of float, None, optional @@ -66,8 +67,8 @@ def make_output_wcs( ------- output_wcs : object WCS object, with defined domain, covering entire set of input frames - """ + wcslist = [i.meta.wcs for i in input_models] for w, i in zip(wcslist, input_models): if w.bounding_box is None: @@ -109,7 +110,9 @@ def build_driz_weight(model, weight_type=None, good_bits=None): ): with np.errstate(divide="ignore", invalid="ignore"): inv_variance = model.var_rnoise**-1 - inv_variance[~np.isfinite(inv_variance)] = 1 * u.s**2 / u.electron**2 + inv_variance[~np.isfinite(inv_variance)] = ( + 1 * u.s**2 / u.electron**2 + ) else: warnings.warn( "var_rnoise array not available. Setting drizzle weight map to 1", @@ -127,8 +130,8 @@ def build_driz_weight(model, weight_type=None, good_bits=None): def build_mask(dqarr, bitvalue): - """Build a bit mask from an input DQ array and a bitvalue flag - + """ + Build a bit mask from an input DQ array and a bitvalue flag. In the returned bit mask, 1 is good, 0 is bad """ bitvalue = interpret_bit_flags(bitvalue, flag_name_map=pixel) @@ -139,7 +142,9 @@ def build_mask(dqarr, bitvalue): def calc_gwcs_pixmap(in_wcs, out_wcs, shape=None): - """Return a pixel grid map from input frame to output frame.""" + """ + Return a pixel grid map from input frame to output frame. + """ if shape: bb = wcs_bbox_from_shape(shape) log.debug(f"Bounding box from data shape: {bb}") @@ -162,14 +167,14 @@ def reproject(wcs1, wcs2): Parameters ---------- - wcs1, wcs2 : `~astropy.wcs.WCS` or `~gwcs.wcs.WCS` or `~astropy.modeling.Model` + wcs1, wcs2 : `astropy.wcs.WCS` or `gwcs.wcs.WCS` or `astropy.modeling.Model` WCS objects. Returns ------- - _reproject : func - Function to compute the transformations. It takes x, y - positions in ``wcs1`` and returns x, y positions in ``wcs2``. + : func + Function to compute the transformations. It takes `(x, y)` + positions in ``wcs1`` and returns `(x, y)` positions in ``wcs2``. """ if isinstance(wcs1, fitswcs.WCS): @@ -180,7 +185,7 @@ def reproject(wcs1, wcs2): forward_transform = wcs1 else: raise TypeError( - "Expected input to be astropy.wcs.WCS or gwcs.WCS " + "Expected input to be astropy.wcs.WCS or gwcs.wcs.WCS " "object or astropy.modeling.Model subclass" ) @@ -192,7 +197,7 @@ def reproject(wcs1, wcs2): backward_transform = wcs2.inverse else: raise TypeError( - "Expected input to be astropy.wcs.WCS or gwcs.WCS " + "Expected input to be astropy.wcs.WCS or gwcs.wcs.WCS " "object or astropy.modeling.Model subclass" ) @@ -211,3 +216,86 @@ def _reproject(x, y): return tuple(det_reshaped) return _reproject + + +def decode_context(context, x, y): + """ + Get 0-based indices of input images that contributed to (resampled) + output pixel with coordinates ``x`` and ``y``. + + Parameters + ---------- + context: numpy.ndarray + A 3D `~numpy.ndarray` of integral data type. + + x: int, list of integers, numpy.ndarray of integers + X-coordinate of pixels to decode (3rd index into the ``context`` array) + + y: int, list of integers, numpy.ndarray of integers + Y-coordinate of pixels to decode (2nd index into the ``context`` array) + + Returns + ------- + + A list of `numpy.ndarray` objects each containing indices of input images + that have contributed to an output pixel with coordinates ``x`` and ``y``. + The length of returned list is equal to the number of input coordinate + arrays ``x`` and ``y``. + + Examples + -------- + + An example context array for an output image of array shape ``(5, 6)`` + obtained by resampling 80 input images. + + >>> import numpy as np + >>> from jwst.resample.resample_utils import decode_context + >>> con = np.array( + ... [[[0, 0, 0, 0, 0, 0], + ... [0, 0, 0, 36196864, 0, 0], + ... [0, 0, 0, 0, 0, 0], + ... [0, 0, 0, 0, 0, 0], + ... [0, 0, 537920000, 0, 0, 0]], + ... [[0, 0, 0, 0, 0, 0,], + ... [0, 0, 0, 67125536, 0, 0], + ... [0, 0, 0, 0, 0, 0], + ... [0, 0, 0, 0, 0, 0], + ... [0, 0, 163856, 0, 0, 0]], + ... [[0, 0, 0, 0, 0, 0], + ... [0, 0, 0, 8203, 0, 0], + ... [0, 0, 0, 0, 0, 0], + ... [0, 0, 0, 0, 0, 0], + ... [0, 0, 32865, 0, 0, 0]]], + ... dtype=np.int32 + ... ) + >>> decode_context(con, [3, 2], [1, 4]) + [array([ 9, 12, 14, 19, 21, 25, 37, 40, 46, 58, 64, 65, 67, 77]), + array([ 9, 20, 29, 36, 47, 49, 64, 69, 70, 79])] + + """ + if context.ndim != 3: + raise ValueError("'context' must be a 3D array.") + + x = np.atleast_1d(x) + y = np.atleast_1d(y) + + if x.size != y.size: + raise ValueError("Coordinate arrays must have equal length.") + + if x.ndim != 1: + raise ValueError("Coordinates must be scalars or 1D arrays.") + + if not ( + np.issubdtype(x.dtype, np.integer) + and np.issubdtype(y.dtype, np.integer) + ): + raise ValueError("Pixel coordinates must be integer values") + + nbits = 8 * context.dtype.itemsize + + return [ + np.flatnonzero( + [v & (1 << k) for v in context[:, yi, xi] for k in range(nbits)] + ) + for xi, yi in zip(x, y) + ] From b5a48cbf5f353897756145465ed84f4554f853b5 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Wed, 27 Sep 2023 10:18:19 -0400 Subject: [PATCH 44/82] Style check fixes. --- CHANGES.rst | 8 ++-- docs/conf.py | 16 ++----- docs/roman/resample/main.rst | 64 +++++++++++++------------- docs/roman/resample/resample_utils.rst | 1 - romancal/resample/resample.py | 16 ++----- romancal/resample/resample_step.py | 16 +++---- romancal/resample/resample_utils.py | 13 ++---- 7 files changed, 53 insertions(+), 81 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 773effab4..bccdd9692 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -21,14 +21,14 @@ ramp_fitting - Fix opening mode for references to be read-only [#854] - -0.12.0 (2023-08-18) -=================== - resample -------- - Implement resampling step. [#787] + +0.12.0 (2023-08-18) +=================== + source_detection ---------------- - Skip the step if the data is not imaging mode. [#798] diff --git a/docs/conf.py b/docs/conf.py index 58228b539..f49764ae1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -41,9 +41,7 @@ def setup(app): sys.path.insert(0, os.path.abspath("exts/")) # -- General configuration ------------------------------------------------ -with open( - Path(__file__).parent.parent / "pyproject.toml", "rb" -) as configuration_file: +with open(Path(__file__).parent.parent / "pyproject.toml", "rb") as configuration_file: conf = tomllib.load(configuration_file) setup_cfg = conf["project"] @@ -78,9 +76,7 @@ def check_sphinx_version(expected_version): intersphinx_mapping["pythonloc"] = ( "http://docs.python.org/", os.path.abspath( - os.path.join( - os.path.dirname(__file__), "local/python2_local_links.inv" - ) + os.path.join(os.path.dirname(__file__), "local/python2_local_links.inv") ), ) @@ -141,9 +137,7 @@ def check_sphinx_version(expected_version): # General information about the project project = setup_cfg["name"] -author = ( - f'{setup_cfg["authors"][0]["name"]} <{setup_cfg["authors"][0]["email"]}>' -) +author = f'{setup_cfg["authors"][0]["name"]} <{setup_cfg["authors"][0]["email"]}>' copyright = f"{datetime.datetime.now().year}, {author}" # The version info for the project you're documenting, acts as replacement for @@ -369,9 +363,7 @@ def check_sphinx_version(expected_version): # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - ("index", "romancal", "Roman Pipeline Documentation", ["romancal"], 1) -] +man_pages = [("index", "romancal", "Roman Pipeline Documentation", ["romancal"], 1)] # If true, show URL addresses after external links. man_show_urls = True diff --git a/docs/roman/resample/main.rst b/docs/roman/resample/main.rst index 9b1f3d06b..536185624 100644 --- a/docs/roman/resample/main.rst +++ b/docs/roman/resample/main.rst @@ -12,16 +12,16 @@ step. The ``resample`` step can take: - * a single 2D input image (in the format of either a string with the full - path and filename of an ASDF file or a Roman + * a single 2D input image (in the format of either a string with the full + path and filename of an ASDF file or a Roman Datamodel/:py:class:`~romancal.datamodels.container.ModelContainer`); * an association table (in JSON format). -The parameters for the drizzle operation itself are set by -:py:func:`~romancal.resample.resample_step.ResampleStep.set_drizzle_defaults`. -The exact values used depends on the number of input images being combined -and the filter being used. Other information may be added as selection criteria -later, but during the :py:class:`~romancal.resample.resample_step.ResampleStep` +The parameters for the drizzle operation itself are set by +:py:func:`~romancal.resample.resample_step.ResampleStep.set_drizzle_defaults`. +The exact values used depends on the number of input images being combined +and the filter being used. Other information may be added as selection criteria +later, but during the :py:class:`~romancal.resample.resample_step.ResampleStep` instantiation, only basic information is set. The output product is determined by using the WCS information of all inputs, @@ -31,38 +31,38 @@ of all the input images with the same orientation and plate scale as the first listed input image. This step uses the interface to the C-based `cdriz` routine to do the -resampling via the -drizzle method (`Fruchter and Hook, PASP 2002`_). -The input-to-output pixel mapping is determined via a mapping function -derived from the WCS of each input image and the WCS of the defined -output product. The mapping function is created by -:py:func:`~romancal.resample.resample_utils.reproject` and passed on to +resampling via the +drizzle method (`Fruchter and Hook, PASP 2002`_). +The input-to-output pixel mapping is determined via a mapping function +derived from the WCS of each input image and the WCS of the defined +output product. The mapping function is created by +:py:func:`~romancal.resample.resample_utils.reproject` and passed on to `cdriz` to drive the actual drizzling to create the output product. Context Image ------------- -In addition to the resampled image data, resample step also creates a -"context image" stored in the ``con`` attribute in the output data model. +In addition to the resampled image data, resample step also creates a +"context image" stored in the ``con`` attribute in the output data model. Each pixel in the context image is a bit field that encodes information about which input image has contributed to the corresponding pixel in the resampled data array. Context image uses 32 bit integers to encode this information and hence it can keep track of 32 input images at most. -For any given pixel, the first bit corresponds to the first input image, -the second bit corrsponds to the second input image, and so on. -If the number of input images is larger than 32, then it is necessary to -have multiple context images ("planes") to hold information about all input -images with the first plane encoding which of the first 32 images -(indexed from 0 through 32) contributed to the output data pixel, second plane -representing next 32 input images (indexed from 33 through 64), etc. -For this reason, the context image is a 3D array of type `numpy.int32` and shape -``(np, ny, nx)``, where ``nx`` and ``ny`` are the dimensions of the image's data -and ``np`` is the number of "planes", which is equal to -``(number of input images - 1) // 32 + 1``. If a bit at position ``k`` in a pixel -with coordinates ``(p, y, x)`` is 0 then input image number ``32 * p + k`` -(0-indexed) did not contribute to the output data pixel with array coordinates -``(y, x)``, and if that bit is 1 then input image number ``32 * p + k`` did +For any given pixel, the first bit corresponds to the first input image, +the second bit corrsponds to the second input image, and so on. +If the number of input images is larger than 32, then it is necessary to +have multiple context images ("planes") to hold information about all input +images with the first plane encoding which of the first 32 images +(indexed from 0 through 32) contributed to the output data pixel, second plane +representing next 32 input images (indexed from 33 through 64), etc. +For this reason, the context image is a 3D array of type `numpy.int32` and shape +``(np, ny, nx)``, where ``nx`` and ``ny`` are the dimensions of the image's data +and ``np`` is the number of "planes", which is equal to +``(number of input images - 1) // 32 + 1``. If a bit at position ``k`` in a pixel +with coordinates ``(p, y, x)`` is 0 then input image number ``32 * p + k`` +(0-indexed) did not contribute to the output data pixel with array coordinates +``(y, x)``, and if that bit is 1 then input image number ``32 * p + k`` did contribute to the pixel ``(y, x)`` in the resampled image. As an example, let's assume we have 8 input images. Then, when ``con`` pixel @@ -106,13 +106,13 @@ References * `Fruchter and Hook, PASP 2002`_: full description of the drizzling algorithm. -* `Casertano et al., AJ 2000`_ (Appendix A2): description of the inverse variance +* `Casertano et al., AJ 2000`_ (Appendix A2): description of the inverse variance map method. -* `DrizzlePac Handbook`_: description of the drizzle parameters and other useful +* `DrizzlePac Handbook`_: description of the drizzle parameters and other useful drizzle-related resources. .. _Fruchter and Hook, PASP 2002: https://doi.org/10.1086/338393 .. _Casertano et al., AJ 2000: https://doi.org/10.1086/316851 -.. _DrizzlePac Handbook: http://drizzlepac.stsci.edu \ No newline at end of file +.. _DrizzlePac Handbook: http://drizzlepac.stsci.edu diff --git a/docs/roman/resample/resample_utils.rst b/docs/roman/resample/resample_utils.rst index 14627b8a8..5b845e910 100644 --- a/docs/roman/resample/resample_utils.rst +++ b/docs/roman/resample/resample_utils.rst @@ -5,4 +5,3 @@ Resample Utilities .. automodule:: romancal.resample.resample_utils :members: - diff --git a/romancal/resample/resample.py b/romancal/resample/resample.py index 0bc3ffa76..56e9bec6b 100644 --- a/romancal/resample/resample.py +++ b/romancal/resample/resample.py @@ -183,13 +183,9 @@ def resample_many_to_many(self): indx = exposure[0].meta.filename.rfind(".") output_type = exposure[0].meta.filename[indx:] output_root = "_".join( - exposure[0] - .meta.filename.replace(output_type, "") - .split("_")[:-1] - ) - output_model.meta.filename = ( - f"{output_root}_outlier_i2d{output_type}" + exposure[0].meta.filename.replace(output_type, "").split("_")[:-1] ) + output_model.meta.filename = f"{output_root}_outlier_i2d{output_type}" # Initialize the output with the wcs driz = gwcs_drizzle.GWCSDrizzle( @@ -240,9 +236,7 @@ def resample_many_to_one(self): output_model.meta["resample"] = mk_resample() output_model.meta.resample["members"] = [] output_model.meta.resample.weight_type = self.weight_type - output_model.meta.resample.pointings = len( - self.input_models.models_grouped - ) + output_model.meta.resample.pointings = len(self.input_models.models_grouped) if self.blendheaders: log.info("Skipping blendheaders for now.") @@ -546,9 +540,7 @@ def drizzle_arrays( # Compute the mapping between the input and output pixel coordinates # for use in drizzle.cdrizzle.tdriz - pixmap = resample_utils.calc_gwcs_pixmap( - input_wcs, output_wcs, insci.shape - ) + pixmap = resample_utils.calc_gwcs_pixmap(input_wcs, output_wcs, insci.shape) log.debug(f"Pixmap shape: {pixmap[:,:,0].shape}") log.debug(f"Input Sci shape: {insci.shape}") diff --git a/romancal/resample/resample_step.py b/romancal/resample/resample_step.py index 3f5e370ab..e12aa4945 100755 --- a/romancal/resample/resample_step.py +++ b/romancal/resample/resample_step.py @@ -83,9 +83,7 @@ def process(self, input): except Exception: # single ASDF filename input_models = ModelContainer([input]) - if hasattr(input_models, "asn_table") and len( - input_models.asn_table - ): + if hasattr(input_models, "asn_table") and len(input_models.asn_table): # set output filename from ASN table output = input_models.asn_table["products"][0]["name"] elif hasattr(input_models[0], "meta"): @@ -94,7 +92,9 @@ def process(self, input): elif isinstance(input, ModelContainer): input_models = input # set output filename using the common prefix of all datamodels - output = f"{os.path.commonprefix([x.meta.filename for x in input_models])}.asdf" + output = ( + f"{os.path.commonprefix([x.meta.filename for x in input_models])}.asdf" + ) if len(output) == 0: # set default filename if no common prefix can be determined output = "resample_output.asdf" @@ -116,9 +116,7 @@ def process(self, input): # Issue a warning about the use of exptime weighting if self.wht_type == "exptime": - self.log.warning( - "Use of EXPTIME weighting will result in incorrect" - ) + self.log.warning("Use of EXPTIME weighting will result in incorrect") self.log.warning("propagated errors in the resampled product") # Custom output WCS parameters. @@ -216,9 +214,7 @@ def _check_list_pars(vals, name, min_vals=None): ) return list(vals) else: - raise ValueError( - f"Both '{name}' values must be either None or not None." - ) + raise ValueError(f"Both '{name}' values must be either None or not None.") @staticmethod def _load_custom_wcs(asdf_wcs_file, output_shape): diff --git a/romancal/resample/resample_utils.py b/romancal/resample/resample_utils.py index a05422450..675bf3457 100644 --- a/romancal/resample/resample_utils.py +++ b/romancal/resample/resample_utils.py @@ -110,9 +110,7 @@ def build_driz_weight(model, weight_type=None, good_bits=None): ): with np.errstate(divide="ignore", invalid="ignore"): inv_variance = model.var_rnoise**-1 - inv_variance[~np.isfinite(inv_variance)] = ( - 1 * u.s**2 / u.electron**2 - ) + inv_variance[~np.isfinite(inv_variance)] = 1 * u.s**2 / u.electron**2 else: warnings.warn( "var_rnoise array not available. Setting drizzle weight map to 1", @@ -285,17 +283,12 @@ def decode_context(context, x, y): if x.ndim != 1: raise ValueError("Coordinates must be scalars or 1D arrays.") - if not ( - np.issubdtype(x.dtype, np.integer) - and np.issubdtype(y.dtype, np.integer) - ): + if not (np.issubdtype(x.dtype, np.integer) and np.issubdtype(y.dtype, np.integer)): raise ValueError("Pixel coordinates must be integer values") nbits = 8 * context.dtype.itemsize return [ - np.flatnonzero( - [v & (1 << k) for v in context[:, yi, xi] for k in range(nbits)] - ) + np.flatnonzero([v & (1 << k) for v in context[:, yi, xi] for k in range(nbits)]) for xi, yi in zip(x, y) ] From 91d86481876727ae27e37e2ccc8a5ffcc4e9a6a3 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Wed, 27 Sep 2023 14:56:00 -0400 Subject: [PATCH 45/82] Small refactoring to account for review comments. --- romancal/resample/resample.py | 71 +++++++++++++++++++++-------- romancal/resample/resample_utils.py | 34 ++++++++++++-- 2 files changed, 81 insertions(+), 24 deletions(-) diff --git a/romancal/resample/resample.py b/romancal/resample/resample.py index 56e9bec6b..658d70b85 100644 --- a/romancal/resample/resample.py +++ b/romancal/resample/resample.py @@ -15,7 +15,7 @@ __all__ = ["OutputTooLargeError", "ResampleData"] -class OutputTooLargeError(RuntimeError): +class OutputTooLargeError(MemoryError): """Raised when the output is too large for in-memory instantiation""" @@ -183,9 +183,13 @@ def resample_many_to_many(self): indx = exposure[0].meta.filename.rfind(".") output_type = exposure[0].meta.filename[indx:] output_root = "_".join( - exposure[0].meta.filename.replace(output_type, "").split("_")[:-1] + exposure[0] + .meta.filename.replace(output_type, "") + .split("_")[:-1] + ) + output_model.meta.filename = ( + f"{output_root}_outlier_i2d{output_type}" ) - output_model.meta.filename = f"{output_root}_outlier_i2d{output_type}" # Initialize the output with the wcs driz = gwcs_drizzle.GWCSDrizzle( @@ -210,7 +214,19 @@ def resample_many_to_many(self): else: data = img.data - driz.add_image(data, img.meta.wcs, inwht=inwht) + xmin, xmax, ymin, ymax = resample_utils.resample_range( + data.shape, img.meta.wcs.bounding_box + ) + + driz.add_image( + data, + img.meta.wcs, + inwht=inwht, + xmin=xmin, + xmax=xmax, + ymin=ymin, + ymax=ymax, + ) del data img.close() @@ -236,7 +252,9 @@ def resample_many_to_one(self): output_model.meta["resample"] = mk_resample() output_model.meta.resample["members"] = [] output_model.meta.resample.weight_type = self.weight_type - output_model.meta.resample.pointings = len(self.input_models.models_grouped) + output_model.meta.resample.pointings = len( + self.input_models.models_grouped + ) if self.blendheaders: log.info("Skipping blendheaders for now.") @@ -265,7 +283,19 @@ def resample_many_to_one(self): else: data = img.data - driz.add_image(data, img.meta.wcs, inwht=inwht) + xmin, xmax, ymin, ymax = resample_utils.resample_range( + data.shape, img.meta.wcs.bounding_box + ) + + driz.add_image( + data, + img.meta.wcs, + inwht=inwht, + xmin=xmin, + xmax=xmax, + ymin=ymin, + ymax=ymax, + ) del data, inwht output_model.meta.resample.members.append(img.meta.filename) @@ -334,6 +364,10 @@ def resample_variance_array(self, name, output_model): outwht = np.zeros_like(output_model.data) outcon = np.zeros_like(output_model.context) + xmin, xmax, ymin, ymax = resample_utils.resample_range( + variance.shape, model.meta.wcs.bounding_box + ) + # resample the variance array (fill "unpopulated" pixels with NaNs) self.drizzle_arrays( variance, @@ -346,6 +380,10 @@ def resample_variance_array(self, name, output_model): pixfrac=self.pixfrac, kernel=self.kernel, fillval=np.nan, + xmin=xmin, + xmax=xmax, + ymin=ymin, + ymax=ymax, ) # Add the inverse of the resampled variance to a running sum. @@ -449,8 +487,7 @@ def drizzle_arrays( this function is called and incremented by one on each subsequent call. - xmin : float, optional - This and the following three parameters set a bounding rectangle + xmin : float, None, optional on the input image. Only pixels on the input image inside this rectangle will have their flux added to the output image. Xmin sets the minimum value of the x dimension. The x dimension is the @@ -458,19 +495,19 @@ def drizzle_arrays( no minimum will be set in the x dimension. All four parameters are zero based, counting starts at zero. - xmax : float, optional + xmax : float, None, optional Sets the maximum value of the x dimension on the bounding box of the input image. If the value is zero, no maximum will be set in the x dimension, the full x dimension of the output image is the bounding box. - ymin : float, optional + ymin : float, None, optional Sets the minimum value in the y dimension on the bounding box. The y dimension varies less rapidly than the x and represents the line index on the input image. If the value is zero, no minimum will be set in the y dimension. - ymax : float, optional + ymax : float, None, optional Sets the maximum value in the y dimension. If the value is zero, no maximum will be set in the y dimension, the full x dimension of the output image is the bounding box. @@ -530,17 +567,11 @@ def drizzle_arrays( if outcon.ndim == 3: outcon = outcon[planeid] - if xmin is xmax is ymin is ymax is None: - bb = input_wcs.bounding_box - ((x1, x2), (y1, y2)) = bb - xmin = int(min(x1, x2)) - ymin = int(min(y1, y2)) - xmax = int(max(x1, x2)) - ymax = int(max(y1, y2)) - # Compute the mapping between the input and output pixel coordinates # for use in drizzle.cdrizzle.tdriz - pixmap = resample_utils.calc_gwcs_pixmap(input_wcs, output_wcs, insci.shape) + pixmap = resample_utils.calc_gwcs_pixmap( + input_wcs, output_wcs, insci.shape + ) log.debug(f"Pixmap shape: {pixmap[:,:,0].shape}") log.debug(f"Input Sci shape: {insci.shape}") diff --git a/romancal/resample/resample_utils.py b/romancal/resample/resample_utils.py index 675bf3457..caa7ae9c6 100644 --- a/romancal/resample/resample_utils.py +++ b/romancal/resample/resample_utils.py @@ -110,7 +110,9 @@ def build_driz_weight(model, weight_type=None, good_bits=None): ): with np.errstate(divide="ignore", invalid="ignore"): inv_variance = model.var_rnoise**-1 - inv_variance[~np.isfinite(inv_variance)] = 1 * u.s**2 / u.electron**2 + inv_variance[~np.isfinite(inv_variance)] = ( + 1 * u.s**2 / u.electron**2 + ) else: warnings.warn( "var_rnoise array not available. Setting drizzle weight map to 1", @@ -122,7 +124,10 @@ def build_driz_weight(model, weight_type=None, good_bits=None): exptime = model.meta.exposure.exposure_time result = exptime * dqmask else: - result = np.ones(model.data.shape, dtype=model.data.dtype) * dqmask + raise ValueError( + f"Invalid weight type: {weight_type}." + "Allowed weight types are 'ivm' or 'exptime'." + ) return result.astype(np.float32) @@ -283,12 +288,33 @@ def decode_context(context, x, y): if x.ndim != 1: raise ValueError("Coordinates must be scalars or 1D arrays.") - if not (np.issubdtype(x.dtype, np.integer) and np.issubdtype(y.dtype, np.integer)): + if not ( + np.issubdtype(x.dtype, np.integer) + and np.issubdtype(y.dtype, np.integer) + ): raise ValueError("Pixel coordinates must be integer values") nbits = 8 * context.dtype.itemsize return [ - np.flatnonzero([v & (1 << k) for v in context[:, yi, xi] for k in range(nbits)]) + np.flatnonzero( + [v & (1 << k) for v in context[:, yi, xi] for k in range(nbits)] + ) for xi, yi in zip(x, y) ] + + +def resample_range(data_shape, bbox=None): + # Find range of input pixels to resample: + if bbox is None: + xmin = ymin = 0 + xmax = data_shape[1] - 1 + ymax = data_shape[0] - 1 + else: + ((x1, x2), (y1, y2)) = bbox + xmin = max(0, int(x1 + 0.5)) + ymin = max(0, int(y1 + 0.5)) + xmax = min(data_shape[1] - 1, int(x2 + 0.5)) + ymax = min(data_shape[0] - 1, int(y2 + 0.5)) + + return xmin, xmax, ymin, ymax From 50a545fe2137f017436031a9705ea742ad007f66 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Wed, 27 Sep 2023 14:56:44 -0400 Subject: [PATCH 46/82] Check style fixes. --- romancal/resample/resample.py | 16 ++++------------ romancal/resample/resample_utils.py | 13 +++---------- 2 files changed, 7 insertions(+), 22 deletions(-) diff --git a/romancal/resample/resample.py b/romancal/resample/resample.py index 658d70b85..cbb0b0af8 100644 --- a/romancal/resample/resample.py +++ b/romancal/resample/resample.py @@ -183,13 +183,9 @@ def resample_many_to_many(self): indx = exposure[0].meta.filename.rfind(".") output_type = exposure[0].meta.filename[indx:] output_root = "_".join( - exposure[0] - .meta.filename.replace(output_type, "") - .split("_")[:-1] - ) - output_model.meta.filename = ( - f"{output_root}_outlier_i2d{output_type}" + exposure[0].meta.filename.replace(output_type, "").split("_")[:-1] ) + output_model.meta.filename = f"{output_root}_outlier_i2d{output_type}" # Initialize the output with the wcs driz = gwcs_drizzle.GWCSDrizzle( @@ -252,9 +248,7 @@ def resample_many_to_one(self): output_model.meta["resample"] = mk_resample() output_model.meta.resample["members"] = [] output_model.meta.resample.weight_type = self.weight_type - output_model.meta.resample.pointings = len( - self.input_models.models_grouped - ) + output_model.meta.resample.pointings = len(self.input_models.models_grouped) if self.blendheaders: log.info("Skipping blendheaders for now.") @@ -569,9 +563,7 @@ def drizzle_arrays( # Compute the mapping between the input and output pixel coordinates # for use in drizzle.cdrizzle.tdriz - pixmap = resample_utils.calc_gwcs_pixmap( - input_wcs, output_wcs, insci.shape - ) + pixmap = resample_utils.calc_gwcs_pixmap(input_wcs, output_wcs, insci.shape) log.debug(f"Pixmap shape: {pixmap[:,:,0].shape}") log.debug(f"Input Sci shape: {insci.shape}") diff --git a/romancal/resample/resample_utils.py b/romancal/resample/resample_utils.py index caa7ae9c6..8c5de9b54 100644 --- a/romancal/resample/resample_utils.py +++ b/romancal/resample/resample_utils.py @@ -110,9 +110,7 @@ def build_driz_weight(model, weight_type=None, good_bits=None): ): with np.errstate(divide="ignore", invalid="ignore"): inv_variance = model.var_rnoise**-1 - inv_variance[~np.isfinite(inv_variance)] = ( - 1 * u.s**2 / u.electron**2 - ) + inv_variance[~np.isfinite(inv_variance)] = 1 * u.s**2 / u.electron**2 else: warnings.warn( "var_rnoise array not available. Setting drizzle weight map to 1", @@ -288,18 +286,13 @@ def decode_context(context, x, y): if x.ndim != 1: raise ValueError("Coordinates must be scalars or 1D arrays.") - if not ( - np.issubdtype(x.dtype, np.integer) - and np.issubdtype(y.dtype, np.integer) - ): + if not (np.issubdtype(x.dtype, np.integer) and np.issubdtype(y.dtype, np.integer)): raise ValueError("Pixel coordinates must be integer values") nbits = 8 * context.dtype.itemsize return [ - np.flatnonzero( - [v & (1 << k) for v in context[:, yi, xi] for k in range(nbits)] - ) + np.flatnonzero([v & (1 << k) for v in context[:, yi, xi] for k in range(nbits)]) for xi, yi in zip(x, y) ] From 28f37fd1a4065e38f55d09ca7d2040c4b937b9f5 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Wed, 27 Sep 2023 16:08:32 -0400 Subject: [PATCH 47/82] Style check fixes and docstring improvements. --- romancal/resample/resample_utils.py | 128 ++++++++++++++++++----- romancal/resample/tests/test_resample.py | 4 +- 2 files changed, 101 insertions(+), 31 deletions(-) diff --git a/romancal/resample/resample_utils.py b/romancal/resample/resample_utils.py index 8c5de9b54..0f9ef7639 100644 --- a/romancal/resample/resample_utils.py +++ b/romancal/resample/resample_utils.py @@ -99,7 +99,38 @@ def make_output_wcs( def build_driz_weight(model, weight_type=None, good_bits=None): - """Create a weight map for use by drizzle""" + """ + Builds the drizzle weight map for resampling. + + Parameters + ---------- + model : object + The input model. + weight_type : str, optional + The type of weight to use. Allowed values are 'ivm' or 'exptime'. + Defaults to None. + good_bits : int or list of int, optional + The good bits to use for building the mask. Defaults to None. + + Returns + ------- + numpy.ndarray + The drizzle weight map. + + Raises + ------ + ValueError + If an invalid weight type is provided. + + Examples + -------- + .. code-block:: none + + model = get_input_model() + weight_map = build_driz_weight(model, weight_type='ivm', good_bits=[1, 2, 3]) + print(weight_map) + """ + dqmask = build_mask(model.dq, good_bits) if weight_type == "ivm": @@ -121,6 +152,8 @@ def build_driz_weight(model, weight_type=None, good_bits=None): elif weight_type == "exptime": exptime = model.meta.exposure.exposure_time result = exptime * dqmask + elif weight_type is None: + result = np.ones(model.data.shape, dtype=model.data.dtype) * dqmask else: raise ValueError( f"Invalid weight type: {weight_type}." @@ -133,7 +166,29 @@ def build_driz_weight(model, weight_type=None, good_bits=None): def build_mask(dqarr, bitvalue): """ Build a bit mask from an input DQ array and a bitvalue flag. - In the returned bit mask, 1 is good, 0 is bad + + Parameters + ---------- + dqarr : numpy.ndarray + Input DQ array. + bitvalue : int + Bitvalue flag. + + Returns + ------- + ndarray + Bit mask where 1 represents good and 0 represents bad. + + Notes + ----- + - The function interprets the bitvalue flag using the + `astropy.nddata.bitmask.interpret_bit_flags` function. + - If the bitvalue is None, the function returns a bit mask with all elements + set to 1. + - Otherwise, the function performs a bitwise AND operation between the dqarr and + the complement of the bitvalue, and then applies a logical NOT operation to + obtain the bit mask. + - The resulting bit mask is returned as an ndarray of dtype `numpy.uint8`. """ bitvalue = interpret_bit_flags(bitvalue, flag_name_map=pixel) @@ -144,7 +199,24 @@ def build_mask(dqarr, bitvalue): def calc_gwcs_pixmap(in_wcs, out_wcs, shape=None): """ - Return a pixel grid map from input frame to output frame. + Generate a pixel map grid using the input and output WCS. + + Parameters + ---------- + in_wcs : `~astropy.wcs.WCS` + Input WCS. + out_wcs : `~astropy.wcs.WCS` + Output WCS. + shape : tuple, optional + Shape of the data. If provided, the bounding box will be calculated + from the shape. If not provided, the bounding box will be calculated + from the input WCS. + + Returns + ------- + pixmap : `~numpy.ndarray` + The calculated pixel map grid. + """ if shape: bb = wcs_bbox_from_shape(shape) @@ -154,9 +226,7 @@ def calc_gwcs_pixmap(in_wcs, out_wcs, shape=None): log.debug(f"Bounding box from WCS: {in_wcs.bounding_box}") grid = gwcs.wcstools.grid_from_bounding_box(bb) - pixmap = np.dstack(reproject(in_wcs, out_wcs)(grid[0], grid[1])) - - return pixmap + return np.dstack(reproject(in_wcs, out_wcs)(grid[0], grid[1])) def reproject(wcs1, wcs2): @@ -249,29 +319,29 @@ def decode_context(context, x, y): An example context array for an output image of array shape ``(5, 6)`` obtained by resampling 80 input images. - >>> import numpy as np - >>> from jwst.resample.resample_utils import decode_context - >>> con = np.array( - ... [[[0, 0, 0, 0, 0, 0], - ... [0, 0, 0, 36196864, 0, 0], - ... [0, 0, 0, 0, 0, 0], - ... [0, 0, 0, 0, 0, 0], - ... [0, 0, 537920000, 0, 0, 0]], - ... [[0, 0, 0, 0, 0, 0,], - ... [0, 0, 0, 67125536, 0, 0], - ... [0, 0, 0, 0, 0, 0], - ... [0, 0, 0, 0, 0, 0], - ... [0, 0, 163856, 0, 0, 0]], - ... [[0, 0, 0, 0, 0, 0], - ... [0, 0, 0, 8203, 0, 0], - ... [0, 0, 0, 0, 0, 0], - ... [0, 0, 0, 0, 0, 0], - ... [0, 0, 32865, 0, 0, 0]]], - ... dtype=np.int32 - ... ) - >>> decode_context(con, [3, 2], [1, 4]) - [array([ 9, 12, 14, 19, 21, 25, 37, 40, 46, 58, 64, 65, 67, 77]), - array([ 9, 20, 29, 36, 47, 49, 64, 69, 70, 79])] + .. code-block:: none + + con = np.array( + [[[0, 0, 0, 0, 0, 0], + [0, 0, 0, 36196864, 0, 0], + [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [0, 0, 537920000, 0, 0, 0]], + [[0, 0, 0, 0, 0, 0,], + [0, 0, 0, 67125536, 0, 0], + [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [0, 0, 163856, 0, 0, 0]], + [[0, 0, 0, 0, 0, 0], + [0, 0, 0, 8203, 0, 0], + [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [0, 0, 32865, 0, 0, 0]]], + dtype=np.int32 + ) + decode_context(con, [3, 2], [1, 4]) + [array([ 9, 12, 14, 19, 21, 25, 37, 40, 46, 58, 64, 65, 67, 77]), + array([ 9, 20, 29, 36, 47, 49, 64, 69, 70, 79])] """ if context.ndim != 3: diff --git a/romancal/resample/tests/test_resample.py b/romancal/resample/tests/test_resample.py index fc420053a..523e9013e 100644 --- a/romancal/resample/tests/test_resample.py +++ b/romancal/resample/tests/test_resample.py @@ -556,7 +556,7 @@ def test_custom_wcs_input_small_overlap_no_rotation(wfi_sca1, wfi_sca3): # pixel scale in RA (N.B.: there's no shift in Dec.) pixel_scale = np.abs(wfi_sca3.meta.wcs(0, 0)[0] - wfi_sca3.meta.wcs(1, 0)[0]) # overlap size in RA (N.B.: there's no shift in Dec.) - ra_overlap_size = np.ceil( + ra_overlap_size = np.round( input_models[0].shape[0] - np.abs(input_models[0].meta.wcs(0, 0)[0] - wfi_sca3.meta.wcs(0, 0)[0]) / pixel_scale @@ -564,7 +564,7 @@ def test_custom_wcs_input_small_overlap_no_rotation(wfi_sca1, wfi_sca3): # determine the size of the region in the output that contains data # (which should have come from the overlap with the input datamodel) output_nonzero_region = np.nonzero(output_models[0].data) - ra_output_nonzero_size = np.ceil(len(set(output_nonzero_region[1]))) + ra_output_nonzero_size = np.round(len(set(output_nonzero_region[1]))) assert ra_output_nonzero_size == ra_overlap_size From 029afbfe01aa7b86e8b8b1337cb23cc4f3060da3 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Wed, 27 Sep 2023 16:20:24 -0400 Subject: [PATCH 48/82] Update requirements-dev with necessary packages. --- requirements-dev-st.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements-dev-st.txt b/requirements-dev-st.txt index f913d6e6e..3aaea5e61 100644 --- a/requirements-dev-st.txt +++ b/requirements-dev-st.txt @@ -3,7 +3,7 @@ git+https://github.com/spacetelescope/roman_datamodels git+https://github.com/spacetelescope/rad # shared upstream packages -git+https://github.com/spacetelescope/stcal +git+https://github.com/mairanteodoro/stcal.git@stcal-alignment git+https://github.com/spacetelescope/stpipe # Other important upstream packages @@ -11,3 +11,4 @@ git+https://github.com/spacetelescope/crds git+https://github.com/spacetelescope/gwcs git+https://github.com/spacetelescope/metrics_logger git+https://github.com/spacetelescope/tweakwcs +git+https://github.com/astropy/photutils.git \ No newline at end of file From 1ed4d19730ad46219eb585c8a9093d617ecf0154 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 27 Sep 2023 20:20:56 +0000 Subject: [PATCH 49/82] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- requirements-dev-st.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev-st.txt b/requirements-dev-st.txt index 3aaea5e61..7b31b7564 100644 --- a/requirements-dev-st.txt +++ b/requirements-dev-st.txt @@ -11,4 +11,4 @@ git+https://github.com/spacetelescope/crds git+https://github.com/spacetelescope/gwcs git+https://github.com/spacetelescope/metrics_logger git+https://github.com/spacetelescope/tweakwcs -git+https://github.com/astropy/photutils.git \ No newline at end of file +git+https://github.com/astropy/photutils.git From 144152c9506a7914506e9b5c410da4c09476fa84 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Wed, 27 Sep 2023 16:49:07 -0400 Subject: [PATCH 50/82] Remove overlap comparison to avoid rounding issues --- romancal/resample/tests/test_resample.py | 29 +++++++++--------------- 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/romancal/resample/tests/test_resample.py b/romancal/resample/tests/test_resample.py index 523e9013e..ffba53160 100644 --- a/romancal/resample/tests/test_resample.py +++ b/romancal/resample/tests/test_resample.py @@ -472,12 +472,18 @@ def test_update_exposure_times_different_sca_same_exposure(exposure_1): == exposure_1[0].meta.exposure.exposure_time ) assert ( - output_model.meta.exposure.start_time == exposure_1[0].meta.exposure.start_time + output_model.meta.exposure.start_time + == exposure_1[0].meta.exposure.start_time + ) + assert ( + output_model.meta.exposure.end_time + == exposure_1[0].meta.exposure.end_time ) - assert output_model.meta.exposure.end_time == exposure_1[0].meta.exposure.end_time -def test_update_exposure_times_same_sca_different_exposures(exposure_1, exposure_2): +def test_update_exposure_times_same_sca_different_exposures( + exposure_1, exposure_2 +): """Test that update_exposure_times is properly updating the exposure parameters for a set of the same SCA but belonging to different exposures.""" input_models = ModelContainer([exposure_1[0], exposure_2[0]]) @@ -553,22 +559,9 @@ def test_custom_wcs_input_small_overlap_no_rotation(wfi_sca1, wfi_sca3): output_models = resample_data.resample_many_to_one() - # pixel scale in RA (N.B.: there's no shift in Dec.) - pixel_scale = np.abs(wfi_sca3.meta.wcs(0, 0)[0] - wfi_sca3.meta.wcs(1, 0)[0]) - # overlap size in RA (N.B.: there's no shift in Dec.) - ra_overlap_size = np.round( - input_models[0].shape[0] - - np.abs(input_models[0].meta.wcs(0, 0)[0] - wfi_sca3.meta.wcs(0, 0)[0]) - / pixel_scale + np.testing.assert_allclose( + output_models[0].meta.wcs(0, 0), wfi_sca3.meta.wcs(0, 0) ) - # determine the size of the region in the output that contains data - # (which should have come from the overlap with the input datamodel) - output_nonzero_region = np.nonzero(output_models[0].data) - ra_output_nonzero_size = np.round(len(set(output_nonzero_region[1]))) - - assert ra_output_nonzero_size == ra_overlap_size - - np.testing.assert_allclose(output_models[0].meta.wcs(0, 0), wfi_sca3.meta.wcs(0, 0)) def test_custom_wcs_input_entire_field_no_rotation(multiple_exposures): From 600ad76a5d5e5005a03325b1d4a3209822269984 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 27 Sep 2023 20:49:29 +0000 Subject: [PATCH 51/82] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- romancal/resample/tests/test_resample.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/romancal/resample/tests/test_resample.py b/romancal/resample/tests/test_resample.py index ffba53160..51b5246a9 100644 --- a/romancal/resample/tests/test_resample.py +++ b/romancal/resample/tests/test_resample.py @@ -472,18 +472,12 @@ def test_update_exposure_times_different_sca_same_exposure(exposure_1): == exposure_1[0].meta.exposure.exposure_time ) assert ( - output_model.meta.exposure.start_time - == exposure_1[0].meta.exposure.start_time - ) - assert ( - output_model.meta.exposure.end_time - == exposure_1[0].meta.exposure.end_time + output_model.meta.exposure.start_time == exposure_1[0].meta.exposure.start_time ) + assert output_model.meta.exposure.end_time == exposure_1[0].meta.exposure.end_time -def test_update_exposure_times_same_sca_different_exposures( - exposure_1, exposure_2 -): +def test_update_exposure_times_same_sca_different_exposures(exposure_1, exposure_2): """Test that update_exposure_times is properly updating the exposure parameters for a set of the same SCA but belonging to different exposures.""" input_models = ModelContainer([exposure_1[0], exposure_2[0]]) @@ -559,9 +553,7 @@ def test_custom_wcs_input_small_overlap_no_rotation(wfi_sca1, wfi_sca3): output_models = resample_data.resample_many_to_one() - np.testing.assert_allclose( - output_models[0].meta.wcs(0, 0), wfi_sca3.meta.wcs(0, 0) - ) + np.testing.assert_allclose(output_models[0].meta.wcs(0, 0), wfi_sca3.meta.wcs(0, 0)) def test_custom_wcs_input_entire_field_no_rotation(multiple_exposures): From 40eb864e61cc18a28bfb103932036f2beadffb48 Mon Sep 17 00:00:00 2001 From: D Davis <49163225+ddavis-stsci@users.noreply.github.com> Date: Wed, 11 Oct 2023 09:42:03 -0400 Subject: [PATCH 52/82] RCAL-641 Add FOV association generation (#931) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- CHANGES.rst | 5 ++ romancal/associations/generate.py | 11 ++- romancal/associations/lib/dms_base.py | 87 ++++++------------- romancal/associations/lib/rules_elpp_base.py | 84 +++++++++++++++++- romancal/associations/lib/rules_level2.py | 46 ++++++++-- .../associations/tests/test_level2_basics.py | 5 +- .../tests/test_level2_candidates.py | 4 +- 7 files changed, 165 insertions(+), 77 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 0e4392f22..7a107763d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,11 @@ 0.13.0 (unreleased) =================== +associations +------------ + +- Add FOV associations to the code [#931] + general ------- diff --git a/romancal/associations/generate.py b/romancal/associations/generate.py index b7184da9d..ed35dff30 100644 --- a/romancal/associations/generate.py +++ b/romancal/associations/generate.py @@ -98,10 +98,13 @@ def generate(pool, rules, version_id=None, finalize=True): # Finalize found associations logger.debug("# associations before finalization: %d", len(associations)) - try: - finalized_asns = rules.callback.reduce("finalize", associations) - except KeyError: - finalized_asns = associations + finalized_asns = associations + if finalize: + logger.debug("Performing association finalization.") + try: + finalized_asns = rules.callback.reduce("finalize", associations) + except KeyError as exception: + logger.debug("Finalization failed for reason: %s", exception) return finalized_asns diff --git a/romancal/associations/lib/dms_base.py b/romancal/associations/lib/dms_base.py index bf10623ec..88bde16e3 100644 --- a/romancal/associations/lib/dms_base.py +++ b/romancal/associations/lib/dms_base.py @@ -21,96 +21,59 @@ # Acquisition and Confirmation images ACQ_EXP_TYPES = ( - "mir_tacq", - "mir_taconfirm", - "nis_taconfirm", - "nis_tacq", "nrc_taconfirm", "nrc_tacq", - "nrs_confirm", - "nrs_msata", - "nrs_taconfirm", - "nrs_tacq", - "nrs_taslit", - "nrs_verify", - "nrs_wata", ) # Exposure EXP_TYPE to Association EXPTYPE mapping # flake8: noqa: E241 EXPTYPE_MAP = { - "mir_darkall": "dark", - "mir_darkimg": "dark", - "mir_darkmrs": "dark", - "mir_flatimage": "flat", - "mir_flatmrs": "flat", - "mir_flatimage-ext": "flat", - "mir_flatmrs-ext": "flat", - "mir_tacq": "target_acquisition", - "mir_taconfirm": "target_acquisition", - "nis_dark": "dark", - "nis_focus": "engineering", - "nis_lamp": "engineering", - "nis_tacq": "target_acquisition", - "nis_taconfirm": "target_acquisition", "nrc_dark": "dark", "nrc_flat": "flat", "nrc_focus": "engineering", "nrc_led": "engineering", "nrc_tacq": "target_acquisition", "nrc_taconfirm": "target_acquisition", - "nrs_autoflat": "autoflat", - "nrs_autowave": "autowave", - "nrs_confirm": "target_acquisition", - "nrs_dark": "dark", - "nrs_focus": "engineering", - "nrs_image": "engineering", - "nrs_lamp": "engineering", - "nrs_msata": "target_acquisition", - "nrs_tacq": "target_acquisition", - "nrs_taconfirm": "target_acquisition", - "nrs_taslit": "target_acquisition", - "nrs_wata": "target_acquisition", } # Coronographic exposures CORON_EXP_TYPES = ["mir_4qpm", "mir_lyot", "nrc_coron"] +# Roman WFI detectors +WFI_DETECTORS = [ + "wfi01", + "wfi02", + "wfi03", + "wfi04", + "wfi05", + "wfi06", + "wfi07", + "wfi08", + "wfi09", + "wfi10", + "wfi11", + "wfi12", + "wfi13", + "wfi14", + "wfi15", + "wfi16", + "wfi17", + "wfi18", +] + # Exposures that get Level2b processing IMAGE2_SCIENCE_EXP_TYPES = [ "wfi_image", - "mir_4qpm", - "mir_image", - "mir_lyot", - "nis_ami", - "nis_image", - "nrc_coron", - "nrc_image", - "nrs_mimf", - "nrc_tsimage", ] IMAGE2_NONSCIENCE_EXP_TYPES = [ - "mir_coroncal", - "nis_focus", - "nrc_focus", - "nrs_focus", - "nrs_image", + "wfi_focus", ] IMAGE2_NONSCIENCE_EXP_TYPES.extend(ACQ_EXP_TYPES) SPEC2_SCIENCE_EXP_TYPES = [ - "mir_lrs-fixedslit", - "mir_lrs-slitless", - "mir_mrs", - "nis_soss", - "nis_wfss", - "nrc_tsgrism", - "nrc_wfss", - "nrs_fixedslit", - "nrs_ifu", - "nrs_msaspec", - "nrs_brightobj", + "wfi_grism", + "wfi_prism", ] SPECIAL_EXPOSURE_MODIFIERS = { diff --git a/romancal/associations/lib/rules_elpp_base.py b/romancal/associations/lib/rules_elpp_base.py index 8cbf7d87d..cf5917ea7 100644 --- a/romancal/associations/lib/rules_elpp_base.py +++ b/romancal/associations/lib/rules_elpp_base.py @@ -1,9 +1,10 @@ """Base classes which define the ELPP Associations""" +import copy import logging import re from collections import defaultdict -from os.path import basename +from os.path import basename, split, splitext from stpipe.format_template import FormatTemplate @@ -18,6 +19,7 @@ IMAGE2_NONSCIENCE_EXP_TYPES, IMAGE2_SCIENCE_EXP_TYPES, SPEC2_SCIENCE_EXP_TYPES, + WFI_DETECTORS, DMSAttrConstraint, DMSBaseMixin, ) @@ -36,6 +38,7 @@ "AsnMixin_AuxData", "AsnMixin_Science", "AsnMixin_Spectrum", + "AsnMixin_Lv2FOV", "AsnMixin_Lv2Image", "AsnMixin_Lv2GBTDSfull", "AsnMixin_Lv2GBTDSpass", @@ -56,6 +59,7 @@ "Constraint_Single_Science", "Constraint_Spectral_Science", "Constraint_Target", + "Constraint_Filename", "DMS_ELPP_Base", "DMSAttrConstraint", "ProcessList", @@ -318,6 +322,43 @@ def make_member(self, item): ) return member + def make_fov_asn(self): + """Take the association with an single exposure with _WFI_ in the name + and expand that to include all 18 detectors. + + Returns + ------- + associations : [association[, ...]] + List of new members to be used in place of + the current one. + """ + results = [] + + # expand the products from _wfi_ to _wfi{det}_ + for product in self["products"]: + for member in product["members"]: + asn = copy.deepcopy(self) + asn.data["products"] = None + product_name = ( + splitext( + split(self.data["products"][0]["members"][0]["expname"])[1] + )[0].rsplit("_", 1)[0] + + "_drzl" + ) + asn.new_product(product_name) + new_members = asn.current_product["members"] + if "_wfi_" in member["expname"]: + # Make and add a member for each detector + for det in WFI_DETECTORS: + new_member = copy.deepcopy(member) + new_member["expname"] = member["expname"].replace("wfi", det) + new_members.append(new_member) + if asn.is_valid: + results.append(asn) + return results + else: + return None + def _init_hook(self, item): """Post-check and pre-add initialization""" super()._init_hook(item) @@ -646,6 +687,16 @@ def __init__(self): ) +class Constraint_Filename(DMSAttrConstraint): + """Select on visit number""" + + def __init__(self): + super().__init__( + name="Filename", + sources=["filename"], + ) + + class Constraint_Expos(DMSAttrConstraint): """Select on exposure number""" @@ -653,8 +704,8 @@ def __init__(self): super().__init__( name="exposure_number", sources=["nexpsur"], - # force_unique=True, - # required=True, + force_unique=True, + required=True, ) @@ -957,6 +1008,33 @@ def _init_hook(self, item): # --------------------------------------------- # Mixins to define the broad category of rules. # --------------------------------------------- +class AsnMixin_Lv2FOV: + """Level 2 Image association base""" + + def _init_hook(self, item): + """Post-check and pre-add initialization""" + + super()._init_hook(item) + self.data["asn_type"] = "FOV" + + def finalize(self): + """Finalize association + + + Returns + ------- + associations: [association[, ...]] or None + List of fully-qualified associations that this association + represents. + `None` if a complete association cannot be produced. + + """ + if self.is_valid: + return self.make_fov_asn() + else: + return None + + class AsnMixin_Lv2Image: """Level 2 Image association base""" diff --git a/romancal/associations/lib/rules_level2.py b/romancal/associations/lib/rules_level2.py index 03fb7dbee..80a324909 100644 --- a/romancal/associations/lib/rules_level2.py +++ b/romancal/associations/lib/rules_level2.py @@ -6,7 +6,14 @@ from romancal.associations.lib.rules_elpp_base import * from romancal.associations.registry import RegistryMarker -__all__ = ["Asn_Lv2Image", "Asn_Lv2GBTDSPass", "Asn_Lv2GBTDSFull", "AsnMixin_Lv2Image"] +__all__ = [ + "Asn_Lv2FOV", + "Asn_Lv2Image", + "Asn_Lv2GBTDSPass", + "Asn_Lv2GBTDSFull", + "AsnMixin_Lv2Image", + "AsnMinxin_Lv2FOV", +] # Configure logging logger = logging.getLogger(__name__) @@ -17,16 +24,40 @@ # -------------------------------- # Start of the User-level rules # -------------------------------- +@RegistryMarker.rule +class Asn_Lv2FOV(AsnMixin_Lv2FOV, DMS_ELPP_Base): + """Level2b Non-TSO Science Image Association + + Characteristics: + - Association type: ``FOV`` + - Pipeline: ``mosaic`` + - Image-based science exposures + - Science exposures for all 18 detectors + """ + + def __init__(self, *args, **kwargs): + # Setup constraints + self.constraints = Constraint( + [ + Constraint_Base(), + Constraint_Target(), + Constraint_Filename(), + ] + ) + + # Now check and continue initialization. + super().__init__(*args, **kwargs) + + @RegistryMarker.rule class Asn_Lv2Image(AsnMixin_Lv2Image, DMS_ELPP_Base): """Level2b Non-TSO Science Image Association Characteristics: - - Association type: ``image2`` - - Pipeline: ``calwebb_image2`` + - Association type: ``image`` + - Pipeline: ``ELPP`` - Image-based science exposures - Single science exposure - - Non-TSO """ def __init__(self, *args, **kwargs): @@ -35,7 +66,12 @@ def __init__(self, *args, **kwargs): [ Constraint_Base(), Constraint_Target(), - Constraint_Expos(), + Constraint( + [ + Constraint_Expos(), + ], + reduce=Constraint.any, + ), Constraint_Optical_Path(), Constraint_Sequence(), Constraint_Pass(), diff --git a/romancal/associations/tests/test_level2_basics.py b/romancal/associations/tests/test_level2_basics.py index 125c6f4d4..663452150 100644 --- a/romancal/associations/tests/test_level2_basics.py +++ b/romancal/associations/tests/test_level2_basics.py @@ -59,7 +59,10 @@ def test_level2_productname(): for member in product["members"] if member["exptype"] == "science" or member["exptype"] == "wfi_image" ] - assert len(science) == 2 + if asn["asn_rule"] == "Asn_Lv2Image": + assert len(science) == 2 + if asn["asn_rule"] == "Asn_Lv2FOV": + assert len(science) == 18 # match = re.match(REGEX_LEVEL2, science[0]['expname']) diff --git a/romancal/associations/tests/test_level2_candidates.py b/romancal/associations/tests/test_level2_candidates.py index 6110114ef..bb85000a5 100644 --- a/romancal/associations/tests/test_level2_candidates.py +++ b/romancal/associations/tests/test_level2_candidates.py @@ -17,11 +17,11 @@ # Basic observation ACIDs (["-i", "o001"], 0), # Whole program - ([], 2), + ([], 5), # Discovered only (["--discover"], 0), # Candidates only - (["--all-candidates"], 2), + (["--all-candidates"], 5), ], ) def test_candidate_observation(partial_args, n_asns): From 5ade0af6c571f851823d05637b4086d3b89ad941 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Mon, 2 Oct 2023 10:24:38 -0400 Subject: [PATCH 53/82] Fix TweakReg's catalog metadata. --- romancal/tweakreg/tweakreg_step.py | 43 ++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/romancal/tweakreg/tweakreg_step.py b/romancal/tweakreg/tweakreg_step.py index 7db401a4c..0c3b4981e 100644 --- a/romancal/tweakreg/tweakreg_step.py +++ b/romancal/tweakreg/tweakreg_step.py @@ -31,7 +31,9 @@ def _oxford_or_str_join(str_list): elif nelem == 2: return f"{str_list[0]} or {str_list[1]}" else: - return ", ".join(map(repr, str_list[:-1])) + ", or " + repr(str_list[-1]) + return ( + ", ".join(map(repr, str_list[:-1])) + ", or " + repr(str_list[-1]) + ) SINGLE_GROUP_REFCAT = ["GAIADR3", "GAIADR2", "GAIADR1"] @@ -144,7 +146,9 @@ def process(self, input): self.catalog_path = os.getcwd() self.catalog_path = Path(self.catalog_path).as_posix() - self.log.info(f"All source catalogs will be saved to: {self.catalog_path}") + self.log.info( + f"All source catalogs will be saved to: {self.catalog_path}" + ) if self.abs_refcat is None or len(self.abs_refcat.strip()) == 0: self.abs_refcat = DEFAULT_ABS_REFCAT @@ -236,7 +240,9 @@ def process(self, input): grp_img = list(images.models_grouped) self.log.info("") - self.log.info(f"Number of image groups to be aligned: {len(grp_img):d}.") + self.log.info( + f"Number of image groups to be aligned: {len(grp_img):d}." + ) self.log.info("Image groups:") if len(grp_img) == 1 and not ALIGN_TO_ABS_REFCAT: @@ -246,7 +252,9 @@ def process(self, input): self.log.info("") # we need at least two exposures to perform image alignment - self.log.warning("At least two exposures are required for image alignment.") + self.log.warning( + "At least two exposures are required for image alignment." + ) self.log.warning("Nothing to do. Skipping 'TweakRegStep'...") self.skip = True for model in images: @@ -320,7 +328,8 @@ def process(self, input): except ValueError as e: msg = e.args[0] if ( - msg == "Too few input images (or groups of images) with non-empty" + msg + == "Too few input images (or groups of images) with non-empty" " catalogs." ): # we need at least two exposures to perform image alignment @@ -328,7 +337,9 @@ def process(self, input): self.log.warning( "At least two exposures are required for image alignment." ) - self.log.warning("Nothing to do. Skipping 'TweakRegStep'...") + self.log.warning( + "Nothing to do. Skipping 'TweakRegStep'..." + ) for model in images: model.meta.cal_step["tweakreg"] = "SKIPPED" if not ALIGN_TO_ABS_REFCAT: @@ -354,7 +365,9 @@ def process(self, input): for model in images: model.meta.cal_step["tweakreg"] = "SKIPPED" if ALIGN_TO_ABS_REFCAT: - self.log.warning("Skipping relative alignment (stage 1)...") + self.log.warning( + "Skipping relative alignment (stage 1)..." + ) else: self.log.warning("Skipping 'TweakRegStep'...") self.skip = True @@ -482,7 +495,9 @@ def process(self, input): # (typecasting numpy objects to python types so that it doesn't cause an # issue when saving datamodel to ASDF) wcs_fit_results = { - k: v.tolist() if isinstance(v, (np.ndarray, np.bool_)) else v + k: v.tolist() + if isinstance(v, (np.ndarray, np.bool_)) + else v for k, v in imcat.meta["fit_info"].items() } # add fit results and new WCS to datamodel @@ -529,10 +544,14 @@ def _imodel2wcsim(self, image_model): catalog_format = self.catalog_format else: catalog_format = "ascii.ecsv" - cat_name = str(catalog) + if isinstance(catalog, str): + # a string with the name of the catalog was provided catalog = Table.read(catalog, format=catalog_format) - catalog.meta["name"] = cat_name + + catalog.meta["name"] = ( + str(catalog) if isinstance(catalog, str) else model_name + ) except OSError: self.log.error(f"Cannot read catalog {catalog}") @@ -572,7 +591,9 @@ def _common_name(group): file_names = [] for im in group: if isinstance(im, rdm.DataModel): - file_names.append(os.path.splitext(im.meta.filename)[0].strip("_- ")) + file_names.append( + os.path.splitext(im.meta.filename)[0].strip("_- ") + ) else: raise TypeError("Input must be a list of datamodels list.") From 05b1eea99c4d71982794f902a004194f7b2b08b2 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Mon, 2 Oct 2023 10:25:49 -0400 Subject: [PATCH 54/82] Update TweakReg's regression test data. --- romancal/regtest/test_tweakreg.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/romancal/regtest/test_tweakreg.py b/romancal/regtest/test_tweakreg.py index d532d9bb8..a17580dd1 100644 --- a/romancal/regtest/test_tweakreg.py +++ b/romancal/regtest/test_tweakreg.py @@ -28,7 +28,7 @@ def create_asn_file(): "name": "files.asdf", "members": [ { - "expname": "r0000401001001001001_01101_0001_WFI01_cal_tweakreg.asdf", + "expname": "r0000501001001001001_01101_0001_WFI02_cal_tweakreg.asdf", "exptype": "science" } ] @@ -53,9 +53,9 @@ def test_tweakreg(rtdata, ignore_asdf_paths, tmp_path): # - assign_wcs; # - photom; # - source_detection. - input_data = "r0000401001001001001_01101_0001_WFI01_cal_tweakreg.asdf" - output_data = "r0000401001001001001_01101_0001_WFI01_output.asdf" - truth_data = "r0000401001001001001_01101_0001_WFI01_cal_twkreg_proc.asdf" + input_data = "r0000501001001001001_01101_0001_WFI02_cal_tweakreg.asdf" + output_data = "r0000501001001001001_01101_0001_WFI02_output.asdf" + truth_data = "r0000501001001001001_01101_0001_WFI02_tweakreg.asdf" rtdata.get_data(f"WFI/image/{input_data}") rtdata.get_truth(f"truth/WFI/image/{truth_data}") From ba7fec28f51ce64b526f20022dc02b17b03040cd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 2 Oct 2023 20:38:29 +0000 Subject: [PATCH 55/82] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- romancal/tweakreg/tweakreg_step.py | 35 ++++++++---------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/romancal/tweakreg/tweakreg_step.py b/romancal/tweakreg/tweakreg_step.py index 0c3b4981e..fcf5f3b06 100644 --- a/romancal/tweakreg/tweakreg_step.py +++ b/romancal/tweakreg/tweakreg_step.py @@ -31,9 +31,7 @@ def _oxford_or_str_join(str_list): elif nelem == 2: return f"{str_list[0]} or {str_list[1]}" else: - return ( - ", ".join(map(repr, str_list[:-1])) + ", or " + repr(str_list[-1]) - ) + return ", ".join(map(repr, str_list[:-1])) + ", or " + repr(str_list[-1]) SINGLE_GROUP_REFCAT = ["GAIADR3", "GAIADR2", "GAIADR1"] @@ -146,9 +144,7 @@ def process(self, input): self.catalog_path = os.getcwd() self.catalog_path = Path(self.catalog_path).as_posix() - self.log.info( - f"All source catalogs will be saved to: {self.catalog_path}" - ) + self.log.info(f"All source catalogs will be saved to: {self.catalog_path}") if self.abs_refcat is None or len(self.abs_refcat.strip()) == 0: self.abs_refcat = DEFAULT_ABS_REFCAT @@ -240,9 +236,7 @@ def process(self, input): grp_img = list(images.models_grouped) self.log.info("") - self.log.info( - f"Number of image groups to be aligned: {len(grp_img):d}." - ) + self.log.info(f"Number of image groups to be aligned: {len(grp_img):d}.") self.log.info("Image groups:") if len(grp_img) == 1 and not ALIGN_TO_ABS_REFCAT: @@ -252,9 +246,7 @@ def process(self, input): self.log.info("") # we need at least two exposures to perform image alignment - self.log.warning( - "At least two exposures are required for image alignment." - ) + self.log.warning("At least two exposures are required for image alignment.") self.log.warning("Nothing to do. Skipping 'TweakRegStep'...") self.skip = True for model in images: @@ -328,8 +320,7 @@ def process(self, input): except ValueError as e: msg = e.args[0] if ( - msg - == "Too few input images (or groups of images) with non-empty" + msg == "Too few input images (or groups of images) with non-empty" " catalogs." ): # we need at least two exposures to perform image alignment @@ -337,9 +328,7 @@ def process(self, input): self.log.warning( "At least two exposures are required for image alignment." ) - self.log.warning( - "Nothing to do. Skipping 'TweakRegStep'..." - ) + self.log.warning("Nothing to do. Skipping 'TweakRegStep'...") for model in images: model.meta.cal_step["tweakreg"] = "SKIPPED" if not ALIGN_TO_ABS_REFCAT: @@ -365,9 +354,7 @@ def process(self, input): for model in images: model.meta.cal_step["tweakreg"] = "SKIPPED" if ALIGN_TO_ABS_REFCAT: - self.log.warning( - "Skipping relative alignment (stage 1)..." - ) + self.log.warning("Skipping relative alignment (stage 1)...") else: self.log.warning("Skipping 'TweakRegStep'...") self.skip = True @@ -495,9 +482,7 @@ def process(self, input): # (typecasting numpy objects to python types so that it doesn't cause an # issue when saving datamodel to ASDF) wcs_fit_results = { - k: v.tolist() - if isinstance(v, (np.ndarray, np.bool_)) - else v + k: v.tolist() if isinstance(v, (np.ndarray, np.bool_)) else v for k, v in imcat.meta["fit_info"].items() } # add fit results and new WCS to datamodel @@ -591,9 +576,7 @@ def _common_name(group): file_names = [] for im in group: if isinstance(im, rdm.DataModel): - file_names.append( - os.path.splitext(im.meta.filename)[0].strip("_- ") - ) + file_names.append(os.path.splitext(im.meta.filename)[0].strip("_- ")) else: raise TypeError("Input must be a list of datamodels list.") From 1ae9286c1cb009b9361f68d71f2d0113c6e7c492 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Thu, 5 Oct 2023 06:49:24 -0400 Subject: [PATCH 56/82] Include parallax correction to unit tests. --- .../tweakreg/tests/test_astrometric_utils.py | 184 +++++++++++++++--- 1 file changed, 160 insertions(+), 24 deletions(-) diff --git a/romancal/tweakreg/tests/test_astrometric_utils.py b/romancal/tweakreg/tests/test_astrometric_utils.py index ad887a28f..4fa0ed56e 100644 --- a/romancal/tweakreg/tests/test_astrometric_utils.py +++ b/romancal/tweakreg/tests/test_astrometric_utils.py @@ -8,6 +8,7 @@ from astropy import coordinates as coord from astropy import table from astropy import units as u +from astropy.time import Time from astropy.modeling import models from astropy.modeling.models import RotationSequence3D, Scale, Shift from gwcs import coordinate_frames as cf @@ -29,6 +30,123 @@ def __init__(self, *args, **kwargs): raise requests.exceptions.ConnectionError +def get_proper_motion_correction(epoch, gaia_ref_epoch_coords, gaia_ref_epoch): + """ + Calculates the proper motion correction for a given epoch and Gaia reference epoch coordinates. + + Parameters + ---------- + epoch : float + The epoch for which the proper motion correction is calculated. + gaia_ref_epoch_coords : dict + A dictionary containing Gaia reference epoch coordinates. + gaia_ref_epoch : float + The Gaia reference epoch. + + Returns + ------- + None + + Examples + -------- + .. code-block:: python + epoch = 2022.5 + gaia_coords = { + "ra": 180.0, + "dec": 45.0, + "pmra": 2.0, + "pmdec": 1.5 + } + gaia_ref_epoch = 2020.0 + get_proper_motion_correction(epoch, gaia_coords, gaia_ref_epoch) + """ + + expected_new_dec = ( + np.array( + gaia_ref_epoch_coords["dec"] * 3600 + + (epoch - gaia_ref_epoch) * gaia_ref_epoch_coords["pmdec"] / 1000 + ) + / 3600 + ) + average_dec = np.array( + [ + np.mean([new, old]) + for new, old in zip(expected_new_dec, gaia_ref_epoch_coords["dec"]) + ] + ) + pmra = gaia_ref_epoch_coords["pmra"] / np.cos(np.deg2rad(average_dec)) + + # angular displacement components + gaia_ref_epoch_coords["pm_delta_dec"] = u.Quantity( + (epoch - gaia_ref_epoch) * gaia_ref_epoch_coords["pmdec"] / 1000, + unit=u.arcsec, + ).to(u.deg) + gaia_ref_epoch_coords["pm_delta_ra"] = u.Quantity( + (epoch - gaia_ref_epoch) * (pmra / 1000), unit=u.arcsec + ).to(u.deg) + + +def get_parallax_correction(epoch, gaia_ref_epoch_coords): + """ + Calculates the parallax correction for a given epoch and Gaia reference epoch + coordinates. + Calculations based on Chapter 8 of "Spherical Astronomy" by Robin M. Green. + + Parameters + ---------- + epoch : float + The epoch for which the parallax correction is calculated. + gaia_ref_epoch_coords : dict + A dictionary containing Gaia reference epoch coordinates. + + Returns + ------- + None + + Examples + -------- + .. code-block :: python + epoch = 2022.5 + gaia_coords = { + "ra": 180.0, + "dec": 45.0, + "parallax": 10.0 + } + get_parallax_correction(epoch, gaia_coords) + """ + + obs_date = Time(epoch, format="decimalyear") + earths_center_barycentric_coords = coord.get_body_barycentric( + "earth", obs_date + ) + earth_X = earths_center_barycentric_coords.x + earth_Y = earths_center_barycentric_coords.y + earth_Z = earths_center_barycentric_coords.z + + # angular displacement components + # (see eq. 8.15 of "Spherical Astronomy" by Robert M. Green) + gaia_ref_epoch_coords["parallax_delta_ra"] = ( + u.Quantity(gaia_ref_epoch_coords["parallax"], unit="mas").to(u.rad) + * (1 / np.cos(gaia_ref_epoch_coords["dec"] / 180 * np.pi)) + * ( + earth_X.value * np.sin(gaia_ref_epoch_coords["ra"] / 180 * np.pi) + - earth_Y.value * np.cos(gaia_ref_epoch_coords["ra"] / 180 * np.pi) + ) + ).to("deg") + gaia_ref_epoch_coords["parallax_delta_dec"] = ( + u.Quantity(gaia_ref_epoch_coords["parallax"], unit="mas").to(u.rad) + * ( + earth_X.value + * np.cos(gaia_ref_epoch_coords["ra"] / 180 * np.pi) + * np.sin(gaia_ref_epoch_coords["dec"] / 180 * np.pi) + + earth_Y.value + * np.sin(gaia_ref_epoch_coords["ra"] / 180 * np.pi) + * np.sin(gaia_ref_epoch_coords["dec"] / 180 * np.pi) + - earth_Z.value * np.cos(gaia_ref_epoch_coords["dec"] / 180 * np.pi) + ) + ).to("deg") + + def update_wcsinfo(input_dm): """ Update WCSInfo with realistic data (i.e. obtained from romanisim simulations). @@ -80,7 +198,9 @@ def create_wcs_for_tweakreg_pipeline(input_dm, shift_1=0, shift_2=0): tel2sky = _create_tel2sky_model(input_dm) # create required frames - detector = cf.Frame2D(name="detector", axes_order=(0, 1), unit=(u.pix, u.pix)) + detector = cf.Frame2D( + name="detector", axes_order=(0, 1), unit=(u.pix, u.pix) + ) v2v3 = cf.Frame2D( name="v2v3", axes_order=(0, 1), @@ -338,7 +458,9 @@ def test_create_astrometric_catalog_write_results_to_disk(tmp_path, base_image): ("GAIADR3", None), ], ) -def test_create_astrometric_catalog_using_epoch(tmp_path, catalog, epoch, request): +def test_create_astrometric_catalog_using_epoch( + tmp_path, catalog, epoch, request +): """Test fetching data from supported catalogs for a specific epoch.""" output_filename = "ref_cat.ecsv" img = request.getfixturevalue("base_image")(shift_1=1000, shift_2=1000) @@ -481,39 +603,53 @@ def test_get_catalog_valid_parameters_but_no_sources_returned(): def test_get_catalog_using_epoch(ra, dec, epoch): """Test that get_catalog returns coordinates corrected by proper motion.""" - result = get_catalog(ra, dec, epoch=epoch) + result_all = get_catalog(ra, dec, epoch=epoch) + + # select sources with reliable astrometric solutions based on the + # parallax_over_error parameter as discussed in Fabricius et al. 2021 + # (https://www.aanda.org/articles/aa/full_html/2021/05/aa39834-20/aa39834-20.html) + mask = result_all["parallax_over_error"] > 5 + + result = result_all[mask] returned_ra = np.array(result["ra"]) returned_dec = np.array(result["dec"]) # get GAIA data and update coords to requested epoch using pm measurements gaia_ref_epoch = 2016.0 - gaia_ref_epoch_coords = get_catalog(ra, dec, epoch=gaia_ref_epoch) + gaia_ref_epoch_coords_all = get_catalog(ra, dec, epoch=gaia_ref_epoch) - expected_new_dec = ( - np.array( - gaia_ref_epoch_coords["dec"] * 3600 - + (epoch - gaia_ref_epoch) * gaia_ref_epoch_coords["pmdec"] / 1000 - ) - / 3600 + gaia_ref_epoch_coords = gaia_ref_epoch_coords_all[mask] + + # calculate proper motion corrections + get_proper_motion_correction( + epoch=epoch, + gaia_ref_epoch_coords=gaia_ref_epoch_coords, + gaia_ref_epoch=gaia_ref_epoch, ) - average_dec = np.array( - [ - np.mean([new, old]) - for new, old in zip(expected_new_dec, gaia_ref_epoch_coords["dec"]) - ] + # calculate parallax corrections + get_parallax_correction( + epoch=epoch, gaia_ref_epoch_coords=gaia_ref_epoch_coords ) - pmra = gaia_ref_epoch_coords["pmra"] / np.cos(np.deg2rad(average_dec)) - expected_new_ra = ( - np.array( - gaia_ref_epoch_coords["ra"] * 3600 - + (epoch - gaia_ref_epoch) * (pmra / 1000) - ) - / 3600 + + # calculate the expected coordinates value after corrections have been applied to + # Gaia's reference epoch coordinates + expected_ra = ( + gaia_ref_epoch_coords["ra"] + + gaia_ref_epoch_coords["pm_delta_ra"] + + gaia_ref_epoch_coords["parallax_delta_ra"] + ) + expected_dec = ( + gaia_ref_epoch_coords["dec"] + + gaia_ref_epoch_coords["pm_delta_dec"] + + gaia_ref_epoch_coords["parallax_delta_dec"] ) assert len(result) > 0 - assert np.isclose(returned_ra, expected_new_ra, atol=1e-10, rtol=1e-9).all() - assert np.isclose(returned_dec, expected_new_dec, atol=1e-10, rtol=1e-9).all() + # choosing atol=0 corresponds to abs(a - b) / abs(b) <= rtol, + # where a is the returned value from the VO API and b is the expected + # value from applying the calculated PM and parallax corrections. + assert np.isclose(returned_ra, expected_ra, atol=0, rtol=1e-5).all() + assert np.isclose(returned_dec, expected_dec, atol=0, rtol=1e-5).all() def test_get_catalog_timeout(): From f8cddf980e47c1f8c3118aae63e4ff6f80fce345 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 5 Oct 2023 10:57:43 +0000 Subject: [PATCH 57/82] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../tweakreg/tests/test_astrometric_utils.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/romancal/tweakreg/tests/test_astrometric_utils.py b/romancal/tweakreg/tests/test_astrometric_utils.py index 4fa0ed56e..f26708a8e 100644 --- a/romancal/tweakreg/tests/test_astrometric_utils.py +++ b/romancal/tweakreg/tests/test_astrometric_utils.py @@ -8,9 +8,9 @@ from astropy import coordinates as coord from astropy import table from astropy import units as u -from astropy.time import Time from astropy.modeling import models from astropy.modeling.models import RotationSequence3D, Scale, Shift +from astropy.time import Time from gwcs import coordinate_frames as cf from gwcs import wcs from gwcs.geometry import CartesianToSpherical, SphericalToCartesian @@ -116,9 +116,7 @@ def get_parallax_correction(epoch, gaia_ref_epoch_coords): """ obs_date = Time(epoch, format="decimalyear") - earths_center_barycentric_coords = coord.get_body_barycentric( - "earth", obs_date - ) + earths_center_barycentric_coords = coord.get_body_barycentric("earth", obs_date) earth_X = earths_center_barycentric_coords.x earth_Y = earths_center_barycentric_coords.y earth_Z = earths_center_barycentric_coords.z @@ -198,9 +196,7 @@ def create_wcs_for_tweakreg_pipeline(input_dm, shift_1=0, shift_2=0): tel2sky = _create_tel2sky_model(input_dm) # create required frames - detector = cf.Frame2D( - name="detector", axes_order=(0, 1), unit=(u.pix, u.pix) - ) + detector = cf.Frame2D(name="detector", axes_order=(0, 1), unit=(u.pix, u.pix)) v2v3 = cf.Frame2D( name="v2v3", axes_order=(0, 1), @@ -458,9 +454,7 @@ def test_create_astrometric_catalog_write_results_to_disk(tmp_path, base_image): ("GAIADR3", None), ], ) -def test_create_astrometric_catalog_using_epoch( - tmp_path, catalog, epoch, request -): +def test_create_astrometric_catalog_using_epoch(tmp_path, catalog, epoch, request): """Test fetching data from supported catalogs for a specific epoch.""" output_filename = "ref_cat.ecsv" img = request.getfixturevalue("base_image")(shift_1=1000, shift_2=1000) @@ -627,9 +621,7 @@ def test_get_catalog_using_epoch(ra, dec, epoch): gaia_ref_epoch=gaia_ref_epoch, ) # calculate parallax corrections - get_parallax_correction( - epoch=epoch, gaia_ref_epoch_coords=gaia_ref_epoch_coords - ) + get_parallax_correction(epoch=epoch, gaia_ref_epoch_coords=gaia_ref_epoch_coords) # calculate the expected coordinates value after corrections have been applied to # Gaia's reference epoch coordinates From 6b4631dce356c1d5c493829b7ce650e339c4b5e3 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Thu, 5 Oct 2023 07:00:02 -0400 Subject: [PATCH 58/82] Fix style checks. --- romancal/tweakreg/tests/test_astrometric_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/romancal/tweakreg/tests/test_astrometric_utils.py b/romancal/tweakreg/tests/test_astrometric_utils.py index f26708a8e..fa9d767ef 100644 --- a/romancal/tweakreg/tests/test_astrometric_utils.py +++ b/romancal/tweakreg/tests/test_astrometric_utils.py @@ -32,7 +32,8 @@ def __init__(self, *args, **kwargs): def get_proper_motion_correction(epoch, gaia_ref_epoch_coords, gaia_ref_epoch): """ - Calculates the proper motion correction for a given epoch and Gaia reference epoch coordinates. + Calculates the proper motion correction for a given epoch and Gaia reference epoch + coordinates. Parameters ---------- From 7fa5c6b7d0673f5258b4af631be076b219899e7d Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Thu, 5 Oct 2023 09:25:59 -0400 Subject: [PATCH 59/82] Add atol parameter to compare_asdf method. --- romancal/regtest/test_tweakreg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/romancal/regtest/test_tweakreg.py b/romancal/regtest/test_tweakreg.py index a17580dd1..7cfd30cf5 100644 --- a/romancal/regtest/test_tweakreg.py +++ b/romancal/regtest/test_tweakreg.py @@ -93,7 +93,7 @@ def test_tweakreg(rtdata, ignore_asdf_paths, tmp_path): ) assert "v2v3corr" in tweakreg_out.meta.wcs.available_frames - diff = compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) + diff = compare_asdf(rtdata.output, rtdata.truth, atol=1e-3, **ignore_asdf_paths) step.log.info( f"DMS280 MSG: Was the proper TweakReg data produced? : {diff.identical}" ) From 5897061f2861acf8062702a0f92fa7bd71824695 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Thu, 5 Oct 2023 14:10:22 -0400 Subject: [PATCH 60/82] Use SCA 01 instead of SCA 02 in resample regtest. --- romancal/regtest/test_tweakreg.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/romancal/regtest/test_tweakreg.py b/romancal/regtest/test_tweakreg.py index 7cfd30cf5..a46bfdde2 100644 --- a/romancal/regtest/test_tweakreg.py +++ b/romancal/regtest/test_tweakreg.py @@ -28,7 +28,7 @@ def create_asn_file(): "name": "files.asdf", "members": [ { - "expname": "r0000501001001001001_01101_0001_WFI02_cal_tweakreg.asdf", + "expname": "r0000501001001001001_01101_0001_WFI01_cal_tweakreg.asdf", "exptype": "science" } ] @@ -53,9 +53,9 @@ def test_tweakreg(rtdata, ignore_asdf_paths, tmp_path): # - assign_wcs; # - photom; # - source_detection. - input_data = "r0000501001001001001_01101_0001_WFI02_cal_tweakreg.asdf" - output_data = "r0000501001001001001_01101_0001_WFI02_output.asdf" - truth_data = "r0000501001001001001_01101_0001_WFI02_tweakreg.asdf" + input_data = "r0000501001001001001_01101_0001_WFI01_cal_tweakreg.asdf" + output_data = "r0000501001001001001_01101_0001_WFI01_output.asdf" + truth_data = "r0000501001001001001_01101_0001_WFI01_tweakreg.asdf" rtdata.get_data(f"WFI/image/{input_data}") rtdata.get_truth(f"truth/WFI/image/{truth_data}") From 18e21df061edd0ecca3e668fd86d7be363dee1fe Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Thu, 5 Oct 2023 14:14:48 -0400 Subject: [PATCH 61/82] Set atol=1e-5 and rtol=0 in astrometric_utils unit tests. --- .../tweakreg/tests/test_astrometric_utils.py | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/romancal/tweakreg/tests/test_astrometric_utils.py b/romancal/tweakreg/tests/test_astrometric_utils.py index fa9d767ef..6b0388993 100644 --- a/romancal/tweakreg/tests/test_astrometric_utils.py +++ b/romancal/tweakreg/tests/test_astrometric_utils.py @@ -117,7 +117,9 @@ def get_parallax_correction(epoch, gaia_ref_epoch_coords): """ obs_date = Time(epoch, format="decimalyear") - earths_center_barycentric_coords = coord.get_body_barycentric("earth", obs_date) + earths_center_barycentric_coords = coord.get_body_barycentric( + "earth", obs_date + ) earth_X = earths_center_barycentric_coords.x earth_Y = earths_center_barycentric_coords.y earth_Z = earths_center_barycentric_coords.z @@ -197,7 +199,9 @@ def create_wcs_for_tweakreg_pipeline(input_dm, shift_1=0, shift_2=0): tel2sky = _create_tel2sky_model(input_dm) # create required frames - detector = cf.Frame2D(name="detector", axes_order=(0, 1), unit=(u.pix, u.pix)) + detector = cf.Frame2D( + name="detector", axes_order=(0, 1), unit=(u.pix, u.pix) + ) v2v3 = cf.Frame2D( name="v2v3", axes_order=(0, 1), @@ -455,7 +459,9 @@ def test_create_astrometric_catalog_write_results_to_disk(tmp_path, base_image): ("GAIADR3", None), ], ) -def test_create_astrometric_catalog_using_epoch(tmp_path, catalog, epoch, request): +def test_create_astrometric_catalog_using_epoch( + tmp_path, catalog, epoch, request +): """Test fetching data from supported catalogs for a specific epoch.""" output_filename = "ref_cat.ecsv" img = request.getfixturevalue("base_image")(shift_1=1000, shift_2=1000) @@ -622,7 +628,9 @@ def test_get_catalog_using_epoch(ra, dec, epoch): gaia_ref_epoch=gaia_ref_epoch, ) # calculate parallax corrections - get_parallax_correction(epoch=epoch, gaia_ref_epoch_coords=gaia_ref_epoch_coords) + get_parallax_correction( + epoch=epoch, gaia_ref_epoch_coords=gaia_ref_epoch_coords + ) # calculate the expected coordinates value after corrections have been applied to # Gaia's reference epoch coordinates @@ -638,11 +646,9 @@ def test_get_catalog_using_epoch(ra, dec, epoch): ) assert len(result) > 0 - # choosing atol=0 corresponds to abs(a - b) / abs(b) <= rtol, - # where a is the returned value from the VO API and b is the expected - # value from applying the calculated PM and parallax corrections. - assert np.isclose(returned_ra, expected_ra, atol=0, rtol=1e-5).all() - assert np.isclose(returned_dec, expected_dec, atol=0, rtol=1e-5).all() + + assert np.isclose(returned_ra, expected_ra, atol=1e-5, rtol=0).all() + assert np.isclose(returned_dec, expected_dec, atol=1e-5, rtol=0).all() def test_get_catalog_timeout(): From 462a93eccea4f23caa8c2967059275826a0fe149 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 5 Oct 2023 18:15:34 +0000 Subject: [PATCH 62/82] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../tweakreg/tests/test_astrometric_utils.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/romancal/tweakreg/tests/test_astrometric_utils.py b/romancal/tweakreg/tests/test_astrometric_utils.py index 6b0388993..b9bfcb19c 100644 --- a/romancal/tweakreg/tests/test_astrometric_utils.py +++ b/romancal/tweakreg/tests/test_astrometric_utils.py @@ -117,9 +117,7 @@ def get_parallax_correction(epoch, gaia_ref_epoch_coords): """ obs_date = Time(epoch, format="decimalyear") - earths_center_barycentric_coords = coord.get_body_barycentric( - "earth", obs_date - ) + earths_center_barycentric_coords = coord.get_body_barycentric("earth", obs_date) earth_X = earths_center_barycentric_coords.x earth_Y = earths_center_barycentric_coords.y earth_Z = earths_center_barycentric_coords.z @@ -199,9 +197,7 @@ def create_wcs_for_tweakreg_pipeline(input_dm, shift_1=0, shift_2=0): tel2sky = _create_tel2sky_model(input_dm) # create required frames - detector = cf.Frame2D( - name="detector", axes_order=(0, 1), unit=(u.pix, u.pix) - ) + detector = cf.Frame2D(name="detector", axes_order=(0, 1), unit=(u.pix, u.pix)) v2v3 = cf.Frame2D( name="v2v3", axes_order=(0, 1), @@ -459,9 +455,7 @@ def test_create_astrometric_catalog_write_results_to_disk(tmp_path, base_image): ("GAIADR3", None), ], ) -def test_create_astrometric_catalog_using_epoch( - tmp_path, catalog, epoch, request -): +def test_create_astrometric_catalog_using_epoch(tmp_path, catalog, epoch, request): """Test fetching data from supported catalogs for a specific epoch.""" output_filename = "ref_cat.ecsv" img = request.getfixturevalue("base_image")(shift_1=1000, shift_2=1000) @@ -628,9 +622,7 @@ def test_get_catalog_using_epoch(ra, dec, epoch): gaia_ref_epoch=gaia_ref_epoch, ) # calculate parallax corrections - get_parallax_correction( - epoch=epoch, gaia_ref_epoch_coords=gaia_ref_epoch_coords - ) + get_parallax_correction(epoch=epoch, gaia_ref_epoch_coords=gaia_ref_epoch_coords) # calculate the expected coordinates value after corrections have been applied to # Gaia's reference epoch coordinates From a3dc698cdb5cd42563458574991b8eb99bd9b335 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Wed, 11 Oct 2023 19:00:53 -0400 Subject: [PATCH 63/82] Fix astrometric_utils tests. --- .../tweakreg/tests/test_astrometric_utils.py | 248 +++++++++++++----- 1 file changed, 189 insertions(+), 59 deletions(-) diff --git a/romancal/tweakreg/tests/test_astrometric_utils.py b/romancal/tweakreg/tests/test_astrometric_utils.py index b9bfcb19c..3e8901c0e 100644 --- a/romancal/tweakreg/tests/test_astrometric_utils.py +++ b/romancal/tweakreg/tests/test_astrometric_utils.py @@ -24,12 +24,160 @@ get_catalog, ) +ARAD = np.pi / 180.0 + class MockConnectionError: def __init__(self, *args, **kwargs): raise requests.exceptions.ConnectionError +def get_earth_sun_orbit_elem(epoch): + amjd = (epoch - 2012.0) * 365.25 + 55927.0 + imjd = int(amjd) + mjd = float(imjd) + + time_argument = mjd - 51544.5 + fmean_longitude = 280.460 + 0.9856474 * time_argument + fmean_anomaly = 357.528 + 0.9856003 * time_argument + iremainder = int(fmean_longitude / 360.0) + fmean_longitude = fmean_longitude - (iremainder * 360.0) + if fmean_longitude < 0: + fmean_longitude = fmean_longitude + 360.0 + iremainder = int(fmean_anomaly / 360.0) + fmean_anomaly = fmean_anomaly - (iremainder * 360.0) + if fmean_anomaly < 0: + fmean_anomaly = fmean_anomaly + 360.0 + ecliptic_longitude = ( + fmean_longitude + + 1.915 * np.sin(fmean_anomaly * ARAD) + + 0.02 * np.sin(2.0 * fmean_anomaly * ARAD) + ) + ecliptic_obliquity = 23.439 - 4.0 * 0.0000001 * time_argument + + # radius vector (in AU) + # (approximation using an algorithm from USNO) + earth_sun_distance = ( + 1.00014 + - 0.01671 * np.cos(fmean_anomaly * ARAD) + - 0.00014 * np.cos(2.0 * fmean_anomaly * ARAD) + ) + + earth_sun_distance = u.Quantity(earth_sun_distance, unit="AU") + + return { + "earth_sun_distance": earth_sun_distance, + "ecliptic_longitude": ecliptic_longitude, + "ecliptic_obliquity": ecliptic_obliquity, + } + + +def get_parallax_correction_mast(epoch, gaia_ref_epoch_coords): + orbit_elem = get_earth_sun_orbit_elem(epoch) + earth_sun_distance = orbit_elem["earth_sun_distance"] + ecliptic_longitude = orbit_elem["ecliptic_longitude"] + ecliptic_obliquity = orbit_elem["ecliptic_obliquity"] + + # cartesian coordinates of the sun + xsun = earth_sun_distance * np.cos(ecliptic_longitude * ARAD) + ysun = ( + earth_sun_distance + * np.cos(ecliptic_obliquity * ARAD) + * np.sin(ecliptic_longitude * ARAD) + ) + zsun = ( + earth_sun_distance + * np.sin(ecliptic_obliquity * ARAD) + * np.sin(ecliptic_longitude * ARAD) + ) + + # angular displacement components + delta_ra = ( + u.Quantity(gaia_ref_epoch_coords["parallax"], unit="mas") + / np.cos(gaia_ref_epoch_coords["dec"] * ARAD) + * ( + -xsun.value * np.sin(gaia_ref_epoch_coords["ra"] * ARAD) + + ysun.value * np.cos(gaia_ref_epoch_coords["ra"] * ARAD) + ) + ).to("deg") + delta_dec = ( + u.Quantity(gaia_ref_epoch_coords["parallax"], unit="mas") + * ( + -xsun.value + * np.cos(gaia_ref_epoch_coords["ra"] * ARAD) + * np.sin(gaia_ref_epoch_coords["dec"] * ARAD) + - ysun.value + * np.sin(gaia_ref_epoch_coords["ra"] * ARAD) + * np.sin(gaia_ref_epoch_coords["dec"] * ARAD) + + zsun.value * np.cos(gaia_ref_epoch_coords["dec"] * ARAD) + ) + ).to("deg") + + return delta_ra, delta_dec + + +def get_parallax_correction_barycenter(epoch, gaia_ref_epoch_coords): + """ + Calculates the parallax correction in the Earth barycenter frame for a given epoch + and Gaia reference epoch coordinates (i.e. Gaia coordinates at the reference epoch). + + Parameters + ---------- + epoch : float + The epoch for which the parallax correction is calculated. + gaia_ref_epoch_coords : dict + The Gaia reference epoch coordinates, including 'ra', 'dec', and 'parallax'. + + Returns + ------- + tuple + A tuple containing the delta_ra and delta_dec values of the parallax correction + in degrees. + + Examples + -------- + .. code-block :: python + epoch = 2022.5 + gaia_coords = {'ra': 180.0, 'dec': 45.0, 'parallax': 10.0} + correction = get_parallax_correction_earth_barycenter(epoch, gaia_coords) + print(correction) + (0.001, -0.002) + """ # noqa: E501 + + obs_date = Time(epoch, format="decimalyear") + earths_center_barycentric_coords = coord.get_body_barycentric( + "earth", obs_date, ephemeris="builtin" + ) + earth_X = earths_center_barycentric_coords.x + earth_Y = earths_center_barycentric_coords.y + earth_Z = earths_center_barycentric_coords.z + + # angular displacement components + # (see eq. 8.15 of "Spherical Astronomy" by Robert M. Green) + delta_ra = ( + u.Quantity(gaia_ref_epoch_coords["parallax"], unit="mas").to(u.rad) + * (1 / np.cos(gaia_ref_epoch_coords["dec"] * ARAD)) + * ( + earth_X.value * np.sin(gaia_ref_epoch_coords["ra"] * ARAD) + - earth_Y.value * np.cos(gaia_ref_epoch_coords["ra"] * ARAD) + ) + ).to("deg") + delta_dec = ( + u.Quantity(gaia_ref_epoch_coords["parallax"], unit="mas").to(u.rad) + * ( + earth_X.value + * np.cos(gaia_ref_epoch_coords["ra"] * ARAD) + * np.sin(gaia_ref_epoch_coords["dec"] * ARAD) + + earth_Y.value + * np.sin(gaia_ref_epoch_coords["ra"] * ARAD) + * np.sin(gaia_ref_epoch_coords["dec"] * ARAD) + - earth_Z.value * np.cos(gaia_ref_epoch_coords["dec"] * ARAD) + ) + ).to("deg") + + return delta_ra, delta_dec + + def get_proper_motion_correction(epoch, gaia_ref_epoch_coords, gaia_ref_epoch): """ Calculates the proper motion correction for a given epoch and Gaia reference epoch @@ -88,62 +236,22 @@ def get_proper_motion_correction(epoch, gaia_ref_epoch_coords, gaia_ref_epoch): def get_parallax_correction(epoch, gaia_ref_epoch_coords): - """ - Calculates the parallax correction for a given epoch and Gaia reference epoch - coordinates. - Calculations based on Chapter 8 of "Spherical Astronomy" by Robin M. Green. - - Parameters - ---------- - epoch : float - The epoch for which the parallax correction is calculated. - gaia_ref_epoch_coords : dict - A dictionary containing Gaia reference epoch coordinates. - - Returns - ------- - None - - Examples - -------- - .. code-block :: python - epoch = 2022.5 - gaia_coords = { - "ra": 180.0, - "dec": 45.0, - "parallax": 10.0 - } - get_parallax_correction(epoch, gaia_coords) - """ + # get parallax correction using textbook calculations (i.e. Earth's barycenter) + parallax_corr = get_parallax_correction_barycenter( + epoch=epoch, gaia_ref_epoch_coords=gaia_ref_epoch_coords + ) - obs_date = Time(epoch, format="decimalyear") - earths_center_barycentric_coords = coord.get_body_barycentric("earth", obs_date) - earth_X = earths_center_barycentric_coords.x - earth_Y = earths_center_barycentric_coords.y - earth_Z = earths_center_barycentric_coords.z + # get parallax using the same coordinates frame as MAST + # (i.e. Sun's geocentric coordinates) + parallax_corr_mast = get_parallax_correction_mast( + epoch=epoch, gaia_ref_epoch_coords=gaia_ref_epoch_coords + ) - # angular displacement components - # (see eq. 8.15 of "Spherical Astronomy" by Robert M. Green) - gaia_ref_epoch_coords["parallax_delta_ra"] = ( - u.Quantity(gaia_ref_epoch_coords["parallax"], unit="mas").to(u.rad) - * (1 / np.cos(gaia_ref_epoch_coords["dec"] / 180 * np.pi)) - * ( - earth_X.value * np.sin(gaia_ref_epoch_coords["ra"] / 180 * np.pi) - - earth_Y.value * np.cos(gaia_ref_epoch_coords["ra"] / 180 * np.pi) - ) - ).to("deg") - gaia_ref_epoch_coords["parallax_delta_dec"] = ( - u.Quantity(gaia_ref_epoch_coords["parallax"], unit="mas").to(u.rad) - * ( - earth_X.value - * np.cos(gaia_ref_epoch_coords["ra"] / 180 * np.pi) - * np.sin(gaia_ref_epoch_coords["dec"] / 180 * np.pi) - + earth_Y.value - * np.sin(gaia_ref_epoch_coords["ra"] / 180 * np.pi) - * np.sin(gaia_ref_epoch_coords["dec"] / 180 * np.pi) - - earth_Z.value * np.cos(gaia_ref_epoch_coords["dec"] / 180 * np.pi) - ) - ).to("deg") + # add parallax corrections columns to the main table + gaia_ref_epoch_coords["parallax_delta_ra"] = parallax_corr[0] + gaia_ref_epoch_coords["parallax_delta_dec"] = parallax_corr[1] + gaia_ref_epoch_coords["parallax_delta_ra_mast"] = parallax_corr_mast[0] + gaia_ref_epoch_coords["parallax_delta_dec_mast"] = parallax_corr_mast[1] def update_wcsinfo(input_dm): @@ -198,6 +306,7 @@ def create_wcs_for_tweakreg_pipeline(input_dm, shift_1=0, shift_2=0): # create required frames detector = cf.Frame2D(name="detector", axes_order=(0, 1), unit=(u.pix, u.pix)) + detector = cf.Frame2D(name="detector", axes_order=(0, 1), unit=(u.pix, u.pix)) v2v3 = cf.Frame2D( name="v2v3", axes_order=(0, 1), @@ -455,6 +564,7 @@ def test_create_astrometric_catalog_write_results_to_disk(tmp_path, base_image): ("GAIADR3", None), ], ) +def test_create_astrometric_catalog_using_epoch(tmp_path, catalog, epoch, request): def test_create_astrometric_catalog_using_epoch(tmp_path, catalog, epoch, request): """Test fetching data from supported catalogs for a specific epoch.""" output_filename = "ref_cat.ecsv" @@ -591,12 +701,11 @@ def test_get_catalog_valid_parameters_but_no_sources_returned(): (0, 0, 2000), (0, 0, 2010.3), (0, 0, 2030), - (269.4521, 4.6933, 2030), - (89, 80, 2010), ], ) def test_get_catalog_using_epoch(ra, dec, epoch): - """Test that get_catalog returns coordinates corrected by proper motion.""" + """Test that get_catalog returns coordinates corrected by proper motion + and parallax.""" result_all = get_catalog(ra, dec, epoch=epoch) @@ -606,6 +715,8 @@ def test_get_catalog_using_epoch(ra, dec, epoch): mask = result_all["parallax_over_error"] > 5 result = result_all[mask] + + # updated coordinates at the provided epoch returned_ra = np.array(result["ra"]) returned_dec = np.array(result["dec"]) @@ -623,9 +734,12 @@ def test_get_catalog_using_epoch(ra, dec, epoch): ) # calculate parallax corrections get_parallax_correction(epoch=epoch, gaia_ref_epoch_coords=gaia_ref_epoch_coords) + get_parallax_correction(epoch=epoch, gaia_ref_epoch_coords=gaia_ref_epoch_coords) # calculate the expected coordinates value after corrections have been applied to # Gaia's reference epoch coordinates + + # textbook (barycentric frame) expected_ra = ( gaia_ref_epoch_coords["ra"] + gaia_ref_epoch_coords["pm_delta_ra"] @@ -637,10 +751,26 @@ def test_get_catalog_using_epoch(ra, dec, epoch): + gaia_ref_epoch_coords["parallax_delta_dec"] ) + # mast (geocentric frame) + expected_ra_mast = ( + gaia_ref_epoch_coords["ra"] + + gaia_ref_epoch_coords["pm_delta_ra"] + + gaia_ref_epoch_coords["parallax_delta_ra_mast"] + ) + expected_dec_mast = ( + gaia_ref_epoch_coords["dec"] + + gaia_ref_epoch_coords["pm_delta_dec"] + + gaia_ref_epoch_coords["parallax_delta_dec_mast"] + ) + assert len(result) > 0 - assert np.isclose(returned_ra, expected_ra, atol=1e-5, rtol=0).all() - assert np.isclose(returned_dec, expected_dec, atol=1e-5, rtol=0).all() + # N.B.: atol=1e-8 (in deg) corresponds to a coordinate difference of ~ 40 uas + assert np.isclose(expected_ra, returned_ra, atol=1e-8, rtol=0).all() + assert np.isclose(expected_dec, returned_dec, atol=1e-8, rtol=0).all() + + assert np.isclose(expected_ra_mast, returned_ra, atol=5e-10, rtol=0).all() + assert np.isclose(expected_dec_mast, returned_dec, atol=5e-10, rtol=0).all() def test_get_catalog_timeout(): From deebd841c6fd4213dbe7c793230fbcaaf88e3527 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Wed, 11 Oct 2023 19:28:35 -0400 Subject: [PATCH 64/82] Fix check-styles. --- .../tweakreg/tests/test_astrometric_utils.py | 142 +++++++++++++++++- 1 file changed, 138 insertions(+), 4 deletions(-) diff --git a/romancal/tweakreg/tests/test_astrometric_utils.py b/romancal/tweakreg/tests/test_astrometric_utils.py index 3e8901c0e..82f316ea1 100644 --- a/romancal/tweakreg/tests/test_astrometric_utils.py +++ b/romancal/tweakreg/tests/test_astrometric_utils.py @@ -33,6 +33,41 @@ def __init__(self, *args, **kwargs): def get_earth_sun_orbit_elem(epoch): + """ + Calculates the Earth-Sun orbit elements for a given epoch. + + Parameters + ---------- + epoch : float + The epoch for which to calculate the Earth-Sun orbit elements. + + Returns + ------- + dict + A dictionary containing the following orbit elements: + - "earth_sun_distance" : `Quantity` + The Earth-Sun distance in astronomical units (AU). + - "ecliptic_longitude" : float + The ecliptic longitude in degrees. + - "ecliptic_obliquity" : float + The ecliptic obliquity in degrees. + + Notes + ----- + This function calculates various parameters related to the Earth-Sun orbit based on the provided epoch. + + Examples + -------- + .. code-block:: python + + epoch = 2023.5 + orbit_elements = get_earth_sun_orbit_elem(epoch) + print(orbit_elements) + + # Output: + # {'earth_sun_distance': , 'ecliptic_longitude': 189.123 degrees, 'ecliptic_obliquity': 23.437 degrees} + """ # noqa: E501 + amjd = (epoch - 2012.0) * 365.25 + 55927.0 imjd = int(amjd) mjd = float(imjd) @@ -73,6 +108,55 @@ def get_earth_sun_orbit_elem(epoch): def get_parallax_correction_mast(epoch, gaia_ref_epoch_coords): + """ + Calculates the parallax correction for MAST coordinates based on the Earth-Sun orbit elements and Gaia reference epoch coordinates. + + Parameters + ---------- + epoch : float + The epoch for which to calculate the parallax correction. + gaia_ref_epoch_coords : dict + A dictionary containing the Gaia reference epoch coordinates: + - "ra" : float + The right ascension in degrees. + - "dec" : float + The declination in degrees. + - "parallax" : float + The parallax in milliarcseconds (mas). + + Returns + ------- + tuple + A tuple containing the following parallax correction components: + - delta_ra : `Quantity` + The correction in right ascension in degrees. + - delta_dec : `Quantity` + The correction in declination in degrees. + + Notes + ----- + This function calculates the parallax correction for MAST coordinates based on the + Earth-Sun orbit elements and Gaia reference epoch coordinates. It uses the + Earth-Sun distance, ecliptic longitude, and ecliptic obliquity obtained from + the `get_earth_sun_orbit_elem` function. + + Examples + -------- + .. code-block:: python + + epoch = 2023.5 + gaia_coords = { + "ra": 180.0, + "dec": 30.0, + "parallax": 2.5 + } + delta_ra, delta_dec = get_parallax_correction_mast(epoch, gaia_coords) + print(delta_ra, delta_dec) + + # Output: + # -0.001234 deg, 0.002345 deg + """ # noqa: E501 + orbit_elem = get_earth_sun_orbit_elem(epoch) earth_sun_distance = orbit_elem["earth_sun_distance"] ecliptic_longitude = orbit_elem["ecliptic_longitude"] @@ -141,7 +225,7 @@ def get_parallax_correction_barycenter(epoch, gaia_ref_epoch_coords): gaia_coords = {'ra': 180.0, 'dec': 45.0, 'parallax': 10.0} correction = get_parallax_correction_earth_barycenter(epoch, gaia_coords) print(correction) - (0.001, -0.002) + (0.001, -0.002) """ # noqa: E501 obs_date = Time(epoch, format="decimalyear") @@ -208,7 +292,7 @@ def get_proper_motion_correction(epoch, gaia_ref_epoch_coords, gaia_ref_epoch): } gaia_ref_epoch = 2020.0 get_proper_motion_correction(epoch, gaia_coords, gaia_ref_epoch) - """ + """ # noqa: E501 expected_new_dec = ( np.array( @@ -236,6 +320,50 @@ def get_proper_motion_correction(epoch, gaia_ref_epoch_coords, gaia_ref_epoch): def get_parallax_correction(epoch, gaia_ref_epoch_coords): + """ + Calculates the parallax correction for a given epoch and Gaia reference epoch + coordinates. + + Parameters + ---------- + epoch : float + The epoch for which to calculate the parallax correction. + gaia_ref_epoch_coords : dict + A dictionary containing the Gaia reference epoch coordinates: + - "ra" : float + The right ascension in degrees. + - "dec" : float + The declination in degrees. + - "parallax" : float + The parallax in milliarcseconds (mas). + + Returns + ------- + None + + Notes + ----- + This function calculates the parallax correction for a given epoch and Gaia + reference epoch coordinates. It uses the `get_parallax_correction_barycenter` + and `get_parallax_correction_mast` functions to obtain the parallax corrections + based on different coordinate frames. + + Examples + -------- + This function is typically used to add parallax correction columns to a main table + of Gaia reference epoch coordinates. + + .. code-block:: python + + epoch = 2023.5 + gaia_coords = { + "ra": 180.0, + "dec": 30.0, + "parallax": 2.5 + } + get_parallax_correction(epoch, gaia_coords) + """ # noqa: E501 + # get parallax correction using textbook calculations (i.e. Earth's barycenter) parallax_corr = get_parallax_correction_barycenter( epoch=epoch, gaia_ref_epoch_coords=gaia_ref_epoch_coords @@ -564,7 +692,6 @@ def test_create_astrometric_catalog_write_results_to_disk(tmp_path, base_image): ("GAIADR3", None), ], ) -def test_create_astrometric_catalog_using_epoch(tmp_path, catalog, epoch, request): def test_create_astrometric_catalog_using_epoch(tmp_path, catalog, epoch, request): """Test fetching data from supported catalogs for a specific epoch.""" output_filename = "ref_cat.ecsv" @@ -705,7 +832,14 @@ def test_get_catalog_valid_parameters_but_no_sources_returned(): ) def test_get_catalog_using_epoch(ra, dec, epoch): """Test that get_catalog returns coordinates corrected by proper motion - and parallax.""" + and parallax. The idea is to fetch data for a specific epoch from the MAST VO API + and compare them with the expected coordinates for that epoch. + First, the data for a specific coordinates and epoch are fetched from the MAST VO + API. Then, the data for the same coordinates are fetched for the Gaia's reference + epoch of 2016.0, and corrected for proper motion and parallax using explicit + calculations for the initially specified epoch. We then compare the results between + the returned coordinates from the MAST VO API and the manually updated + coordinates.""" result_all = get_catalog(ra, dec, epoch=epoch) From ec656788a6b06b7e28bac5487e276307800c3f07 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Thu, 12 Oct 2023 11:03:45 -0400 Subject: [PATCH 65/82] Remove MAST code and decrease catalog comparison tolerance. --- .../tweakreg/tests/test_astrometric_utils.py | 211 +----------------- 1 file changed, 8 insertions(+), 203 deletions(-) diff --git a/romancal/tweakreg/tests/test_astrometric_utils.py b/romancal/tweakreg/tests/test_astrometric_utils.py index 82f316ea1..18f27988d 100644 --- a/romancal/tweakreg/tests/test_astrometric_utils.py +++ b/romancal/tweakreg/tests/test_astrometric_utils.py @@ -10,6 +10,7 @@ from astropy import units as u from astropy.modeling import models from astropy.modeling.models import RotationSequence3D, Scale, Shift +from astropy.stats import mad_std from astropy.time import Time from gwcs import coordinate_frames as cf from gwcs import wcs @@ -32,174 +33,6 @@ def __init__(self, *args, **kwargs): raise requests.exceptions.ConnectionError -def get_earth_sun_orbit_elem(epoch): - """ - Calculates the Earth-Sun orbit elements for a given epoch. - - Parameters - ---------- - epoch : float - The epoch for which to calculate the Earth-Sun orbit elements. - - Returns - ------- - dict - A dictionary containing the following orbit elements: - - "earth_sun_distance" : `Quantity` - The Earth-Sun distance in astronomical units (AU). - - "ecliptic_longitude" : float - The ecliptic longitude in degrees. - - "ecliptic_obliquity" : float - The ecliptic obliquity in degrees. - - Notes - ----- - This function calculates various parameters related to the Earth-Sun orbit based on the provided epoch. - - Examples - -------- - .. code-block:: python - - epoch = 2023.5 - orbit_elements = get_earth_sun_orbit_elem(epoch) - print(orbit_elements) - - # Output: - # {'earth_sun_distance': , 'ecliptic_longitude': 189.123 degrees, 'ecliptic_obliquity': 23.437 degrees} - """ # noqa: E501 - - amjd = (epoch - 2012.0) * 365.25 + 55927.0 - imjd = int(amjd) - mjd = float(imjd) - - time_argument = mjd - 51544.5 - fmean_longitude = 280.460 + 0.9856474 * time_argument - fmean_anomaly = 357.528 + 0.9856003 * time_argument - iremainder = int(fmean_longitude / 360.0) - fmean_longitude = fmean_longitude - (iremainder * 360.0) - if fmean_longitude < 0: - fmean_longitude = fmean_longitude + 360.0 - iremainder = int(fmean_anomaly / 360.0) - fmean_anomaly = fmean_anomaly - (iremainder * 360.0) - if fmean_anomaly < 0: - fmean_anomaly = fmean_anomaly + 360.0 - ecliptic_longitude = ( - fmean_longitude - + 1.915 * np.sin(fmean_anomaly * ARAD) - + 0.02 * np.sin(2.0 * fmean_anomaly * ARAD) - ) - ecliptic_obliquity = 23.439 - 4.0 * 0.0000001 * time_argument - - # radius vector (in AU) - # (approximation using an algorithm from USNO) - earth_sun_distance = ( - 1.00014 - - 0.01671 * np.cos(fmean_anomaly * ARAD) - - 0.00014 * np.cos(2.0 * fmean_anomaly * ARAD) - ) - - earth_sun_distance = u.Quantity(earth_sun_distance, unit="AU") - - return { - "earth_sun_distance": earth_sun_distance, - "ecliptic_longitude": ecliptic_longitude, - "ecliptic_obliquity": ecliptic_obliquity, - } - - -def get_parallax_correction_mast(epoch, gaia_ref_epoch_coords): - """ - Calculates the parallax correction for MAST coordinates based on the Earth-Sun orbit elements and Gaia reference epoch coordinates. - - Parameters - ---------- - epoch : float - The epoch for which to calculate the parallax correction. - gaia_ref_epoch_coords : dict - A dictionary containing the Gaia reference epoch coordinates: - - "ra" : float - The right ascension in degrees. - - "dec" : float - The declination in degrees. - - "parallax" : float - The parallax in milliarcseconds (mas). - - Returns - ------- - tuple - A tuple containing the following parallax correction components: - - delta_ra : `Quantity` - The correction in right ascension in degrees. - - delta_dec : `Quantity` - The correction in declination in degrees. - - Notes - ----- - This function calculates the parallax correction for MAST coordinates based on the - Earth-Sun orbit elements and Gaia reference epoch coordinates. It uses the - Earth-Sun distance, ecliptic longitude, and ecliptic obliquity obtained from - the `get_earth_sun_orbit_elem` function. - - Examples - -------- - .. code-block:: python - - epoch = 2023.5 - gaia_coords = { - "ra": 180.0, - "dec": 30.0, - "parallax": 2.5 - } - delta_ra, delta_dec = get_parallax_correction_mast(epoch, gaia_coords) - print(delta_ra, delta_dec) - - # Output: - # -0.001234 deg, 0.002345 deg - """ # noqa: E501 - - orbit_elem = get_earth_sun_orbit_elem(epoch) - earth_sun_distance = orbit_elem["earth_sun_distance"] - ecliptic_longitude = orbit_elem["ecliptic_longitude"] - ecliptic_obliquity = orbit_elem["ecliptic_obliquity"] - - # cartesian coordinates of the sun - xsun = earth_sun_distance * np.cos(ecliptic_longitude * ARAD) - ysun = ( - earth_sun_distance - * np.cos(ecliptic_obliquity * ARAD) - * np.sin(ecliptic_longitude * ARAD) - ) - zsun = ( - earth_sun_distance - * np.sin(ecliptic_obliquity * ARAD) - * np.sin(ecliptic_longitude * ARAD) - ) - - # angular displacement components - delta_ra = ( - u.Quantity(gaia_ref_epoch_coords["parallax"], unit="mas") - / np.cos(gaia_ref_epoch_coords["dec"] * ARAD) - * ( - -xsun.value * np.sin(gaia_ref_epoch_coords["ra"] * ARAD) - + ysun.value * np.cos(gaia_ref_epoch_coords["ra"] * ARAD) - ) - ).to("deg") - delta_dec = ( - u.Quantity(gaia_ref_epoch_coords["parallax"], unit="mas") - * ( - -xsun.value - * np.cos(gaia_ref_epoch_coords["ra"] * ARAD) - * np.sin(gaia_ref_epoch_coords["dec"] * ARAD) - - ysun.value - * np.sin(gaia_ref_epoch_coords["ra"] * ARAD) - * np.sin(gaia_ref_epoch_coords["dec"] * ARAD) - + zsun.value * np.cos(gaia_ref_epoch_coords["dec"] * ARAD) - ) - ).to("deg") - - return delta_ra, delta_dec - - def get_parallax_correction_barycenter(epoch, gaia_ref_epoch_coords): """ Calculates the parallax correction in the Earth barycenter frame for a given epoch @@ -369,17 +202,9 @@ def get_parallax_correction(epoch, gaia_ref_epoch_coords): epoch=epoch, gaia_ref_epoch_coords=gaia_ref_epoch_coords ) - # get parallax using the same coordinates frame as MAST - # (i.e. Sun's geocentric coordinates) - parallax_corr_mast = get_parallax_correction_mast( - epoch=epoch, gaia_ref_epoch_coords=gaia_ref_epoch_coords - ) - # add parallax corrections columns to the main table gaia_ref_epoch_coords["parallax_delta_ra"] = parallax_corr[0] gaia_ref_epoch_coords["parallax_delta_dec"] = parallax_corr[1] - gaia_ref_epoch_coords["parallax_delta_ra_mast"] = parallax_corr_mast[0] - gaia_ref_epoch_coords["parallax_delta_dec_mast"] = parallax_corr_mast[1] def update_wcsinfo(input_dm): @@ -841,14 +666,7 @@ def test_get_catalog_using_epoch(ra, dec, epoch): the returned coordinates from the MAST VO API and the manually updated coordinates.""" - result_all = get_catalog(ra, dec, epoch=epoch) - - # select sources with reliable astrometric solutions based on the - # parallax_over_error parameter as discussed in Fabricius et al. 2021 - # (https://www.aanda.org/articles/aa/full_html/2021/05/aa39834-20/aa39834-20.html) - mask = result_all["parallax_over_error"] > 5 - - result = result_all[mask] + result = get_catalog(ra, dec, epoch=epoch) # updated coordinates at the provided epoch returned_ra = np.array(result["ra"]) @@ -858,7 +676,7 @@ def test_get_catalog_using_epoch(ra, dec, epoch): gaia_ref_epoch = 2016.0 gaia_ref_epoch_coords_all = get_catalog(ra, dec, epoch=gaia_ref_epoch) - gaia_ref_epoch_coords = gaia_ref_epoch_coords_all[mask] + gaia_ref_epoch_coords = gaia_ref_epoch_coords_all # [mask] # calculate proper motion corrections get_proper_motion_correction( @@ -868,7 +686,6 @@ def test_get_catalog_using_epoch(ra, dec, epoch): ) # calculate parallax corrections get_parallax_correction(epoch=epoch, gaia_ref_epoch_coords=gaia_ref_epoch_coords) - get_parallax_correction(epoch=epoch, gaia_ref_epoch_coords=gaia_ref_epoch_coords) # calculate the expected coordinates value after corrections have been applied to # Gaia's reference epoch coordinates @@ -885,26 +702,14 @@ def test_get_catalog_using_epoch(ra, dec, epoch): + gaia_ref_epoch_coords["parallax_delta_dec"] ) - # mast (geocentric frame) - expected_ra_mast = ( - gaia_ref_epoch_coords["ra"] - + gaia_ref_epoch_coords["pm_delta_ra"] - + gaia_ref_epoch_coords["parallax_delta_ra_mast"] - ) - expected_dec_mast = ( - gaia_ref_epoch_coords["dec"] - + gaia_ref_epoch_coords["pm_delta_dec"] - + gaia_ref_epoch_coords["parallax_delta_dec_mast"] - ) - assert len(result) > 0 - # N.B.: atol=1e-8 (in deg) corresponds to a coordinate difference of ~ 40 uas - assert np.isclose(expected_ra, returned_ra, atol=1e-8, rtol=0).all() - assert np.isclose(expected_dec, returned_dec, atol=1e-8, rtol=0).all() + # adopted tolerance: 2.8e-9 deg -> 10 uas (~0.0001 pix) + assert np.median(returned_ra - expected_ra) < 2.8e-9 + assert np.median(returned_dec - expected_dec) < 2.8e-9 - assert np.isclose(expected_ra_mast, returned_ra, atol=5e-10, rtol=0).all() - assert np.isclose(expected_dec_mast, returned_dec, atol=5e-10, rtol=0).all() + assert mad_std(returned_ra - expected_ra) < 2.8e-9 + assert mad_std(returned_dec - expected_dec) < 2.8e-9 def test_get_catalog_timeout(): From 08af6699cfdd02115477c5796b058333d4014e2c Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Mon, 2 Oct 2023 10:24:38 -0400 Subject: [PATCH 66/82] Fix TweakReg's catalog metadata. --- romancal/tweakreg/tweakreg_step.py | 35 ++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/romancal/tweakreg/tweakreg_step.py b/romancal/tweakreg/tweakreg_step.py index fcf5f3b06..0c3b4981e 100644 --- a/romancal/tweakreg/tweakreg_step.py +++ b/romancal/tweakreg/tweakreg_step.py @@ -31,7 +31,9 @@ def _oxford_or_str_join(str_list): elif nelem == 2: return f"{str_list[0]} or {str_list[1]}" else: - return ", ".join(map(repr, str_list[:-1])) + ", or " + repr(str_list[-1]) + return ( + ", ".join(map(repr, str_list[:-1])) + ", or " + repr(str_list[-1]) + ) SINGLE_GROUP_REFCAT = ["GAIADR3", "GAIADR2", "GAIADR1"] @@ -144,7 +146,9 @@ def process(self, input): self.catalog_path = os.getcwd() self.catalog_path = Path(self.catalog_path).as_posix() - self.log.info(f"All source catalogs will be saved to: {self.catalog_path}") + self.log.info( + f"All source catalogs will be saved to: {self.catalog_path}" + ) if self.abs_refcat is None or len(self.abs_refcat.strip()) == 0: self.abs_refcat = DEFAULT_ABS_REFCAT @@ -236,7 +240,9 @@ def process(self, input): grp_img = list(images.models_grouped) self.log.info("") - self.log.info(f"Number of image groups to be aligned: {len(grp_img):d}.") + self.log.info( + f"Number of image groups to be aligned: {len(grp_img):d}." + ) self.log.info("Image groups:") if len(grp_img) == 1 and not ALIGN_TO_ABS_REFCAT: @@ -246,7 +252,9 @@ def process(self, input): self.log.info("") # we need at least two exposures to perform image alignment - self.log.warning("At least two exposures are required for image alignment.") + self.log.warning( + "At least two exposures are required for image alignment." + ) self.log.warning("Nothing to do. Skipping 'TweakRegStep'...") self.skip = True for model in images: @@ -320,7 +328,8 @@ def process(self, input): except ValueError as e: msg = e.args[0] if ( - msg == "Too few input images (or groups of images) with non-empty" + msg + == "Too few input images (or groups of images) with non-empty" " catalogs." ): # we need at least two exposures to perform image alignment @@ -328,7 +337,9 @@ def process(self, input): self.log.warning( "At least two exposures are required for image alignment." ) - self.log.warning("Nothing to do. Skipping 'TweakRegStep'...") + self.log.warning( + "Nothing to do. Skipping 'TweakRegStep'..." + ) for model in images: model.meta.cal_step["tweakreg"] = "SKIPPED" if not ALIGN_TO_ABS_REFCAT: @@ -354,7 +365,9 @@ def process(self, input): for model in images: model.meta.cal_step["tweakreg"] = "SKIPPED" if ALIGN_TO_ABS_REFCAT: - self.log.warning("Skipping relative alignment (stage 1)...") + self.log.warning( + "Skipping relative alignment (stage 1)..." + ) else: self.log.warning("Skipping 'TweakRegStep'...") self.skip = True @@ -482,7 +495,9 @@ def process(self, input): # (typecasting numpy objects to python types so that it doesn't cause an # issue when saving datamodel to ASDF) wcs_fit_results = { - k: v.tolist() if isinstance(v, (np.ndarray, np.bool_)) else v + k: v.tolist() + if isinstance(v, (np.ndarray, np.bool_)) + else v for k, v in imcat.meta["fit_info"].items() } # add fit results and new WCS to datamodel @@ -576,7 +591,9 @@ def _common_name(group): file_names = [] for im in group: if isinstance(im, rdm.DataModel): - file_names.append(os.path.splitext(im.meta.filename)[0].strip("_- ")) + file_names.append( + os.path.splitext(im.meta.filename)[0].strip("_- ") + ) else: raise TypeError("Input must be a list of datamodels list.") From be6f85c87fce9a8d6a88987c634ae4a2420eeba2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 2 Oct 2023 20:38:29 +0000 Subject: [PATCH 67/82] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- romancal/tweakreg/tweakreg_step.py | 35 ++++++++---------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/romancal/tweakreg/tweakreg_step.py b/romancal/tweakreg/tweakreg_step.py index 0c3b4981e..fcf5f3b06 100644 --- a/romancal/tweakreg/tweakreg_step.py +++ b/romancal/tweakreg/tweakreg_step.py @@ -31,9 +31,7 @@ def _oxford_or_str_join(str_list): elif nelem == 2: return f"{str_list[0]} or {str_list[1]}" else: - return ( - ", ".join(map(repr, str_list[:-1])) + ", or " + repr(str_list[-1]) - ) + return ", ".join(map(repr, str_list[:-1])) + ", or " + repr(str_list[-1]) SINGLE_GROUP_REFCAT = ["GAIADR3", "GAIADR2", "GAIADR1"] @@ -146,9 +144,7 @@ def process(self, input): self.catalog_path = os.getcwd() self.catalog_path = Path(self.catalog_path).as_posix() - self.log.info( - f"All source catalogs will be saved to: {self.catalog_path}" - ) + self.log.info(f"All source catalogs will be saved to: {self.catalog_path}") if self.abs_refcat is None or len(self.abs_refcat.strip()) == 0: self.abs_refcat = DEFAULT_ABS_REFCAT @@ -240,9 +236,7 @@ def process(self, input): grp_img = list(images.models_grouped) self.log.info("") - self.log.info( - f"Number of image groups to be aligned: {len(grp_img):d}." - ) + self.log.info(f"Number of image groups to be aligned: {len(grp_img):d}.") self.log.info("Image groups:") if len(grp_img) == 1 and not ALIGN_TO_ABS_REFCAT: @@ -252,9 +246,7 @@ def process(self, input): self.log.info("") # we need at least two exposures to perform image alignment - self.log.warning( - "At least two exposures are required for image alignment." - ) + self.log.warning("At least two exposures are required for image alignment.") self.log.warning("Nothing to do. Skipping 'TweakRegStep'...") self.skip = True for model in images: @@ -328,8 +320,7 @@ def process(self, input): except ValueError as e: msg = e.args[0] if ( - msg - == "Too few input images (or groups of images) with non-empty" + msg == "Too few input images (or groups of images) with non-empty" " catalogs." ): # we need at least two exposures to perform image alignment @@ -337,9 +328,7 @@ def process(self, input): self.log.warning( "At least two exposures are required for image alignment." ) - self.log.warning( - "Nothing to do. Skipping 'TweakRegStep'..." - ) + self.log.warning("Nothing to do. Skipping 'TweakRegStep'...") for model in images: model.meta.cal_step["tweakreg"] = "SKIPPED" if not ALIGN_TO_ABS_REFCAT: @@ -365,9 +354,7 @@ def process(self, input): for model in images: model.meta.cal_step["tweakreg"] = "SKIPPED" if ALIGN_TO_ABS_REFCAT: - self.log.warning( - "Skipping relative alignment (stage 1)..." - ) + self.log.warning("Skipping relative alignment (stage 1)...") else: self.log.warning("Skipping 'TweakRegStep'...") self.skip = True @@ -495,9 +482,7 @@ def process(self, input): # (typecasting numpy objects to python types so that it doesn't cause an # issue when saving datamodel to ASDF) wcs_fit_results = { - k: v.tolist() - if isinstance(v, (np.ndarray, np.bool_)) - else v + k: v.tolist() if isinstance(v, (np.ndarray, np.bool_)) else v for k, v in imcat.meta["fit_info"].items() } # add fit results and new WCS to datamodel @@ -591,9 +576,7 @@ def _common_name(group): file_names = [] for im in group: if isinstance(im, rdm.DataModel): - file_names.append( - os.path.splitext(im.meta.filename)[0].strip("_- ") - ) + file_names.append(os.path.splitext(im.meta.filename)[0].strip("_- ")) else: raise TypeError("Input must be a list of datamodels list.") From 1ebfe54f714980685400a134e0c7b1fb3359faf9 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Thu, 5 Oct 2023 06:49:24 -0400 Subject: [PATCH 68/82] Include parallax correction to unit tests. --- romancal/tweakreg/tests/test_astrometric_utils.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/romancal/tweakreg/tests/test_astrometric_utils.py b/romancal/tweakreg/tests/test_astrometric_utils.py index 18f27988d..c5ac02655 100644 --- a/romancal/tweakreg/tests/test_astrometric_utils.py +++ b/romancal/tweakreg/tests/test_astrometric_utils.py @@ -8,6 +8,7 @@ from astropy import coordinates as coord from astropy import table from astropy import units as u +from astropy.time import Time from astropy.modeling import models from astropy.modeling.models import RotationSequence3D, Scale, Shift from astropy.stats import mad_std @@ -258,8 +259,9 @@ def create_wcs_for_tweakreg_pipeline(input_dm, shift_1=0, shift_2=0): tel2sky = _create_tel2sky_model(input_dm) # create required frames - detector = cf.Frame2D(name="detector", axes_order=(0, 1), unit=(u.pix, u.pix)) - detector = cf.Frame2D(name="detector", axes_order=(0, 1), unit=(u.pix, u.pix)) + detector = cf.Frame2D( + name="detector", axes_order=(0, 1), unit=(u.pix, u.pix) + ) v2v3 = cf.Frame2D( name="v2v3", axes_order=(0, 1), @@ -517,7 +519,9 @@ def test_create_astrometric_catalog_write_results_to_disk(tmp_path, base_image): ("GAIADR3", None), ], ) -def test_create_astrometric_catalog_using_epoch(tmp_path, catalog, epoch, request): +def test_create_astrometric_catalog_using_epoch( + tmp_path, catalog, epoch, request +): """Test fetching data from supported catalogs for a specific epoch.""" output_filename = "ref_cat.ecsv" img = request.getfixturevalue("base_image")(shift_1=1000, shift_2=1000) From f40dccad06b5636ab650558cc673165a760cc700 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 5 Oct 2023 10:57:43 +0000 Subject: [PATCH 69/82] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- romancal/tweakreg/tests/test_astrometric_utils.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/romancal/tweakreg/tests/test_astrometric_utils.py b/romancal/tweakreg/tests/test_astrometric_utils.py index c5ac02655..f69b9b660 100644 --- a/romancal/tweakreg/tests/test_astrometric_utils.py +++ b/romancal/tweakreg/tests/test_astrometric_utils.py @@ -8,7 +8,6 @@ from astropy import coordinates as coord from astropy import table from astropy import units as u -from astropy.time import Time from astropy.modeling import models from astropy.modeling.models import RotationSequence3D, Scale, Shift from astropy.stats import mad_std @@ -259,9 +258,7 @@ def create_wcs_for_tweakreg_pipeline(input_dm, shift_1=0, shift_2=0): tel2sky = _create_tel2sky_model(input_dm) # create required frames - detector = cf.Frame2D( - name="detector", axes_order=(0, 1), unit=(u.pix, u.pix) - ) + detector = cf.Frame2D(name="detector", axes_order=(0, 1), unit=(u.pix, u.pix)) v2v3 = cf.Frame2D( name="v2v3", axes_order=(0, 1), @@ -519,9 +516,7 @@ def test_create_astrometric_catalog_write_results_to_disk(tmp_path, base_image): ("GAIADR3", None), ], ) -def test_create_astrometric_catalog_using_epoch( - tmp_path, catalog, epoch, request -): +def test_create_astrometric_catalog_using_epoch(tmp_path, catalog, epoch, request): """Test fetching data from supported catalogs for a specific epoch.""" output_filename = "ref_cat.ecsv" img = request.getfixturevalue("base_image")(shift_1=1000, shift_2=1000) From 51feaf7ceab4b6b2c6b7fae10c739a9c01fe0c18 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Thu, 5 Oct 2023 14:14:48 -0400 Subject: [PATCH 70/82] Set atol=1e-5 and rtol=0 in astrometric_utils unit tests. --- romancal/tweakreg/tests/test_astrometric_utils.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/romancal/tweakreg/tests/test_astrometric_utils.py b/romancal/tweakreg/tests/test_astrometric_utils.py index f69b9b660..7678547a9 100644 --- a/romancal/tweakreg/tests/test_astrometric_utils.py +++ b/romancal/tweakreg/tests/test_astrometric_utils.py @@ -258,7 +258,9 @@ def create_wcs_for_tweakreg_pipeline(input_dm, shift_1=0, shift_2=0): tel2sky = _create_tel2sky_model(input_dm) # create required frames - detector = cf.Frame2D(name="detector", axes_order=(0, 1), unit=(u.pix, u.pix)) + detector = cf.Frame2D( + name="detector", axes_order=(0, 1), unit=(u.pix, u.pix) + ) v2v3 = cf.Frame2D( name="v2v3", axes_order=(0, 1), @@ -516,7 +518,9 @@ def test_create_astrometric_catalog_write_results_to_disk(tmp_path, base_image): ("GAIADR3", None), ], ) -def test_create_astrometric_catalog_using_epoch(tmp_path, catalog, epoch, request): +def test_create_astrometric_catalog_using_epoch( + tmp_path, catalog, epoch, request +): """Test fetching data from supported catalogs for a specific epoch.""" output_filename = "ref_cat.ecsv" img = request.getfixturevalue("base_image")(shift_1=1000, shift_2=1000) @@ -684,7 +688,9 @@ def test_get_catalog_using_epoch(ra, dec, epoch): gaia_ref_epoch=gaia_ref_epoch, ) # calculate parallax corrections - get_parallax_correction(epoch=epoch, gaia_ref_epoch_coords=gaia_ref_epoch_coords) + get_parallax_correction( + epoch=epoch, gaia_ref_epoch_coords=gaia_ref_epoch_coords + ) # calculate the expected coordinates value after corrections have been applied to # Gaia's reference epoch coordinates From b4d3e712d1caefc2c20e8acb8edddc8b46a4e61c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 5 Oct 2023 18:15:34 +0000 Subject: [PATCH 71/82] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- romancal/tweakreg/tests/test_astrometric_utils.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/romancal/tweakreg/tests/test_astrometric_utils.py b/romancal/tweakreg/tests/test_astrometric_utils.py index 7678547a9..f69b9b660 100644 --- a/romancal/tweakreg/tests/test_astrometric_utils.py +++ b/romancal/tweakreg/tests/test_astrometric_utils.py @@ -258,9 +258,7 @@ def create_wcs_for_tweakreg_pipeline(input_dm, shift_1=0, shift_2=0): tel2sky = _create_tel2sky_model(input_dm) # create required frames - detector = cf.Frame2D( - name="detector", axes_order=(0, 1), unit=(u.pix, u.pix) - ) + detector = cf.Frame2D(name="detector", axes_order=(0, 1), unit=(u.pix, u.pix)) v2v3 = cf.Frame2D( name="v2v3", axes_order=(0, 1), @@ -518,9 +516,7 @@ def test_create_astrometric_catalog_write_results_to_disk(tmp_path, base_image): ("GAIADR3", None), ], ) -def test_create_astrometric_catalog_using_epoch( - tmp_path, catalog, epoch, request -): +def test_create_astrometric_catalog_using_epoch(tmp_path, catalog, epoch, request): """Test fetching data from supported catalogs for a specific epoch.""" output_filename = "ref_cat.ecsv" img = request.getfixturevalue("base_image")(shift_1=1000, shift_2=1000) @@ -688,9 +684,7 @@ def test_get_catalog_using_epoch(ra, dec, epoch): gaia_ref_epoch=gaia_ref_epoch, ) # calculate parallax corrections - get_parallax_correction( - epoch=epoch, gaia_ref_epoch_coords=gaia_ref_epoch_coords - ) + get_parallax_correction(epoch=epoch, gaia_ref_epoch_coords=gaia_ref_epoch_coords) # calculate the expected coordinates value after corrections have been applied to # Gaia's reference epoch coordinates From d0e204afaf339ea6841fb643fad14e34df015c7d Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Wed, 11 Oct 2023 19:00:53 -0400 Subject: [PATCH 72/82] Fix astrometric_utils tests. --- romancal/tweakreg/tests/test_astrometric_utils.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/romancal/tweakreg/tests/test_astrometric_utils.py b/romancal/tweakreg/tests/test_astrometric_utils.py index f69b9b660..07dcbbfd9 100644 --- a/romancal/tweakreg/tests/test_astrometric_utils.py +++ b/romancal/tweakreg/tests/test_astrometric_utils.py @@ -259,6 +259,7 @@ def create_wcs_for_tweakreg_pipeline(input_dm, shift_1=0, shift_2=0): # create required frames detector = cf.Frame2D(name="detector", axes_order=(0, 1), unit=(u.pix, u.pix)) + detector = cf.Frame2D(name="detector", axes_order=(0, 1), unit=(u.pix, u.pix)) v2v3 = cf.Frame2D( name="v2v3", axes_order=(0, 1), @@ -516,6 +517,7 @@ def test_create_astrometric_catalog_write_results_to_disk(tmp_path, base_image): ("GAIADR3", None), ], ) +def test_create_astrometric_catalog_using_epoch(tmp_path, catalog, epoch, request): def test_create_astrometric_catalog_using_epoch(tmp_path, catalog, epoch, request): """Test fetching data from supported catalogs for a specific epoch.""" output_filename = "ref_cat.ecsv" @@ -685,6 +687,7 @@ def test_get_catalog_using_epoch(ra, dec, epoch): ) # calculate parallax corrections get_parallax_correction(epoch=epoch, gaia_ref_epoch_coords=gaia_ref_epoch_coords) + get_parallax_correction(epoch=epoch, gaia_ref_epoch_coords=gaia_ref_epoch_coords) # calculate the expected coordinates value after corrections have been applied to # Gaia's reference epoch coordinates @@ -701,6 +704,18 @@ def test_get_catalog_using_epoch(ra, dec, epoch): + gaia_ref_epoch_coords["parallax_delta_dec"] ) + # mast (geocentric frame) + expected_ra_mast = ( + gaia_ref_epoch_coords["ra"] + + gaia_ref_epoch_coords["pm_delta_ra"] + + gaia_ref_epoch_coords["parallax_delta_ra_mast"] + ) + expected_dec_mast = ( + gaia_ref_epoch_coords["dec"] + + gaia_ref_epoch_coords["pm_delta_dec"] + + gaia_ref_epoch_coords["parallax_delta_dec_mast"] + ) + assert len(result) > 0 # adopted tolerance: 2.8e-9 deg -> 10 uas (~0.0001 pix) From a42155677bc73abfc6311c6d5310aeb3fb7a43fe Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Wed, 11 Oct 2023 19:28:35 -0400 Subject: [PATCH 73/82] Fix check-styles. --- romancal/tweakreg/tests/test_astrometric_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/romancal/tweakreg/tests/test_astrometric_utils.py b/romancal/tweakreg/tests/test_astrometric_utils.py index 07dcbbfd9..6be1d8c87 100644 --- a/romancal/tweakreg/tests/test_astrometric_utils.py +++ b/romancal/tweakreg/tests/test_astrometric_utils.py @@ -517,7 +517,6 @@ def test_create_astrometric_catalog_write_results_to_disk(tmp_path, base_image): ("GAIADR3", None), ], ) -def test_create_astrometric_catalog_using_epoch(tmp_path, catalog, epoch, request): def test_create_astrometric_catalog_using_epoch(tmp_path, catalog, epoch, request): """Test fetching data from supported catalogs for a specific epoch.""" output_filename = "ref_cat.ecsv" From 584016424f6d8c4033e796ef1f97d2c8ebc18a0e Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Thu, 12 Oct 2023 11:03:45 -0400 Subject: [PATCH 74/82] Remove MAST code and decrease catalog comparison tolerance. --- romancal/tweakreg/tests/test_astrometric_utils.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/romancal/tweakreg/tests/test_astrometric_utils.py b/romancal/tweakreg/tests/test_astrometric_utils.py index 6be1d8c87..18f27988d 100644 --- a/romancal/tweakreg/tests/test_astrometric_utils.py +++ b/romancal/tweakreg/tests/test_astrometric_utils.py @@ -686,7 +686,6 @@ def test_get_catalog_using_epoch(ra, dec, epoch): ) # calculate parallax corrections get_parallax_correction(epoch=epoch, gaia_ref_epoch_coords=gaia_ref_epoch_coords) - get_parallax_correction(epoch=epoch, gaia_ref_epoch_coords=gaia_ref_epoch_coords) # calculate the expected coordinates value after corrections have been applied to # Gaia's reference epoch coordinates @@ -703,18 +702,6 @@ def test_get_catalog_using_epoch(ra, dec, epoch): + gaia_ref_epoch_coords["parallax_delta_dec"] ) - # mast (geocentric frame) - expected_ra_mast = ( - gaia_ref_epoch_coords["ra"] - + gaia_ref_epoch_coords["pm_delta_ra"] - + gaia_ref_epoch_coords["parallax_delta_ra_mast"] - ) - expected_dec_mast = ( - gaia_ref_epoch_coords["dec"] - + gaia_ref_epoch_coords["pm_delta_dec"] - + gaia_ref_epoch_coords["parallax_delta_dec_mast"] - ) - assert len(result) > 0 # adopted tolerance: 2.8e-9 deg -> 10 uas (~0.0001 pix) From f0b491f3cd25e062870f130da7d72ec7d641d41f Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Fri, 13 Oct 2023 08:15:28 -0400 Subject: [PATCH 75/82] add instructions for downloading WebbPSF data (#937) --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 852037c76..a10982257 100644 --- a/README.md +++ b/README.md @@ -185,6 +185,8 @@ $ crds sync --contexts roman-edit The CRDS_READONLY_CACHE variable should not be set, since references will need to be downloaded to your local cache as they are requested. +Additionally, currently WebbPSF data is also required. Follow [these instructions to download the data files / point to existing files on the shared internal network](https://webbpsf.readthedocs.io/en/latest/installation.html#data-install). + ### Running tests Unit tests can be run via `pytest`. Within the top level of your local `roman` repo checkout: From 9ecc872cb48ab037e2ec89505796d96e68cb3bbd Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Fri, 13 Oct 2023 09:20:38 -0400 Subject: [PATCH 76/82] add `.git` to dev dependency specifications (#932) --- requirements-dev-st.txt | 4 ++-- requirements-dev-thirdparty.txt | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/requirements-dev-st.txt b/requirements-dev-st.txt index 7b31b7564..62b46a481 100644 --- a/requirements-dev-st.txt +++ b/requirements-dev-st.txt @@ -1,6 +1,6 @@ # Roman upstream packages -git+https://github.com/spacetelescope/roman_datamodels -git+https://github.com/spacetelescope/rad +git+https://github.com/spacetelescope/roman_datamodels.git +git+https://github.com/spacetelescope/rad.git # shared upstream packages git+https://github.com/mairanteodoro/stcal.git@stcal-alignment diff --git a/requirements-dev-thirdparty.txt b/requirements-dev-thirdparty.txt index 0568dde4b..7e3c8d34f 100644 --- a/requirements-dev-thirdparty.txt +++ b/requirements-dev-thirdparty.txt @@ -1,12 +1,12 @@ # ASDF upstream packages -git+https://github.com/asdf-format/asdf-standard -git+https://github.com/asdf-format/asdf -git+https://github.com/asdf-format/asdf-transform-schemas -git+https://github.com/asdf-format/asdf-coordinates-schemas -git+https://github.com/asdf-format/asdf-wcs-schemas +git+https://github.com/asdf-format/asdf-standard.git +git+https://github.com/asdf-format/asdf.git +git+https://github.com/asdf-format/asdf-transform-schemas.git +git+https://github.com/asdf-format/asdf-coordinates-schemas.git +git+https://github.com/asdf-format/asdf-wcs-schemas.git # Use weekly astropy dev build -git+https://github.com/astropy/asdf-astropy +git+https://github.com/astropy/asdf-astropy.git --extra-index-url https://pypi.anaconda.org/astropy/simple astropy --pre git+https://github.com/astropy/photutils.git From 5d2ffbe2f2ff5068d556deb793685fba954fd17f Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Mon, 16 Oct 2023 16:24:24 -0400 Subject: [PATCH 77/82] Updates. --- romancal/regtest/regtestdata.py | 38 ++++++++++++++++--------------- romancal/regtest/test_resample.py | 25 +++++--------------- 2 files changed, 26 insertions(+), 37 deletions(-) diff --git a/romancal/regtest/regtestdata.py b/romancal/regtest/regtestdata.py index 8a737e297..51442297f 100644 --- a/romancal/regtest/regtestdata.py +++ b/romancal/regtest/regtestdata.py @@ -583,7 +583,11 @@ def _compare_arrays(self, a, b): difference["abs_diff"] = np.nansum(np.abs(a - b)) difference["n_diffs"] = np.count_nonzero( np.isclose( - a, b, rtol=self.rtol, atol=self.atol, equal_nan=self.equal_nan + a, + b, + rtol=self.rtol, + atol=self.atol, + equal_nan=self.equal_nan, ) ) return difference @@ -633,27 +637,22 @@ def _wcs_to_ra_dec(wcs): return wcs(x, y) -class WCSOperator(BaseOperator): +class WCSOperator(NDArrayTypeOperator): def give_up_diffing(self, level, diff_instance): # for comparing wcs instances this function evaluates # each wcs and compares the resulting ra and dec outputs # TODO should we compare the bounding boxes? ra_a, dec_a = _wcs_to_ra_dec(level.t1) ra_b, dec_b = _wcs_to_ra_dec(level.t2) - meta = {} - for name, a, b in [("ra", ra_a, ra_b), ("dec", dec_a, dec_b)]: - # TODO do we want to do something fancier than allclose? - if not np.allclose(a, b): - meta[name] = { - "abs_diff": np.abs(a - b), - } - if meta: - diff_instance.custom_report_result( - "wcs_differ", - level, - meta, - ) - return True + ra_diff = self._compare_arrays(ra_a, ra_b) + dec_diff = self._compare_arrays(dec_a, dec_b) + difference = {} + if ra_diff: + difference["ra"] = ra_diff + if dec_diff: + difference["dec"] = dec_diff + if difference: + diff_instance.custom_report_result("wcs_differ", level, difference) class DiffResult: @@ -708,11 +707,14 @@ def compare_asdf(result, truth, ignore=None, rtol=1e-05, atol=1e-08, equal_nan=T exclude_paths.append(f"root{key_path}") operators = [ NDArrayTypeOperator( - rtol, atol, equal_nan, types=[asdf.tags.core.NDArrayType, np.ndarray] + rtol, + atol, + equal_nan, + types=[asdf.tags.core.NDArrayType, np.ndarray], ), TimeOperator(types=[astropy.time.Time]), TableOperator(rtol, atol, equal_nan, types=[astropy.table.Table]), - WCSOperator(types=[gwcs.WCS]), + WCSOperator(rtol, atol, equal_nan, types=[gwcs.WCS]), ] # warnings can be seen in regtest runs which indicate # that ddtrace logs are evaluated at times after the below diff --git a/romancal/regtest/test_resample.py b/romancal/regtest/test_resample.py index e14aeafa9..106e03ad6 100644 --- a/romancal/regtest/test_resample.py +++ b/romancal/regtest/test_resample.py @@ -51,19 +51,14 @@ def create_asn_file( @pytest.mark.bigdata def test_resample_single_file(rtdata, ignore_asdf_paths): input_data = [ - "r0000501001001001001_01101_0001_WFI02_cal_proc.asdf", - "r0000501001001001001_01101_0002_WFI02_cal_proc.asdf", + "r0000501001001001001_01101_0001_WFI02_cal_proc_resample.asdf", + "r0000501001001001001_01101_0002_WFI02_cal_proc_resample.asdf", ] output_data = "resample_output_resamplestep.asdf" - # truth_data = "r0000401001001001001_01101_0001_WFI01_cal_twkreg_proc.asdf" + truth_data = "resample_truth_resamplestep.asdf" - [ - rtdata.get_data( - f"/Users/mteodoro/ROMAN/SYNTHETIC_IMAGES/IMAGES/23Q4_B11/L2/PROC/{data}" - ) - for data in input_data - ] - # rtdata.get_truth(f"truth/WFI/image/{truth_data}") + [rtdata.get_data(f"WFI/image/{data}") for data in input_data] + rtdata.get_truth(f"truth/WFI/image/{truth_data}") rtdata.input = create_asn_file(members_filename_list=input_data) rtdata.output = output_data @@ -76,19 +71,11 @@ def test_resample_single_file(rtdata, ignore_asdf_paths): rtdata.input, "--rotation=0", f"--output_file='{rtdata.output}'", + # "--wht_type='exptime'" ] RomanStep.from_cmdline(args) resample_out = rdm.open(rtdata.output) - # Can you add the metrics_logger decorator - # from metrics_logger.decorators import metrics_logger - # and indicate the this satisfies the requirement SOC-581 (RSOCREQ-28) - - # Also can you add SOC-582 (RSOCREQ-73) and this should test - # (a) Data quality and uncertainty information (DMS343) - # (b) Total exposure time (DMS344) - # (c) Metadata used in the mosaic generation process (DMS345) - step.log.info( "ResampleStep recorded as complete? :" f' {resample_out.meta.cal_step.resample == "COMPLETE"}' From a2a1f447f10a9244ccdce79853c2d16a36d41e65 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Tue, 17 Oct 2023 15:53:02 -0400 Subject: [PATCH 78/82] Bug fix. --- romancal/regtest/regtestdata.py | 1 + 1 file changed, 1 insertion(+) diff --git a/romancal/regtest/regtestdata.py b/romancal/regtest/regtestdata.py index 51442297f..cb41bc313 100644 --- a/romancal/regtest/regtestdata.py +++ b/romancal/regtest/regtestdata.py @@ -653,6 +653,7 @@ def give_up_diffing(self, level, diff_instance): difference["dec"] = dec_diff if difference: diff_instance.custom_report_result("wcs_differ", level, difference) + return True class DiffResult: From 918f176e6f4f06a59ab8cd4f3068045b5348727f Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Wed, 18 Oct 2023 17:23:41 -0400 Subject: [PATCH 79/82] Refactor and removal of units from weighted image. --- romancal/regtest/test_resample.py | 1 - romancal/resample/gwcs_drizzle.py | 4 ++-- romancal/resample/resample_utils.py | 5 ++--- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/romancal/regtest/test_resample.py b/romancal/regtest/test_resample.py index 106e03ad6..3b1e3026a 100644 --- a/romancal/regtest/test_resample.py +++ b/romancal/regtest/test_resample.py @@ -71,7 +71,6 @@ def test_resample_single_file(rtdata, ignore_asdf_paths): rtdata.input, "--rotation=0", f"--output_file='{rtdata.output}'", - # "--wht_type='exptime'" ] RomanStep.from_cmdline(args) resample_out = rdm.open(rtdata.output) diff --git a/romancal/resample/gwcs_drizzle.py b/romancal/resample/gwcs_drizzle.py index a23e0eac7..8d5fc157d 100644 --- a/romancal/resample/gwcs_drizzle.py +++ b/romancal/resample/gwcs_drizzle.py @@ -431,8 +431,8 @@ def dodrizzle( log.info(f"Drizzling {insci.shape} --> {outsci.shape}") _vers, nmiss, nskip = cdrizzle.tdriz( - insci.astype(np.float32).value, - inwht.astype(np.float32).value, + insci.astype(np.float32), + inwht.astype(np.float32), pixmap, outsci, outwht, diff --git a/romancal/resample/resample_utils.py b/romancal/resample/resample_utils.py index 0f9ef7639..6464156d1 100644 --- a/romancal/resample/resample_utils.py +++ b/romancal/resample/resample_utils.py @@ -4,7 +4,6 @@ import gwcs import numpy as np -from astropy import units as u from astropy import wcs as fitswcs from astropy.modeling import Model from astropy.nddata.bitmask import interpret_bit_flags @@ -141,13 +140,13 @@ def build_driz_weight(model, weight_type=None, good_bits=None): ): with np.errstate(divide="ignore", invalid="ignore"): inv_variance = model.var_rnoise**-1 - inv_variance[~np.isfinite(inv_variance)] = 1 * u.s**2 / u.electron**2 + inv_variance[~np.isfinite(inv_variance)] = 1 else: warnings.warn( "var_rnoise array not available. Setting drizzle weight map to 1", RuntimeWarning, ) - inv_variance = 1.0 * u.s**2 / u.electron**2 + inv_variance = 1.0 result = inv_variance * dqmask elif weight_type == "exptime": exptime = model.meta.exposure.exposure_time From abe44c3fdfa3b5c60bffead6c49a03301f340015 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Thu, 19 Oct 2023 09:32:11 -0400 Subject: [PATCH 80/82] Remove units from weighted image. --- romancal/resample/gwcs_drizzle.py | 2 +- romancal/resample/resample_utils.py | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/romancal/resample/gwcs_drizzle.py b/romancal/resample/gwcs_drizzle.py index 8d5fc157d..dc7999507 100644 --- a/romancal/resample/gwcs_drizzle.py +++ b/romancal/resample/gwcs_drizzle.py @@ -431,7 +431,7 @@ def dodrizzle( log.info(f"Drizzling {insci.shape} --> {outsci.shape}") _vers, nmiss, nskip = cdrizzle.tdriz( - insci.astype(np.float32), + insci.astype(np.float32).value, inwht.astype(np.float32), pixmap, outsci, diff --git a/romancal/resample/resample_utils.py b/romancal/resample/resample_utils.py index 6464156d1..213949e1d 100644 --- a/romancal/resample/resample_utils.py +++ b/romancal/resample/resample_utils.py @@ -4,6 +4,7 @@ import gwcs import numpy as np +from astropy import units as u from astropy import wcs as fitswcs from astropy.modeling import Model from astropy.nddata.bitmask import interpret_bit_flags @@ -139,7 +140,7 @@ def build_driz_weight(model, weight_type=None, good_bits=None): and model.var_rnoise.shape == model.data.shape ): with np.errstate(divide="ignore", invalid="ignore"): - inv_variance = model.var_rnoise**-1 + inv_variance = model.var_rnoise.value**-1 inv_variance[~np.isfinite(inv_variance)] = 1 else: warnings.warn( @@ -355,13 +356,18 @@ def decode_context(context, x, y): if x.ndim != 1: raise ValueError("Coordinates must be scalars or 1D arrays.") - if not (np.issubdtype(x.dtype, np.integer) and np.issubdtype(y.dtype, np.integer)): + if not ( + np.issubdtype(x.dtype, np.integer) + and np.issubdtype(y.dtype, np.integer) + ): raise ValueError("Pixel coordinates must be integer values") nbits = 8 * context.dtype.itemsize return [ - np.flatnonzero([v & (1 << k) for v in context[:, yi, xi] for k in range(nbits)]) + np.flatnonzero( + [v & (1 << k) for v in context[:, yi, xi] for k in range(nbits)] + ) for xi, yi in zip(x, y) ] From d1c220a87254ccecc889122f5c61be3fd51e322b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 19 Oct 2023 13:32:32 +0000 Subject: [PATCH 81/82] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- romancal/resample/resample_utils.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/romancal/resample/resample_utils.py b/romancal/resample/resample_utils.py index 213949e1d..964b7298e 100644 --- a/romancal/resample/resample_utils.py +++ b/romancal/resample/resample_utils.py @@ -4,7 +4,6 @@ import gwcs import numpy as np -from astropy import units as u from astropy import wcs as fitswcs from astropy.modeling import Model from astropy.nddata.bitmask import interpret_bit_flags @@ -356,18 +355,13 @@ def decode_context(context, x, y): if x.ndim != 1: raise ValueError("Coordinates must be scalars or 1D arrays.") - if not ( - np.issubdtype(x.dtype, np.integer) - and np.issubdtype(y.dtype, np.integer) - ): + if not (np.issubdtype(x.dtype, np.integer) and np.issubdtype(y.dtype, np.integer)): raise ValueError("Pixel coordinates must be integer values") nbits = 8 * context.dtype.itemsize return [ - np.flatnonzero( - [v & (1 << k) for v in context[:, yi, xi] for k in range(nbits)] - ) + np.flatnonzero([v & (1 << k) for v in context[:, yi, xi] for k in range(nbits)]) for xi, yi in zip(x, y) ] From 9928f9f0ab7349d0affecca2d183de33cf7fba12 Mon Sep 17 00:00:00 2001 From: "M. Teodoro" Date: Thu, 19 Oct 2023 13:38:37 -0400 Subject: [PATCH 82/82] Revert stcal to main branch. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 63864ef3a..05d908ccb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ # 'roman_datamodels @ git+https://github.com/spacetelescope/roman_datamodels.git@main', 'scipy >=1.11', # 'stcal >=1.4.0', - 'stcal @ git+https://github.com/mairanteodoro/stcal.git@stcal-alignment', + 'stcal @ git+https://github.com/spacetelescope/stcal.git@main', 'stpipe >=0.5.0', 'tweakwcs >=0.8.0', 'spherical-geometry >= 1.2.22',