From cc4911d8e2274194f66a5f824e4ae9b3861a3678 Mon Sep 17 00:00:00 2001 From: Viraj Karambelkar Date: Thu, 10 Aug 2023 10:44:52 -0700 Subject: [PATCH] Ultra deterministic WINTER stackID (#608) --- mirar/pipelines/winter/blocks.py | 11 ++- mirar/pipelines/winter/generator.py | 51 ++++++------ mirar/pipelines/winter/load_winter_image.py | 77 ++++++++++--------- .../sources/source_table_modifier.py | 5 +- mirar/processors/utils/__init__.py | 2 +- mirar/processors/utils/image_modifier.py | 17 ++-- 6 files changed, 89 insertions(+), 74 deletions(-) diff --git a/mirar/pipelines/winter/blocks.py b/mirar/pipelines/winter/blocks.py index 610f70b76..69a47f52e 100644 --- a/mirar/pipelines/winter/blocks.py +++ b/mirar/pipelines/winter/blocks.py @@ -98,7 +98,7 @@ ) from mirar.processors.split import SUB_ID_KEY, SplitImage from mirar.processors.utils import ( - CustomImageModifier, + CustomImageBatchModifier, HeaderAnnotator, ImageBatcher, ImageDebatcher, @@ -214,7 +214,7 @@ MaskDatasecPixels(), MaskPixelsFromFunction(mask_function=get_raw_winter_mask), SplitImage(n_x=NXSPLIT, n_y=NYSPLIT), - CustomImageModifier(annotate_winter_subdet_headers), + CustomImageBatchModifier(annotate_winter_subdet_headers), ] # Save raw images @@ -222,6 +222,10 @@ save_raw = [ ImageSaver(output_dir_name="raw_unpacked", write_mask=False), DatabaseImageInserter(db_table=Raw, duplicate_protocol="replace"), + ImageDebatcher(), + ImageBatcher(["BOARD_ID", "FILTER", "EXPTIME", TARGET_KEY, "SUBCOORD"]), + CustomImageBatchModifier(winter_stackid_annotator), + ImageSaver(output_dir_name="raw_unpacked", write_mask=False), ] # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -259,7 +263,7 @@ ImageSaver(output_dir_name="skysub"), ] -fourier_filter = [CustomImageModifier(winter_fourier_filtered_image_generator)] +fourier_filter = [CustomImageBatchModifier(winter_fourier_filtered_image_generator)] astrometry = [ ImageDebatcher(), @@ -324,7 +328,6 @@ temp_output_sub_dir="stack_all", header_keys_to_combine=["RAWID"], ), - CustomImageModifier(winter_stackid_annotator), ImageSaver(output_dir_name="stack"), ] diff --git a/mirar/pipelines/winter/generator.py b/mirar/pipelines/winter/generator.py index 31f99781f..b12478a2c 100644 --- a/mirar/pipelines/winter/generator.py +++ b/mirar/pipelines/winter/generator.py @@ -11,7 +11,7 @@ from mirar.catalog import Gaia2Mass from mirar.catalog.vizier import PS1 -from mirar.data import Image +from mirar.data import Image, ImageBatch from mirar.data.utils.compress import decode_img from mirar.database.constraints import DBQueryConstraints from mirar.database.transactions import select_from_table @@ -207,16 +207,18 @@ def winter_astrometry_sextractor_catalog_purifier(catalog: Table, _) -> Table: return clean_catalog -def winter_stackid_annotator(image: Image) -> Image: +def winter_stackid_annotator(batch: ImageBatch) -> ImageBatch: """ - Generates a stack id for WINTER images + Generates a stack id for WINTER images as the minimum of the RAWID of the + images for which the stack was requested. - :param image: Image - :return: stack id + :param batch: ImageBatch + :return: ImageBatch with stackid added to the header """ - first_rawid = np.min([int(x) for x in image["RAWID"].split(",")]) - image["STACKID"] = int(first_rawid) - return image + first_rawid = np.min([int(image["RAWID"]) for image in batch]) + for image in batch: + image["STACKID"] = int(first_rawid) + return batch def winter_candidate_annotator_filterer(src_df: pd.DataFrame): @@ -383,27 +385,30 @@ def winter_reference_generator(image: Image): ) -def winter_fourier_filtered_image_generator(image: Image) -> Image: +def winter_fourier_filtered_image_generator(batch: ImageBatch) -> ImageBatch: """ Generates a fourier filtered image for the winter data """ - # First, set the nans in the raw_data to the median value - raw_data = image.get_data() - replace_value = np.nanmedian(raw_data) # 0.0 - - mask = image.get_mask() # 0 is masked, 1 is unmasked + new_batch = [] + for image in batch: + # First, set the nans in the raw_data to the median value + raw_data = image.get_data() + replace_value = np.nanmedian(raw_data) # 0.0 - raw_data[~mask] = replace_value + mask = image.get_mask() # 0 is masked, 1 is unmasked - filtered_data, sky_model = subtract_fourier_background_model(raw_data) + raw_data[~mask] = replace_value - # mask the data back - filtered_data[~mask] = np.nan + filtered_data, sky_model = subtract_fourier_background_model(raw_data) - image.set_data(filtered_data) + # mask the data back + filtered_data[~mask] = np.nan - # Update the header - image.header["MEDCOUNT"] = np.nanmedian(filtered_data) - image.header[SATURATE_KEY] -= np.nanmedian(sky_model) + image.set_data(filtered_data) - return image + # Update the header + image.header["MEDCOUNT"] = np.nanmedian(filtered_data) + image.header[SATURATE_KEY] -= np.nanmedian(sky_model) + new_batch.append(image) + new_batch = ImageBatch(new_batch) + return new_batch diff --git a/mirar/pipelines/winter/load_winter_image.py b/mirar/pipelines/winter/load_winter_image.py index d4f4f5a58..700e1f7a4 100644 --- a/mirar/pipelines/winter/load_winter_image.py +++ b/mirar/pipelines/winter/load_winter_image.py @@ -13,7 +13,7 @@ from astropy.time import Time from astropy.utils.exceptions import AstropyWarning -from mirar.data import Image +from mirar.data import Image, ImageBatch from mirar.io import ( open_fits, open_mef_fits, @@ -292,49 +292,54 @@ def load_winter_mef_image( return images -def annotate_winter_subdet_headers(image: Image) -> Image: +def annotate_winter_subdet_headers(batch: ImageBatch) -> ImageBatch: """ Annotate winter header with information on the subdetector - :param image: Image to annotate - :return: Image with updated header + :param batch: ImageBatch to annotate + :return: ImageBatch where images have the updated header """ - data = image.get_data() + new_batch = [] + for image in batch: + data = image.get_data() + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", AstropyWarning) + _, med, std = sigma_clipped_stats(data, sigma=3.0, maxiters=5) + image["MEDCOUNT"] = med + image["STDDEV"] = std + + subnx, subny, subnxtot, subnytot = ( + image["SUBNX"], + image["SUBNY"], + image["SUBNXTOT"], + image["SUBNYTOT"], + ) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", AstropyWarning) - _, med, std = sigma_clipped_stats(data, sigma=3.0, maxiters=5) - image["MEDCOUNT"] = med - image["STDDEV"] = std - - subnx, subny, subnxtot, subnytot = ( - image["SUBNX"], - image["SUBNY"], - image["SUBNXTOT"], - image["SUBNYTOT"], - ) + mask = ( + (subdets["nx"] == subnx) + & (subdets["ny"] == subny) + & (subdets["nxtot"] == subnxtot) + & (subdets["nytot"] == subnytot) + & (subdets["boardid"] == image["BOARD_ID"]) + ) + assert np.sum(mask) == 1, ( + f"Subdet not found for nx={subnx}, ny={subny}, " + f"nxtot={subnxtot}, nytot={subnytot} and boardid={image['BOARD_ID']}" + ) + image["SUBDETID"] = int(subdets[mask]["subdetid"].iloc[0]) + image["RAWID"] = int(f"{image['EXPID']}_{str(image['SUBDETID']).rjust(2, '0')}") + image["USTACKID"] = None - mask = ( - (subdets["nx"] == subnx) - & (subdets["ny"] == subny) - & (subdets["nxtot"] == subnxtot) - & (subdets["nytot"] == subnytot) - & (subdets["boardid"] == image["BOARD_ID"]) - ) - assert np.sum(mask) == 1, ( - f"Subdet not found for nx={subnx}, ny={subny}, " - f"nxtot={subnxtot}, nytot={subnytot} and boardid={image['BOARD_ID']}" - ) - image["SUBDETID"] = int(subdets[mask]["subdetid"].iloc[0]) - image["RAWID"] = int(f"{image['EXPID']}_{str(image['SUBDETID']).rjust(2, '0')}") - image["USTACKID"] = None + if "DATASEC" in image.keys(): + del image["DATASEC"] - if "DATASEC" in image.keys(): - del image["DATASEC"] + # TODO: Write a little snippet to estimate the central RA/Dec from the pointing + # RA/Dec, BOARD_ID, SUBCOORD, and PA - # TODO: Write a little snippet to estimate the central RA/Dec from the pointing - # RA/Dec, BOARD_ID, SUBCOORD, and PA - return image + new_batch.append(image) + new_batch = ImageBatch(new_batch) + return new_batch def get_raw_winter_mask(image: Image) -> np.ndarray: diff --git a/mirar/processors/sources/source_table_modifier.py b/mirar/processors/sources/source_table_modifier.py index 791d0c8b5..0a16089ec 100644 --- a/mirar/processors/sources/source_table_modifier.py +++ b/mirar/processors/sources/source_table_modifier.py @@ -24,7 +24,10 @@ def __init__(self, modifier_function: Callable[[pd.DataFrame], pd.DataFrame]): self.modifier_function = modifier_function def __str__(self) -> str: - return "Processor to modify a source dataframe based on a function." + return ( + f"Processor to modify a source dataframe using the" + f" {self.modifier_function.__name__} function." + ) def _apply_to_sources(self, batch: SourceBatch) -> SourceBatch: modified_batch = SourceBatch() diff --git a/mirar/processors/utils/__init__.py b/mirar/processors/utils/__init__.py index 1f567e37b..40a0a0c2d 100644 --- a/mirar/processors/utils/__init__.py +++ b/mirar/processors/utils/__init__.py @@ -4,7 +4,7 @@ from mirar.processors.utils.header_annotate import HeaderAnnotator from mirar.processors.utils.header_reader import HeaderReader from mirar.processors.utils.image_loader import ImageLoader, MEFLoader -from mirar.processors.utils.image_modifier import CustomImageModifier +from mirar.processors.utils.image_modifier import CustomImageBatchModifier from mirar.processors.utils.image_saver import ImageSaver from mirar.processors.utils.image_selector import ( ImageBatcher, diff --git a/mirar/processors/utils/image_modifier.py b/mirar/processors/utils/image_modifier.py index 2a5f302f3..08f0f3946 100644 --- a/mirar/processors/utils/image_modifier.py +++ b/mirar/processors/utils/image_modifier.py @@ -3,11 +3,11 @@ """ from typing import Callable -from mirar.data import Image, ImageBatch +from mirar.data import ImageBatch from mirar.processors.base_processor import BaseImageProcessor -class CustomImageModifier(BaseImageProcessor): +class CustomImageBatchModifier(BaseImageProcessor): """ Class to modify an image using a custom user-defined function """ @@ -16,21 +16,20 @@ class CustomImageModifier(BaseImageProcessor): def __init__( self, - image_modifier: Callable[[Image], Image], + image_batch_modifier: Callable[[ImageBatch], ImageBatch], ): super().__init__() - self.image_modifier = image_modifier + self.image_batch_modifier = image_batch_modifier def __str__(self): return ( - f"Processor to modify images using " - f"'{self.image_modifier.__name__}' function." + f"Processor to modify image batches using " + f"'{self.image_batch_modifier.__name__}' function." ) def _apply_to_images( self, batch: ImageBatch, ) -> ImageBatch: - for i, image in enumerate(batch): - batch[i] = self.image_modifier(image) - return batch + new_batch = self.image_batch_modifier(batch) + return new_batch