Ultra deterministic WINTER stackID

mirar/pipelines/winter/blocks.py

@@ -98,7 +98,7 @@
 )
 from mirar.processors.split import SUB_ID_KEY, SplitImage
 from mirar.processors.utils import (
-    CustomImageModifier,
+    CustomImageBatchModifier,
     HeaderAnnotator,
     ImageBatcher,
     ImageDebatcher,
@@ -214,14 +214,18 @@
     MaskDatasecPixels(),
     MaskPixelsFromFunction(mask_function=get_raw_winter_mask),
     SplitImage(n_x=NXSPLIT, n_y=NYSPLIT),
-    CustomImageModifier(annotate_winter_subdet_headers),
+    CustomImageBatchModifier(annotate_winter_subdet_headers),
 ]
 
 # Save raw images
 
 save_raw = [
     ImageSaver(output_dir_name="raw_unpacked", write_mask=False),
     DatabaseImageInserter(db_table=Raw, duplicate_protocol="replace"),
+    ImageDebatcher(),
+    ImageBatcher(["BOARD_ID", "FILTER", "EXPTIME", TARGET_KEY, "SUBCOORD"]),
+    CustomImageBatchModifier(winter_stackid_annotator),
+    ImageSaver(output_dir_name="raw_unpacked", write_mask=False),
 ]
 
 # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
@@ -259,7 +263,7 @@
     ImageSaver(output_dir_name="skysub"),
 ]
 
-fourier_filter = [CustomImageModifier(winter_fourier_filtered_image_generator)]
+fourier_filter = [CustomImageBatchModifier(winter_fourier_filtered_image_generator)]
 
 astrometry = [
     ImageDebatcher(),
@@ -324,7 +328,6 @@
         temp_output_sub_dir="stack_all",
         header_keys_to_combine=["RAWID"],
     ),
-    CustomImageModifier(winter_stackid_annotator),
     ImageSaver(output_dir_name="stack"),
 ]
 

mirar/pipelines/winter/generator.py

@@ -11,7 +11,7 @@
 
 from mirar.catalog import Gaia2Mass
 from mirar.catalog.vizier import PS1
-from mirar.data import Image
+from mirar.data import Image, ImageBatch
 from mirar.data.utils.compress import decode_img
 from mirar.database.constraints import DBQueryConstraints
 from mirar.database.transactions import select_from_table
@@ -207,16 +207,18 @@ def winter_astrometry_sextractor_catalog_purifier(catalog: Table, _) -> Table:
     return clean_catalog
 
 
-def winter_stackid_annotator(image: Image) -> Image:
+def winter_stackid_annotator(batch: ImageBatch) -> ImageBatch:
     """
-    Generates a stack id for WINTER images
+    Generates a stack id for WINTER images as the minimum of the RAWID of the
+    images for which the stack was requested.
 
-    :param image: Image
-    :return: stack id
+    :param batch: ImageBatch
+    :return: ImageBatch with stackid added to the header
     """
-    first_rawid = np.min([int(x) for x in image["RAWID"].split(",")])
-    image["STACKID"] = int(first_rawid)
-    return image
+    first_rawid = np.min([int(image["RAWID"]) for image in batch])
+    for image in batch:
+        image["STACKID"] = int(first_rawid)
+    return batch
 
 
 def winter_candidate_annotator_filterer(src_df: pd.DataFrame):
@@ -383,27 +385,30 @@ def winter_reference_generator(image: Image):
     )
 
 
-def winter_fourier_filtered_image_generator(image: Image) -> Image:
+def winter_fourier_filtered_image_generator(batch: ImageBatch) -> ImageBatch:
     """
     Generates a fourier filtered image for the winter data
     """
-    # First, set the nans in the raw_data to the median value
-    raw_data = image.get_data()
-    replace_value = np.nanmedian(raw_data)  # 0.0
-
-    mask = image.get_mask()  # 0 is masked, 1 is unmasked
+    new_batch = []
+    for image in batch:
+        # First, set the nans in the raw_data to the median value
+        raw_data = image.get_data()
+        replace_value = np.nanmedian(raw_data)  # 0.0
 
-    raw_data[~mask] = replace_value
+        mask = image.get_mask()  # 0 is masked, 1 is unmasked
 
-    filtered_data, sky_model = subtract_fourier_background_model(raw_data)
+        raw_data[~mask] = replace_value
 
-    # mask the data back
-    filtered_data[~mask] = np.nan
+        filtered_data, sky_model = subtract_fourier_background_model(raw_data)
 
-    image.set_data(filtered_data)
+        # mask the data back
+        filtered_data[~mask] = np.nan
 
-    # Update the header
-    image.header["MEDCOUNT"] = np.nanmedian(filtered_data)
-    image.header[SATURATE_KEY] -= np.nanmedian(sky_model)
+        image.set_data(filtered_data)
 
-    return image
+        # Update the header
+        image.header["MEDCOUNT"] = np.nanmedian(filtered_data)
+        image.header[SATURATE_KEY] -= np.nanmedian(sky_model)
+        new_batch.append(image)
+    new_batch = ImageBatch(new_batch)
+    return new_batch

mirar/pipelines/winter/load_winter_image.py

@@ -13,7 +13,7 @@
 from astropy.time import Time
 from astropy.utils.exceptions import AstropyWarning
 
-from mirar.data import Image
+from mirar.data import Image, ImageBatch
 from mirar.io import (
     open_fits,
     open_mef_fits,
@@ -292,49 +292,54 @@ def load_winter_mef_image(
     return images
 
 
-def annotate_winter_subdet_headers(image: Image) -> Image:
+def annotate_winter_subdet_headers(batch: ImageBatch) -> ImageBatch:
     """
     Annotate winter header with information on the subdetector
 
-    :param image: Image to annotate
-    :return: Image with updated header
+    :param batch: ImageBatch to annotate
+    :return: ImageBatch where images have the updated header
     """
-    data = image.get_data()
+    new_batch = []
+    for image in batch:
+        data = image.get_data()
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", AstropyWarning)
+            _, med, std = sigma_clipped_stats(data, sigma=3.0, maxiters=5)
+            image["MEDCOUNT"] = med
+            image["STDDEV"] = std
+
+        subnx, subny, subnxtot, subnytot = (
+            image["SUBNX"],
+            image["SUBNY"],
+            image["SUBNXTOT"],
+            image["SUBNYTOT"],
+        )
 
-    with warnings.catch_warnings():
-        warnings.simplefilter("ignore", AstropyWarning)
-        _, med, std = sigma_clipped_stats(data, sigma=3.0, maxiters=5)
-        image["MEDCOUNT"] = med
-        image["STDDEV"] = std
-
-    subnx, subny, subnxtot, subnytot = (
-        image["SUBNX"],
-        image["SUBNY"],
-        image["SUBNXTOT"],
-        image["SUBNYTOT"],
-    )
+        mask = (
+            (subdets["nx"] == subnx)
+            & (subdets["ny"] == subny)
+            & (subdets["nxtot"] == subnxtot)
+            & (subdets["nytot"] == subnytot)
+            & (subdets["boardid"] == image["BOARD_ID"])
+        )
+        assert np.sum(mask) == 1, (
+            f"Subdet not found for nx={subnx}, ny={subny}, "
+            f"nxtot={subnxtot}, nytot={subnytot} and boardid={image['BOARD_ID']}"
+        )
+        image["SUBDETID"] = int(subdets[mask]["subdetid"].iloc[0])
+        image["RAWID"] = int(f"{image['EXPID']}_{str(image['SUBDETID']).rjust(2, '0')}")
+        image["USTACKID"] = None
 
-    mask = (
-        (subdets["nx"] == subnx)
-        & (subdets["ny"] == subny)
-        & (subdets["nxtot"] == subnxtot)
-        & (subdets["nytot"] == subnytot)
-        & (subdets["boardid"] == image["BOARD_ID"])
-    )
-    assert np.sum(mask) == 1, (
-        f"Subdet not found for nx={subnx}, ny={subny}, "
-        f"nxtot={subnxtot}, nytot={subnytot} and boardid={image['BOARD_ID']}"
-    )
-    image["SUBDETID"] = int(subdets[mask]["subdetid"].iloc[0])
-    image["RAWID"] = int(f"{image['EXPID']}_{str(image['SUBDETID']).rjust(2, '0')}")
-    image["USTACKID"] = None
+        if "DATASEC" in image.keys():
+            del image["DATASEC"]
 
-    if "DATASEC" in image.keys():
-        del image["DATASEC"]
+        # TODO: Write a little snippet to estimate the central RA/Dec from the pointing
+        #  RA/Dec, BOARD_ID, SUBCOORD, and PA
 
-    # TODO: Write a little snippet to estimate the central RA/Dec from the pointing
-    #  RA/Dec, BOARD_ID, SUBCOORD, and PA
-    return image
+        new_batch.append(image)
+    new_batch = ImageBatch(new_batch)
+    return new_batch
 
 
 def get_raw_winter_mask(image: Image) -> np.ndarray:

mirar/processors/sources/source_table_modifier.py

@@ -24,7 +24,10 @@ def __init__(self, modifier_function: Callable[[pd.DataFrame], pd.DataFrame]):
         self.modifier_function = modifier_function
 
     def __str__(self) -> str:
-        return "Processor to modify a source dataframe based on a function."
+        return (
+            f"Processor to modify a source dataframe using the"
+            f" {self.modifier_function.__name__} function."
+        )
 
     def _apply_to_sources(self, batch: SourceBatch) -> SourceBatch:
         modified_batch = SourceBatch()

mirar/processors/utils/__init__.py

@@ -4,7 +4,7 @@
 from mirar.processors.utils.header_annotate import HeaderAnnotator
 from mirar.processors.utils.header_reader import HeaderReader
 from mirar.processors.utils.image_loader import ImageLoader, MEFLoader
-from mirar.processors.utils.image_modifier import CustomImageModifier
+from mirar.processors.utils.image_modifier import CustomImageBatchModifier
 from mirar.processors.utils.image_saver import ImageSaver
 from mirar.processors.utils.image_selector import (
     ImageBatcher,

mirar/processors/utils/image_modifier.py

@@ -3,11 +3,11 @@
 """
 from typing import Callable
 
-from mirar.data import Image, ImageBatch
+from mirar.data import ImageBatch
 from mirar.processors.base_processor import BaseImageProcessor
 
 
-class CustomImageModifier(BaseImageProcessor):
+class CustomImageBatchModifier(BaseImageProcessor):
     """
     Class to modify an image using a custom user-defined function
     """
@@ -16,21 +16,20 @@ class CustomImageModifier(BaseImageProcessor):
 
     def __init__(
         self,
-        image_modifier: Callable[[Image], Image],
+        image_batch_modifier: Callable[[ImageBatch], ImageBatch],
     ):
         super().__init__()
-        self.image_modifier = image_modifier
+        self.image_batch_modifier = image_batch_modifier
 
     def __str__(self):
         return (
-            f"Processor to modify images using "
-            f"'{self.image_modifier.__name__}' function."
+            f"Processor to modify image batches using "
+            f"'{self.image_batch_modifier.__name__}' function."
         )
 
     def _apply_to_images(
         self,
         batch: ImageBatch,
     ) -> ImageBatch:
-        for i, image in enumerate(batch):
-            batch[i] = self.image_modifier(image)
-        return batch
+        new_batch = self.image_batch_modifier(batch)
+        return new_batch