update to reflect changes, change stackid generator

winter-telescope · Aug 10, 2023 · cbcb557 · cbcb557
1 parent d3d8805
commit cbcb557
Show file tree

Hide file tree

Showing 4 changed files with 77 additions and 64 deletions.
diff --git a/mirar/pipelines/winter/blocks.py b/mirar/pipelines/winter/blocks.py
@@ -98,7 +98,7 @@
 )
 from mirar.processors.split import SUB_ID_KEY, SplitImage
 from mirar.processors.utils import (
-    CustomImageModifier,
+    CustomImageBatchModifier,
     HeaderAnnotator,
     ImageBatcher,
     ImageDebatcher,
@@ -214,14 +214,18 @@
     MaskDatasecPixels(),
     MaskPixelsFromFunction(mask_function=get_raw_winter_mask),
     SplitImage(n_x=NXSPLIT, n_y=NYSPLIT),
-    CustomImageModifier(annotate_winter_subdet_headers),
+    CustomImageBatchModifier(annotate_winter_subdet_headers),
 ]
 
 # Save raw images
 
 save_raw = [
     ImageSaver(output_dir_name="raw_unpacked", write_mask=False),
     DatabaseImageInserter(db_table=Raw, duplicate_protocol="replace"),
+    ImageDebatcher(),
+    ImageBatcher(["BOARD_ID", "FILTER", "EXPTIME", TARGET_KEY, "SUBCOORD"]),
+    CustomImageBatchModifier(winter_stackid_annotator),
+    ImageSaver(output_dir_name="raw_unpacked", write_mask=False),
 ]
 
 # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
@@ -259,7 +263,7 @@
     ImageSaver(output_dir_name="skysub"),
 ]
 
-fourier_filter = [CustomImageModifier(winter_fourier_filtered_image_generator)]
+fourier_filter = [CustomImageBatchModifier(winter_fourier_filtered_image_generator)]
 
 astrometry = [
     ImageDebatcher(),
@@ -324,7 +328,6 @@
         temp_output_sub_dir="stack_all",
         header_keys_to_combine=["RAWID"],
     ),
-    CustomImageModifier(winter_stackid_annotator),
     ImageSaver(output_dir_name="stack"),
 ]
 

diff --git a/mirar/pipelines/winter/generator.py b/mirar/pipelines/winter/generator.py
@@ -11,7 +11,7 @@
 
 from mirar.catalog import Gaia2Mass
 from mirar.catalog.vizier import PS1
-from mirar.data import Image
+from mirar.data import Image, ImageBatch
 from mirar.data.utils.compress import decode_img
 from mirar.database.constraints import DBQueryConstraints
 from mirar.database.transactions import select_from_table
@@ -207,16 +207,18 @@ def winter_astrometry_sextractor_catalog_purifier(catalog: Table, _) -> Table:
     return clean_catalog
 
 
-def winter_stackid_annotator(image: Image) -> Image:
+def winter_stackid_annotator(batch: ImageBatch) -> ImageBatch:
     """
-    Generates a stack id for WINTER images
+    Generates a stack id for WINTER images as the minimum of the RAWID of the
+    images for which the stack was requested.
 
-    :param image: Image
-    :return: stack id
+    :param batch: ImageBatch
+    :return: ImageBatch with stackid added to the header
     """
-    first_rawid = np.min([int(x) for x in image["RAWID"].split(",")])
-    image["STACKID"] = int(first_rawid)
-    return image
+    first_rawid = np.min([int(image["RAWID"]) for image in batch])
+    for image in batch:
+        image["STACKID"] = int(first_rawid)
+    return batch
 
 
 def winter_candidate_annotator_filterer(src_df: pd.DataFrame):
@@ -383,27 +385,30 @@ def winter_reference_generator(image: Image):
     )
 
 
-def winter_fourier_filtered_image_generator(image: Image) -> Image:
+def winter_fourier_filtered_image_generator(batch: ImageBatch) -> ImageBatch:
     """
     Generates a fourier filtered image for the winter data
     """
-    # First, set the nans in the raw_data to the median value
-    raw_data = image.get_data()
-    replace_value = np.nanmedian(raw_data)  # 0.0
-
-    mask = image.get_mask()  # 0 is masked, 1 is unmasked
+    new_batch = []
+    for image in batch:
+        # First, set the nans in the raw_data to the median value
+        raw_data = image.get_data()
+        replace_value = np.nanmedian(raw_data)  # 0.0
 
-    raw_data[~mask] = replace_value
+        mask = image.get_mask()  # 0 is masked, 1 is unmasked
 
-    filtered_data, sky_model = subtract_fourier_background_model(raw_data)
+        raw_data[~mask] = replace_value
 
-    # mask the data back
-    filtered_data[~mask] = np.nan
+        filtered_data, sky_model = subtract_fourier_background_model(raw_data)
 
-    image.set_data(filtered_data)
+        # mask the data back
+        filtered_data[~mask] = np.nan
 
-    # Update the header
-    image.header["MEDCOUNT"] = np.nanmedian(filtered_data)
-    image.header[SATURATE_KEY] -= np.nanmedian(sky_model)
+        image.set_data(filtered_data)
 
-    return image
+        # Update the header
+        image.header["MEDCOUNT"] = np.nanmedian(filtered_data)
+        image.header[SATURATE_KEY] -= np.nanmedian(sky_model)
+        new_batch.append(image)
+    new_batch = ImageBatch(new_batch)
+    return new_batch
diff --git a/mirar/pipelines/winter/load_winter_image.py b/mirar/pipelines/winter/load_winter_image.py
@@ -13,7 +13,7 @@
 from astropy.time import Time
 from astropy.utils.exceptions import AstropyWarning
 
-from mirar.data import Image
+from mirar.data import Image, ImageBatch
 from mirar.io import (
     open_fits,
     open_mef_fits,
@@ -292,49 +292,54 @@ def load_winter_mef_image(
     return images
 
 
-def annotate_winter_subdet_headers(image: Image) -> Image:
+def annotate_winter_subdet_headers(batch: ImageBatch) -> ImageBatch:
     """
     Annotate winter header with information on the subdetector
 
-    :param image: Image to annotate
-    :return: Image with updated header
+    :param batch: ImageBatch to annotate
+    :return: ImageBatch where images have the updated header
     """
-    data = image.get_data()
+    new_batch = []
+    for image in batch:
+        data = image.get_data()
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", AstropyWarning)
+            _, med, std = sigma_clipped_stats(data, sigma=3.0, maxiters=5)
+            image["MEDCOUNT"] = med
+            image["STDDEV"] = std
+
+        subnx, subny, subnxtot, subnytot = (
+            image["SUBNX"],
+            image["SUBNY"],
+            image["SUBNXTOT"],
+            image["SUBNYTOT"],
+        )
 
-    with warnings.catch_warnings():
-        warnings.simplefilter("ignore", AstropyWarning)
-        _, med, std = sigma_clipped_stats(data, sigma=3.0, maxiters=5)
-        image["MEDCOUNT"] = med
-        image["STDDEV"] = std
-
-    subnx, subny, subnxtot, subnytot = (
-        image["SUBNX"],
-        image["SUBNY"],
-        image["SUBNXTOT"],
-        image["SUBNYTOT"],
-    )
+        mask = (
+            (subdets["nx"] == subnx)
+            & (subdets["ny"] == subny)
+            & (subdets["nxtot"] == subnxtot)
+            & (subdets["nytot"] == subnytot)
+            & (subdets["boardid"] == image["BOARD_ID"])
+        )
+        assert np.sum(mask) == 1, (
+            f"Subdet not found for nx={subnx}, ny={subny}, "
+            f"nxtot={subnxtot}, nytot={subnytot} and boardid={image['BOARD_ID']}"
+        )
+        image["SUBDETID"] = int(subdets[mask]["subdetid"].iloc[0])
+        image["RAWID"] = int(f"{image['EXPID']}_{str(image['SUBDETID']).rjust(2, '0')}")
+        image["USTACKID"] = None
 
-    mask = (
-        (subdets["nx"] == subnx)
-        & (subdets["ny"] == subny)
-        & (subdets["nxtot"] == subnxtot)
-        & (subdets["nytot"] == subnytot)
-        & (subdets["boardid"] == image["BOARD_ID"])
-    )
-    assert np.sum(mask) == 1, (
-        f"Subdet not found for nx={subnx}, ny={subny}, "
-        f"nxtot={subnxtot}, nytot={subnytot} and boardid={image['BOARD_ID']}"
-    )
-    image["SUBDETID"] = int(subdets[mask]["subdetid"].iloc[0])
-    image["RAWID"] = int(f"{image['EXPID']}_{str(image['SUBDETID']).rjust(2, '0')}")
-    image["USTACKID"] = None
+        if "DATASEC" in image.keys():
+            del image["DATASEC"]
 
-    if "DATASEC" in image.keys():
-        del image["DATASEC"]
+        # TODO: Write a little snippet to estimate the central RA/Dec from the pointing
+        #  RA/Dec, BOARD_ID, SUBCOORD, and PA
 
-    # TODO: Write a little snippet to estimate the central RA/Dec from the pointing
-    #  RA/Dec, BOARD_ID, SUBCOORD, and PA
-    return image
+        new_batch.append(image)
+    new_batch = ImageBatch(new_batch)
+    return new_batch
 
 
 def get_raw_winter_mask(image: Image) -> np.ndarray:

diff --git a/mirar/processors/utils/__init__.py b/mirar/processors/utils/__init__.py
@@ -4,7 +4,7 @@
 from mirar.processors.utils.header_annotate import HeaderAnnotator
 from mirar.processors.utils.header_reader import HeaderReader
 from mirar.processors.utils.image_loader import ImageLoader, MEFLoader
-from mirar.processors.utils.image_modifier import CustomImageModifier
+from mirar.processors.utils.image_modifier import CustomImageBatchModifier
 from mirar.processors.utils.image_saver import ImageSaver
 from mirar.processors.utils.image_selector import (
     ImageBatcher,