Merge branch 'main' into dependabot/pip/pydantic-1.10.11

mirar/io.py

@@ -4,14 +4,22 @@
 All opening/writing of fits files should run via this script.
 """
 
+import logging
 import warnings
 from pathlib import Path
 
 import numpy as np
 from astropy.io import fits
 from astropy.utils.exceptions import AstropyUserWarning
 
-from mirar.paths import BASE_NAME_KEY, RAW_IMG_KEY
+from mirar.data import Image
+from mirar.paths import BASE_NAME_KEY, RAW_IMG_KEY, core_fields
+
+logger = logging.getLogger(__name__)
+
+
+class MissingCoreFieldError(KeyError):
+    """Base class for missing core field errors"""
 
 
 def create_fits(data: np.ndarray, header: fits.Header | None) -> fits.PrimaryHDU:
@@ -30,7 +38,7 @@ def create_fits(data: np.ndarray, header: fits.Header | None) -> fits.PrimaryHDU
 
 def save_hdu_as_fits(hdu: fits.PrimaryHDU, path: str | Path, overwrite: bool = True):
     """
-    Wrapper hunction to save an astropy hdu to file
+    Wrapper function to save an astropy hdu to file
 
     :param hdu: hdu to save
     :param path: path to save
@@ -70,6 +78,7 @@ def open_fits(path: str | Path) -> tuple[np.ndarray, fits.Header]:
     """
     with fits.open(path) as img:
         hdu = img.pop(0)
+        hdu.verify("silentfix+ignore")
         data = hdu.data
         header = hdu.header
 
@@ -110,3 +119,23 @@ def check_file_is_complete(path: str) -> bool:
             pass
 
     return check
+
+
+def check_image_has_core_fields(img: Image):
+    """
+    Function to ensure that an image has all the core fields
+
+    :param img: Image object to check
+    :return: None
+    """
+    for key in core_fields:
+        if key not in img.keys():
+            if BASE_NAME_KEY in img.keys():
+                msg = f"({img[BASE_NAME_KEY]}) "
+
+            err = (
+                f"New image {msg}is missing the core field {key}. "
+                f"Available fields are {list(img.keys())}."
+            )
+            logger.error(err)
+            raise MissingCoreFieldError(err)

mirar/paths.py

@@ -263,7 +263,7 @@ def get_astrometry_keys() -> list:
 DARK_FRAME_KEY = "DARKNAME"
 COADD_KEY = "COADDS"
 GAIN_KEY = "GAIN"
-EXPTIME_KEY = "AEXPTIME"
+EXPTIME_KEY = "EXPTIME"
 ZP_KEY = "ZP"
 SATURATE_KEY = "SATURATE"
 PSF_FLUX_KEY = "psf_flux"
@@ -288,10 +288,16 @@ def get_astrometry_keys() -> list:
 APMAG_PREFIX_KEY = "magap"
 APMAGUNC_PREFIX_KEY = "sigmagap"
 
+TIME_KEY = "DATE-OBS"
+OBSCLASS_KEY = "OBSCLASS"
+TARGET_KEY = "TARGET"
+DITHER_N_KEY = "DITHNUM"
+MAX_DITHER_KEY = "NUMDITHS"
+
 core_fields = [
-    "OBSCLASS",
-    "TARGET",
-    "UTCTIME",
+    OBSCLASS_KEY,
+    TARGET_KEY,
+    TIME_KEY,
     COADD_KEY,
     PROC_HISTORY_KEY,
     PROC_FAIL_KEY,

mirar/pipelines/summer/blocks.py

@@ -229,6 +229,7 @@
     ZOGYPrepare(
         output_sub_dir="subtract",
         sci_zp_header_key="ZP_AUTO",
+        ref_zp_header_key="ZP",
         catalog_purifier=default_summer_catalog_purifier,
     ),
     ZOGY(output_sub_dir="subtract"),

mirar/pipelines/summer/generator.py

@@ -116,7 +116,7 @@ def summer_reference_image_generator(image: Image) -> BaseReferenceGenerator:
     :return: Reference image generator
     """
     filter_name = image["FILTER"]
-    logger.info(f"Filter is {filter_name}")
+    logger.debug(f"Filter is {filter_name}")
 
     if filter_name in ["u", "U"]:
         if in_sdss(image["RA"], image["DEC"]):

mirar/pipelines/summer/load_summer_image.py

@@ -95,6 +95,7 @@ def load_raw_summer_image(path: str) -> tuple[np.array, astropy.io.fits.Header]:
 
         header["NIGHTDATE"] = obstime.to_datetime().strftime("%Y%m%d")
         header["EXPMJD"] = header["OBSMJD"]
+        header["OBS-DATE"] = Time(header["OBSMJD"], format="mjd").isot
 
         default_id = 0
 

mirar/pipelines/winter/blocks.py

@@ -2,11 +2,13 @@
 Module for WINTER data reduction
 """
 from mirar.io import open_fits
-from mirar.paths import FITS_MASK_KEY
+from mirar.paths import BASE_NAME_KEY, DITHER_N_KEY, FITS_MASK_KEY, MAX_DITHER_KEY
 from mirar.pipelines.winter.config import (
+    psfex_path,
     sextractor_anet_config,
     sextractor_autoastrometry_config,
     sextractor_photometry_config,
+    sextractor_reference_config,
     swarp_config_path,
 )
 from mirar.pipelines.winter.generator import (
@@ -15,13 +17,16 @@
     winter_astrostat_catalog_purifier,
     winter_photometric_catalog_generator,
     winter_reference_generator,
+    winter_reference_image_resampler,
+    winter_reference_psfex,
+    winter_reference_sextractor,
 )
 from mirar.pipelines.winter.load_winter_image import (
     load_proc_winter_image,
     load_raw_winter_image,
     load_stacked_winter_image,
 )
-from mirar.processors.astromatic import Scamp
+from mirar.processors.astromatic import PSFex, Scamp
 from mirar.processors.astromatic.sextractor.sextractor import (
     Sextractor,
     sextractor_checkimg_map,
@@ -33,17 +38,19 @@
 from mirar.processors.dark import DarkCalibrator
 from mirar.processors.mask import MaskPixelsFromPath, WriteMaskedCoordsToFile
 from mirar.processors.photcal import PhotCalibrator
-from mirar.processors.reference import GetReferenceImage
+from mirar.processors.reference import GetReferenceImage, ProcessReference
 from mirar.processors.sky import NightSkyMedianCalibrator, SkyFlatCalibrator
-from mirar.processors.split import SplitImage
+from mirar.processors.split import SUB_ID_KEY, SplitImage
 from mirar.processors.utils import (
+    HeaderAnnotator,
     ImageBatcher,
     ImageDebatcher,
     ImageLoader,
     ImageSaver,
     ImageSelector,
 )
 from mirar.processors.utils.multi_ext_parser import MultiExtParser
+from mirar.processors.zogy.zogy import ZOGY, ZOGYPrepare
 
 refbuild = [
     ImageDebatcher(),
@@ -259,8 +266,8 @@
 process_proc_all_boards = [
     ImageDebatcher(),
     ImageBatcher(["UTCTIME", "BOARD_ID", "SUBCOORD"]),
-    ImageSelector(("FIELDID", ["2789", "0697", "9170"])),
-    # ImageSelector(("FIELDID", "2789")),
+    # ImageSelector(("FIELDID", ["2789", "0697", "9170"])),
+    # # ImageSelector(("FIELDID", "2789")),
     ImageSaver(output_dir_name="pre_anet"),
     AstrometryNet(
         output_sub_dir="anet",
@@ -452,18 +459,14 @@
 ]
 
 split_indiv = [
-    SplitImage(n_x=1, n_y=2),
     ImageDebatcher(),
-]
-
-select_split_subset = [ImageSelector(("SUBCOORD", "0_0"))]
-
-make_log_and_save = [
     CSVLog(
         export_keys=[
-            "SUBCOORD",
-            "FILTER",
             "UTCTIME",
+            "PROGNAME",
+            DITHER_N_KEY,
+            MAX_DITHER_KEY,
+            "FILTER",
             "EXPTIME",
             "OBSTYPE",
             "UNIQTYPE",
@@ -480,20 +483,52 @@
             "T_ROIC",
         ]
     ),
+    SplitImage(n_x=1, n_y=2),
+]
+
+save_raw = [
     ImageSaver(output_dir_name="raw_unpacked", write_mask=False),
 ]
 
-unpack_subset = extract_subset + split_indiv + select_split_subset + make_log_and_save
-unpack_all = extract_all + split_indiv + make_log_and_save
+imsub = [
+    ImageLoader(input_sub_dir="photcal", load_image=open_fits),
+    HeaderAnnotator(input_keys=[SUB_ID_KEY], output_key="SUBDETID"),
+    ProcessReference(
+        ref_image_generator=winter_reference_generator,
+        swarp_resampler=winter_reference_image_resampler,
+        sextractor=winter_reference_sextractor,
+        ref_psfex=winter_reference_psfex,
+    ),
+    Sextractor(**sextractor_reference_config, output_sub_dir="subtract", cache=False),
+    PSFex(config_path=psfex_path, output_sub_dir="subtract", norm_fits=True),
+    ImageSaver(output_dir_name="presubtract"),
+    ZOGYPrepare(output_sub_dir="subtract", sci_zp_header_key="ZP_AUTO"),
+    ImageSaver(output_dir_name="prezogy"),
+    ZOGY(output_sub_dir="subtract", sci_zp_header_key="ZP_AUTO"),
+    ImageSaver(output_dir_name="diffs"),
+]
+
+final = [
+    ImageLoader(input_sub_dir="prezogy", load_image=open_fits),
+    ZOGY(output_sub_dir="subtract", sci_zp_header_key="ZP_AUTO"),
+    ImageSaver(output_dir_name="diffs"),
+]
+
+unpack_subset = extract_subset + split_indiv + save_raw
+unpack_all = extract_all + split_indiv + save_raw
 
 load_unpacked = [
     ImageLoader(input_sub_dir="raw_unpacked", load_image=open_fits),
 ]
 
 full_commissioning = load_unpacked + process + process_proc  # + stack_proc
-full_commissioning_all_boards = (
-    load_unpacked + dark_cal_all_boards + flat_cal_all_boards + process_proc_all_boards
+
+full_commissioning_proc = (
+    dark_cal_all_boards + flat_cal_all_boards + process_proc_all_boards + photcal
 )
+
+full_commissioning_all_boards = load_unpacked + full_commissioning_proc
+
 commissioning_split_single_board = (
     log
     + split_images
@@ -510,12 +545,5 @@
     + process_proc_all_boards
     + photcal
 )
-# commissioning_split = load_all_boards + split_images + process + \
-# process_proc_all_boards + photcal
-commissioning_split = (
-    load_unpacked
-    + dark_cal_all_boards
-    + flat_cal_all_boards
-    + process_proc_all_boards
-    + photcal
-)
+
+reduce = unpack_all + full_commissioning_proc

mirar/pipelines/winter/build_references.py

@@ -15,6 +15,7 @@
 from mirar.paths import BASE_NAME_KEY, core_fields, get_output_dir
 from mirar.pipelines.winter.winter_pipeline import WINTERPipeline
 from mirar.processors.candidates.utils import get_corners_ra_dec_from_header
+from mirar.processors.split import SUB_ID_KEY
 
 logger = logging.getLogger(__name__)
 
@@ -83,7 +84,7 @@ def dummy_split_image_batch_generator(
 
             hdu.header["FIELDID"] = int(fieldid)
             subdet = int(iind * len(np.arange(-ny + 1, ny + 1, 2)) + jind)
-            hdu.header["SUBDETID"] = subdet
+            hdu.header[SUB_ID_KEY] = subdet
             hdu.header[BASE_NAME_KEY] = f"field{fieldid}_subdet{subdet}.fits"
 
             image = Image(header=hdu.header, data=data)
@@ -163,6 +164,7 @@ def run_winter_reference_build_pipeline(
     full_dec_size_deg: float = 1.2,
     field_id: int | None = None,
     subdet_id: int | None = None,
+    catch_all_errors: bool = True,
 ):
     """
     Run the reference build pipeline on the winter fields
@@ -173,6 +175,7 @@ def run_winter_reference_build_pipeline(
         full_dec_size_deg: Full declination size of the field in degrees
         field_id: Run only for this fieldid (for debugging)
         subdet_id: Run only for this subdetid (for debugging)
+        catch_all_errors: Catch all errors and continue
     Returns:
 
     """
@@ -207,18 +210,18 @@ def run_winter_reference_build_pipeline(
             full_dec_size_deg=full_dec_size_deg,
         )
 
-        subdetids = np.array([x.header["SUBDETID"] for x in split_image_batch])
+        subdetids = np.array([x.header[SUB_ID_KEY] for x in split_image_batch])
         if subdet_id is not None:
             split_image_batch = [
                 split_image_batch[np.where(subdetids == subdet_id)[0][0]]
             ]
-        subdetids = np.array([x.header["SUBDETID"] for x in split_image_batch])
+        subdetids = np.array([x.header[SUB_ID_KEY] for x in split_image_batch])
 
         dataset = Dataset([ImageBatch(x) for x in split_image_batch])
 
         res, errorstack = pipeline.reduce_images(
             dataset=dataset,
-            catch_all_errors=True,
+            catch_all_errors=catch_all_errors,
         )
 
         if len(res) == 0:
@@ -236,7 +239,7 @@ def run_winter_reference_build_pipeline(
                 plots_dir.mkdir(parents=True)
 
             for res_image in res[0]:
-                subdet_id = np.where(subdetids == int(res_image.header["SUBDETID"]))[0][
+                subdet_id = np.where(subdetids == int(res_image.header[SUB_ID_KEY]))[0][
                     0
                 ]
                 split_image = split_image_batch[subdet_id]

mirar/pipelines/winter/config/__init__.py

@@ -37,4 +37,15 @@
     "starnnw_path": winter_file_dir.joinpath("default.nnw"),
 }
 
-swarp_config_path = winter_file_dir.joinpath("swarp.config")
+sextractor_reference_config = {
+    "config_path": winter_file_dir.joinpath("photomCat.sex"),
+    "parameter_path": winter_file_dir.joinpath("photom.param"),
+    "filter_path": winter_file_dir.joinpath("default.conv"),
+    "starnnw_path": winter_file_dir.joinpath("default.nnw"),
+}
+
+swarp_config_path = winter_file_dir.joinpath("config.swarp")
+scamp_config_path = winter_file_dir.joinpath("scamp.conf")
+winter_mask_path = winter_file_dir.joinpath("winter_mask.fits")
+
+psfex_path = winter_file_dir.joinpath("photom.psfex")