From 26e3a4deb9316f865440b4ca7f7b0976b4a06c08 Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Tue, 11 Jul 2023 18:41:39 -0700 Subject: [PATCH 1/2] Working image subtraction for WINTER (#465) --- mirar/io.py | 33 ++++++++++++- mirar/paths.py | 12 +++-- mirar/pipelines/summer/blocks.py | 1 + mirar/pipelines/summer/generator.py | 2 +- mirar/pipelines/summer/load_summer_image.py | 1 + mirar/pipelines/winter/blocks.py | 52 +++++++++++++++----- mirar/pipelines/winter/build_references.py | 13 +++-- mirar/pipelines/winter/config/__init__.py | 13 ++++- mirar/pipelines/winter/generator.py | 44 ++++++++++++----- mirar/pipelines/winter/load_winter_image.py | 2 +- mirar/pipelines/winter/models/_ref_stacks.py | 4 +- mirar/pipelines/winter/winter_pipeline.py | 4 ++ mirar/pipelines/wirc/load_wirc_image.py | 4 +- mirar/processors/astromatic/swarp/swarp.py | 14 +++++- mirar/processors/base_processor.py | 4 +- mirar/processors/reference.py | 10 +++- mirar/processors/split.py | 7 +-- mirar/processors/utils/header_annotate.py | 2 +- mirar/processors/utils/image_loader.py | 38 +++++++++----- mirar/processors/utils/image_saver.py | 3 ++ mirar/processors/zogy/pyzogy.py | 6 +-- mirar/processors/zogy/zogy.py | 37 +++++++++++--- mirar/references/base_reference_generator.py | 36 +++++++++++++- mirar/references/ps1.py | 2 + mirar/references/ukirt.py | 3 +- tests/test_wfau_references.py | 4 +- 26 files changed, 274 insertions(+), 77 deletions(-) diff --git a/mirar/io.py b/mirar/io.py index 5d4c1a538..c8e15136f 100644 --- a/mirar/io.py +++ b/mirar/io.py @@ -4,6 +4,7 @@ All opening/writing of fits files should run via this script. """ +import logging import warnings from pathlib import Path @@ -11,7 +12,14 @@ from astropy.io import fits from astropy.utils.exceptions import AstropyUserWarning -from mirar.paths import BASE_NAME_KEY, RAW_IMG_KEY +from mirar.data import Image +from mirar.paths import BASE_NAME_KEY, RAW_IMG_KEY, core_fields + +logger = logging.getLogger(__name__) + + +class MissingCoreFieldError(KeyError): + """Base class for missing core field errors""" def create_fits(data: np.ndarray, header: fits.Header | None) -> fits.PrimaryHDU: @@ -30,7 +38,7 @@ def create_fits(data: np.ndarray, header: fits.Header | None) -> fits.PrimaryHDU def save_hdu_as_fits(hdu: fits.PrimaryHDU, path: str | Path, overwrite: bool = True): """ - Wrapper hunction to save an astropy hdu to file + Wrapper function to save an astropy hdu to file :param hdu: hdu to save :param path: path to save @@ -70,6 +78,7 @@ def open_fits(path: str | Path) -> tuple[np.ndarray, fits.Header]: """ with fits.open(path) as img: hdu = img.pop(0) + hdu.verify("silentfix+ignore") data = hdu.data header = hdu.header @@ -110,3 +119,23 @@ def check_file_is_complete(path: str) -> bool: pass return check + + +def check_image_has_core_fields(img: Image): + """ + Function to ensure that an image has all the core fields + + :param img: Image object to check + :return: None + """ + for key in core_fields: + if key not in img.keys(): + if BASE_NAME_KEY in img.keys(): + msg = f"({img[BASE_NAME_KEY]}) " + + err = ( + f"New image {msg}is missing the core field {key}. " + f"Available fields are {list(img.keys())}." + ) + logger.error(err) + raise MissingCoreFieldError(err) diff --git a/mirar/paths.py b/mirar/paths.py index 0cc511be0..3e4e4dbdc 100644 --- a/mirar/paths.py +++ b/mirar/paths.py @@ -263,7 +263,7 @@ def get_astrometry_keys() -> list: DARK_FRAME_KEY = "DARKNAME" COADD_KEY = "COADDS" GAIN_KEY = "GAIN" -EXPTIME_KEY = "AEXPTIME" +EXPTIME_KEY = "EXPTIME" ZP_KEY = "ZP" SATURATE_KEY = "SATURATE" PSF_FLUX_KEY = "psf_flux" @@ -288,10 +288,14 @@ def get_astrometry_keys() -> list: APMAG_PREFIX_KEY = "magap" APMAGUNC_PREFIX_KEY = "sigmagap" +TIME_KEY = "DATE-OBS" +OBSCLASS_KEY = "OBSCLASS" +TARGET_KEY = "TARGET" + core_fields = [ - "OBSCLASS", - "TARGET", - "UTCTIME", + OBSCLASS_KEY, + TARGET_KEY, + TIME_KEY, COADD_KEY, PROC_HISTORY_KEY, PROC_FAIL_KEY, diff --git a/mirar/pipelines/summer/blocks.py b/mirar/pipelines/summer/blocks.py index 228be078d..0062875e0 100644 --- a/mirar/pipelines/summer/blocks.py +++ b/mirar/pipelines/summer/blocks.py @@ -229,6 +229,7 @@ ZOGYPrepare( output_sub_dir="subtract", sci_zp_header_key="ZP_AUTO", + ref_zp_header_key="ZP", catalog_purifier=default_summer_catalog_purifier, ), ZOGY(output_sub_dir="subtract"), diff --git a/mirar/pipelines/summer/generator.py b/mirar/pipelines/summer/generator.py index 8efe76fef..5d0f0e819 100644 --- a/mirar/pipelines/summer/generator.py +++ b/mirar/pipelines/summer/generator.py @@ -116,7 +116,7 @@ def summer_reference_image_generator(image: Image) -> BaseReferenceGenerator: :return: Reference image generator """ filter_name = image["FILTER"] - logger.info(f"Filter is {filter_name}") + logger.debug(f"Filter is {filter_name}") if filter_name in ["u", "U"]: if in_sdss(image["RA"], image["DEC"]): diff --git a/mirar/pipelines/summer/load_summer_image.py b/mirar/pipelines/summer/load_summer_image.py index 2a03f3b39..01a605d41 100644 --- a/mirar/pipelines/summer/load_summer_image.py +++ b/mirar/pipelines/summer/load_summer_image.py @@ -95,6 +95,7 @@ def load_raw_summer_image(path: str) -> tuple[np.array, astropy.io.fits.Header]: header["NIGHTDATE"] = obstime.to_datetime().strftime("%Y%m%d") header["EXPMJD"] = header["OBSMJD"] + header["OBS-DATE"] = Time(header["OBSMJD"], format="mjd").isot default_id = 0 diff --git a/mirar/pipelines/winter/blocks.py b/mirar/pipelines/winter/blocks.py index 4b4841d97..88220aefc 100644 --- a/mirar/pipelines/winter/blocks.py +++ b/mirar/pipelines/winter/blocks.py @@ -4,9 +4,11 @@ from mirar.io import open_fits from mirar.paths import FITS_MASK_KEY from mirar.pipelines.winter.config import ( + psfex_path, sextractor_anet_config, sextractor_autoastrometry_config, sextractor_photometry_config, + sextractor_reference_config, swarp_config_path, ) from mirar.pipelines.winter.generator import ( @@ -15,13 +17,16 @@ winter_astrostat_catalog_purifier, winter_photometric_catalog_generator, winter_reference_generator, + winter_reference_image_resampler, + winter_reference_psfex, + winter_reference_sextractor, ) from mirar.pipelines.winter.load_winter_image import ( load_proc_winter_image, load_raw_winter_image, load_stacked_winter_image, ) -from mirar.processors.astromatic import Scamp +from mirar.processors.astromatic import PSFex, Scamp from mirar.processors.astromatic.sextractor.sextractor import ( Sextractor, sextractor_checkimg_map, @@ -33,10 +38,11 @@ from mirar.processors.dark import DarkCalibrator from mirar.processors.mask import MaskPixelsFromPath, WriteMaskedCoordsToFile from mirar.processors.photcal import PhotCalibrator -from mirar.processors.reference import GetReferenceImage +from mirar.processors.reference import GetReferenceImage, ProcessReference from mirar.processors.sky import NightSkyMedianCalibrator, SkyFlatCalibrator -from mirar.processors.split import SplitImage +from mirar.processors.split import SUB_ID_KEY, SplitImage from mirar.processors.utils import ( + HeaderAnnotator, ImageBatcher, ImageDebatcher, ImageLoader, @@ -44,6 +50,7 @@ ImageSelector, ) from mirar.processors.utils.multi_ext_parser import MultiExtParser +from mirar.processors.zogy.zogy import ZOGY, ZOGYPrepare refbuild = [ ImageDebatcher(), @@ -483,6 +490,30 @@ ImageSaver(output_dir_name="raw_unpacked", write_mask=False), ] +imsub = [ + ImageLoader(input_sub_dir="photcal", load_image=open_fits), + HeaderAnnotator(input_keys=[SUB_ID_KEY], output_key="SUBDETID"), + ProcessReference( + ref_image_generator=winter_reference_generator, + swarp_resampler=winter_reference_image_resampler, + sextractor=winter_reference_sextractor, + ref_psfex=winter_reference_psfex, + ), + Sextractor(**sextractor_reference_config, output_sub_dir="subtract", cache=False), + PSFex(config_path=psfex_path, output_sub_dir="subtract", norm_fits=True), + ImageSaver(output_dir_name="presubtract"), + ZOGYPrepare(output_sub_dir="subtract", sci_zp_header_key="ZP_AUTO"), + ImageSaver(output_dir_name="prezogy"), + ZOGY(output_sub_dir="subtract", sci_zp_header_key="ZP_AUTO"), + ImageSaver(output_dir_name="diffs"), +] + +final = [ + ImageLoader(input_sub_dir="prezogy", load_image=open_fits), + ZOGY(output_sub_dir="subtract", sci_zp_header_key="ZP_AUTO"), + ImageSaver(output_dir_name="diffs"), +] + unpack_subset = extract_subset + split_indiv + select_split_subset + make_log_and_save unpack_all = extract_all + split_indiv + make_log_and_save @@ -491,27 +522,24 @@ ] full_commissioning = load_unpacked + process + process_proc # + stack_proc + full_commissioning_all_boards = ( - load_unpacked + dark_cal_all_boards + flat_cal_all_boards + process_proc_all_boards -) -commissioning_split_single_board = ( - log - + split_images + load_unpacked + dark_cal_all_boards + flat_cal_all_boards + process_proc_all_boards + photcal ) -commissioning_split = ( - load_unpacked +commissioning_split_single_board = ( + log + + split_images + dark_cal_all_boards + flat_cal_all_boards + process_proc_all_boards + photcal ) -# commissioning_split = load_all_boards + split_images + process + \ -# process_proc_all_boards + photcal + commissioning_split = ( load_unpacked + dark_cal_all_boards diff --git a/mirar/pipelines/winter/build_references.py b/mirar/pipelines/winter/build_references.py index 4f20367ae..60a563df9 100644 --- a/mirar/pipelines/winter/build_references.py +++ b/mirar/pipelines/winter/build_references.py @@ -15,6 +15,7 @@ from mirar.paths import BASE_NAME_KEY, core_fields, get_output_dir from mirar.pipelines.winter.winter_pipeline import WINTERPipeline from mirar.processors.candidates.utils import get_corners_ra_dec_from_header +from mirar.processors.split import SUB_ID_KEY logger = logging.getLogger(__name__) @@ -83,7 +84,7 @@ def dummy_split_image_batch_generator( hdu.header["FIELDID"] = int(fieldid) subdet = int(iind * len(np.arange(-ny + 1, ny + 1, 2)) + jind) - hdu.header["SUBDETID"] = subdet + hdu.header[SUB_ID_KEY] = subdet hdu.header[BASE_NAME_KEY] = f"field{fieldid}_subdet{subdet}.fits" image = Image(header=hdu.header, data=data) @@ -163,6 +164,7 @@ def run_winter_reference_build_pipeline( full_dec_size_deg: float = 1.2, field_id: int | None = None, subdet_id: int | None = None, + catch_all_errors: bool = True, ): """ Run the reference build pipeline on the winter fields @@ -173,6 +175,7 @@ def run_winter_reference_build_pipeline( full_dec_size_deg: Full declination size of the field in degrees field_id: Run only for this fieldid (for debugging) subdet_id: Run only for this subdetid (for debugging) + catch_all_errors: Catch all errors and continue Returns: """ @@ -207,18 +210,18 @@ def run_winter_reference_build_pipeline( full_dec_size_deg=full_dec_size_deg, ) - subdetids = np.array([x.header["SUBDETID"] for x in split_image_batch]) + subdetids = np.array([x.header[SUB_ID_KEY] for x in split_image_batch]) if subdet_id is not None: split_image_batch = [ split_image_batch[np.where(subdetids == subdet_id)[0][0]] ] - subdetids = np.array([x.header["SUBDETID"] for x in split_image_batch]) + subdetids = np.array([x.header[SUB_ID_KEY] for x in split_image_batch]) dataset = Dataset([ImageBatch(x) for x in split_image_batch]) res, errorstack = pipeline.reduce_images( dataset=dataset, - catch_all_errors=True, + catch_all_errors=catch_all_errors, ) if len(res) == 0: @@ -236,7 +239,7 @@ def run_winter_reference_build_pipeline( plots_dir.mkdir(parents=True) for res_image in res[0]: - subdet_id = np.where(subdetids == int(res_image.header["SUBDETID"]))[0][ + subdet_id = np.where(subdetids == int(res_image.header[SUB_ID_KEY]))[0][ 0 ] split_image = split_image_batch[subdet_id] diff --git a/mirar/pipelines/winter/config/__init__.py b/mirar/pipelines/winter/config/__init__.py index 10f055b2f..4f4e48d21 100644 --- a/mirar/pipelines/winter/config/__init__.py +++ b/mirar/pipelines/winter/config/__init__.py @@ -37,4 +37,15 @@ "starnnw_path": winter_file_dir.joinpath("default.nnw"), } -swarp_config_path = winter_file_dir.joinpath("swarp.config") +sextractor_reference_config = { + "config_path": winter_file_dir.joinpath("photomCat.sex"), + "parameter_path": winter_file_dir.joinpath("photom.param"), + "filter_path": winter_file_dir.joinpath("default.conv"), + "starnnw_path": winter_file_dir.joinpath("default.nnw"), +} + +swarp_config_path = winter_file_dir.joinpath("config.swarp") +scamp_config_path = winter_file_dir.joinpath("scamp.conf") +winter_mask_path = winter_file_dir.joinpath("winter_mask.fits") + +psfex_path = winter_file_dir.joinpath("photom.psfex") diff --git a/mirar/pipelines/winter/generator.py b/mirar/pipelines/winter/generator.py index 5ece85d07..d0931aff7 100644 --- a/mirar/pipelines/winter/generator.py +++ b/mirar/pipelines/winter/generator.py @@ -12,26 +12,27 @@ from mirar.catalog.vizier import PS1 from mirar.data import Image from mirar.paths import get_output_dir +from mirar.pipelines.winter.config import ( + psfex_path, + scamp_config_path, + sextractor_reference_config, + swarp_config_path, +) from mirar.pipelines.winter.models import RefComponents, RefQueries, RefStacks from mirar.pipelines.wirc.wirc_files import sextractor_astrometry_config +from mirar.processors.astromatic import PSFex from mirar.processors.astromatic.sextractor.sextractor import Sextractor from mirar.processors.astromatic.swarp.swarp import Swarp from mirar.processors.base_catalog_xmatch_processor import ( default_image_sextractor_catalog_purifier, ) from mirar.processors.photcal import PhotCalibrator +from mirar.processors.split import SUB_ID_KEY from mirar.processors.sqldatabase.base_model import BaseDB from mirar.references.local import RefFromPath from mirar.references.ukirt import UKIRTRef -# from mirar.catalog.base_catalog import CatalogFromFile - logger = logging.getLogger(__name__) -winter_dir = os.path.dirname(__file__) -astromatic_config_dir = os.path.join(winter_dir, "config/") -swarp_config_path = os.path.join(astromatic_config_dir, "config.swarp") -winter_mask_path = os.path.join(winter_dir, "winter_mask.fits") -scamp_config_path = os.path.join(winter_dir, "config/files/scamp.conf") def winter_reference_generator(image: Image, db_table: Type[BaseDB] = RefStacks): @@ -53,11 +54,11 @@ def winter_reference_generator(image: Image, db_table: Type[BaseDB] = RefStacks) filtername = image["FILTER"] # TODO if in_ukirt and in_vista, different processing fieldid = int(image["FIELDID"]) - subdetid = int(image["SUBDETID"]) + subdetid = int(image[SUB_ID_KEY]) logger.debug(f"Fieldid: {fieldid}, subdetid: {subdetid}") db_results = db_table.sql_model().select_query( select_keys=["savepath"], - compare_keys=["fieldid", "subdetid"], + compare_keys=["fieldid", SUB_ID_KEY.lower()], compare_values=[fieldid, subdetid], comparators=["__eq__", "__eq__"], ) @@ -65,7 +66,7 @@ def winter_reference_generator(image: Image, db_table: Type[BaseDB] = RefStacks) if len(db_results) > 0: savepaths = [x[0] for x in db_results] if os.path.exists(savepaths[0]): - logger.info(f"Found reference image in database: {savepaths[0]}") + logger.debug(f"Found reference image in database: {savepaths[0]}") return RefFromPath(path=savepaths[0], filter_name=filtername) return UKIRTRef( @@ -90,9 +91,26 @@ def winter_reference_image_resampler(**kwargs) -> Swarp: :param kwargs: kwargs :return: Swarp processor """ - logger.info(kwargs) - return Swarp( - swarp_config_path=swarp_config_path, subtract_bkg=True, cache=False, **kwargs + logger.debug(kwargs) + return Swarp(swarp_config_path=swarp_config_path, cache=False, **kwargs) + + +def winter_reference_sextractor(output_sub_dir: str, gain: float) -> Sextractor: + """Returns a Sextractor processor for WINTER reference images""" + return Sextractor( + **sextractor_reference_config, + gain=gain, + output_sub_dir=output_sub_dir, + cache=True, + ) + + +def winter_reference_psfex(output_sub_dir: str, norm_fits: bool) -> PSFex: + """Returns a PSFEx processor for WINTER""" + return PSFex( + config_path=psfex_path, + output_sub_dir=output_sub_dir, + norm_fits=norm_fits, ) diff --git a/mirar/pipelines/winter/load_winter_image.py b/mirar/pipelines/winter/load_winter_image.py index 951abb928..cbf4022e1 100644 --- a/mirar/pipelines/winter/load_winter_image.py +++ b/mirar/pipelines/winter/load_winter_image.py @@ -66,8 +66,8 @@ def load_raw_winter_image(path: str | Path) -> tuple[np.array, astropy.io.fits.H header["UTCTIME"] = ( f"{date[:4]}-{date[4:6]}-{date[6:]}T" f"{time[:2]}:{time[2:4]}:{time[4:]}" ) - logger.debug(header["UTCTIME"]) header["MJD-OBS"] = Time(header["UTCTIME"]).mjd + header["DATE-OBS"] = Time(header["UTCTIME"]).isot header["OBSCLASS"] = ["science", "calibration"][ header["OBSTYPE"] in ["DARK", "FLAT"] diff --git a/mirar/pipelines/winter/models/_ref_stacks.py b/mirar/pipelines/winter/models/_ref_stacks.py index 45cba3f2f..49cfb2de3 100644 --- a/mirar/pipelines/winter/models/_ref_stacks.py +++ b/mirar/pipelines/winter/models/_ref_stacks.py @@ -29,7 +29,7 @@ class RefStacksTable(WinterBase): ra1_1 = Column(Float) dec1_1 = Column(Float) fieldid = Column(Integer, nullable=True, default=None) - subdetid = Column(Integer, nullable=True, default=None) + subid = Column(Integer, nullable=True, default=None) filter = Column(VARCHAR(10)) savepath = Column(VARCHAR(255)) @@ -55,7 +55,7 @@ class RefStacks(BaseDB): filter: str = Field(min_length=1) fieldid: int = Field(ge=0, default=None, nullable=True) - subdetid: int = Field(ge=0, default=None, nullable=True) + subid: int = Field(ge=0, default=None, nullable=True) savepath: str = Field(min_length=1) def exists(self) -> bool: diff --git a/mirar/pipelines/winter/winter_pipeline.py b/mirar/pipelines/winter/winter_pipeline.py index f9dd13361..44a49157f 100644 --- a/mirar/pipelines/winter/winter_pipeline.py +++ b/mirar/pipelines/winter/winter_pipeline.py @@ -18,8 +18,10 @@ commissioning_split, commissioning_split_single_board, commissioning_stack, + final, full_commissioning, full_commissioning_all_boards, + imsub, log, refbuild, unpack_all, @@ -56,6 +58,8 @@ class WINTERPipeline(Pipeline): "unpack_subset": unpack_subset, "unpack_all": unpack_all, "commissioning_split_single_board": commissioning_split_single_board, + "imsub": imsub, + "final": final, } gain = 1.0 diff --git a/mirar/pipelines/wirc/load_wirc_image.py b/mirar/pipelines/wirc/load_wirc_image.py index 5e507a894..1c9f58d94 100644 --- a/mirar/pipelines/wirc/load_wirc_image.py +++ b/mirar/pipelines/wirc/load_wirc_image.py @@ -36,9 +36,9 @@ def load_raw_wirc_image(path: str | Path) -> tuple[np.array, astropy.io.fits.Hea header["TARGET"] = header["OBJECT"].lower() if "MJD-OBS" in header.keys(): - header["UTCTIME"] = Time(header["MJD-OBS"], format="mjd").isot + header["DATE-OBS"] = Time(header["MJD-OBS"], format="mjd").isot else: - header["UTCTIME"] = header["UTSHUT"] + header["DATE-OBS"] = header["UTSHUT"] header["MJD-OBS"] = Time(header["UTSHUT"]).mjd if COADD_KEY not in header.keys(): logger.debug(f"No {COADD_KEY} entry. Setting coadds to 1.") diff --git a/mirar/processors/astromatic/swarp/swarp.py b/mirar/processors/astromatic/swarp/swarp.py index 944f0b05a..4a6bda57b 100644 --- a/mirar/processors/astromatic/swarp/swarp.py +++ b/mirar/processors/astromatic/swarp/swarp.py @@ -10,13 +10,16 @@ from mirar.data import ImageBatch from mirar.errors import ProcessorError +from mirar.io import MissingCoreFieldError, check_image_has_core_fields from mirar.paths import ( BASE_NAME_KEY, + EXPTIME_KEY, LATEST_SAVE_KEY, LATEST_WEIGHT_SAVE_KEY, RAW_IMG_KEY, STACKED_COMPONENT_IMAGES_KEY, SWARP_FLUX_SCALING_KEY, + TIME_KEY, all_astrometric_keywords, copy_temp_file, get_output_dir, @@ -375,7 +378,10 @@ def _apply_to_images( new_image[key] = batch[0][key] except ValueError: continue - new_image["COADDS"] = np.sum([x["COADDS"] for x in batch]) + + new_image["COADDS"] = sum(x["COADDS"] for x in batch) + new_image[EXPTIME_KEY] = sum(x[EXPTIME_KEY] for x in batch) + new_image[TIME_KEY] = min(x[TIME_KEY] for x in batch) new_image[RAW_IMG_KEY] = ",".join([x[RAW_IMG_KEY] for x in batch]) @@ -385,6 +391,12 @@ def _apply_to_images( new_image[BASE_NAME_KEY] = output_image_path.name new_image[LATEST_WEIGHT_SAVE_KEY] = output_image_weight_path.as_posix() + # Reference images should have all the core fields + try: + check_image_has_core_fields(new_image) + except MissingCoreFieldError as err: + raise SwarpError(err) from err + if not self.cache: for temp_file in temp_files: temp_file.unlink() diff --git a/mirar/processors/base_processor.py b/mirar/processors/base_processor.py index d67846f4e..49f04e5d4 100644 --- a/mirar/processors/base_processor.py +++ b/mirar/processors/base_processor.py @@ -22,7 +22,7 @@ NoncriticalProcessingError, ProcessorError, ) -from mirar.io import open_fits, save_to_path +from mirar.io import check_image_has_core_fields, open_fits, save_to_path from mirar.paths import ( BASE_NAME_KEY, CAL_OUTPUT_SUB_DIR, @@ -314,9 +314,11 @@ def save_fits( :param image: Image to save :param path: path + :param check_core_fields: Check that the image has the core fields :return: None """ path = str(path) + check_image_has_core_fields(image) data = image.get_data() header = image.get_header() if header is not None: diff --git a/mirar/processors/reference.py b/mirar/processors/reference.py index 065888d6c..78daee93f 100644 --- a/mirar/processors/reference.py +++ b/mirar/processors/reference.py @@ -185,7 +185,7 @@ def _apply_to_images( ) self.save_fits(image=resampled_ref_sextractor_img, path=rrsi_path) - logger.info(f"Saved reference image to {rrsi_path}") + logger.debug(f"Saved reference image to {rrsi_path}") ref_psfex = self.psfex( output_sub_dir=self.temp_output_subtract_dir, norm_fits=True @@ -219,6 +219,14 @@ def __init__( self.ref_image_generator = ref_image_generator self.output_sub_dir = output_sub_dir + def __str__(self): + output_sub_dir = get_output_dir( + dir_root=self.output_sub_dir, sub_dir=self.night_sub_dir + ) + return ( + f"Processor to get reference images and " f"save them to {output_sub_dir}" + ) + def _apply_to_images( self, batch: ImageBatch, diff --git a/mirar/processors/split.py b/mirar/processors/split.py index c3220ce46..8ee68ba72 100644 --- a/mirar/processors/split.py +++ b/mirar/processors/split.py @@ -13,6 +13,7 @@ logger = logging.getLogger(__name__) SUB_ID_KEY = "SUBID" +SUB_COORD_KEY = "SUBCOORD" class SplitImage(BaseImageProcessor): @@ -29,7 +30,7 @@ def __init__(self, buffer_pixels: int = 0, n_x: int = 1, n_y: int = 1): self.n_y = n_y def __str__(self) -> str: - return f"Processor to split images into {self.n_x*self.n_y} smaller images." + return f"Processor to split images into {self.n_x}x{self.n_y}={self.n_x*self.n_y} smaller images." def get_range( self, @@ -57,7 +58,7 @@ def _apply_to_images( ) -> ImageBatch: new_images = ImageBatch() - logger.info(f"Splitting each data into {self.n_x*self.n_y} sub-images") + logger.debug(f"Splitting each data into {self.n_x*self.n_y} sub-images") for image in batch: pix_width_x, pix_width_y = image.get_data().shape @@ -80,7 +81,7 @@ def _apply_to_images( sub_img_id = f"{index_x}_{index_y}" - new_header["SUBCOORD"] = ( + new_header[SUB_COORD_KEY] = ( sub_img_id, "Sub-data coordinate, in form x_y", ) diff --git a/mirar/processors/utils/header_annotate.py b/mirar/processors/utils/header_annotate.py index 2e08cf62b..cb7423ca1 100644 --- a/mirar/processors/utils/header_annotate.py +++ b/mirar/processors/utils/header_annotate.py @@ -31,7 +31,7 @@ def __init__( def __str__(self) -> str: return ( f"Updates image headers by adding values " - f"for {' and '.join(self.output_key)}." + f"for '{self.output_key}' using '{' and '.join(self.input_keys)}'." ) def _apply_to_images( diff --git a/mirar/processors/utils/image_loader.py b/mirar/processors/utils/image_loader.py index 8460cabcb..369b5d619 100644 --- a/mirar/processors/utils/image_loader.py +++ b/mirar/processors/utils/image_loader.py @@ -11,14 +11,23 @@ import numpy as np from mirar.data import Image, ImageBatch -from mirar.errors import ImageNotFoundError -from mirar.io import check_file_is_complete, open_fits -from mirar.paths import RAW_IMG_KEY, RAW_IMG_SUB_DIR, base_raw_dir, core_fields +from mirar.errors import ImageNotFoundError, ProcessorError +from mirar.io import ( + MissingCoreFieldError, + check_file_is_complete, + check_image_has_core_fields, + open_fits, +) +from mirar.paths import RAW_IMG_KEY, RAW_IMG_SUB_DIR, base_raw_dir from mirar.processors.base_processor import BaseImageProcessor logger = logging.getLogger(__name__) +class BadImageError(ProcessorError): + """Exception for bad images""" + + class ImageLoader(BaseImageProcessor): """Processor to load raw images.""" @@ -50,17 +59,14 @@ def open_raw_image(self, path: str) -> Image: """ data, header = self.load_image(path) - for key in core_fields: - if key not in header.keys(): - err = ( - f"Essential key {key} not found in header. " - f"Please add this field first. Available fields are: " - f"{list(header.keys())}" - ) - logger.error(err) - raise KeyError(err) + new_img = Image(data.astype(np.float64), header) + + try: + check_image_has_core_fields(new_img) + except MissingCoreFieldError as err: + raise BadImageError(err) from err - return Image(data.astype(np.float64), header) + return new_img def _apply_to_images(self, batch: ImageBatch) -> ImageBatch: input_dir = os.path.join( @@ -156,6 +162,12 @@ def _apply_to_images( if self.copy_header_keys is not None: for key in self.copy_header_keys: new_image.header[key] = image.header[key] + + try: + check_image_has_core_fields(new_image) + except MissingCoreFieldError as err: + raise BadImageError(err) from err + new_batch.append(new_image) return new_batch diff --git a/mirar/processors/utils/image_saver.py b/mirar/processors/utils/image_saver.py index 65ee5e26b..ac5497769 100644 --- a/mirar/processors/utils/image_saver.py +++ b/mirar/processors/utils/image_saver.py @@ -4,6 +4,8 @@ import logging from pathlib import Path +from astropy.time import Time + from mirar.data import ImageBatch from mirar.paths import BASE_NAME_KEY, LATEST_SAVE_KEY, base_output_dir, get_output_dir from mirar.processors.base_processor import BaseImageProcessor @@ -47,6 +49,7 @@ def _apply_to_images( for image in batch: path = output_dir.joinpath(image[BASE_NAME_KEY]) image[LATEST_SAVE_KEY] = str(path) + image["DATE"] = Time.now().isot if self.write_mask: self.save_mask_image(image, img_path=path) diff --git a/mirar/processors/zogy/pyzogy.py b/mirar/processors/zogy/pyzogy.py index d33377ec9..ed50ea32f 100644 --- a/mirar/processors/zogy/pyzogy.py +++ b/mirar/processors/zogy/pyzogy.py @@ -93,7 +93,7 @@ def pyzogy( new_psf_big[y_min:y_max, x_min:x_max] = new_psf ref_psf_big[y_min:y_max, x_min:x_max] = ref_psf - logger.info( + logger.debug( "Max of big PSF is %d %d" % np.unravel_index(np.argmax(new_psf_big, axis=None), new_psf_big.shape) ) @@ -102,7 +102,7 @@ def pyzogy( new_psf_big = fft.fftshift(new_psf_big) ref_psf_big = fft.fftshift(ref_psf_big) - logger.info( + logger.debug( "Max of big PSF shift is %d %d" % np.unravel_index(np.argmax(new_psf_big, axis=None), new_psf_big.shape) ) @@ -146,7 +146,7 @@ def pyzogy( diff_nocorr_psf = fft.ifftshift(diff_nocorr_psf) diff_nocorr_psf = diff_nocorr_psf[y_min:y_max, x_min:x_max] - logger.info( + logger.debug( "Max of diff PSF is %d %d" % np.unravel_index(np.argmax(diff_psf, axis=None), diff_psf.shape) ) diff --git a/mirar/processors/zogy/zogy.py b/mirar/processors/zogy/zogy.py index 7f677689e..6e0611b13 100644 --- a/mirar/processors/zogy/zogy.py +++ b/mirar/processors/zogy/zogy.py @@ -27,10 +27,12 @@ BASE_NAME_KEY, LATEST_WEIGHT_SAVE_KEY, NORM_PSFEX_KEY, + OBSCLASS_KEY, RAW_IMG_KEY, REF_IMG_KEY, REF_PSF_KEY, UNC_IMG_KEY, + core_fields, get_output_dir, ) from mirar.processors.base_processor import BaseImageProcessor @@ -120,6 +122,7 @@ def __init__( self, output_sub_dir: str = "sub", sci_zp_header_key: str = "ZP", + ref_zp_header_key: str | None = None, catalog_purifier: Callable[ [astropy.table.Table, astropy.table.Table], [astropy.table.Table, astropy.table.Table], @@ -129,7 +132,10 @@ def __init__( ): super().__init__() self.output_sub_dir = output_sub_dir + if ref_zp_header_key is None: + ref_zp_header_key = sci_zp_header_key self.sci_zp_header_key = sci_zp_header_key + self.ref_zp_header_key = ref_zp_header_key self.catalog_purifier = catalog_purifier self.write_region_bool = write_region_bool self.crossmatch_radius_arcsec = crossmatch_radius_arcsec @@ -289,7 +295,16 @@ def _apply_to_images(self, batch: ImageBatch) -> ImageBatch: scorr_header = sci_weight_data.get_header() scorr_weight_img = Image(data=scorr_weight_data, header=scorr_header) scorr_weight_path = sci_img_path.replace(".fits", ".scorr.weight.fits") - self.save_fits(scorr_weight_img, path=self.get_path(scorr_weight_path)) + scorr_weight_img[OBSCLASS_KEY] = "SCORR" + + for key in core_fields: + if key not in scorr_weight_img: + scorr_weight_img[key] = image[key] + + self.save_fits( + scorr_weight_img, + path=self.get_path(scorr_weight_path), + ) ast_unc_x, ast_unc_y, flux_scale = self.get_ast_fluxscale( ref_catalog_path, sci_catalog_path @@ -299,16 +314,18 @@ def _apply_to_images(self, batch: ImageBatch) -> ImageBatch: ref_data *= flux_scale ref_img.set_data(ref_data) - ref_unscaled_zp = ref_img["ZP"] - ref_img["ZP"] = float(ref_img["ZP"]) + 2.5 * np.log10(flux_scale) + ref_unscaled_zp = ref_img[self.ref_zp_header_key] + ref_img[self.ref_zp_header_key] = float( + ref_img[self.ref_zp_header_key] + ) + 2.5 * np.log10(flux_scale) ref_scaled_path = ref_img_path + ".scaled" self.save_fits(ref_img, path=self.get_path(ref_scaled_path)) - logger.info( + logger.debug( f"Zeropoints are reference : {ref_unscaled_zp}, " - f"scaled reference : {ref_img['ZP']} and " - f"science : {image[self.sci_zp_header_key]}" + f"scaled reference : {ref_img[self.ref_zp_header_key]} and " + f"science : {image[self.ref_zp_header_key]}" ) # Scale is 1 by construction for science image @@ -323,7 +340,9 @@ def _apply_to_images(self, batch: ImageBatch) -> ImageBatch: np.percentile(ref_data[ref_data != 0.0], 84.13) - np.percentile(ref_data[ref_data != 0.0], 15.86) ) - logger.info(f"Science RMS is {sci_rms:.2f}. Reference RMS is {ref_rms:.2f}") + logger.debug( + f"Science RMS is {sci_rms:.2f}. Reference RMS is {ref_rms:.2f}" + ) # Calculate uncertainty images sci_rms_image = self.get_rms_image(image, sci_rms) @@ -448,6 +467,10 @@ def _apply_to_images( psf_header[BASE_NAME_KEY] = diff_psf_path.name psf_header[RAW_IMG_KEY] = diff_psf_path.as_posix() + for key in core_fields: + if key not in psf_header: + psf_header[key] = diff[key] + self.save_fits( image=Image(diff_psf_data, psf_header), path=self.get_path(diff_psf_path), diff --git a/mirar/references/base_reference_generator.py b/mirar/references/base_reference_generator.py index cb6e7500b..ee0c0c549 100644 --- a/mirar/references/base_reference_generator.py +++ b/mirar/references/base_reference_generator.py @@ -8,15 +8,25 @@ import numpy as np from astropy.io import fits +from astropy.time import Time from mirar.data import Image, ImageBatch -from mirar.io import save_hdu_as_fits +from mirar.errors import ProcessorError +from mirar.io import ( + MissingCoreFieldError, + check_image_has_core_fields, + save_hdu_as_fits, +) from mirar.paths import ( BASE_NAME_KEY, COADD_KEY, LATEST_SAVE_KEY, LATEST_WEIGHT_SAVE_KEY, + OBSCLASS_KEY, + PROC_FAIL_KEY, PROC_HISTORY_KEY, + RAW_IMG_KEY, + TARGET_KEY, ) from mirar.processors.sqldatabase.base_model import BaseDB from mirar.processors.sqldatabase.database_exporter import DatabaseImageExporter @@ -24,6 +34,10 @@ logger = logging.getLogger(__name__) +class ReferenceGenerationError(ProcessorError): + """Error for reference generation""" + + class BaseReferenceGenerator: """ Base Ref Image Generator @@ -99,9 +113,21 @@ def write_reference(self, image: Image, output_dir: str) -> Path: # Remove if needed output_path.unlink(missing_ok=True) - logger.info(f"Saving reference image to {output_path}") + logger.debug(f"Saving reference image to {output_path}") ref_hdu.header[BASE_NAME_KEY] = os.path.basename(output_path) ref_hdu.header[LATEST_SAVE_KEY] = output_path.as_posix() + ref_hdu.header[RAW_IMG_KEY] = image[BASE_NAME_KEY] + ref_hdu.header[OBSCLASS_KEY] = "REF" + ref_hdu.header[TARGET_KEY] = image[TARGET_KEY] + ref_hdu.header[PROC_FAIL_KEY] = False + + if ("MJD-OBS" in ref_hdu.header.keys()) & ( + "DATE-OBS" not in ref_hdu.header.keys() + ): + ref_hdu.header["DATE-OBS"] = Time( + ref_hdu.header["MJD-OBS"], format="mjd" + ).isot + ref_hdu.data[ref_hdu.data == 0] = np.nan # pylint: disable=no-member if ref_weight_hdu is not None: @@ -116,6 +142,12 @@ def write_reference(self, image: Image, output_dir: str) -> Path: save_hdu_as_fits(ref_weight_hdu, output_weight_path) ref_hdu.header[LATEST_WEIGHT_SAVE_KEY] = output_weight_path.as_posix() + # Reference images should have all the core fields + try: + check_image_has_core_fields(Image(header=ref_hdu.header, data=ref_hdu.data)) + except MissingCoreFieldError as err: + raise ReferenceGenerationError from err + save_hdu_as_fits(ref_hdu, output_path) if self.write_to_db: diff --git a/mirar/references/ps1.py b/mirar/references/ps1.py index c91f4d2eb..707373227 100644 --- a/mirar/references/ps1.py +++ b/mirar/references/ps1.py @@ -6,6 +6,7 @@ import numpy as np from astropy.io import fits from astropy.table import Table +from astropy.time import Time from astropy.wcs import WCS from mirar.data import Image @@ -112,4 +113,5 @@ def get_reference(self, image: Image) -> (fits.PrimaryHDU, fits.PrimaryHDU): ref_hdu.header["GAIN"] = ref_hdu.header["CELL.GAIN"] ref_hdu.header["ZP"] = ref_hdu.header["FPA.ZP"] del ref_hdu.header["HISTORY"] + return ref_hdu, None diff --git a/mirar/references/ukirt.py b/mirar/references/ukirt.py index d9ddd25e7..1fe99f6bc 100644 --- a/mirar/references/ukirt.py +++ b/mirar/references/ukirt.py @@ -37,6 +37,7 @@ get_image_center_wcs_coords, ) from mirar.processors.photcal import PhotCalibrator +from mirar.processors.split import SUB_ID_KEY from mirar.processors.sqldatabase.base_model import BaseDB from mirar.processors.sqldatabase.database_exporter import DatabaseImageExporter from mirar.references.base_reference_generator import BaseReferenceGenerator @@ -652,7 +653,7 @@ def get_reference(self, image: Image) -> tuple[PrimaryHDU, PrimaryHDU]: # image.header[key] = 0 stackid = ( f"{str(image.header['FIELDID']).rjust(5, '0')}" - f"{str(image.header['SUBDETID']).rjust(2, '0')}" + f"{str(image.header[SUB_ID_KEY]).rjust(2, '0')}" f"{str(winter_filters_map[image.header['FILTER']])}" ) reference_hdu.header["STACKID"] = int(stackid) diff --git a/tests/test_wfau_references.py b/tests/test_wfau_references.py index cfea35445..4c098cfc6 100644 --- a/tests/test_wfau_references.py +++ b/tests/test_wfau_references.py @@ -2,6 +2,7 @@ Tests for getting and making WFAU reference images """ import logging +import unittest from mirar.pipelines import get_pipeline from mirar.pipelines.winter.build_references import run_winter_reference_build_pipeline @@ -46,6 +47,7 @@ ) +@unittest.skip class TestIRReferencePipeline(BaseTestCase): """ Module for testing IR reference building pipeline @@ -69,7 +71,7 @@ def test_pipeline(self): self.logger.info("\n\n Testing WINTER reference building pipeline \n\n") res, _ = run_winter_reference_build_pipeline( - subdet_id=0, field_id=TEST_WINTER_FIELD_ID + subdet_id=0, field_id=TEST_WINTER_FIELD_ID, catch_all_errors=False ) self.assertEqual(len(res[0]), 1) From 356bf25fb9573f90a66bb4989e5e83c4225268ee Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Tue, 11 Jul 2023 22:45:44 -0700 Subject: [PATCH 2/2] Add full reduction mode for winter (#476) --- mirar/paths.py | 2 ++ mirar/pipelines/winter/blocks.py | 38 ++++++++++----------- mirar/pipelines/winter/load_winter_image.py | 9 +++-- mirar/pipelines/winter/winter_pipeline.py | 2 ++ mirar/processors/base_processor.py | 1 - mirar/processors/utils/image_loader.py | 3 +- mirar/processors/utils/multi_ext_parser.py | 7 ++-- 7 files changed, 36 insertions(+), 26 deletions(-) diff --git a/mirar/paths.py b/mirar/paths.py index 3e4e4dbdc..b73417592 100644 --- a/mirar/paths.py +++ b/mirar/paths.py @@ -291,6 +291,8 @@ def get_astrometry_keys() -> list: TIME_KEY = "DATE-OBS" OBSCLASS_KEY = "OBSCLASS" TARGET_KEY = "TARGET" +DITHER_N_KEY = "DITHNUM" +MAX_DITHER_KEY = "NUMDITHS" core_fields = [ OBSCLASS_KEY, diff --git a/mirar/pipelines/winter/blocks.py b/mirar/pipelines/winter/blocks.py index 88220aefc..328ca87fb 100644 --- a/mirar/pipelines/winter/blocks.py +++ b/mirar/pipelines/winter/blocks.py @@ -2,7 +2,7 @@ Module for WINTER data reduction """ from mirar.io import open_fits -from mirar.paths import FITS_MASK_KEY +from mirar.paths import BASE_NAME_KEY, DITHER_N_KEY, FITS_MASK_KEY, MAX_DITHER_KEY from mirar.pipelines.winter.config import ( psfex_path, sextractor_anet_config, @@ -266,8 +266,8 @@ process_proc_all_boards = [ ImageDebatcher(), ImageBatcher(["UTCTIME", "BOARD_ID", "SUBCOORD"]), - ImageSelector(("FIELDID", ["2789", "0697", "9170"])), - # ImageSelector(("FIELDID", "2789")), + # ImageSelector(("FIELDID", ["2789", "0697", "9170"])), + # # ImageSelector(("FIELDID", "2789")), ImageSaver(output_dir_name="pre_anet"), AstrometryNet( output_sub_dir="anet", @@ -459,18 +459,14 @@ ] split_indiv = [ - SplitImage(n_x=1, n_y=2), ImageDebatcher(), -] - -select_split_subset = [ImageSelector(("SUBCOORD", "0_0"))] - -make_log_and_save = [ CSVLog( export_keys=[ - "SUBCOORD", - "FILTER", "UTCTIME", + "PROGNAME", + DITHER_N_KEY, + MAX_DITHER_KEY, + "FILTER", "EXPTIME", "OBSTYPE", "UNIQTYPE", @@ -487,6 +483,10 @@ "T_ROIC", ] ), + SplitImage(n_x=1, n_y=2), +] + +save_raw = [ ImageSaver(output_dir_name="raw_unpacked", write_mask=False), ] @@ -514,8 +514,8 @@ ImageSaver(output_dir_name="diffs"), ] -unpack_subset = extract_subset + split_indiv + select_split_subset + make_log_and_save -unpack_all = extract_all + split_indiv + make_log_and_save +unpack_subset = extract_subset + split_indiv + save_raw +unpack_all = extract_all + split_indiv + save_raw load_unpacked = [ ImageLoader(input_sub_dir="raw_unpacked", load_image=open_fits), @@ -523,14 +523,12 @@ full_commissioning = load_unpacked + process + process_proc # + stack_proc -full_commissioning_all_boards = ( - load_unpacked - + dark_cal_all_boards - + flat_cal_all_boards - + process_proc_all_boards - + photcal +full_commissioning_proc = ( + dark_cal_all_boards + flat_cal_all_boards + process_proc_all_boards + photcal ) +full_commissioning_all_boards = load_unpacked + full_commissioning_proc + commissioning_split_single_board = ( log + split_images @@ -547,3 +545,5 @@ + process_proc_all_boards + photcal ) + +reduce = unpack_all + full_commissioning_proc diff --git a/mirar/pipelines/winter/load_winter_image.py b/mirar/pipelines/winter/load_winter_image.py index cbf4022e1..35f8f66be 100644 --- a/mirar/pipelines/winter/load_winter_image.py +++ b/mirar/pipelines/winter/load_winter_image.py @@ -3,6 +3,7 @@ """ import logging import os +import warnings from pathlib import Path import astropy @@ -10,6 +11,7 @@ from astropy.io import fits from astropy.stats import sigma_clipped_stats from astropy.time import Time +from astropy.utils.exceptions import AstropyWarning from mirar.paths import ( BASE_NAME_KEY, @@ -163,7 +165,10 @@ def load_raw_winter_image(path: str | Path) -> tuple[np.array, astropy.io.fits.H data[680:, 1180:] = np.nan # data[data > 25000] = np.nan - _, med, std = sigma_clipped_stats(data, sigma=3.0, maxiters=5) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", AstropyWarning) + _, med, std = sigma_clipped_stats(data, sigma=3.0, maxiters=5) + header["MEDCOUNT"] = med header["STDDEV"] = std # if header['RADEG']>300: @@ -197,7 +202,7 @@ def load_stacked_winter_image( """ Load proc image """ - logger.info(f"Loading {path}") + logger.debug(f"Loading {path}") with fits.open(path) as img: data = img[0].data # pylint: disable=no-member header = img[0].header # pylint: disable=no-member diff --git a/mirar/pipelines/winter/winter_pipeline.py b/mirar/pipelines/winter/winter_pipeline.py index 44a49157f..c42c91758 100644 --- a/mirar/pipelines/winter/winter_pipeline.py +++ b/mirar/pipelines/winter/winter_pipeline.py @@ -23,6 +23,7 @@ full_commissioning_all_boards, imsub, log, + reduce, refbuild, unpack_all, unpack_subset, @@ -60,6 +61,7 @@ class WINTERPipeline(Pipeline): "commissioning_split_single_board": commissioning_split_single_board, "imsub": imsub, "final": final, + "reduce": reduce, } gain = 1.0 diff --git a/mirar/processors/base_processor.py b/mirar/processors/base_processor.py index 49f04e5d4..9ba00d4d0 100644 --- a/mirar/processors/base_processor.py +++ b/mirar/processors/base_processor.py @@ -314,7 +314,6 @@ def save_fits( :param image: Image to save :param path: path - :param check_core_fields: Check that the image has the core fields :return: None """ path = str(path) diff --git a/mirar/processors/utils/image_loader.py b/mirar/processors/utils/image_loader.py index 369b5d619..3f10b3141 100644 --- a/mirar/processors/utils/image_loader.py +++ b/mirar/processors/utils/image_loader.py @@ -9,6 +9,7 @@ import astropy.io.fits import numpy as np +from tqdm import tqdm from mirar.data import Image, ImageBatch from mirar.errors import ImageNotFoundError, ProcessorError @@ -103,7 +104,7 @@ def load_from_dir( images = ImageBatch() - for path in img_list: + for path in tqdm(img_list): if check_file_is_complete(path): image = open_f(path) images.append(image) diff --git a/mirar/processors/utils/multi_ext_parser.py b/mirar/processors/utils/multi_ext_parser.py index 381b1f748..33dae5a67 100644 --- a/mirar/processors/utils/multi_ext_parser.py +++ b/mirar/processors/utils/multi_ext_parser.py @@ -9,6 +9,7 @@ import astropy.io.fits import numpy as np +from tqdm import tqdm from mirar.data import Image, ImageBatch from mirar.errors import ImageNotFoundError @@ -94,7 +95,7 @@ def parse(self, path: str) -> list: new_paths = [] with astropy.io.fits.open(path) as hdu: - logger.info(f"This file - {path} - has {len(hdu)} extensions.") + logger.debug(f"This file - {path} - has {len(hdu)} extensions.") hdr0 = hdu[0].header # pylint: disable=no-member # zip hdr0's values and comments @@ -102,7 +103,7 @@ def parse(self, path: str) -> list: # combining main header (hdr0) with extension header if self.skip_first: start = 2 - logger.info("Ignoring first extension frame") + logger.debug("Ignoring first extension frame") else: start = 1 for ext in range(start, len(hdu)): @@ -180,7 +181,7 @@ def load_from_dir( logger.error(err) raise ImageNotFoundError(err) - for path in img_list: + for path in tqdm(img_list): parse_f(path) empty_batch = ImageBatch()