From 97124671b49b98f4eb61516415f7bcde73978d33 Mon Sep 17 00:00:00 2001 From: Alfredo Mejia-Narvaez Date: Fri, 18 Oct 2024 10:51:23 -0600 Subject: [PATCH 1/3] implementing QA flags to filter out bad exposures in science reductions --- python/lvmdrp/main.py | 12 ++++++++++-- python/lvmdrp/utils/bitmask.py | 2 +- python/lvmdrp/utils/hdrfix.py | 8 ++++++++ python/lvmdrp/utils/metadata.py | 22 ++++++++++++---------- 4 files changed, 31 insertions(+), 13 deletions(-) diff --git a/python/lvmdrp/main.py b/python/lvmdrp/main.py index 18d86208..a65864e7 100644 --- a/python/lvmdrp/main.py +++ b/python/lvmdrp/main.py @@ -1497,6 +1497,11 @@ def science_reduction(expnum: int, use_longterm_cals: bool = False, sci_metadata = get_frames_metadata(mjd=sci_mjd) sci_metadata.query("expnum == @expnum", inplace=True) sci_metadata.sort_values("expnum", ascending=False, inplace=True) + sci_metadata.query("qaqual == 'good'", inplace=True) + + if sci_metadata.empty: + log.error(f"exposure {expnum = } was flagged as 'BAD' by the raw data quality pipeline") + return # define general metadata sci_tileid = sci_metadata["tileid"].unique()[0] @@ -1513,7 +1518,7 @@ def science_reduction(expnum: int, use_longterm_cals: bool = False, calibs = get_calib_paths(mjd=cals_mjd, version=drpver, longterm_cals=use_longterm_cals, from_sanbox=True) # make sure only one exposure number is being reduced - sci_metadata.query("expnum == @sci_expnum", inplace=True) + # sci_metadata.query("expnum == @sci_expnum", inplace=True) sci_metadata.sort_values("camera", inplace=True) # detrend science exposure @@ -1737,7 +1742,10 @@ def run_drp(mjd: Union[int, str, list], expnum: Union[int, str, list] = None, sub = frames.copy() # remove bad or test quality frames - sub = sub[~(sub['quality'] != 'excellent')] + sub = sub[~(sub['qaqual'] != 'good')] + if sub.empty: + log.error(f'no good exposure found with parameters: {mjd = }, {expnum = }') + return # filter on exposure number if expnum: diff --git a/python/lvmdrp/utils/bitmask.py b/python/lvmdrp/utils/bitmask.py index eb2aa0a9..1325ef2d 100644 --- a/python/lvmdrp/utils/bitmask.py +++ b/python/lvmdrp/utils/bitmask.py @@ -99,8 +99,8 @@ def __contains__(self, flag): class RawFrameQuality(BaseBitmask): GOOD = auto() # bit whether a raw frame is good for reduction - TEST = auto() # bit whether a raw frame is for instrument testing purposes BAD = auto() # bit whether a raw frame is bad for reduction + TEST = auto() # bit whether a raw frame is for instrument testing purposes class ReductionStatus(BaseBitmask): diff --git a/python/lvmdrp/utils/hdrfix.py b/python/lvmdrp/utils/hdrfix.py index 0929890b..8920acdd 100644 --- a/python/lvmdrp/utils/hdrfix.py +++ b/python/lvmdrp/utils/hdrfix.py @@ -168,6 +168,10 @@ def apply_hdrfix(mjd: int, camera: str = None, expnum: int = None, tileid = 11111 if tileid in (-999, 999, None) else tileid hdr['TILE_ID'] = tileid + # add QA header keywords default values + hdr['QAQUAL'] = ('GOOD', 'string value for raw data quality flag') + hdr['QAFLAG'] = ('0000', 'bitmask value for raw data quality flag') + # read the hdr fix file fix = read_hdrfix_file(mjd) @@ -185,4 +189,8 @@ def apply_hdrfix(mjd: int, camera: str = None, expnum: int = None, hdr[row['keyword']] = row['value'] log.info(f'Applying header fix on {current_file} for key: {row["keyword"]}, value: {row["value"]}.') + # fix typing in QAFLAG keyword + # hdr['QAQUAL'] = ('GOOD', 'string value for raw data quality flag') + # hdr.set('QAFLAG', QAFlag(int(hdr['QAFLAG'], base=2)), 'bitmask value for raw data quality flag') + return hdr diff --git a/python/lvmdrp/utils/metadata.py b/python/lvmdrp/utils/metadata.py index 0c29da41..cefef8df 100644 --- a/python/lvmdrp/utils/metadata.py +++ b/python/lvmdrp/utils/metadata.py @@ -22,7 +22,6 @@ from lvmdrp.core.constants import FRAMES_CALIB_NEEDS, CAMERAS from lvmdrp.utils.bitmask import ( QualityFlag, - RawFrameQuality, ReductionStage, ReductionStatus, ) @@ -55,8 +54,7 @@ ("ldls", bool), ("quartz", bool), ("hartmann", str), - ("quality", str), - ("qual", RawFrameQuality), + ("qaqual", str), ("stage", ReductionStage), ("status", ReductionStatus), ("drpqual", QualityFlag), @@ -79,8 +77,7 @@ ("ldls", bool), ("quartz", bool), ("hartmann", str), - ("quality", str), - ("qual", RawFrameQuality), + ("qaqual", str), ("stage", ReductionStage), ("status", ReductionStatus), ("drpqual", QualityFlag), @@ -607,8 +604,11 @@ def extract_metadata(frames_paths: list, kind: str = "raw") -> pd.DataFrame: header.get("LDLS", "OFF") in onlamp, header.get("QUARTZ", "OFF") in onlamp, header.get("HARTMANN", "0 0"), - header.get("QUALITY", "excellent"), - header.get("QUAL", RawFrameQuality(0)), + # header.get("QUALITY", "excellent"), + # QC pipeline keywords + header.get("QAQUAL", "GOOD"), + # header.get("QAFLAG", QAFlag(0)), + # DRP quality keywords header.get("DRPSTAGE", ReductionStage.UNREDUCED), header.get("DRPSTAT", ReductionStatus(0)), header.get("DRPQUAL", QualityFlag(0)), @@ -632,8 +632,10 @@ def extract_metadata(frames_paths: list, kind: str = "raw") -> pd.DataFrame: header.get("LDLS", "OFF") in onlamp, header.get("QUARTZ", "OFF") in onlamp, header.get("HARTMANN", "0 0"), - header.get("QUALITY", "excellent"), - header.get("QUAL", RawFrameQuality(0)), + # TODO: QUALITY may be redundant, double check and remove if it is + # header.get("QUALITY", "excellent"), + header.get("QAQUAL", "GOOD"), + # header.get("QAFLAG", QAFlag(0)), header.get("DRPSTAGE", ReductionStage.UNREDUCED), header.get("DRPSTAT", ReductionStatus(0)), header.get("DRPQUAL", QualityFlag(0)), @@ -1448,7 +1450,7 @@ def _collect_header_data(filename: str) -> dict: 'skyw_ra': 'TESKYWRA', 'skyw_dec': 'TESKYWDE', 'skyw_amass': 'TESKYWAM', 'skyw_kmpos': 'TESKYWKM', 'skyw_focpos': 'TESKYWFO', 'skyw_name': 'SKYWNAME', 'skyw_sh_hght': 'GEOCORONAL SKYW_SH_HGHT', 'skyw_moon_sep': 'SKYMODEL SKYW_RHO', - # sky parameters + # sky parameters 'moon_ra': 'SKYMODEL MOON_RA', 'moon_dec': 'SKYMODEL MOON_DEC', 'moon_phase': 'SKYMODEL MOON_PHASE', 'moon_fli': 'SKYMODEL MOON_FLI', 'sun_alt': 'SKYMODEL SUNALT', 'moon_alt': 'SKYMODEL MOONALT' From 685ee22a9ac3cbff5089f4df02c7c2f3ab8ccdbc Mon Sep 17 00:00:00 2001 From: Alfredo Mejia-Narvaez Date: Mon, 11 Nov 2024 12:23:39 -0300 Subject: [PATCH 2/3] consistent messaging on bad exposures --- python/lvmdrp/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lvmdrp/main.py b/python/lvmdrp/main.py index a65864e7..bc78570a 100644 --- a/python/lvmdrp/main.py +++ b/python/lvmdrp/main.py @@ -1744,7 +1744,7 @@ def run_drp(mjd: Union[int, str, list], expnum: Union[int, str, list] = None, # remove bad or test quality frames sub = sub[~(sub['qaqual'] != 'good')] if sub.empty: - log.error(f'no good exposure found with parameters: {mjd = }, {expnum = }') + log.error(f"exposure {expnum = } was flagged as 'BAD' by the raw data quality pipeline") return # filter on exposure number From 7ebabdbb5063160e3cdb0e64a4fec7d5e4face9b Mon Sep 17 00:00:00 2001 From: Alfredo Mejia-Narvaez Date: Mon, 11 Nov 2024 12:51:56 -0300 Subject: [PATCH 3/3] fixing typo in QA flag check --- python/lvmdrp/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/lvmdrp/main.py b/python/lvmdrp/main.py index bc78570a..a9645d16 100644 --- a/python/lvmdrp/main.py +++ b/python/lvmdrp/main.py @@ -1497,7 +1497,7 @@ def science_reduction(expnum: int, use_longterm_cals: bool = False, sci_metadata = get_frames_metadata(mjd=sci_mjd) sci_metadata.query("expnum == @expnum", inplace=True) sci_metadata.sort_values("expnum", ascending=False, inplace=True) - sci_metadata.query("qaqual == 'good'", inplace=True) + sci_metadata.query("qaqual == 'GOOD'", inplace=True) if sci_metadata.empty: log.error(f"exposure {expnum = } was flagged as 'BAD' by the raw data quality pipeline") @@ -1742,7 +1742,7 @@ def run_drp(mjd: Union[int, str, list], expnum: Union[int, str, list] = None, sub = frames.copy() # remove bad or test quality frames - sub = sub[~(sub['qaqual'] != 'good')] + sub = sub[~(sub['qaqual'] != 'GOOD')] if sub.empty: log.error(f"exposure {expnum = } was flagged as 'BAD' by the raw data quality pipeline") return