diff --git a/python/lvmdrp/main.py b/python/lvmdrp/main.py index 3e7c981f..ba330a11 100644 --- a/python/lvmdrp/main.py +++ b/python/lvmdrp/main.py @@ -1497,6 +1497,13 @@ def science_reduction(expnum: int, use_longterm_cals: bool = False, sci_metadata = get_frames_metadata(mjd=sci_mjd) sci_metadata.query("expnum == @expnum", inplace=True) sci_metadata.sort_values("expnum", ascending=False, inplace=True) + sci_metadata.query("qaqual == 'GOOD'", inplace=True) + + if sci_metadata.empty: + log.error(f"exposure {expnum = } was flagged as 'BAD' by the raw data quality pipeline") + return + + # define general metadata sci_tileid = sci_metadata["tileid"].unique()[0] sci_mjd = sci_metadata["mjd"].unique()[0] sci_expnum = sci_metadata["expnum"].unique()[0] @@ -1513,7 +1520,7 @@ def science_reduction(expnum: int, use_longterm_cals: bool = False, calibs = get_calib_paths(mjd=cals_mjd, version=drpver, longterm_cals=use_longterm_cals, from_sanbox=True) # make sure only one exposure number is being reduced - sci_metadata.query("expnum == @sci_expnum", inplace=True) + # sci_metadata.query("expnum == @sci_expnum", inplace=True) sci_metadata.sort_values("camera", inplace=True) # detrend science exposure @@ -1745,7 +1752,10 @@ def run_drp(mjd: Union[int, str, list], expnum: Union[int, str, list] = None, sub = frames.copy() # remove bad or test quality frames - sub = sub[~(sub['quality'] != 'excellent')] + sub = sub[~(sub['qaqual'] != 'GOOD')] + if sub.empty: + log.error(f"exposure {expnum = } was flagged as 'BAD' by the raw data quality pipeline") + return # filter on exposure number if expnum: diff --git a/python/lvmdrp/utils/bitmask.py b/python/lvmdrp/utils/bitmask.py index eb2aa0a9..1325ef2d 100644 --- a/python/lvmdrp/utils/bitmask.py +++ b/python/lvmdrp/utils/bitmask.py @@ -99,8 +99,8 @@ def __contains__(self, flag): class RawFrameQuality(BaseBitmask): GOOD = auto() # bit whether a raw frame is good for reduction - TEST = auto() # bit whether a raw frame is for instrument testing purposes BAD = auto() # bit whether a raw frame is bad for reduction + TEST = auto() # bit whether a raw frame is for instrument testing purposes class ReductionStatus(BaseBitmask): diff --git a/python/lvmdrp/utils/hdrfix.py b/python/lvmdrp/utils/hdrfix.py index 0929890b..8920acdd 100644 --- a/python/lvmdrp/utils/hdrfix.py +++ b/python/lvmdrp/utils/hdrfix.py @@ -168,6 +168,10 @@ def apply_hdrfix(mjd: int, camera: str = None, expnum: int = None, tileid = 11111 if tileid in (-999, 999, None) else tileid hdr['TILE_ID'] = tileid + # add QA header keywords default values + hdr['QAQUAL'] = ('GOOD', 'string value for raw data quality flag') + hdr['QAFLAG'] = ('0000', 'bitmask value for raw data quality flag') + # read the hdr fix file fix = read_hdrfix_file(mjd) @@ -185,4 +189,8 @@ def apply_hdrfix(mjd: int, camera: str = None, expnum: int = None, hdr[row['keyword']] = row['value'] log.info(f'Applying header fix on {current_file} for key: {row["keyword"]}, value: {row["value"]}.') + # fix typing in QAFLAG keyword + # hdr['QAQUAL'] = ('GOOD', 'string value for raw data quality flag') + # hdr.set('QAFLAG', QAFlag(int(hdr['QAFLAG'], base=2)), 'bitmask value for raw data quality flag') + return hdr diff --git a/python/lvmdrp/utils/metadata.py b/python/lvmdrp/utils/metadata.py index 0c29da41..cefef8df 100644 --- a/python/lvmdrp/utils/metadata.py +++ b/python/lvmdrp/utils/metadata.py @@ -22,7 +22,6 @@ from lvmdrp.core.constants import FRAMES_CALIB_NEEDS, CAMERAS from lvmdrp.utils.bitmask import ( QualityFlag, - RawFrameQuality, ReductionStage, ReductionStatus, ) @@ -55,8 +54,7 @@ ("ldls", bool), ("quartz", bool), ("hartmann", str), - ("quality", str), - ("qual", RawFrameQuality), + ("qaqual", str), ("stage", ReductionStage), ("status", ReductionStatus), ("drpqual", QualityFlag), @@ -79,8 +77,7 @@ ("ldls", bool), ("quartz", bool), ("hartmann", str), - ("quality", str), - ("qual", RawFrameQuality), + ("qaqual", str), ("stage", ReductionStage), ("status", ReductionStatus), ("drpqual", QualityFlag), @@ -607,8 +604,11 @@ def extract_metadata(frames_paths: list, kind: str = "raw") -> pd.DataFrame: header.get("LDLS", "OFF") in onlamp, header.get("QUARTZ", "OFF") in onlamp, header.get("HARTMANN", "0 0"), - header.get("QUALITY", "excellent"), - header.get("QUAL", RawFrameQuality(0)), + # header.get("QUALITY", "excellent"), + # QC pipeline keywords + header.get("QAQUAL", "GOOD"), + # header.get("QAFLAG", QAFlag(0)), + # DRP quality keywords header.get("DRPSTAGE", ReductionStage.UNREDUCED), header.get("DRPSTAT", ReductionStatus(0)), header.get("DRPQUAL", QualityFlag(0)), @@ -632,8 +632,10 @@ def extract_metadata(frames_paths: list, kind: str = "raw") -> pd.DataFrame: header.get("LDLS", "OFF") in onlamp, header.get("QUARTZ", "OFF") in onlamp, header.get("HARTMANN", "0 0"), - header.get("QUALITY", "excellent"), - header.get("QUAL", RawFrameQuality(0)), + # TODO: QUALITY may be redundant, double check and remove if it is + # header.get("QUALITY", "excellent"), + header.get("QAQUAL", "GOOD"), + # header.get("QAFLAG", QAFlag(0)), header.get("DRPSTAGE", ReductionStage.UNREDUCED), header.get("DRPSTAT", ReductionStatus(0)), header.get("DRPQUAL", QualityFlag(0)), @@ -1448,7 +1450,7 @@ def _collect_header_data(filename: str) -> dict: 'skyw_ra': 'TESKYWRA', 'skyw_dec': 'TESKYWDE', 'skyw_amass': 'TESKYWAM', 'skyw_kmpos': 'TESKYWKM', 'skyw_focpos': 'TESKYWFO', 'skyw_name': 'SKYWNAME', 'skyw_sh_hght': 'GEOCORONAL SKYW_SH_HGHT', 'skyw_moon_sep': 'SKYMODEL SKYW_RHO', - # sky parameters + # sky parameters 'moon_ra': 'SKYMODEL MOON_RA', 'moon_dec': 'SKYMODEL MOON_DEC', 'moon_phase': 'SKYMODEL MOON_PHASE', 'moon_fli': 'SKYMODEL MOON_FLI', 'sun_alt': 'SKYMODEL SUNALT', 'moon_alt': 'SKYMODEL MOONALT'