Merge pull request #170 from sdss/addqc

Implementing QC flags to filter out bad exposures

python/lvmdrp/main.py

@@ -1497,6 +1497,13 @@ def science_reduction(expnum: int, use_longterm_cals: bool = False,
     sci_metadata = get_frames_metadata(mjd=sci_mjd)
     sci_metadata.query("expnum == @expnum", inplace=True)
     sci_metadata.sort_values("expnum", ascending=False, inplace=True)
+    sci_metadata.query("qaqual == 'GOOD'", inplace=True)
+
+    if sci_metadata.empty:
+        log.error(f"exposure {expnum = } was flagged as 'BAD' by the raw data quality pipeline")
+        return
+
+    # define general metadata
     sci_tileid = sci_metadata["tileid"].unique()[0]
     sci_mjd = sci_metadata["mjd"].unique()[0]
     sci_expnum = sci_metadata["expnum"].unique()[0]
@@ -1513,7 +1520,7 @@ def science_reduction(expnum: int, use_longterm_cals: bool = False,
     calibs = get_calib_paths(mjd=cals_mjd, version=drpver, longterm_cals=use_longterm_cals, from_sanbox=True)
 
     # make sure only one exposure number is being reduced
-    sci_metadata.query("expnum == @sci_expnum", inplace=True)
+    # sci_metadata.query("expnum == @sci_expnum", inplace=True)
     sci_metadata.sort_values("camera", inplace=True)
 
     # detrend science exposure
@@ -1745,7 +1752,10 @@ def run_drp(mjd: Union[int, str, list], expnum: Union[int, str, list] = None,
     sub = frames.copy()
 
     # remove bad or test quality frames
-    sub = sub[~(sub['quality'] != 'excellent')]
+    sub = sub[~(sub['qaqual'] != 'GOOD')]
+    if sub.empty:
+        log.error(f"exposure {expnum = } was flagged as 'BAD' by the raw data quality pipeline")
+        return
 
     # filter on exposure number
     if expnum:

python/lvmdrp/utils/bitmask.py

@@ -99,8 +99,8 @@ def __contains__(self, flag):
 
 class RawFrameQuality(BaseBitmask):
     GOOD = auto()  # bit whether a raw frame is good for reduction
-    TEST = auto()  # bit whether a raw frame is for instrument testing purposes
     BAD = auto()  # bit whether a raw frame is bad for reduction
+    TEST = auto()  # bit whether a raw frame is for instrument testing purposes
 
 
 class ReductionStatus(BaseBitmask):

python/lvmdrp/utils/hdrfix.py

@@ -168,6 +168,10 @@ def apply_hdrfix(mjd: int, camera: str = None, expnum: int = None,
     tileid = 11111 if tileid in (-999, 999, None) else tileid
     hdr['TILE_ID'] = tileid
 
+    # add QA header keywords default values
+    hdr['QAQUAL'] = ('GOOD', 'string value for raw data quality flag')
+    hdr['QAFLAG'] = ('0000', 'bitmask value for raw data quality flag')
+
     # read the hdr fix file
     fix = read_hdrfix_file(mjd)
 
@@ -185,4 +189,8 @@ def apply_hdrfix(mjd: int, camera: str = None, expnum: int = None,
             hdr[row['keyword']] = row['value']
             log.info(f'Applying header fix on {current_file} for key: {row["keyword"]}, value: {row["value"]}.')
 
+    # fix typing in QAFLAG keyword
+    # hdr['QAQUAL'] = ('GOOD', 'string value for raw data quality flag')
+    # hdr.set('QAFLAG', QAFlag(int(hdr['QAFLAG'], base=2)), 'bitmask value for raw data quality flag')
+
     return hdr

python/lvmdrp/utils/metadata.py

@@ -22,7 +22,6 @@
 from lvmdrp.core.constants import FRAMES_CALIB_NEEDS, CAMERAS
 from lvmdrp.utils.bitmask import (
     QualityFlag,
-    RawFrameQuality,
     ReductionStage,
     ReductionStatus,
 )
@@ -55,8 +54,7 @@
     ("ldls", bool),
     ("quartz", bool),
     ("hartmann", str),
-    ("quality", str),
-    ("qual", RawFrameQuality),
+    ("qaqual", str),
     ("stage", ReductionStage),
     ("status", ReductionStatus),
     ("drpqual", QualityFlag),
@@ -79,8 +77,7 @@
     ("ldls", bool),
     ("quartz", bool),
     ("hartmann", str),
-    ("quality", str),
-    ("qual", RawFrameQuality),
+    ("qaqual", str),
     ("stage", ReductionStage),
     ("status", ReductionStatus),
     ("drpqual", QualityFlag),
@@ -607,8 +604,11 @@ def extract_metadata(frames_paths: list, kind: str = "raw") -> pd.DataFrame:
                 header.get("LDLS", "OFF") in onlamp,
                 header.get("QUARTZ", "OFF") in onlamp,
                 header.get("HARTMANN", "0 0"),
-                header.get("QUALITY", "excellent"),
-                header.get("QUAL", RawFrameQuality(0)),
+                # header.get("QUALITY", "excellent"),
+                # QC pipeline keywords
+                header.get("QAQUAL", "GOOD"),
+                # header.get("QAFLAG", QAFlag(0)),
+                # DRP quality keywords
                 header.get("DRPSTAGE", ReductionStage.UNREDUCED),
                 header.get("DRPSTAT", ReductionStatus(0)),
                 header.get("DRPQUAL", QualityFlag(0)),
@@ -632,8 +632,10 @@ def extract_metadata(frames_paths: list, kind: str = "raw") -> pd.DataFrame:
                 header.get("LDLS", "OFF") in onlamp,
                 header.get("QUARTZ", "OFF") in onlamp,
                 header.get("HARTMANN", "0 0"),
-                header.get("QUALITY", "excellent"),
-                header.get("QUAL", RawFrameQuality(0)),
+                # TODO: QUALITY may be redundant, double check and remove if it is
+                # header.get("QUALITY", "excellent"),
+                header.get("QAQUAL", "GOOD"),
+                # header.get("QAFLAG", QAFlag(0)),
                 header.get("DRPSTAGE", ReductionStage.UNREDUCED),
                 header.get("DRPSTAT", ReductionStatus(0)),
                 header.get("DRPQUAL", QualityFlag(0)),
@@ -1448,7 +1450,7 @@ def _collect_header_data(filename: str) -> dict:
                         'skyw_ra': 'TESKYWRA', 'skyw_dec': 'TESKYWDE', 'skyw_amass': 'TESKYWAM',
                         'skyw_kmpos': 'TESKYWKM', 'skyw_focpos': 'TESKYWFO', 'skyw_name': 'SKYWNAME',
                         'skyw_sh_hght': 'GEOCORONAL SKYW_SH_HGHT', 'skyw_moon_sep': 'SKYMODEL SKYW_RHO',
-                        # sky parameters 
+                        # sky parameters
                         'moon_ra': 'SKYMODEL MOON_RA', 'moon_dec': 'SKYMODEL MOON_DEC',
                         'moon_phase': 'SKYMODEL MOON_PHASE', 'moon_fli': 'SKYMODEL MOON_FLI',
                         'sun_alt': 'SKYMODEL SUNALT', 'moon_alt': 'SKYMODEL MOONALT'