diff --git a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py index 60d5923fd..8f68054f0 100755 --- a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py +++ b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py @@ -99,12 +99,11 @@ from jwql.database.database_interface import NIRSpecBadPixelQueryHistory, NIRSpecBadPixelStats from jwql.database.database_interface import FGSBadPixelQueryHistory, FGSBadPixelStats from jwql.instrument_monitors import pipeline_tools -from jwql.utils import crds_tools, instrument_properties +from jwql.utils import crds_tools, instrument_properties, monitor_utils from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE from jwql.utils.constants import FLAT_EXP_TYPES, DARK_EXP_TYPES from jwql.utils.logging_functions import log_info, log_fail from jwql.utils.mast_utils import mast_query -from jwql.utils.monitor_utils import initialize_instrument_monitor, update_monitor_table from jwql.utils.permissions import set_permissions from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path @@ -1022,10 +1021,17 @@ def run(self): # lists to align. if new_flat_entries: + # Exclude ASIC tuning data + len_new_flats = len(new_flat_entries) + new_flat_entries = monitor_utils.exclude_asic_tuning(new_flat_entries) + len_no_asic = len(new_flat_entries) + num_asic = len_new_flats - len_no_asic + logging.info("\tFiltering out ASIC tuning files removed {} flat files.".format(num_asic)) + new_flat_entries = self.filter_query_results(new_flat_entries, datatype='flat') apcheck_flat_entries = pipeline_tools.aperture_size_check(new_flat_entries, instrument, aperture) lost_to_bad_metadata = len(new_flat_entries) - len(apcheck_flat_entries) - logging.info('{} flat field files ignored due to inconsistency in array size and metadata.'.format(lost_to_bad_metadata)) + logging.info('\t{} flat field files ignored due to inconsistency in array size and metadata.'.format(lost_to_bad_metadata)) flat_uncal_files = locate_uncal_files(apcheck_flat_entries) flat_uncal_files, run_flats = check_for_sufficient_files(flat_uncal_files, instrument, aperture, flat_file_count_threshold, 'flats') flat_rate_files, flat_rate_files_to_copy = locate_rate_files(flat_uncal_files) @@ -1034,10 +1040,17 @@ def run(self): flat_uncal_files, flat_rate_files, flat_rate_files_to_copy = None, None, None if new_dark_entries: + # Exclude ASIC tuning data + len_new_darks = len(new_dark_entries) + new_dark_entries = monitor_utils.exclude_asic_tuning(new_dark_entries) + len_no_asic = len(new_dark_entries) + num_asic = len_new_darks - len_no_asic + logging.info("\tFiltering out ASIC tuning files removed {} dark files.".format(num_asic)) + new_dark_entries = self.filter_query_results(new_dark_entries, datatype='dark') apcheck_dark_entries = pipeline_tools.aperture_size_check(new_dark_entries, instrument, aperture) lost_to_bad_metadata = len(new_dark_entries) - len(apcheck_dark_entries) - logging.info('{} dark files ignored due to inconsistency in array size and metadata.'.format(lost_to_bad_metadata)) + logging.info('\t{} dark files ignored due to inconsistency in array size and metadata.'.format(lost_to_bad_metadata)) dark_uncal_files = locate_uncal_files(apcheck_dark_entries) dark_uncal_files, run_darks = check_for_sufficient_files(dark_uncal_files, instrument, aperture, dark_file_count_threshold, 'darks') dark_rate_files, dark_rate_files_to_copy = locate_rate_files(dark_uncal_files) @@ -1092,9 +1105,9 @@ def run(self): if __name__ == '__main__': module = os.path.basename(__file__).strip('.py') - start_time, log_file = initialize_instrument_monitor(module) + start_time, log_file = monitor_utils.initialize_instrument_monitor(module) monitor = BadPixels() monitor.run() - update_monitor_table(module, start_time, log_file) + monitor_utils.update_monitor_table(module, start_time, log_file) diff --git a/jwql/instrument_monitors/common_monitors/bias_monitor.py b/jwql/instrument_monitors/common_monitors/bias_monitor.py index 920642492..a0194ad44 100755 --- a/jwql/instrument_monitors/common_monitors/bias_monitor.py +++ b/jwql/instrument_monitors/common_monitors/bias_monitor.py @@ -51,13 +51,13 @@ from jwql.database.database_interface import session from jwql.database.database_interface import NIRCamBiasQueryHistory, NIRCamBiasStats, NIRISSBiasQueryHistory, NIRISSBiasStats, NIRSpecBiasQueryHistory, NIRSpecBiasStats from jwql.instrument_monitors import pipeline_tools -from jwql.instrument_monitors.common_monitors.dark_monitor import mast_query_darks -from jwql.utils import instrument_properties +from jwql.utils import instrument_properties, monitor_utils from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE from jwql.utils.logging_functions import log_info, log_fail from jwql.utils.monitor_utils import update_monitor_table from jwql.utils.permissions import set_permissions -from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config, initialize_instrument_monitor +from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config + class Bias(): @@ -466,7 +466,15 @@ def run(self): # Query MAST for new dark files for this instrument/aperture logging.info('\tQuery times: {} {}'.format(self.query_start, self.query_end)) - new_entries = mast_query_darks(instrument, aperture, self.query_start, self.query_end) + new_entries = monitor_utils.mast_query_darks(instrument, aperture, self.query_start, self.query_end) + + # Exclude ASIC tuning data + len_new_darks = len(new_entries) + new_entries = monitor_utils.exclude_asic_tuning(new_entries) + len_no_asic = len(new_entries) + num_asic = len_new_darks - len_no_asic + logging.info("\tFiltering out ASIC tuning files removed {} dark files.".format(num_asic)) + logging.info('\tAperture: {}, new entries: {}'.format(self.aperture, len(new_entries))) # Set up a directory to store the data for this aperture @@ -524,9 +532,9 @@ def run(self): if __name__ == '__main__': module = os.path.basename(__file__).strip('.py') - start_time, log_file = initialize_instrument_monitor(module) + start_time, log_file = monitor_utils.initialize_instrument_monitor(module) monitor = Bias() monitor.run() - update_monitor_table(module, start_time, log_file) + monitor_utils.update_monitor_table(module, start_time, log_file) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index a622ebec5..2c98a1dc1 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -75,86 +75,16 @@ from jwql.database.database_interface import FGSDarkQueryHistory, FGSDarkPixelStats, FGSDarkDarkCurrent from jwql.instrument_monitors import pipeline_tools from jwql.jwql_monitors import monitor_mast -from jwql.utils import calculations, instrument_properties -from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_DATAPRODUCTS, RAPID_READPATTERNS +from jwql.utils import calculations, instrument_properties, monitor_utils +from jwql.utils.constants import ASIC_TEMPLATES, JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_DATAPRODUCTS, \ + RAPID_READPATTERNS from jwql.utils.logging_functions import log_info, log_fail -from jwql.utils.monitor_utils import initialize_instrument_monitor, update_monitor_table from jwql.utils.permissions import set_permissions from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path THRESHOLDS_FILE = os.path.join(os.path.split(__file__)[0], 'dark_monitor_file_thresholds.txt') -def mast_query_darks(instrument, aperture, start_date, end_date, readpatt=None): - """Use ``astroquery`` to search MAST for dark current data - - Parameters - ---------- - instrument : str - Instrument name (e.g. ``nircam``) - - aperture : str - Detector aperture to search for (e.g. ``NRCA1_FULL``) - - start_date : float - Starting date for the search in MJD - - end_date : float - Ending date for the search in MJD - - readpatt : str - Readout pattern to search for (e.g. ``RAPID``). If None, - readout pattern will not be added to the query parameters. - - Returns - ------- - query_results : list - List of dictionaries containing the query results - """ - - # Make sure instrument is correct case - if instrument.lower() == 'nircam': - instrument = 'NIRCam' - dark_template = ['NRC_DARK'] - elif instrument.lower() == 'niriss': - instrument = 'NIRISS' - dark_template = ['NIS_DARK'] - elif instrument.lower() == 'nirspec': - instrument = 'NIRSpec' - dark_template = ['NRS_DARK'] - elif instrument.lower() == 'fgs': - instrument = 'FGS' - dark_template = ['FGS_DARK'] - elif instrument.lower() == 'miri': - instrument = 'MIRI' - dark_template = ['MIR_DARKALL', 'MIR_DARKIMG', 'MIR_DARKMRS'] - - # monitor_mast.instrument_inventory does not allow list inputs to - # the added_filters input (or at least if you do provide a list, then - # it becomes a nested list when it sends the query to MAST. The - # nested list is subsequently ignored by MAST.) - # So query once for each dark template, and combine outputs into a - # single list. - query_results = [] - for template_name in dark_template: - - # Create dictionary of parameters to add - parameters = {"date_obs_mjd": {"min": start_date, "max": end_date}, - "apername": aperture, "exp_type": template_name, - } - - if readpatt is not None: - parameters["readpatt"] = readpatt - - query = monitor_mast.instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, - add_filters=parameters, return_data=True, caom=False) - if 'data' in query.keys(): - if len(query['data']) > 0: - query_results.extend(query['data']) - - return query_results - - class Dark(): """Class for executing the dark current monitor. @@ -768,7 +698,15 @@ def run(self): # Query MAST using the aperture and the time of the # most recent previous search as the starting time - new_entries = mast_query_darks(instrument, aperture, self.query_start, self.query_end, readpatt=self.readpatt) + new_entries = monitor_utils.mast_query_darks(instrument, aperture, self.query_start, self.query_end, readpatt=self.readpatt) + + # Exclude ASIC tuning data + len_new_darks = len(new_entries) + new_entries = monitor_utils.exclude_asic_tuning(new_entries) + len_no_asic = len(new_entries) + num_asic = len_new_darks - len_no_asic + logging.info("\tFiltering out ASIC tuning files removed {} dark files.".format(num_asic)) + logging.info('\tAperture: {}, Readpattern: {}, new entries: {}'.format(self.aperture, self.readpatt, len(new_entries))) @@ -1087,9 +1025,9 @@ def stats_by_amp(self, image, amps): if __name__ == '__main__': module = os.path.basename(__file__).strip('.py') - start_time, log_file = initialize_instrument_monitor(module) + start_time, log_file = monitor_utils.initialize_instrument_monitor(module) monitor = Dark() monitor.run() - update_monitor_table(module, start_time, log_file) + monitor_utils.update_monitor_table(module, start_time, log_file) diff --git a/jwql/instrument_monitors/common_monitors/readnoise_monitor.py b/jwql/instrument_monitors/common_monitors/readnoise_monitor.py index ce8292460..12bd0afc2 100755 --- a/jwql/instrument_monitors/common_monitors/readnoise_monitor.py +++ b/jwql/instrument_monitors/common_monitors/readnoise_monitor.py @@ -56,13 +56,12 @@ from jwql.database.database_interface import NIRSpecReadnoiseQueryHistory, NIRSpecReadnoiseStats from jwql.database.database_interface import session from jwql.instrument_monitors import pipeline_tools -from jwql.instrument_monitors.common_monitors.dark_monitor import mast_query_darks -from jwql.utils import instrument_properties +from jwql.utils import instrument_properties, monitor_utils from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE from jwql.utils.logging_functions import log_info, log_fail from jwql.utils.monitor_utils import update_monitor_table from jwql.utils.permissions import set_permissions -from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config, initialize_instrument_monitor +from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config class Readnoise(): @@ -554,7 +553,15 @@ def run(self): # Query MAST for new dark files for this instrument/aperture logging.info('\tQuery times: {} {}'.format(self.query_start, self.query_end)) - new_entries = mast_query_darks(instrument, aperture, self.query_start, self.query_end) + new_entries = monitor_utils.mast_query_darks(instrument, aperture, self.query_start, self.query_end) + + # Exclude ASIC tuning data + len_new_darks = len(new_entries) + new_entries = monitor_utils.exclude_asic_tuning(new_entries) + len_no_asic = len(new_entries) + num_asic = len_new_darks - len_no_asic + logging.info("\tFiltering out ASIC tuning files removed {} dark files.".format(num_asic)) + logging.info('\tAperture: {}, new entries: {}'.format(self.aperture, len(new_entries))) # Set up a directory to store the data for this aperture @@ -624,9 +631,9 @@ def run(self): if __name__ == '__main__': module = os.path.basename(__file__).strip('.py') - start_time, log_file = initialize_instrument_monitor(module) + start_time, log_file = monitor_utils.initialize_instrument_monitor(module) monitor = Readnoise() monitor.run() - update_monitor_table(module, start_time, log_file) + monitor_utils.update_monitor_table(module, start_time, log_file) diff --git a/jwql/jwql_monitors/monitor_mast.py b/jwql/jwql_monitors/monitor_mast.py index d2e6dca0e..6b9d1545c 100755 --- a/jwql/jwql_monitors/monitor_mast.py +++ b/jwql/jwql_monitors/monitor_mast.py @@ -29,8 +29,8 @@ from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_DATAPRODUCTS from jwql.utils.logging_functions import configure_logging, log_info, log_fail from jwql.utils.permissions import set_permissions -from jwql.utils.utils import get_config, initialize_instrument_monitor -from jwql.utils.monitor_utils import update_monitor_table +from jwql.utils.utils import get_config +from jwql.utils import monitor_utils from jwql.utils.plotting import bar_chart @@ -265,8 +265,8 @@ def monitor_mast(): # Configure logging module = os.path.basename(__file__).strip('.py') - start_time, log_file = initialize_instrument_monitor(module) + start_time, log_file = monitor_utils.initialize_instrument_monitor(module) # Run the monitors monitor_mast() - update_monitor_table(module, start_time, log_file) + monitor_utils.update_monitor_table(module, start_time, log_file) diff --git a/jwql/tests/test_dark_monitor.py b/jwql/tests/test_dark_monitor.py index b5204a44a..c9b877412 100644 --- a/jwql/tests/test_dark_monitor.py +++ b/jwql/tests/test_dark_monitor.py @@ -24,6 +24,7 @@ import numpy as np from jwql.instrument_monitors.common_monitors import dark_monitor +from jwql.utils.monitor_utils import mast_query_darks from jwql.utils.utils import get_config ON_GITHUB_ACTIONS = '/home/runner' in os.path.expanduser('~') or '/Users/runner' in os.path.expanduser('~') @@ -79,7 +80,7 @@ def test_mast_query_darks(): readpatt = 'BRIGHT2' start_date = Time("2016-01-01T00:00:00").mjd end_date = Time("2018-01-01T00:00:00").mjd - query = dark_monitor.mast_query_darks(instrument, aperture, readpatt, start_date, end_date) + query = mast_query_darks(instrument, aperture, readpatt, start_date, end_date) apernames = [entry['apername'] for entry in query] filenames = [entry['filename'] for entry in query] diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index 3523552d6..566470a0a 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -117,6 +117,10 @@ 'MIRIM_BRIGHTSKY', 'MIRIM_SLITLESSPRISM'], 'FGS': ['FGS1_FULL', 'FGS2_FULL']} +# Observing templates used for ASIC tuning. MAST query results that +# have one of these templates will be ignored +ASIC_TEMPLATES = ['ISIM ASIC Tuning'] + # Bad pixel types by the type of data used to find them BAD_PIXEL_TYPES = ['DEAD', 'HOT', 'LOW_QE', 'RC', 'OPEN', 'ADJ_OPEN', 'TELEGRAPH', 'OTHER_BAD_PIXEL'] DARKS_BAD_PIXEL_TYPES = ['HOT', 'RC', 'OTHER_BAD_PIXEL', 'TELEGRAPH'] diff --git a/jwql/utils/monitor_utils.py b/jwql/utils/monitor_utils.py index 16d9c3c17..533eea0e4 100644 --- a/jwql/utils/monitor_utils.py +++ b/jwql/utils/monitor_utils.py @@ -20,9 +20,33 @@ from jwql.database.database_interface import Monitor +from jwql.jwql_monitors import monitor_mast +from jwql.utils.constants import ASIC_TEMPLATES, JWST_DATAPRODUCTS from jwql.utils.logging_functions import configure_logging, get_log_status +def exclude_asic_tuning(mast_results): + """Given a list of file information from a MAST query, filter out + files taken during ASIC tuning, which will have bad data in terms + of results for the instrument monitors. + + Parameters + ---------- + mast_results : list + List of dictionaries containing a MAST query result + + Returns + ------- + filtered_results : list + Modified list with ASIC tuning entries removed + """ + filtered_results = [] + for mast_result in mast_results: + if mast_result['template'] not in ASIC_TEMPLATES: + filtered_results.append(mast_result) + return filtered_results + + def initialize_instrument_monitor(module): """Configures a log file for the instrument monitor run and captures the start time of the monitor @@ -45,6 +69,76 @@ def initialize_instrument_monitor(module): return start_time, log_file +def mast_query_darks(instrument, aperture, start_date, end_date, readpatt=None): + """Use ``astroquery`` to search MAST for dark current data + + Parameters + ---------- + instrument : str + Instrument name (e.g. ``nircam``) + + aperture : str + Detector aperture to search for (e.g. ``NRCA1_FULL``) + + start_date : float + Starting date for the search in MJD + + end_date : float + Ending date for the search in MJD + + readpatt : str + Readout pattern to search for (e.g. ``RAPID``). If None, + readout pattern will not be added to the query parameters. + + Returns + ------- + query_results : list + List of dictionaries containing the query results + """ + + # Make sure instrument is correct case + if instrument.lower() == 'nircam': + instrument = 'NIRCam' + dark_template = ['NRC_DARK'] + elif instrument.lower() == 'niriss': + instrument = 'NIRISS' + dark_template = ['NIS_DARK'] + elif instrument.lower() == 'nirspec': + instrument = 'NIRSpec' + dark_template = ['NRS_DARK'] + elif instrument.lower() == 'fgs': + instrument = 'FGS' + dark_template = ['FGS_DARK'] + elif instrument.lower() == 'miri': + instrument = 'MIRI' + dark_template = ['MIR_DARKALL', 'MIR_DARKIMG', 'MIR_DARKMRS'] + + # monitor_mast.instrument_inventory does not allow list inputs to + # the added_filters input (or at least if you do provide a list, then + # it becomes a nested list when it sends the query to MAST. The + # nested list is subsequently ignored by MAST.) + # So query once for each dark template, and combine outputs into a + # single list. + query_results = [] + for template_name in dark_template: + + # Create dictionary of parameters to add + parameters = {"date_obs_mjd": {"min": start_date, "max": end_date}, + "apername": aperture, "exp_type": template_name, + } + + if readpatt is not None: + parameters["readpatt"] = readpatt + + query = monitor_mast.instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, + add_filters=parameters, return_data=True, caom=False) + if 'data' in query.keys(): + if len(query['data']) > 0: + query_results.extend(query['data']) + + return query_results + + def update_monitor_table(module, start_time, log_file): """Update the ``monitor`` database table with information about the instrument monitor run diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index 507cc4a2e..c5b0a256a 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -541,31 +541,6 @@ def check_config_for_key(key): ) -def initialize_instrument_monitor(module): - """Configures a log file for the instrument monitor run and - captures the start time of the monitor - - Parameters - ---------- - module : str - The module name (e.g. ``dark_monitor``) - - Returns - ------- - start_time : datetime object - The start time of the monitor - log_file : str - The path to where the log file is stored - """ - - from jwql.utils.logging_functions import configure_logging - - start_time = datetime.datetime.now() - log_file = configure_logging(module) - - return start_time, log_file - - def query_format(string): """Take a string of format lower_case and change it to UPPER CASE""" upper_case = string.upper()