CCD-1213/ewsc-detailed-logging

CHANGES.rst

@@ -1,3 +1,10 @@
+11.16.14 (unreleased)
+=====================
+
+General
+-------
+- Equal Weight Special Case log messages include filenames and useafter dates [#901]
+
 11.16.13 (2022-09-20)
 =====================
 

crds/core/selectors.py

@@ -2013,6 +2013,43 @@ def _check_valid_values(self, valid_values_map):
             if name not in valid_values_map:
                 log.verbose_warning(self.short_name, "Parameter ",
                                     repr(name), " is unchecked.")
+
+    def _ambiguous_match_data(self, key, other):
+        """Retrieve filenames and useafter data for the given match case to be included
+        in error/warning log message.
+
+        Parameters
+        ----------
+        key : tuple or tuple of tuples
+            match key parameters (selector parkey values), usually the subset
+        other : tuple or tuple of tuples
+            match key parameters (selector parkey values), usually the superset
+
+        Returns
+        -------
+        strings
+            two pre-formatted strings (subset and superset), each composed of the match key 
+            and first selection's useafter and reference filename. If there is an exception while getting
+            data from the mapping, a warning is raised but the match key tuples are still returned as strings.
+        """
+        kstr, ostr = f"{key}", f"{other}"
+        with log.warn_on_exception("Unable to retrieve filenames - skipping"):
+            k_name = [v.choices()[0] for k,v in self._selections if k == key and isinstance(v, Selector)]
+            o_name = [v.choices()[0] for k,v in self._selections if k == other and isinstance(v, Selector)]
+            if k_name and o_name:
+                uas = [self.file_matches(k_name[0]), self.file_matches(o_name[0])]
+                useafters = []
+                for ua in uas:
+                    if ua and len(ua[0][-1]) == 2:
+                        useafters.append(ua[0][-1][0][1] + ' ' + ua[0][-1][1][1])
+                    elif ua and len(ua[0][-1]) == 1:
+                        useafters.append(ua[0][-1][0][1])
+                if useafters:
+                    kstr += f" : UseAfter({str('{')}\n    '{useafters[0]}'"
+                    ostr += f" : UseAfter({str('{')}\n    '{useafters[1]}'"
+                kstr += ' : ' + k_name[0]
+                ostr += ' : ' + o_name[0]
+        return kstr, ostr
 
     def _validate_raw_key(self, key, valid_values_map):
         """Validate each field of a single Match `key` against the possible
@@ -2038,15 +2075,14 @@ def _validate_raw_key(self, key, valid_values_map):
         for other in self.keys():
             if key != other and match_superset(other, key) and \
                 not different_match_weight(key, other):
+                # include match case filename and useafter data in log message
+                key_data, other_data = self._ambiguous_match_data(key, other)
                 if self._observatory != "hst" or self._merge_overlaps is False:
                     raise AmbiguousMatchError(
-                        f"\n----------------------------------------\nMatch case\n{self.match_item(key)}\nis an equal weight special case of\n{self.match_item(other)}\nCancel the submission and regenerate the reference files\nwith different parameter values which coincide with an existing category.\nFor some parameter sets, CRDS interprets both matches as equally good.\nFor more explanation, see the file submission section of the CRDS server user's guide here:\nhttps://{self._observatory.lower()}-crds.stsci.edu/static/users_guide/index.html\n----------------------------------------"
+                        f"\n----------------------------------------\nMatch case\n{key_data}\nis an equal weight special case of\n{other_data}\nCancel the submission and regenerate the reference files\nwith different parameter values which coincide with an existing category.\nFor some parameter sets, CRDS interprets both matches as equally good.\nFor more explanation, see the file submission section of the CRDS server user's guide here:\nhttps://{self._observatory.lower()}-crds.stsci.edu/static/users_guide/index.html\n----------------------------------------"
                         )
                 else:
-                    log.verbose_warning("-"*40 + "\nMatch case\n",
-                     log.PP(self.match_item(key)),
-                     "\nis an equal weight special case of\n",
-                     log.PP(self.match_item(other)), """
+                    log.verbose_warning("-"*40 + f"\nMatch case\n{key_data}\nis an equal weight special case of\n{other_data}","""
 For some parameter sets, CRDS interprets both matches as equally good.
 See the file submission section of the CRDS server user's guide here:
     https://hst-crds.stsci.edu/static/users_guide/index.html

crds/tests/data/roman_wfi_flat_ewsc.rmap

@@ -0,0 +1,28 @@
+header = {
+    'classes' : ('Match', 'UseAfter'),
+    'derived_from' : 'roman_wfi_flat_0001.rmap',
+    'file_ext' : '.asdf',
+    'filekind' : 'FLAT',
+    'filetype' : 'FLAT',
+    'instrument' : 'WFI',
+    'ld_tpn' : 'wfi_flat_ld.tpn',
+    'mapping' : 'REFERENCE',
+    'name' : 'roman_wfi_flat_ewsc.rmap',
+    'observatory' : 'ROMAN',
+    'parkey' : (('ROMAN.META.INSTRUMENT.DETECTOR', 'ROMAN.META.INSTRUMENT.OPTICAL_ELEMENT'), ('ROMAN.META.EXPOSURE.START_TIME',)),
+    'sha1sum' : 'd34147f4c4a6fd4ffec42e95b5530b8704be19cf',
+    'suffix' : 'flat',
+    'text_descr' : 'Flat Field',
+    'tpn' : 'wfi_flat.tpn',
+}
+
+selector = Match({
+    ('WFI01', 'F158') : UseAfter({
+        '2020-01-01 00:00:00' : 'roman_wfi_flat_0002.asdf',
+        '2021-01-01 00:00:00' : 'roman_wfi_flat_0003.asdf',
+    }),
+    ('WFI01', 'F158|F184|F213') : UseAfter({
+        '2021-08-01 11:11:11' : 'roman_wfi_flat_0004.asdf',
+        '2022-01-01 00:00:00' : 'roman_wfi_flat_0005.asdf',
+    }),
+})

crds/tests/test_certify.py

@@ -10,7 +10,6 @@
 # ==================================================================================
 
 from crds.core import utils, log, exceptions
-from crds import client
 from crds import data_file
 from crds import certify
 from crds.certify import CertifyScript
@@ -1319,9 +1318,11 @@ def test_certify_check_rmap_updates():
     CRDS - DEBUG -  Checking 'DETECTOR' = 'FUV' against ('FUV', 'NUV')
     CRDS - WARNING -  ----------------------------------------
     Match case
-     (('DETECTOR', 'FUV'),)
+    ('FUV',) : UseAfter({
+        '1996-10-01 00:00:00' : s7g1700gl_dead_dup2.fits
     is an equal weight special case of
-     (('DETECTOR', 'FUV|NUV'),)
+    ('FUV|NUV',) : UseAfter({
+        '1996-10-01 00:00:00' : s7g1700gl_dead_overlap.fits 
     For some parameter sets, CRDS interprets both matches as equally good.
     See the file submission section of the CRDS server user's guide here:
         https://hst-crds.stsci.edu/static/users_guide/index.html
@@ -1336,9 +1337,11 @@ def test_certify_check_rmap_updates():
     CRDS - DEBUG -  Checking 'DETECTOR' = 'NUV' against ('FUV', 'NUV')
     CRDS - WARNING -  ----------------------------------------
     Match case
-     (('DETECTOR', 'NUV'),)
+    ('NUV',) : UseAfter({
+        '1996-10-01 00:00:00' : s7g1700ql_dead.fits
     is an equal weight special case of
-     (('DETECTOR', 'FUV|NUV'),)
+    ('FUV|NUV',) : UseAfter({
+        '1996-10-01 00:00:00' : s7g1700gl_dead_overlap.fits 
     For some parameter sets, CRDS interprets both matches as equally good.
     See the file submission section of the CRDS server user's guide here:
         https://hst-crds.stsci.edu/static/users_guide/index.html

crds/tests/test_rmap.py

@@ -6,9 +6,8 @@
 from pprint import pprint as pp
 import pickle
 
-from crds import rmap, log, config, tests, utils
-from crds.client import api
-from crds.exceptions import *
+from crds import rmap, log, config, utils
+from crds.core.exceptions import *
 from crds.tests import test_config
 
 from nose.tools import assert_raises, assert_true
@@ -395,9 +394,11 @@ def dt_validate_mapping_ambiguous():
     CRDS - ERROR -  Match('DETECTOR', 'CCDAMP', 'CCDGAIN') : ('HRC', 'C', '1.0|2.0|4.0|8.0') : 
     ----------------------------------------
     Match case
-    (('DETECTOR', 'HRC'), ('CCDAMP', 'C'), ('CCDGAIN', '1.0|2.0|4.0|8.0'))
+    ('HRC', 'C', '1.0|2.0|4.0|8.0') : UseAfter({
+        '2003-11-06 15:11:06' : nc113178j_drk.fits
     is an equal weight special case of
-    (('DETECTOR', 'HRC'), ('CCDAMP', 'A|ABCD|AD|B|BC|C|D'), ('CCDGAIN', '1.0|2.0|4.0|8.0'))
+    ('HRC', 'A|ABCD|AD|B|BC|C|D', '1.0|2.0|4.0|8.0') : UseAfter({
+        '1992-01-01 00:00:00' : lcb12060j_drk.fits
     Cancel the submission and regenerate the reference files
     with different parameter values which coincide with an existing category.
     For some parameter sets, CRDS interprets both matches as equally good.
@@ -407,9 +408,11 @@ def dt_validate_mapping_ambiguous():
     CRDS - ERROR -  Match('DETECTOR', 'CCDAMP', 'CCDGAIN') : ('HRC', 'C', '2.0') : 
     ----------------------------------------
     Match case
-    (('DETECTOR', 'HRC'), ('CCDAMP', 'C'), ('CCDGAIN', '2.0'))
+    ('HRC', 'C', '2.0') : UseAfter({
+        '2002-03-26 00:00:00' : m3t1633tj_drk.fits
     is an equal weight special case of
-    (('DETECTOR', 'HRC'), ('CCDAMP', 'A|ABCD|AD|B|BC|C|D'), ('CCDGAIN', '1.0|2.0|4.0|8.0'))
+    ('HRC', 'A|ABCD|AD|B|BC|C|D', '1.0|2.0|4.0|8.0') : UseAfter({
+        '1992-01-01 00:00:00' : lcb12060j_drk.fits
     Cancel the submission and regenerate the reference files
     with different parameter values which coincide with an existing category.
     For some parameter sets, CRDS interprets both matches as equally good.
@@ -419,9 +422,11 @@ def dt_validate_mapping_ambiguous():
     CRDS - ERROR -  Match('DETECTOR', 'CCDAMP', 'CCDGAIN') : ('HRC', 'D', '1.0|2.0|4.0|8.0') : 
     ----------------------------------------
     Match case
-    (('DETECTOR', 'HRC'), ('CCDAMP', 'D'), ('CCDGAIN', '1.0|2.0|4.0|8.0'))
+    ('HRC', 'D', '1.0|2.0|4.0|8.0') : UseAfter({
+        '1991-01-01 00:00:00' : j4d1435nj_drk.fits
     is an equal weight special case of
-    (('DETECTOR', 'HRC'), ('CCDAMP', 'A|ABCD|AD|B|BC|C|D'), ('CCDGAIN', '1.0|2.0|4.0|8.0'))
+    ('HRC', 'A|ABCD|AD|B|BC|C|D', '1.0|2.0|4.0|8.0') : UseAfter({
+        '1992-01-01 00:00:00' : lcb12060j_drk.fits
     Cancel the submission and regenerate the reference files
     with different parameter values which coincide with an existing category.
     For some parameter sets, CRDS interprets both matches as equally good.
@@ -431,9 +436,12 @@ def dt_validate_mapping_ambiguous():
     CRDS - ERROR -  Match('DETECTOR', 'CCDAMP', 'CCDGAIN') : ('WFC', 'ABCD', '1.0') : 
     ----------------------------------------
     Match case
+    ('WFC', 'ABCD', '1.0') : UseAfter({
+        '2003-10-25 01:18:03' : nba1143tj_drk.fits
     (('DETECTOR', 'WFC'), ('CCDAMP', 'ABCD'), ('CCDGAIN', '1.0'))
     is an equal weight special case of
-    (('DETECTOR', 'WFC'), ('CCDAMP', 'A|ABCD|AC|AD|B|BC|BD|C|D'), ('CCDGAIN', '1.0'))
+    ('WFC', 'A|ABCD|AC|AD|B|BC|BD|C|D', '1.0') : UseAfter({
+        '2003-09-13 00:48:08' : na11410lj_drk.fits
     Cancel the submission and regenerate the reference files
     with different parameter values which coincide with an existing category.
     For some parameter sets, CRDS interprets both matches as equally good.
@@ -443,9 +451,11 @@ def dt_validate_mapping_ambiguous():
     CRDS - ERROR -  Match('DETECTOR', 'CCDAMP', 'CCDGAIN') : ('WFC', 'A|ABCD|AC|AD|B|BC|BD|C|D', '1.0') : 
     ----------------------------------------
     Match case
-    (('DETECTOR', 'WFC'), ('CCDAMP', 'A|ABCD|AC|AD|B|BC|BD|C|D'), ('CCDGAIN', '1.0'))
+    ('WFC', 'A|ABCD|AC|AD|B|BC|BD|C|D', '1.0') : UseAfter({
+        '2003-09-13 00:48:08' : na11410lj_drk.fits
     is an equal weight special case of
-    (('DETECTOR', 'WFC'), ('CCDAMP', 'A|ABCD|AC|AD|B|BC|BD|C|D'), ('CCDGAIN', '1.0|2.0|4.0|8.0'))
+    ('WFC', 'A|ABCD|AC|AD|B|BC|BD|C|D', '1.0|2.0|4.0|8.0') : UseAfter({
+        '1991-01-01 00:00:00' : lcb1202gj_drk.fits
     Cancel the submission and regenerate the reference files
     with different parameter values which coincide with an existing category.
     For some parameter sets, CRDS interprets both matches as equally good.
@@ -455,9 +465,11 @@ def dt_validate_mapping_ambiguous():
     CRDS - ERROR -  Match('DETECTOR', 'CCDAMP', 'CCDGAIN') : ('WFC', 'A|ABCD|AD|B|BC|C|D', '1.0|2.0|4.0|8.0') : 
     ----------------------------------------
     Match case
-    (('DETECTOR', 'WFC'), ('CCDAMP', 'A|ABCD|AD|B|BC|C|D'), ('CCDGAIN', '1.0|2.0|4.0|8.0'))
+    ('WFC', 'A|ABCD|AD|B|BC|C|D', '1.0|2.0|4.0|8.0') : UseAfter({
+        '2006-07-30 02:04:10' : q9520146j_drk.fits
     is an equal weight special case of
-    (('DETECTOR', 'WFC'), ('CCDAMP', 'A|ABCD|AC|AD|B|BC|BD|C|D'), ('CCDGAIN', '1.0|2.0|4.0|8.0'))
+    ('WFC', 'A|ABCD|AC|AD|B|BC|BD|C|D', '1.0|2.0|4.0|8.0') : UseAfter({
+        '1991-01-01 00:00:00' : lcb1202gj_drk.fits
     Cancel the submission and regenerate the reference files
     with different parameter values which coincide with an existing category.
     For some parameter sets, CRDS interprets both matches as equally good.
@@ -467,6 +479,29 @@ def dt_validate_mapping_ambiguous():
     >>> test_config.cleanup(old_state)
     """
 
+def dt_validate_mapping_ambiguous_roman():
+    """
+    >>> old_state = test_config.setup(url="https://roman-crds-serverless.stsci.edu", observatory="roman")
+    >>> r = rmap.get_cached_mapping("data/roman_wfi_flat_ewsc.rmap")
+    >>> r.validate_mapping()
+    CRDS - ERROR -  Match('ROMAN.META.INSTRUMENT.DETECTOR [DETECTOR]', 'ROMAN.META.INSTRUMENT.OPTICAL_ELEMENT [FITS unknown]') : ('WFI01', 'F158') : 
+    ----------------------------------------
+    Match case
+    ('WFI01', 'F158') : UseAfter({
+        '2020-01-01 00:00:00' : roman_wfi_flat_0002.asdf
+    is an equal weight special case of
+    ('WFI01', 'F158|F184|F213') : UseAfter({
+        '2021-08-01 11:11:11' : roman_wfi_flat_0004.asdf
+    Cancel the submission and regenerate the reference files
+    with different parameter values which coincide with an existing category.
+    For some parameter sets, CRDS interprets both matches as equally good.
+    For more explanation, see the file submission section of the CRDS server user's guide here:
+    https://roman-crds.stsci.edu/static/users_guide/index.html
+    ----------------------------------------
+    >>> test_config.cleanup(old_state)
+    """
+
+
 def dt_validate_mapping_invalid1():
     """
     >>> old_state = test_config.setup()
@@ -610,7 +645,6 @@ def test_rmap_list_references(self):
         self.assertEqual(rmap.list_references("*.r1h", "hst"), ['dbu1405fu.r1h', 'dbu1405iu.r1h', 'e1b09593u.r1h', 'e1b09594u.r1h', 'valid.r1h'])
 
     def test_rmap_get_derived_from(self):
-        # api.dump_mappings("hst.pmap", mappings=["hst_acs_flshfile_0251.rmap"])
         os.environ["CRDS_MAPPATH_SINGLE"] = self.data_dir
         r = rmap.get_cached_mapping("data/hst_acs_flshfile_0252.rmap")
         self.assertEqual(r.get_derived_from().name, 'hst_acs_flshfile_0251.rmap')