Fixed bug when multiply path/rows were passed to make_quicklook_lists.py

The script was only building a lookup table from quicklook image ID to full product ID for the first path/row it found instead of a separate one for each path/row.

VERSION.txt

@@ -1 +1 @@
-0.2.1
+0.2.2

make_quicklook_lists.py

@@ -65,6 +65,7 @@ def main(csv_folder, quicklook_folder, output_folder, wrs2_tiles=None,
     ]
 
     product_id_col = 'LANDSAT_PRODUCT_ID'
+    wrs2_tile_col = 'WRS2_TILE'
 
     quicklook_re = re.compile(
         '(?P<year>\d{4})(?P<month>\d{2})(?P<day>\d{2})_'
@@ -96,8 +97,7 @@ def main(csv_folder, quicklook_folder, output_folder, wrs2_tiles=None,
 
     # Read in metadata CSV files
     logging.info('\nReading metadata CSV files')
-    # product_ids = set()
-    quicklook_ids = dict()
+    quicklook_ids = defaultdict(dict)
     for csv_name in csv_file_list:
         logging.debug('  {}'.format(csv_name))
         try:
@@ -117,15 +117,18 @@ def main(csv_folder, quicklook_folder, output_folder, wrs2_tiles=None,
                 dt.datetime.strptime(x[0][17:25], '%Y%m%d').strftime('%Y%m%d_%j'),
                 x[0][:4]),
             axis=1)
-        input_df.set_index('QUICKLOOK', drop=True, inplace=True)
+        input_df.set_index([wrs2_tile_col, 'QUICKLOOK'], drop=True, inplace=True)
 
         if id_type.lower() == 'short':
             input_df['temp_id'] = input_df[[product_id_col]].apply(
                 lambda x: '{}_{}_{}'.format(x[0][0:4], x[0][10:16], x[0][17:25]),
                 axis=1)
-            quicklook_ids.update(input_df['temp_id'].to_dict())
+            update_dict = input_df['temp_id'].to_dict()
         else:
-            quicklook_ids.update(input_df[product_id_col].to_dict())
+            update_dict = input_df[product_id_col].to_dict()
+
+        for [wrs2_tile, image_id], product_id in update_dict.items():
+            quicklook_ids[wrs2_tile][image_id] = product_id
 
     logging.debug('\nQuicklook PRODUCT_ID lookup:')
     logging.debug(pprint.pformat(quicklook_ids))
@@ -173,13 +176,15 @@ def main(csv_folder, quicklook_folder, output_folder, wrs2_tiles=None,
             try:
                 y, m, d, doy, landsat = quicklook_re.match(name).groups()
             except Exception as e:
+                logging.debug('  {} - quicklook_re exception, skipping'.format(
+                    name))
                 continue
 
             # Look up PRODUCT_ID/SCENE_ID using metadata CSV data
             try:
-                product_id = quicklook_ids[name]
+                product_id = quicklook_ids[wrs2_tile][name]
             except:
-                logging.debug('  {} - skip list, skipping'.format(
+                logging.debug('  {} - quicklook_id exception, skipping'.format(
                     quicklook_ids))
                 continue
             if input_skip_list and product_id in input_skip_list:

metadata_csv_filter.py

@@ -1,8 +1,8 @@
 import argparse
 import logging
 import os
+import pprint
 import re
-# import shutil
 import sys
 
 import pandas as pd
@@ -41,7 +41,6 @@ def main(csv_folder, wrs2_tiles=None, years=None, months=None,
     """
     logging.info('\nFilter/reducing Landsat Metadata CSV files')
 
-
     if wrs2_tiles is not None:
         wrs2_tile_list = sorted([
             x.strip() for w in wrs2_tiles for x in w.split(',') if x.strip()])