From c63f57b9d0d5504ea068eb7e3e29f09bfa74385a Mon Sep 17 00:00:00 2001 From: Jim Hofman Date: Wed, 2 Mar 2022 17:51:30 -0800 Subject: [PATCH 1/3] ticket56 - add ProductCounter --- src/opera/pge/base_pge.py | 6 +++--- src/opera/pge/dswx_pge.py | 7 ++++--- src/opera/test/pge/test_base_pge.py | 21 +++++++++++++++------ src/opera/test/pge/test_dswx_pge.py | 20 +++++++++++++++++++- 4 files changed, 41 insertions(+), 13 deletions(-) diff --git a/src/opera/pge/base_pge.py b/src/opera/pge/base_pge.py index ad656d5d..01a8f2f2 100644 --- a/src/opera/pge/base_pge.py +++ b/src/opera/pge/base_pge.py @@ -276,7 +276,7 @@ def _core_filename(self, inter_filename=None): The core file name component of the Base PGE consists of: - ___ + ____ Callers of this function are responsible for assignment of any other product-specific fields, such as the file extension. @@ -297,7 +297,8 @@ def _core_filename(self, inter_filename=None): """ time_tag = get_time_for_filename(self.production_datetime) - return f"{self.PROJECT}_{self.LEVEL}_{self.NAME}_{time_tag}" + return f"{self.PROJECT}_{self.LEVEL}_{self.NAME}_" \ + f"{time_tag}_{str(self.runconfig.product_counter).zfill(3)}" def _geotiff_filename(self, inter_filename): """ @@ -379,7 +380,6 @@ def _assign_filename(self, input_filepath, output_dir): """ file_extension = splitext(input_filepath)[-1] - # Lookup the specific rename function configured for the current file extension try: rename_function = self.rename_by_extension_map[file_extension] diff --git a/src/opera/pge/dswx_pge.py b/src/opera/pge/dswx_pge.py index 5396447c..fdc41243 100644 --- a/src/opera/pge/dswx_pge.py +++ b/src/opera/pge/dswx_pge.py @@ -180,7 +180,7 @@ def _core_filename(self, inter_filename=None): dataset_fields = get_hls_filename_fields(f"{dataset}.BAND.tif") source = dataset_fields['product'] - sensor = get_geotiff_spacecraft_name(inter_filename) + spacecraft_name = get_geotiff_spacecraft_name(inter_filename) tile_id = dataset_fields['tile_id'] timetag = dataset_fields['acquisition_time'] version = dataset_fields['collection_version'] @@ -188,12 +188,13 @@ def _core_filename(self, inter_filename=None): # Assign the core file to the cached class attribute self._cached_core_filename = ( - f"{self.PROJECT}_{self.LEVEL}_{self.NAME}_{source}_{sensor}_{tile_id}_{timetag}_{version}.{subversion}" + f"{self.PROJECT}_{self.LEVEL}_{self.NAME}_{source}_{spacecraft_name}_" + f"{tile_id}_{timetag}_{str(self.runconfig.product_counter).zfill(3)}_{version}.{subversion}" ) return self._cached_core_filename - def _geotiff_filename(self, inter_filename): + def _geotiff_filename(self, inter_filename=None): """ Returns the file name to use for GeoTIFF's produced by the DSWx PGE. diff --git a/src/opera/test/pge/test_base_pge.py b/src/opera/test/pge/test_base_pge.py index 73ecac5b..870bec18 100644 --- a/src/opera/test/pge/test_base_pge.py +++ b/src/opera/test/pge/test_base_pge.py @@ -21,6 +21,7 @@ Unit tests for the pge/base_pge.py module. """ import os +import re import tempfile import unittest from io import StringIO @@ -35,6 +36,9 @@ class BasePgeTestCase(unittest.TestCase): """Base test class using unittest""" + test_dir = None + starting_dir = None + @classmethod def setUpClass(cls) -> None: """Set up class method: set up directories for testing""" @@ -173,12 +177,6 @@ def test_base_pge_w_failing_sas(self): self.assertIn(f"failed with exit code {expected_error_code}", log_contents) - def test_geotiff_filename(self): - """Test _geotiff_filename() method""" - runconfig_path = join(self.data_dir, 'test_sas_qa_config.yaml') - pge = PgeExecutor(pge_name='PgeQATest', runconfig_path=runconfig_path) - pge._geotiff_filename("TestName.ext") - def test_sas_qa_execution(self): """ Test execution of the PgeExecutor class using a test RunConfig that invokes @@ -211,6 +209,17 @@ def test_sas_qa_execution(self): self.assertIn('sas.elapsed_seconds:', log_contents) self.assertIn('sas.qa.elapsed_seconds:', log_contents) + def test_geotiff_filename(self): + """Test _geotiff_filename() method""" + runconfig_path = join(self.data_dir, 'test_sas_qa_config.yaml') + pge = PgeExecutor(pge_name='BasePgeFilenameTest', runconfig_path=runconfig_path) + pge._initialize_logger() + pge._load_runconfig() + name = "TestName.tif" + file_name = pge._geotiff_filename(name) + file_name_regex = rf'{pge.PROJECT}_{pge.LEVEL}_BasePge_\d{{8}}T\d{{6}}_\d{{3}}_{name}{{1,2}}?' + self.assertEqual(re.match(file_name_regex, file_name).group(), file_name) + if __name__ == "__main__": unittest.main() diff --git a/src/opera/test/pge/test_dswx_pge.py b/src/opera/test/pge/test_dswx_pge.py index 80745039..8a4ab633 100644 --- a/src/opera/test/pge/test_dswx_pge.py +++ b/src/opera/test/pge/test_dswx_pge.py @@ -22,6 +22,7 @@ """ import glob import os +import re import tempfile import unittest from io import StringIO @@ -35,7 +36,7 @@ import opera.util.img_utils from opera.pge import DSWxExecutor, RunConfig from opera.util import PgeLogger -from opera.util.img_utils import MockGdal +from opera.util.img_utils import MockGdal, get_geotiff_metadata class DSWxPgeTestCase(unittest.TestCase): @@ -294,6 +295,23 @@ def test_dswx_pge_output_validation(self): if os.path.exists(test_runconfig_path): os.unlink(test_runconfig_path) + def test_geotiff_filename(self): + """Test _geotiff_filename() method""" + runconfig_path = join(self.data_dir, 'test_dswx_hls_config.yaml') + + pge = DSWxExecutor(pge_name="DSWxPgeTest", runconfig_path=runconfig_path) + + pge.run() + + image_files = glob.glob(join(pge.runconfig.output_product_path, "*.tif")) + for i in range(len(image_files)): + file_name = pge._geotiff_filename(image_files[i]) + md = get_geotiff_metadata(image_files[i]) + file_name_regex = rf"{pge.PROJECT}_{pge.LEVEL}_{md['PRODUCT_TYPE']}_{md['PRODUCT_SOURCE']}_" \ + rf"{md['SPACECRAFT_NAME']}_{md['HLS_DATASET'].split('.')[2]}_\d{{8}}T\d{{6}}_\d{{3}}_" \ + rf"{'.'.join(md['HLS_DATASET'].split('.')[-2:])}.tif?" + self.assertEqual(re.match(file_name_regex, file_name).group(), file_name) + if __name__ == "__main__": unittest.main() From d4f0a0a7716dfaa598cd7fdedfa2f4b341ed16b7 Mon Sep 17 00:00:00 2001 From: Jim Hofman Date: Thu, 3 Mar 2022 10:58:25 -0800 Subject: [PATCH 2/3] changes after code walkthrough --- src/opera/pge/dswx_pge.py | 6 +++--- src/opera/test/pge/test_dswx_pge.py | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/opera/pge/dswx_pge.py b/src/opera/pge/dswx_pge.py index fdc41243..cca426ed 100644 --- a/src/opera/pge/dswx_pge.py +++ b/src/opera/pge/dswx_pge.py @@ -137,7 +137,7 @@ def _core_filename(self, inter_filename=None): The core file name component of the DSWx PGE consists of: - _______ + ________ Callers of this function are responsible for assignment of any other product-specific fields, such as the file extension. @@ -189,12 +189,12 @@ def _core_filename(self, inter_filename=None): # Assign the core file to the cached class attribute self._cached_core_filename = ( f"{self.PROJECT}_{self.LEVEL}_{self.NAME}_{source}_{spacecraft_name}_" - f"{tile_id}_{timetag}_{str(self.runconfig.product_counter).zfill(3)}_{version}.{subversion}" + f"{tile_id}_{timetag}_{version}.{subversion}_{str(self.runconfig.product_counter).zfill(3)}" ) return self._cached_core_filename - def _geotiff_filename(self, inter_filename=None): + def _geotiff_filename(self, inter_filename): """ Returns the file name to use for GeoTIFF's produced by the DSWx PGE. diff --git a/src/opera/test/pge/test_dswx_pge.py b/src/opera/test/pge/test_dswx_pge.py index 8a4ab633..426bec7d 100644 --- a/src/opera/test/pge/test_dswx_pge.py +++ b/src/opera/test/pge/test_dswx_pge.py @@ -295,6 +295,7 @@ def test_dswx_pge_output_validation(self): if os.path.exists(test_runconfig_path): os.unlink(test_runconfig_path) + @patch.object(opera.util.img_utils, "get_geotiff_metadata", get_geotiff_metadata_patch) def test_geotiff_filename(self): """Test _geotiff_filename() method""" runconfig_path = join(self.data_dir, 'test_dswx_hls_config.yaml') @@ -308,8 +309,8 @@ def test_geotiff_filename(self): file_name = pge._geotiff_filename(image_files[i]) md = get_geotiff_metadata(image_files[i]) file_name_regex = rf"{pge.PROJECT}_{pge.LEVEL}_{md['PRODUCT_TYPE']}_{md['PRODUCT_SOURCE']}_" \ - rf"{md['SPACECRAFT_NAME']}_{md['HLS_DATASET'].split('.')[2]}_\d{{8}}T\d{{6}}_\d{{3}}_" \ - rf"{'.'.join(md['HLS_DATASET'].split('.')[-2:])}.tif?" + rf"{md['SPACECRAFT_NAME']}_{md['HLS_DATASET'].split('.')[2]}_\d{{8}}T\d{{6}}_" \ + rf"{'.'.join(md['HLS_DATASET'].split('.')[-2:])}_\d{{3}}.tif?" self.assertEqual(re.match(file_name_regex, file_name).group(), file_name) From 0c2f4d91a2bc9311aee827157931dcec3ae6454e Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Mon, 7 Mar 2022 09:49:58 -0800 Subject: [PATCH 3/3] Fixed issue in test_geotiff_filename so test results are consistent both inside and outside of a docker container --- src/opera/test/pge/test_dswx_pge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/opera/test/pge/test_dswx_pge.py b/src/opera/test/pge/test_dswx_pge.py index 426bec7d..1eb60c24 100644 --- a/src/opera/test/pge/test_dswx_pge.py +++ b/src/opera/test/pge/test_dswx_pge.py @@ -307,7 +307,7 @@ def test_geotiff_filename(self): image_files = glob.glob(join(pge.runconfig.output_product_path, "*.tif")) for i in range(len(image_files)): file_name = pge._geotiff_filename(image_files[i]) - md = get_geotiff_metadata(image_files[i]) + md = self.get_geotiff_metadata_patch() file_name_regex = rf"{pge.PROJECT}_{pge.LEVEL}_{md['PRODUCT_TYPE']}_{md['PRODUCT_SOURCE']}_" \ rf"{md['SPACECRAFT_NAME']}_{md['HLS_DATASET'].split('.')[2]}_\d{{8}}T\d{{6}}_" \ rf"{'.'.join(md['HLS_DATASET'].split('.')[-2:])}_\d{{3}}.tif?"