RTC-S1 Gamma Integration

.ci/scripts/build_rtc_s1.sh

@@ -24,7 +24,7 @@ BUILD_DATE_TIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
 # defaults, SAS image should be updated as necessary for new image releases from ADT
 [ -z "${WORKSPACE}" ] && WORKSPACE=$(realpath $(dirname $(realpath $0))/../..)
 [ -z "${TAG}" ] && TAG="${USER}-dev"
-[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/rtc:beta_0.2.1"
+[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/rtc:gamma_0.3"
 
 echo "WORKSPACE: $WORKSPACE"
 echo "IMAGE: $IMAGE"

examples/rtc_s1_sample_runconfig-v2.0.0-er.5.0.yaml → examples/rtc_s1_sample_runconfig-v2.0.0-rc.1.0.yaml

@@ -1,4 +1,4 @@
-# Sample RunConfig for use with the RTC-S1 PGE v2.0.0-er.5.0
+# Sample RunConfig for use with the RTC-S1 PGE v2.0.0-rc.1.0
 # This RunConfig should require minimal changes in order to be used with the
 # OPERA PCM.
 
@@ -139,14 +139,13 @@ RunConfig:
                     dynamic_ancillary_file_group:
                         # Digital elevation model
                         dem_file: /home/rtc_user/input_dir/dem.tif
+                        dem_description: Digital Elevation Model (DEM) for the NASA OPERA project (v1.0) based on the Copernicus DEM 30-m and Copernicus 90-m referenced to the WGS84 ellipsoid
 
                     static_ancillary_file_group:
                         # burst database sqlite file
                         burst_database_file: /home/rtc_user/input_dir/opera_burst_database_deploy_2022_1212.sqlite3
 
                     product_group:
-                        processing_type: 'NOMINAL'
-
                         product_version: 1.0
 
                         # This should match the path used for OutputProductPath
@@ -161,14 +160,31 @@ RunConfig:
                         # These field determines the intermediate file name used
                         # by the SAS for its output products. These products will
                         # be renamed by the PGE to match the OPERA file name conventions.
-                        product_id: rtc_product
+                        product_id: OPERA_L2_RTC-S1_T{burst_id}
 
+                        # RTC-S1 imagery
                         save_bursts: True
+
+                        # Save mosaic of RTC-S1 bursts
                         save_mosaics: False
+
+                        # Save browse image(s)
+                        save_browse: True
+
                         output_imagery_format: COG
                         output_imagery_compression: ZSTD
                         output_imagery_nbits: 16
 
+                        # Optional. Save secondary layers (e.g., inc. angle) within
+                        # the HDF5 file
+                        save_secondary_layers_as_hdf5: False
+
+                        # Save RTC-S1 metadata in the HDF5 format
+                        # Optional for `output_imagery_format` equal to 'ENVI', 'GTiff', or
+                        # 'COG', and enabled by default for `output_imagery_format` equal
+                        # to 'HDF5' or 'NETCDF' or `save_secondary_layers_as_hdf5` is True
+                        save_metadata: True
+
                     primary_executable:
                         # This should match the value used for ProductIdentifier
                         product_type: RTC_S1
@@ -180,6 +196,7 @@ RunConfig:
                         # Check if ancillary input covers entirely output products
                         check_ancillary_inputs_coverage: True
 
+                        # Polarization channels to process.
                         polarization: dual-pol
 
                         # Options to run geo2rdr
@@ -192,6 +209,9 @@ RunConfig:
                             threshold: 1.0e-7
                             numiter: 25
 
+                        # DEM interpolation method
+                        dem_interpolation_method: biquintic
+
                         # Apply absolute radiometric correction
                         apply_absolute_radiometric_correction: True
 
@@ -208,7 +228,6 @@ RunConfig:
                         apply_dry_tropospheric_delay_correction: True
 
                         # OPTIONAL - to control behavior of RTC module
-                        # (only applicable if geocode.apply_rtc is True)
                         rtc:
                             # OPTIONAL - Choices:
                             # "gamma0" (default)
@@ -226,27 +245,40 @@ RunConfig:
                             input_terrain_radiometry: beta0
 
                             # OPTIONAL - Minimum RTC area factor in dB
-                            rtc_min_value_db:
+                            rtc_min_value_db: -30
 
                             # RTC DEM upsampling
                             dem_upsampling: 2
 
+                        # OPTIONAL - to provide the number of processes when processing the bursts in parallel
+                        # "0" means that the number will be automatically decided based on
+                        # the number of cores, `OMP_NUM_THREADS` in environment setting,
+                        # and the number of burst to process in runconfig
+                        num_workers: 0
+
                         # OPTIONAL - Mechanism to specify output posting and DEM
                         geocoding:
-                            # OPTIONAL -
+                            # OPTIONAL - Apply RSLC metadata valid-samples sub-swath masking
+                            apply_valid_samples_sub_swath_masking: True
+
+                            # OPTIONAL - Apply shadow masking
+                            apply_shadow_masking: True
+
+                            # OPTIONAL - Algorithm type, area projection or
+                            # interpolation: sinc, bilinear, bicubic, nearest, and biquintic
                             algorithm_type: area_projection
 
-                            # OPTIONAL - Choices: "single_block", "geogrid", "geogrid_radargrid", and "auto" (default)
+                            # OPTIONAL - Choices: "single_block", "geogrid", "geogrid_and_radargrid", and "auto" (default)
                             memory_mode: auto
 
                             # OPTIONAL - Processing upsampling factor applied to input geogrid
                             geogrid_upsampling: 1
 
                             # Save the incidence angle
-                            save_incidence_angle: False
+                            save_incidence_angle: True
 
                             # Save the local-incidence angle
-                            save_local_inc_angle: False
+                            save_local_inc_angle: True
 
                             # Save the projection angle
                             save_projection_angle: False
@@ -258,16 +290,16 @@ RunConfig:
                             save_range_slope: False
 
                             # Save the number of looks used to compute GCOV
-                            save_nlooks: False
+                            save_nlooks: True
 
                             # Save the RTC area factor used to compute GCOV
-                            save_rtc_anf: False
+                            save_rtc_anf: True
 
                             # Save interpolated DEM used to compute GCOV
                             save_dem: False
 
                             # Save layover shadow mask
-                            save_layover_shadow_mask: False
+                            save_layover_shadow_mask: True
 
                             # OPTIONAL - Absolute radiometric correction
                             abs_rad_cal: 1
@@ -278,18 +310,32 @@ RunConfig:
                             # OPTIONAL - Clip values below threshold
                             clip_min:
 
-                            # OPTIONAL - Double sampling of the radar-grid
-                            # input sampling in the range direction
+                            # Double SLC sampling in the range direction
                             upsample_radargrid: False
 
-                            output_epsg:
-                            x_posting: 30
-                            y_posting: 30
-                            x_snap: 30
-                            y_snap: 30
                             top_left:
                                 x:
                                 y:
                             bottom_right:
                                 x:
                                 y:
+
+                        browse_image_group:
+
+                            # If neither height or width parameters are provided, the browse
+                            # image is generated with the same pixel spacing of the RTC-S1
+                            # imagery (burst or mosaic).
+
+                            # If the height parameter is provided but the width is not provided,
+                            # a new width is assigned in order to keep the aspect ratio
+                            # of the RTC-S1 geographic grid.
+
+                            # Conversely, if the width parameter is provided but the height is not,
+                            # a new height is assigned in order to keep the aspect ratio
+                            # of the RTC-S1 geographic grid.
+
+                            # Height in pixels for the PNG browse image of RTC-S1 bursts.
+                            browse_image_burst_height: 1024
+
+                            # Height in pixels for the PNG browse image of RTC-S1 mosaics.
+                            browse_image_mosaic_height: 1024

src/opera/pge/rtc_s1/rtc_s1_pge.py

@@ -21,6 +21,7 @@
 from opera.util.error_codes import ErrorCode
 from opera.util.input_validation import validate_slc_s1_inputs
 from opera.util.metadata_utils import get_rtc_s1_product_metadata
+from opera.util.metadata_utils import get_sensor_from_spacecraft_name
 from opera.util.render_jinja2 import render_jinja2
 from opera.util.time import get_time_for_filename
 
@@ -94,7 +95,8 @@ def _validate_output(self):
             if not dirs and scratch_dir not in path:  # Ignore files in 'output_dir' and scratch directory
                 out_dir_walk_dict[basename(path)] = files
 
-        output_format = self.runconfig.sas_config['runconfig']['groups']['product_group']['output_imagery_format']
+        sas_product_group = self.runconfig.sas_config['runconfig']['groups']['product_group']
+        output_format = sas_product_group['output_imagery_format']
 
         if output_format == 'NETCDF':
             expected_ext = ['nc']
@@ -103,6 +105,11 @@ def _validate_output(self):
         elif output_format in ('GTiff', 'COG', 'ENVI'):
             expected_ext = ['tiff', 'tif', 'h5']
 
+        save_browse = sas_product_group['save_browse']
+
+        if save_browse:
+            expected_ext.append('png')
+
         # Verify: files in subdirectories, file length, and proper extension.
         for dir_name_key, file_names in out_dir_walk_dict.items():
             if len(file_names) == 0:
@@ -221,12 +228,16 @@ def _rtc_filename(self, inter_filename):
         # Use doppler start time as the acq time and convert it to our format
         # used for file naming
         acquisition_time = product_metadata['identification']['zeroDopplerStartTime']
+
+        if not acquisition_time.endswith('Z'):
+            acquisition_time += 'Z'
+
         acquisition_time = get_time_for_filename(
-            datetime.strptime(acquisition_time, "%Y-%m-%dT%H:%M:%S.%f")
+            datetime.strptime(acquisition_time, "%Y-%m-%dT%H:%M:%S.%fZ")
         )
 
         # Get the sensor (should be either S1A or S1B)
-        sensor = product_metadata['identification']['missionId']
+        sensor = get_sensor_from_spacecraft_name(product_metadata['identification']['platform'])
 
         # Spacing is assumed to be identical in both X and Y direction
         spacing = int(product_metadata['frequencyA']['xCoordinateSpacing'])
@@ -249,6 +260,54 @@ def _rtc_filename(self, inter_filename):
 
         return rtc_filename
 
+    def _static_layer_filename(self, inter_filename):
+        """
+        Returns the final file name for the static layer RTC product which
+        may be optionally produced by this PGE. There are currently 5 static layer
+        products which may be produced by this PGE, each identified by a
+        static layer name appended to the end of the intermediate filename.
+
+        The filename for static layer RTC products consists of:
+
+            <RTC filename>_<Static layer name>.tif
+
+        Where <RTC filename> is returned by RtcS1PostProcessorMixin._rtc_filename(),
+        and <Static layer name> is the identifier for the specific static layer
+        as parsed from the intermediate filename.
+
+        Parameters
+        ----------
+        inter_filename : str
+            The intermediate filename of the static layer output product to generate
+            a filename for. This parameter may is used to derive the core RTC
+            file name component, to which the particular static layer name (nlooks,
+            shadow_mask, etc...) is appended to denote the type.
+
+        Returns
+        -------
+        static_layer_filename : str
+            The file name to assign to static layer GeoTIFF product(s) created
+            by this PGE.
+
+        """
+        filename, ext = os.path.splitext(basename(inter_filename))
+
+        # The name of the static layer should always follow the product version
+        # within the intermediate filename
+        sas_product_group = self.runconfig.sas_config['runconfig']['groups']['product_group']
+        product_version = str(sas_product_group['product_version'])
+
+        if not product_version.startswith('v'):
+            product_version = f"v{product_version}"
+
+        static_layer_name = filename.split(product_version)[-1]
+
+        rtc_filename = self._rtc_filename(inter_filename)
+
+        static_layer_filename = f"{rtc_filename}{static_layer_name}.tif"
+
+        return static_layer_filename
+
     def _rtc_geotiff_filename(self, inter_filename):
         """
         Returns the file name to use for GeoTIFF format RTC products produced
@@ -285,6 +344,36 @@ def _rtc_geotiff_filename(self, inter_filename):
 
         return f"{rtc_filename}_{polarization}.tif"
 
+    def _browse_filename(self, inter_filename):
+        """
+        Returns the final file name of the PNG browse image product which may
+        be optionally produced by this PGE.
+
+        The filename for RTC metadata products consists of:
+
+            <RTC filename>_BROWSE.png
+
+        Where <RTC filename> is returned by RtcS1PostProcessorMixin._rtc_filename().
+
+        Parameters
+        ----------
+        inter_filename : str
+            The intermediate filename of the output product to generate
+            a filename for. This parameter may be used to inspect the file
+            in order to derive any necessary components of the returned filename.
+
+        Returns
+        -------
+        browse_filename : str
+            The file name to assign to browse product created by this PGE.
+
+        """
+        rtc_filename = self._rtc_filename(inter_filename)
+
+        browse_filename = f"{rtc_filename}_BROWSE.png"
+
+        return browse_filename
+
     def _rtc_metadata_filename(self, inter_filename):
         """
         Returns the file name to use for RTC metadata products produced by this PGE.
@@ -342,7 +431,7 @@ def _ancillary_filename(self):
         product_metadata = list(self._burst_metadata_cache.values())[0]
 
         # Get the sensor (should be either S1A or S1B)
-        sensor = product_metadata['identification']['missionId']
+        sensor = get_sensor_from_spacecraft_name(product_metadata['identification']['platform'])
 
         # Spacing is assumed to be identical in both X and Y direction
         spacing = int(product_metadata['frequencyA']['xCoordinateSpacing'])
@@ -470,21 +559,6 @@ def _collect_rtc_product_metadata(self, metadata_product):
         output_product_metadata['frequencyA']['frequencyALength'] = len(output_product_metadata['frequencyA']
                                                                         ['yCoordinates'])
 
-        # TODO: the following fields seems to be missing in the interface delivery products,
-        #       but are documented, remove these kludges once they are actually available
-        if 'azimuthBandwidth' not in output_product_metadata['frequencyA']:
-            output_product_metadata['frequencyA']['azimuthBandwidth'] = 12345678.9
-
-        if 'noiseCorrectionFlag' not in output_product_metadata['frequencyA']:
-            output_product_metadata['frequencyA']['noiseCorrectionFlag'] = False
-
-        if 'plannedDatatakeId' not in output_product_metadata['identification']:
-            output_product_metadata['identification']['plannedDatatakeId'] = ['datatake1', 'datatake2']
-
-        if 'plannedObservationId' not in output_product_metadata['identification']:
-            output_product_metadata['identification']['plannedObservationId'] = ['obs1', 'obs2']
-        # TODO: end kludges
-
         return output_product_metadata
 
     def _create_custom_metadata(self):
@@ -680,10 +754,10 @@ class RtcS1Executor(RtcS1PreProcessorMixin, RtcS1PostProcessorMixin, PgeExecutor
     LEVEL = "L2"
     """Processing Level for RTC-S1 Products"""
 
-    PGE_VERSION = "2.0.0-er.5.1"
+    PGE_VERSION = "2.0.0-rc.1.0"
     """Version of the PGE (overrides default from base_pge)"""
 
-    SAS_VERSION = "0.2.1"  # Beta release https://github.com/opera-adt/RTC/releases/tag/v0.2.1
+    SAS_VERSION = "0.3"  # Gamma release https://github.com/opera-adt/RTC/releases/tag/v0.3
     """Version of the SAS wrapped by this PGE, should be updated as needed"""
 
     SOURCE = "S1"
@@ -692,7 +766,14 @@ def __init__(self, pge_name, runconfig_path, **kwargs):
         super().__init__(pge_name, runconfig_path, **kwargs)
 
         self.rename_by_pattern_map = {
-            "*.tif": self._rtc_geotiff_filename,
+            "*_VV.tif": self._rtc_geotiff_filename,
+            "*_VH.tif": self._rtc_geotiff_filename,
+            "*_rtc_anf.tif": self._static_layer_filename,
+            "*_nlooks.tif": self._static_layer_filename,
+            "*_local_incidence_angle.tif": self._static_layer_filename,
+            "*_layover_shadow_mask.tif": self._static_layer_filename,
+            "*_incidence_angle.tif": self._static_layer_filename,
+            "*.png": self._browse_filename,
             "*.h5": self._rtc_metadata_filename,
             "*.nc": self._rtc_metadata_filename
         }