Skip to content

Commit

Permalink
Merge pull request #263 from nasa/259_rtc_s1_gamma_integration
Browse files Browse the repository at this point in the history 
RTC-S1 Gamma Integration
  • Loading branch information
@collinss-jpl
collinss-jpl authored Apr 5, 2023
2 parents 3fe4c4f + 12ed47a commit 1002e36
Show file tree
Hide file tree
Showing 9 changed files with 338 additions and 219 deletions.
2 changes: 1 addition & 1 deletion .ci/scripts/build_rtc_s1.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ BUILD_DATE_TIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
# defaults, SAS image should be updated as necessary for new image releases from ADT
[ -z "${WORKSPACE}" ] && WORKSPACE=$(realpath $(dirname $(realpath $0))/../..)
[ -z "${TAG}" ] && TAG="${USER}-dev"
[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/rtc:beta_0.2.1"
[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/rtc:gamma_0.3"

echo "WORKSPACE: $WORKSPACE"
echo "IMAGE: $IMAGE"
Expand Down
86 changes: 66 additions & 20 deletions ...tc_s1_sample_runconfig-v2.0.0-er.5.0.yaml → ...tc_s1_sample_runconfig-v2.0.0-rc.1.0.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Sample RunConfig for use with the RTC-S1 PGE v2.0.0-er.5.0
# Sample RunConfig for use with the RTC-S1 PGE v2.0.0-rc.1.0
# This RunConfig should require minimal changes in order to be used with the
# OPERA PCM.

Expand Down Expand Up @@ -139,14 +139,13 @@ RunConfig:
dynamic_ancillary_file_group:
# Digital elevation model
dem_file: /home/rtc_user/input_dir/dem.tif
dem_description: Digital Elevation Model (DEM) for the NASA OPERA project (v1.0) based on the Copernicus DEM 30-m and Copernicus 90-m referenced to the WGS84 ellipsoid

static_ancillary_file_group:
# burst database sqlite file
burst_database_file: /home/rtc_user/input_dir/opera_burst_database_deploy_2022_1212.sqlite3

product_group:
processing_type: 'NOMINAL'

product_version: 1.0

# This should match the path used for OutputProductPath
Expand All @@ -161,14 +160,31 @@ RunConfig:
# These field determines the intermediate file name used
# by the SAS for its output products. These products will
# be renamed by the PGE to match the OPERA file name conventions.
product_id: rtc_product
product_id: OPERA_L2_RTC-S1_T{burst_id}

# RTC-S1 imagery
save_bursts: True

# Save mosaic of RTC-S1 bursts
save_mosaics: False

# Save browse image(s)
save_browse: True

output_imagery_format: COG
output_imagery_compression: ZSTD
output_imagery_nbits: 16

# Optional. Save secondary layers (e.g., inc. angle) within
# the HDF5 file
save_secondary_layers_as_hdf5: False

# Save RTC-S1 metadata in the HDF5 format
# Optional for `output_imagery_format` equal to 'ENVI', 'GTiff', or
# 'COG', and enabled by default for `output_imagery_format` equal
# to 'HDF5' or 'NETCDF' or `save_secondary_layers_as_hdf5` is True
save_metadata: True

primary_executable:
# This should match the value used for ProductIdentifier
product_type: RTC_S1
Expand All @@ -180,6 +196,7 @@ RunConfig:
# Check if ancillary input covers entirely output products
check_ancillary_inputs_coverage: True

# Polarization channels to process.
polarization: dual-pol

# Options to run geo2rdr
Expand All @@ -192,6 +209,9 @@ RunConfig:
threshold: 1.0e-7
numiter: 25

# DEM interpolation method
dem_interpolation_method: biquintic

# Apply absolute radiometric correction
apply_absolute_radiometric_correction: True

Expand All @@ -208,7 +228,6 @@ RunConfig:
apply_dry_tropospheric_delay_correction: True

# OPTIONAL - to control behavior of RTC module
# (only applicable if geocode.apply_rtc is True)
rtc:
# OPTIONAL - Choices:
# "gamma0" (default)
Expand All @@ -226,27 +245,40 @@ RunConfig:
input_terrain_radiometry: beta0

# OPTIONAL - Minimum RTC area factor in dB
rtc_min_value_db:
rtc_min_value_db: -30

# RTC DEM upsampling
dem_upsampling: 2

# OPTIONAL - to provide the number of processes when processing the bursts in parallel
# "0" means that the number will be automatically decided based on
# the number of cores, `OMP_NUM_THREADS` in environment setting,
# and the number of burst to process in runconfig
num_workers: 0

# OPTIONAL - Mechanism to specify output posting and DEM
geocoding:
# OPTIONAL -
# OPTIONAL - Apply RSLC metadata valid-samples sub-swath masking
apply_valid_samples_sub_swath_masking: True

# OPTIONAL - Apply shadow masking
apply_shadow_masking: True

# OPTIONAL - Algorithm type, area projection or
# interpolation: sinc, bilinear, bicubic, nearest, and biquintic
algorithm_type: area_projection

# OPTIONAL - Choices: "single_block", "geogrid", "geogrid_radargrid", and "auto" (default)
# OPTIONAL - Choices: "single_block", "geogrid", "geogrid_and_radargrid", and "auto" (default)
memory_mode: auto

# OPTIONAL - Processing upsampling factor applied to input geogrid
geogrid_upsampling: 1

# Save the incidence angle
save_incidence_angle: False
save_incidence_angle: True

# Save the local-incidence angle
save_local_inc_angle: False
save_local_inc_angle: True

# Save the projection angle
save_projection_angle: False
Expand All @@ -258,16 +290,16 @@ RunConfig:
save_range_slope: False

# Save the number of looks used to compute GCOV
save_nlooks: False
save_nlooks: True

# Save the RTC area factor used to compute GCOV
save_rtc_anf: False
save_rtc_anf: True

# Save interpolated DEM used to compute GCOV
save_dem: False

# Save layover shadow mask
save_layover_shadow_mask: False
save_layover_shadow_mask: True

# OPTIONAL - Absolute radiometric correction
abs_rad_cal: 1
Expand All @@ -278,18 +310,32 @@ RunConfig:
# OPTIONAL - Clip values below threshold
clip_min:

# OPTIONAL - Double sampling of the radar-grid
# input sampling in the range direction
# Double SLC sampling in the range direction
upsample_radargrid: False

output_epsg:
x_posting: 30
y_posting: 30
x_snap: 30
y_snap: 30
top_left:
x:
y:
bottom_right:
x:
y:

browse_image_group:

# If neither height or width parameters are provided, the browse
# image is generated with the same pixel spacing of the RTC-S1
# imagery (burst or mosaic).

# If the height parameter is provided but the width is not provided,
# a new width is assigned in order to keep the aspect ratio
# of the RTC-S1 geographic grid.

# Conversely, if the width parameter is provided but the height is not,
# a new height is assigned in order to keep the aspect ratio
# of the RTC-S1 geographic grid.

# Height in pixels for the PNG browse image of RTC-S1 bursts.
browse_image_burst_height: 1024

# Height in pixels for the PNG browse image of RTC-S1 mosaics.
browse_image_mosaic_height: 1024
125 changes: 103 additions & 22 deletions src/opera/pge/rtc_s1/rtc_s1_pge.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
from opera.util.error_codes import ErrorCode
from opera.util.input_validation import validate_slc_s1_inputs
from opera.util.metadata_utils import get_rtc_s1_product_metadata
from opera.util.metadata_utils import get_sensor_from_spacecraft_name
from opera.util.render_jinja2 import render_jinja2
from opera.util.time import get_time_for_filename

Expand Down Expand Up @@ -94,7 +95,8 @@ def _validate_output(self):
if not dirs and scratch_dir not in path: # Ignore files in 'output_dir' and scratch directory
out_dir_walk_dict[basename(path)] = files

output_format = self.runconfig.sas_config['runconfig']['groups']['product_group']['output_imagery_format']
sas_product_group = self.runconfig.sas_config['runconfig']['groups']['product_group']
output_format = sas_product_group['output_imagery_format']

if output_format == 'NETCDF':
expected_ext = ['nc']
Expand All @@ -103,6 +105,11 @@ def _validate_output(self):
elif output_format in ('GTiff', 'COG', 'ENVI'):
expected_ext = ['tiff', 'tif', 'h5']

save_browse = sas_product_group['save_browse']

if save_browse:
expected_ext.append('png')

# Verify: files in subdirectories, file length, and proper extension.
for dir_name_key, file_names in out_dir_walk_dict.items():
if len(file_names) == 0:
Expand Down Expand Up @@ -221,12 +228,16 @@ def _rtc_filename(self, inter_filename):
# Use doppler start time as the acq time and convert it to our format
# used for file naming
acquisition_time = product_metadata['identification']['zeroDopplerStartTime']

if not acquisition_time.endswith('Z'):
acquisition_time += 'Z'

acquisition_time = get_time_for_filename(
datetime.strptime(acquisition_time, "%Y-%m-%dT%H:%M:%S.%f")
datetime.strptime(acquisition_time, "%Y-%m-%dT%H:%M:%S.%fZ")
)

# Get the sensor (should be either S1A or S1B)
sensor = product_metadata['identification']['missionId']
sensor = get_sensor_from_spacecraft_name(product_metadata['identification']['platform'])

# Spacing is assumed to be identical in both X and Y direction
spacing = int(product_metadata['frequencyA']['xCoordinateSpacing'])
Expand All @@ -249,6 +260,54 @@ def _rtc_filename(self, inter_filename):

return rtc_filename

def _static_layer_filename(self, inter_filename):
"""
Returns the final file name for the static layer RTC product which
may be optionally produced by this PGE. There are currently 5 static layer
products which may be produced by this PGE, each identified by a
static layer name appended to the end of the intermediate filename.
The filename for static layer RTC products consists of:
<RTC filename>_<Static layer name>.tif
Where <RTC filename> is returned by RtcS1PostProcessorMixin._rtc_filename(),
and <Static layer name> is the identifier for the specific static layer
as parsed from the intermediate filename.
Parameters
----------
inter_filename : str
The intermediate filename of the static layer output product to generate
a filename for. This parameter may is used to derive the core RTC
file name component, to which the particular static layer name (nlooks,
shadow_mask, etc...) is appended to denote the type.
Returns
-------
static_layer_filename : str
The file name to assign to static layer GeoTIFF product(s) created
by this PGE.
"""
filename, ext = os.path.splitext(basename(inter_filename))

# The name of the static layer should always follow the product version
# within the intermediate filename
sas_product_group = self.runconfig.sas_config['runconfig']['groups']['product_group']
product_version = str(sas_product_group['product_version'])

if not product_version.startswith('v'):
product_version = f"v{product_version}"

static_layer_name = filename.split(product_version)[-1]

rtc_filename = self._rtc_filename(inter_filename)

static_layer_filename = f"{rtc_filename}{static_layer_name}.tif"

return static_layer_filename

def _rtc_geotiff_filename(self, inter_filename):
"""
Returns the file name to use for GeoTIFF format RTC products produced
Expand Down Expand Up @@ -285,6 +344,36 @@ def _rtc_geotiff_filename(self, inter_filename):

return f"{rtc_filename}_{polarization}.tif"

def _browse_filename(self, inter_filename):
"""
Returns the final file name of the PNG browse image product which may
be optionally produced by this PGE.
The filename for RTC metadata products consists of:
<RTC filename>_BROWSE.png
Where <RTC filename> is returned by RtcS1PostProcessorMixin._rtc_filename().
Parameters
----------
inter_filename : str
The intermediate filename of the output product to generate
a filename for. This parameter may be used to inspect the file
in order to derive any necessary components of the returned filename.
Returns
-------
browse_filename : str
The file name to assign to browse product created by this PGE.
"""
rtc_filename = self._rtc_filename(inter_filename)

browse_filename = f"{rtc_filename}_BROWSE.png"

return browse_filename

def _rtc_metadata_filename(self, inter_filename):
"""
Returns the file name to use for RTC metadata products produced by this PGE.
Expand Down Expand Up @@ -342,7 +431,7 @@ def _ancillary_filename(self):
product_metadata = list(self._burst_metadata_cache.values())[0]

# Get the sensor (should be either S1A or S1B)
sensor = product_metadata['identification']['missionId']
sensor = get_sensor_from_spacecraft_name(product_metadata['identification']['platform'])

# Spacing is assumed to be identical in both X and Y direction
spacing = int(product_metadata['frequencyA']['xCoordinateSpacing'])
Expand Down Expand Up @@ -470,21 +559,6 @@ def _collect_rtc_product_metadata(self, metadata_product):
output_product_metadata['frequencyA']['frequencyALength'] = len(output_product_metadata['frequencyA']
['yCoordinates'])

# TODO: the following fields seems to be missing in the interface delivery products,
# but are documented, remove these kludges once they are actually available
if 'azimuthBandwidth' not in output_product_metadata['frequencyA']:
output_product_metadata['frequencyA']['azimuthBandwidth'] = 12345678.9

if 'noiseCorrectionFlag' not in output_product_metadata['frequencyA']:
output_product_metadata['frequencyA']['noiseCorrectionFlag'] = False

if 'plannedDatatakeId' not in output_product_metadata['identification']:
output_product_metadata['identification']['plannedDatatakeId'] = ['datatake1', 'datatake2']

if 'plannedObservationId' not in output_product_metadata['identification']:
output_product_metadata['identification']['plannedObservationId'] = ['obs1', 'obs2']
# TODO: end kludges

return output_product_metadata

def _create_custom_metadata(self):
Expand Down Expand Up @@ -680,10 +754,10 @@ class RtcS1Executor(RtcS1PreProcessorMixin, RtcS1PostProcessorMixin, PgeExecutor
LEVEL = "L2"
"""Processing Level for RTC-S1 Products"""

PGE_VERSION = "2.0.0-er.5.1"
PGE_VERSION = "2.0.0-rc.1.0"
"""Version of the PGE (overrides default from base_pge)"""

SAS_VERSION = "0.2.1" # Beta release https://github.com/opera-adt/RTC/releases/tag/v0.2.1
SAS_VERSION = "0.3" # Gamma release https://github.com/opera-adt/RTC/releases/tag/v0.3
"""Version of the SAS wrapped by this PGE, should be updated as needed"""

SOURCE = "S1"
Expand All @@ -692,7 +766,14 @@ def __init__(self, pge_name, runconfig_path, **kwargs):
super().__init__(pge_name, runconfig_path, **kwargs)

self.rename_by_pattern_map = {
"*.tif": self._rtc_geotiff_filename,
"*_VV.tif": self._rtc_geotiff_filename,
"*_VH.tif": self._rtc_geotiff_filename,
"*_rtc_anf.tif": self._static_layer_filename,
"*_nlooks.tif": self._static_layer_filename,
"*_local_incidence_angle.tif": self._static_layer_filename,
"*_layover_shadow_mask.tif": self._static_layer_filename,
"*_incidence_angle.tif": self._static_layer_filename,
"*.png": self._browse_filename,
"*.h5": self._rtc_metadata_filename,
"*.nc": self._rtc_metadata_filename
}
Loading

0 comments on commit 1002e36

Please sign in to comment.