From 2cc0e21e4277ba5a929ec5b95e7c3d054d441cb5 Mon Sep 17 00:00:00 2001 From: kim Date: Thu, 21 Mar 2024 13:17:06 -0800 Subject: [PATCH 1/6] adds shortname search keyword support --- SearchAPI/CMR/Translate/input_map.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/SearchAPI/CMR/Translate/input_map.py b/SearchAPI/CMR/Translate/input_map.py index b3bdfd90..fc879799 100644 --- a/SearchAPI/CMR/Translate/input_map.py +++ b/SearchAPI/CMR/Translate/input_map.py @@ -57,7 +57,8 @@ def input_map(): 'absoluteburstid': ['attribute[]', 'int,BURST_ID_ABSOLUTE,{0}', parse_int_list], 'fullburstid': ['attribute[]', 'string,BURST_ID_FULL,{0}', parse_string_list], 'operaburstid': ['attribute[]', 'string,OPERA_BURST_ID,{0}', parse_string_list], - 'dataset': [None, '{0}', parse_string_list] + 'dataset': [None, '{0}', parse_string_list], + 'shortname': ['shortName', '{0}', parse_string_list] } return parameter_map From d01ea13020ce1772da413ff7d15a1a8c016bebd5 Mon Sep 17 00:00:00 2001 From: kim Date: Thu, 21 Mar 2024 17:15:48 -0800 Subject: [PATCH 2/6] checks length of absolute orbit before assignment in geojson output. check form data for cmr token if missing from args as fallback --- SearchAPI/CMR/Output/geojson.py | 14 ++++++++++---- SearchAPI/CMR/SubQuery.py | 3 +++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/SearchAPI/CMR/Output/geojson.py b/SearchAPI/CMR/Output/geojson.py index 8178162f..981b572c 100644 --- a/SearchAPI/CMR/Output/geojson.py +++ b/SearchAPI/CMR/Output/geojson.py @@ -58,13 +58,19 @@ def getItem(self, p): except TypeError: pass + if p.get('absoluteOrbit') is not None and len(p.get('absoluteOrbit')): + p['absoluteOrbit'] = p['absoluteOrbit'][0] + + coordinates = [] + + if p.get('shape') is not None: + coordinates = [[float(c['lon']), float(c['lat'])] for c in p.get('shape')] + result = { 'type': 'Feature', 'geometry': { 'type': 'Polygon', - 'coordinates': [ - [[float(c['lon']), float(c['lat'])] for c in p['shape']] - ] + 'coordinates': coordinates }, 'properties': { 'beamModeType': p['beamModeType'], @@ -82,7 +88,7 @@ def getItem(self, p): 'insarStackId': p['insarGrouping'], 'md5sum': p['md5sum'], 'offNadirAngle': p['offNadirAngle'], - 'orbit': p['absoluteOrbit'][0], + 'orbit': p['absoluteOrbit'], 'pathNumber': p['relativeOrbit'], 'platform': p['platform'], 'pointingAngle': p['pointingAngle'], diff --git a/SearchAPI/CMR/SubQuery.py b/SearchAPI/CMR/SubQuery.py index 1ac0a4e7..00087a00 100644 --- a/SearchAPI/CMR/SubQuery.py +++ b/SearchAPI/CMR/SubQuery.py @@ -25,6 +25,9 @@ def __init__(self, req_fields, params, extra_params): self.headers = {} token = request.args.get("cmr_token") + if token is None: + token = request.form.get('cmr_token') + if token != None: self.headers['Authorization'] = f'Bearer {token}' From e520313573183c06bc278dd71315ced481aa76b3 Mon Sep 17 00:00:00 2001 From: kim Date: Fri, 22 Mar 2024 11:52:59 -0800 Subject: [PATCH 3/6] shortname will dodge the subquery system --- SearchAPI/CMR/Query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SearchAPI/CMR/Query.py b/SearchAPI/CMR/Query.py index 63d1d6bd..77b3f146 100644 --- a/SearchAPI/CMR/Query.py +++ b/SearchAPI/CMR/Query.py @@ -114,7 +114,7 @@ def chunk_list(source_list, n): if chunk_type in params: params[chunk_type] = chunk_list(list(set(params[chunk_type])), 500) # distinct and split - list_param_names = ['platform', 'collections'] # these parameters will dodge the subquery system + list_param_names = ['platform', 'collections', 'shortname'] # these parameters will dodge the subquery system for k, v in params.items(): if k in list_param_names: From 0e32a0f2cd202a4b1cae04c09af2ab0bd1003837 Mon Sep 17 00:00:00 2001 From: kim Date: Wed, 27 Mar 2024 11:02:47 -0700 Subject: [PATCH 4/6] adds s3urls to jsonlite and jsonlite2 endpoints --- SearchAPI/CMR/Output/jsonlite.py | 6 +++++- SearchAPI/CMR/Output/jsonlite2.py | 3 +++ SearchAPI/CMR/Translate/parse_cmr_response.py | 9 ++++++++- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/SearchAPI/CMR/Output/jsonlite.py b/SearchAPI/CMR/Output/jsonlite.py index 99795463..e6b56117 100644 --- a/SearchAPI/CMR/Output/jsonlite.py +++ b/SearchAPI/CMR/Output/jsonlite.py @@ -42,7 +42,8 @@ def req_fields_jsonlite(): 'subswath', 'pgeVersion', 'operaBurstID', - 'additionalUrls' + 'additionalUrls', + 's3Urls' ] return fields @@ -186,4 +187,7 @@ def getItem(self, p): if p.get('validityStartDate'): result['opera']['validityStartDate'] = p.get('validityStartDate') + if p.get('platform') == 'NISAR': + result['s3Urls'] = p.get('s3Urls', []) + return result diff --git a/SearchAPI/CMR/Output/jsonlite2.py b/SearchAPI/CMR/Output/jsonlite2.py index b403e566..2758f963 100644 --- a/SearchAPI/CMR/Output/jsonlite2.py +++ b/SearchAPI/CMR/Output/jsonlite2.py @@ -61,4 +61,7 @@ def getItem(self, p): if p.get('opera') is not None: result['s1o'] = p['opera'] + if p.get('platform') == 'NISAR': + result['s3du'] = p.get('s3Urls', []) + return result diff --git a/SearchAPI/CMR/Translate/parse_cmr_response.py b/SearchAPI/CMR/Translate/parse_cmr_response.py index 29b2c0e9..c4345cf9 100644 --- a/SearchAPI/CMR/Translate/parse_cmr_response.py +++ b/SearchAPI/CMR/Translate/parse_cmr_response.py @@ -217,7 +217,14 @@ def float_or_none(a): if 'STATIC' in result['processingLevel']: result['validityStartDate'] = get_val('./Temporal/SingleDateTime') - + if result.get('platform', '') == 'NISAR': + accessUrls = [url for url in get_all_vals('./OnlineAccessURLs/OnlineAccessURL/URL') if not url.endswith('.md5') and not url.startswith('s3://') and not 's3credentials' in url] + OnlineResources = [url for url in get_all_vals('./OnlineResources/OnlineResource/URL') if not url.endswith('.md5') and not url.startswith('s3://') and not 's3credentials' in url] + result['additionalUrls'] = list(set([*accessUrls, *OnlineResources])) + result['s3Urls'] = list(set( + *[url for url in get_all_vals('./OnlineResources/OnlineAccessURL/URL') if not url.endswith('.md5') and (url.startswith('s3://') or 's3credentials' in url)], + *[url for url in get_all_vals('./OnlineResources/OnlineResource/URL') if not url.endswith('.md5') and (url.startswith('s3://') or 's3credentials' in url)] + )) return result From ba3bd35e7098d72cb3f05b90e3eeeacd7413be0e Mon Sep 17 00:00:00 2001 From: kim Date: Mon, 1 Apr 2024 08:32:42 -0800 Subject: [PATCH 5/6] reveals s3urls and additionalUrls for nisar products --- SearchAPI/CMR/Output/jsonlite.py | 5 ++++- SearchAPI/CMR/Output/jsonlite2.py | 4 ++-- SearchAPI/CMR/Translate/fields.py | 1 + SearchAPI/CMR/Translate/parse_cmr_response.py | 13 +++++++++++-- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/SearchAPI/CMR/Output/jsonlite.py b/SearchAPI/CMR/Output/jsonlite.py index e6b56117..660440a7 100644 --- a/SearchAPI/CMR/Output/jsonlite.py +++ b/SearchAPI/CMR/Output/jsonlite.py @@ -188,6 +188,9 @@ def getItem(self, p): result['opera']['validityStartDate'] = p.get('validityStartDate') if p.get('platform') == 'NISAR': - result['s3Urls'] = p.get('s3Urls', []) + result['nisar'] = { + 'additionalUrls': p.get('additionalUrls', []), + 's3Urls': p.get('s3Urls', []) + } return result diff --git a/SearchAPI/CMR/Output/jsonlite2.py b/SearchAPI/CMR/Output/jsonlite2.py index 2758f963..5cdff707 100644 --- a/SearchAPI/CMR/Output/jsonlite2.py +++ b/SearchAPI/CMR/Output/jsonlite2.py @@ -61,7 +61,7 @@ def getItem(self, p): if p.get('opera') is not None: result['s1o'] = p['opera'] - if p.get('platform') == 'NISAR': - result['s3du'] = p.get('s3Urls', []) + if p.get('nisar') is not None: + result['nsr'] = p['nisar'] return result diff --git a/SearchAPI/CMR/Translate/fields.py b/SearchAPI/CMR/Translate/fields.py index b9f5bebe..f639ff28 100644 --- a/SearchAPI/CMR/Translate/fields.py +++ b/SearchAPI/CMR/Translate/fields.py @@ -61,6 +61,7 @@ def get_field_paths(): 'track': attr_path('PATH_NUMBER'), 'pgeVersion': "./PGEVersionClass/PGEVersion", 'additionalUrls': "./OnlineAccessURLs", + 's3Urls': "./OnlineAccessURLs", # BURST FIELDS 'absoluteBurstID': attr_path('BURST_ID_ABSOLUTE'), diff --git a/SearchAPI/CMR/Translate/parse_cmr_response.py b/SearchAPI/CMR/Translate/parse_cmr_response.py index c4345cf9..08528328 100644 --- a/SearchAPI/CMR/Translate/parse_cmr_response.py +++ b/SearchAPI/CMR/Translate/parse_cmr_response.py @@ -221,9 +221,18 @@ def float_or_none(a): accessUrls = [url for url in get_all_vals('./OnlineAccessURLs/OnlineAccessURL/URL') if not url.endswith('.md5') and not url.startswith('s3://') and not 's3credentials' in url] OnlineResources = [url for url in get_all_vals('./OnlineResources/OnlineResource/URL') if not url.endswith('.md5') and not url.startswith('s3://') and not 's3credentials' in url] result['additionalUrls'] = list(set([*accessUrls, *OnlineResources])) + + accessUrls = get_all_vals('./OnlineAccessURLs/OnlineAccessURL/URL') + if accessUrls is None: + accessUrls = [] + resourceUrls = get_all_vals('./OnlineResources/OnlineResource/URL') + if resourceUrls is None: + resourceUrls = [] + result['s3Urls'] = list(set( - *[url for url in get_all_vals('./OnlineResources/OnlineAccessURL/URL') if not url.endswith('.md5') and (url.startswith('s3://') or 's3credentials' in url)], - *[url for url in get_all_vals('./OnlineResources/OnlineResource/URL') if not url.endswith('.md5') and (url.startswith('s3://') or 's3credentials' in url)] + [*[url for url in accessUrls if not url.endswith('.md5') and (url.startswith('s3://') or 's3credentials' in url)], + *[url for url in resourceUrls if not url.endswith('.md5') and (url.startswith('s3://') or 's3credentials' in url)] + ] )) return result From 61f9112d67d47b5b295fe2ca78f18f09e9cb17e6 Mon Sep 17 00:00:00 2001 From: kim Date: Tue, 2 Apr 2024 14:34:48 -0800 Subject: [PATCH 6/6] refactors some url parsing logic --- SearchAPI/CMR/Translate/parse_cmr_response.py | 44 +++++++++++-------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/SearchAPI/CMR/Translate/parse_cmr_response.py b/SearchAPI/CMR/Translate/parse_cmr_response.py index 08528328..83c93e31 100644 --- a/SearchAPI/CMR/Translate/parse_cmr_response.py +++ b/SearchAPI/CMR/Translate/parse_cmr_response.py @@ -205,11 +205,31 @@ def float_or_none(a): result['downloadUrl'] = urls[0] result['fileName'] = result['granuleName'] + '.' + urls[0].split('.')[-1] + + + def get_all_urls(): + accessPath = './OnlineAccessURLs/OnlineAccessURL/URL' + resourcesPath = './OnlineResources/OnlineResource/URL' + + access_urls = get_all_vals(accessPath) + if access_urls is None: + access_urls = [] + + resource_urls = get_all_vals(resourcesPath) + if resource_urls is None: + resource_urls = [] + + return list(set([*access_urls, *resource_urls])) + + def get_http_urls(): + return [url for url in get_all_urls() if not url.endswith('.md5') and not url.startswith('s3://') and not 's3credentials' in url] + + def get_s3_urls(): + return [url for url in get_all_urls() if not url.endswith('.md5') and (url.startswith('s3://') or 's3credentials' in url)] + if result.get('product_file_id', '').startswith('OPERA'): result['beamMode'] = get_val(attr_path('BEAM_MODE')) - accessUrls = [url for url in get_all_vals('./OnlineAccessURLs/OnlineAccessURL/URL') if not url.endswith('.md5') and not url.startswith('s3://') and not 's3credentials' in url] - OnlineResources = [url for url in get_all_vals('./OnlineResources/OnlineResource/URL') if not url.endswith('.md5') and not url.startswith('s3://') and not 's3credentials' in url] - result['additionalUrls'] = list(set([*accessUrls, *OnlineResources])) + result['additionalUrls'] = get_http_urls() result['configurationName'] = "Interferometric Wide. 250 km swath, 5 m x 20 m spatial resolution and burst synchronization for interferometry. IW is considered to be the standard mode over land masses." if (providerbrowseUrls := get_all_vals('./AssociatedBrowseImageUrls/ProviderBrowseUrl/URL')): @@ -218,22 +238,8 @@ def float_or_none(a): if 'STATIC' in result['processingLevel']: result['validityStartDate'] = get_val('./Temporal/SingleDateTime') if result.get('platform', '') == 'NISAR': - accessUrls = [url for url in get_all_vals('./OnlineAccessURLs/OnlineAccessURL/URL') if not url.endswith('.md5') and not url.startswith('s3://') and not 's3credentials' in url] - OnlineResources = [url for url in get_all_vals('./OnlineResources/OnlineResource/URL') if not url.endswith('.md5') and not url.startswith('s3://') and not 's3credentials' in url] - result['additionalUrls'] = list(set([*accessUrls, *OnlineResources])) - - accessUrls = get_all_vals('./OnlineAccessURLs/OnlineAccessURL/URL') - if accessUrls is None: - accessUrls = [] - resourceUrls = get_all_vals('./OnlineResources/OnlineResource/URL') - if resourceUrls is None: - resourceUrls = [] - - result['s3Urls'] = list(set( - [*[url for url in accessUrls if not url.endswith('.md5') and (url.startswith('s3://') or 's3credentials' in url)], - *[url for url in resourceUrls if not url.endswith('.md5') and (url.startswith('s3://') or 's3credentials' in url)] - ] - )) + result['additionalUrls'] = get_http_urls() + result['s3Urls'] = get_s3_urls() return result