From 6161eaab65a8739564e92573ab4feb96ee48f550 Mon Sep 17 00:00:00 2001 From: Matt Anikiej Date: Tue, 30 Jul 2024 13:11:34 -0700 Subject: [PATCH 1/3] script to create lola pds4 --- .gitignore | 1 + README.md | 10 ++ setup.cfg | 1 + src/pds/registry/utils/geostac/__init__.py | 1 + .../utils/geostac/create_lola_pds4.py | 165 ++++++++++++++++++ .../utils/geostac/templates/__init__.py | 1 + .../templates/product-browse-template.xml | 41 +++++ .../templates/product-external-template.xml | 87 +++++++++ 8 files changed, 307 insertions(+) create mode 100644 src/pds/registry/utils/geostac/__init__.py create mode 100644 src/pds/registry/utils/geostac/create_lola_pds4.py create mode 100644 src/pds/registry/utils/geostac/templates/__init__.py create mode 100644 src/pds/registry/utils/geostac/templates/product-browse-template.xml create mode 100644 src/pds/registry/utils/geostac/templates/product-external-template.xml diff --git a/.gitignore b/.gitignore index 36d87ee..bc48ac1 100644 --- a/.gitignore +++ b/.gitignore @@ -99,4 +99,5 @@ venv/ # Generated by utility scripts treks_xml/ +lola_xml/ create-treks-pds4-log.log diff --git a/README.md b/README.md index f2311c4..484572a 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,16 @@ All users and developers of the NASA-PDS software are expected to abide by our [ * Run the command: ```create-treks-pds4``` +* GeoSTAC - Lola + * To deploy the package run one of these commands from the root directory: + ```pip install .``` for users + ```pip install -e '.[dev]'``` for developers + * This package is also hosted on the "cheeseshop" and can be installed with + ```pip install pds.registry``` + * The GeoSTAC utilities can be used to create pds4 labels for the Lola point clouds they host + * Run the command: + ```create-lola-pds4``` + ## Development diff --git a/setup.cfg b/setup.cfg index b157c2a..c1910ae 100644 --- a/setup.cfg +++ b/setup.cfg @@ -87,6 +87,7 @@ dev = # Put your entry point scripts here console_scripts = create-treks-pds4 = pds.registry.utils.treks.create_treks_pds4:main + create-lola-pds4 = pds.registry.utils.geostac.create_lola_pds4:main # some_script: ... [options.packages.find] diff --git a/src/pds/registry/utils/geostac/__init__.py b/src/pds/registry/utils/geostac/__init__.py new file mode 100644 index 0000000..ddb552b --- /dev/null +++ b/src/pds/registry/utils/geostac/__init__.py @@ -0,0 +1 @@ +"""Geostac Utilities.""" diff --git a/src/pds/registry/utils/geostac/create_lola_pds4.py b/src/pds/registry/utils/geostac/create_lola_pds4.py new file mode 100644 index 0000000..6165474 --- /dev/null +++ b/src/pds/registry/utils/geostac/create_lola_pds4.py @@ -0,0 +1,165 @@ +"""Script to scrape GeoSTAC for Lola point clouds and make pds4 xml.""" +import argparse +import importlib +from datetime import date +from pathlib import Path + +import requests +from jinja2 import Environment +from pds.registry.utils.geostac import templates + + +def create_product_external(item): + """Creates Product_External for given item. + + :param item: item to fill out pds4 xml information with + + :return: pds4 xml + """ + # create env + env = Environment() + + with importlib.resources.open_text(templates, "product-external-template.xml") as io: + template_text = io.read() + template = env.from_string(template_text) + + item_title = item["assets"]["data"]["title"] + + last_slash_i = item["assets"]["data"]["href"].rfind("/") + file = item["assets"]["data"]["href"][last_slash_i + 1:] + + # fill out template params + lid = "urn:nasa:pds:geostac:external:" + item_title.lower() + title = item_title + modification_date = str(date.today()) + lid_ref = item_title.lower() + bb_west = item["bbox"][0] + bb_south = item["bbox"][1] + bb_east = item["bbox"][2] + bb_north = item["bbox"][3] + file_name = file + lid_file = item_title.lower() + file_date = item["properties"]["datetime"][:10] + file_url = item["assets"]["data"]["href"] + # file_size not in api, but also not required by pds + encoding_standard = item["properties"]["pc:encoding"] + + context = { + "lid": lid, + "title": title, + "modification_date": modification_date, + "lid_ref": lid_ref, + "bb_west": bb_west, + "bb_east": bb_east, + "bb_north": bb_north, + "bb_south": bb_south, + "file_name": file_name, + "lid_file": lid_file, + "file_date": file_date, + "file_url": file_url, + # file_size + "encoding_standard": encoding_standard + } + + return template.render(context) + + +def create_product_browse(item): + """Creates Product_Browse for given item. + + :param item: item to fill out pds4 xml information with + + :return: pds4 xml + """ + # create env + env = Environment() + + with importlib.resources.open_text(templates, "product-browse-template.xml") as io: + template_text = io.read() + template = env.from_string(template_text) + + item_title = item["assets"]["data"]["title"] + + last_slash_i = item["assets"]["thumbnail"]["href"].rfind("/") + file = item["assets"]["thumbnail"]["href"][last_slash_i + 1:] + data_type = item["assets"]["thumbnail"]["type"] + encoding_map = {"image/jpeg": "JPEG"} + + # fill out template params + lid = "urn:nasa:pds:geostac:browse:" + item_title.lower() + "_browse" + title = item_title + lid_ref = item_title.lower() + "_browse" + file_name = file + file_date = item["properties"]["datetime"][:10] + file_url = item["assets"]["thumbnail"]["href"] + encoding_standard = encoding_map[data_type] + + context = { + "lid": lid, + "title": title, + "lid_ref": lid_ref, + "file_name": file_name, + "file_date": file_date, + "file_url": file_url, + "encoding_standard": encoding_standard + } + + return template.render(context) + + +def parse_args(): + """Parses arguments of command. + + :return: args dictionary + """ + # set up command line args + parser = argparse.ArgumentParser(description="Create and save pds4 xml labels created for Lola point clouds in GeoSTAC", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + parser.add_argument("-s", + "--save-xml", + action="store_true", + help="True if you want the xml files to be saved", + default=True) + parser.add_argument("-d", + "--destination-directory", + help="Directory to save pds4 xml files", + default="lola_xml") + + args = parser.parse_args() + + return args + + +def main(): + """Main function of script.""" + # parse args + args = parse_args() + dest = args.destination_directory + save_xml = args.save_xml + + # get json from url + url = "https://stac.astrogeology.usgs.gov/api//collections/lunar_orbiter_laser_altimeter/items?limit=100000" + response = requests.get(url, timeout=30) + json_data = response.json() + features = json_data["features"] + + for item in features: + external_pds4 = create_product_external(item) + browse_pds4 = create_product_browse(item) + + if save_xml: + Path(dest + "/product_external").mkdir(parents=True, exist_ok=True) + Path(dest + "/product_browse").mkdir(parents=True, exist_ok=True) + + ex_path = dest + "/product_external/" + item["assets"]["data"]["title"] + "_product_external.xml" + with open(ex_path, "w") as f: + f.write(external_pds4) + + br_path = dest + "/product_browse/" + item["assets"]["data"]["title"] + "_product_browse.xml" + with open(br_path, "w") as f: + f.write(browse_pds4) + + +if __name__ == "__main__": + main() diff --git a/src/pds/registry/utils/geostac/templates/__init__.py b/src/pds/registry/utils/geostac/templates/__init__.py new file mode 100644 index 0000000..96cb55c --- /dev/null +++ b/src/pds/registry/utils/geostac/templates/__init__.py @@ -0,0 +1 @@ +"""GEOSTAC PDS4 XML Templates.""" diff --git a/src/pds/registry/utils/geostac/templates/product-browse-template.xml b/src/pds/registry/utils/geostac/templates/product-browse-template.xml new file mode 100644 index 0000000..5217ab1 --- /dev/null +++ b/src/pds/registry/utils/geostac/templates/product-browse-template.xml @@ -0,0 +1,41 @@ + + + + + {{lid}} + + 1.0 + + {{title}} + + 1.21.0.0 + Product_Browse + + + + lid_reference is incorrect, used as filler right now + + {{lid_ref}} + + browse_to_data + + This is a reference to LOLA RDR point observations as scraped from the NASA PDS and converted to LOLA/COPC format. + + + + + + {{file_name}} + BROWSE_FILE + {{file_date}} + {{file_url}} + + + BROWSE_IMAGE + 0 + {{encoding}} + + + diff --git a/src/pds/registry/utils/geostac/templates/product-external-template.xml b/src/pds/registry/utils/geostac/templates/product-external-template.xml new file mode 100644 index 0000000..4142aa0 --- /dev/null +++ b/src/pds/registry/utils/geostac/templates/product-external-template.xml @@ -0,0 +1,87 @@ + + + + + + + + {{lid}} + 1.0 + {{title}} + 1.21.0.0 + Product_External + + + {{modification_date}} + 1.0 + + Derived release of the LOLA LIDAR shots in a stretched LAZ/COPC format for streaming services + + + + + + + + + {{lid_ref}} + cartography_parameters_to_service + + + + {{bb_west}} + {{bb_east}} + {{bb_north}} + {{bb_south}} + + + + + + + The actual shots in the LAZ/COPC file are in geocentric meters. To use in a lat/lon viewer, + the X,Y,Z data will need to be converted to Longitude,Latitude. The Z can be maintained as radius + or elevation using "radius - 1737400". + + + + + Planetocentric + MOON + 1737400 + 1737400 + 1737400 + Positive East + + + + + + + + + {{file_name}} + {{lid_file}} + {{file_date}} + {{file_url}} + + + A Cloud Optimized Point Cloud (COpCG) is a regular, but generally compressed, LAZ file, + aimed at being hosted on a HTTPS file server, with an internal organization that + enables more efficient workflows on the cloud. + + + + 0 + {{encoding_standard}} + + + From b3b8b39b65108d7f964ead0a511e14e178d367f9 Mon Sep 17 00:00:00 2001 From: Matt Anikiej Date: Wed, 31 Jul 2024 08:29:55 -0700 Subject: [PATCH 2/3] comments to lola script --- src/pds/registry/utils/geostac/create_lola_pds4.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pds/registry/utils/geostac/create_lola_pds4.py b/src/pds/registry/utils/geostac/create_lola_pds4.py index 6165474..082949b 100644 --- a/src/pds/registry/utils/geostac/create_lola_pds4.py +++ b/src/pds/registry/utils/geostac/create_lola_pds4.py @@ -149,6 +149,7 @@ def main(): browse_pds4 = create_product_browse(item) if save_xml: + # create destination directoru if they don't exist Path(dest + "/product_external").mkdir(parents=True, exist_ok=True) Path(dest + "/product_browse").mkdir(parents=True, exist_ok=True) From ce056b44f84a7da56d7a54e61207f1a63c8bbf17 Mon Sep 17 00:00:00 2001 From: Matt Anikiej Date: Fri, 2 Aug 2024 10:57:50 -0700 Subject: [PATCH 3/3] add target to template --- .../utils/geostac/templates/product-external-template.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/pds/registry/utils/geostac/templates/product-external-template.xml b/src/pds/registry/utils/geostac/templates/product-external-template.xml index 4142aa0..026282e 100644 --- a/src/pds/registry/utils/geostac/templates/product-external-template.xml +++ b/src/pds/registry/utils/geostac/templates/product-external-template.xml @@ -28,6 +28,10 @@ + + Moon + Satellite +