Merge pull request #306 from NASA-PDS/stac_utility

Stac utility

.gitignore

@@ -99,4 +99,5 @@ venv/
 
 # Generated by utility scripts
 treks_xml/
+lola_xml/
 create-treks-pds4-log.log

README.md

@@ -36,6 +36,16 @@ All users and developers of the NASA-PDS software are expected to abide by our [
     * Run the command:
         ```create-treks-pds4```
 
+* GeoSTAC - Lola
+    * To deploy the package run one of these commands from the root directory:
+        ```pip install .``` for users
+        ```pip install -e '.[dev]'``` for developers
+        * This package is also hosted on the "cheeseshop" and can be installed with
+            ```pip install pds.registry```
+    * The GeoSTAC utilities can be used to create pds4 labels for the Lola point clouds they host
+    * Run the command:
+        ```create-lola-pds4```
+
 
 ## Development
 

setup.cfg

@@ -87,6 +87,7 @@ dev =
 # Put your entry point scripts here
 console_scripts =
     create-treks-pds4 = pds.registry.utils.treks.create_treks_pds4:main
+    create-lola-pds4 = pds.registry.utils.geostac.create_lola_pds4:main
     # some_script: ...
 
 [options.packages.find]

src/pds/registry/utils/geostac/__init__.py

@@ -0,0 +1 @@
+"""Geostac Utilities."""

src/pds/registry/utils/geostac/create_lola_pds4.py

@@ -0,0 +1,166 @@
+"""Script to scrape GeoSTAC for Lola point clouds and make pds4 xml."""
+import argparse
+import importlib
+from datetime import date
+from pathlib import Path
+
+import requests
+from jinja2 import Environment
+from pds.registry.utils.geostac import templates
+
+
+def create_product_external(item):
+    """Creates Product_External for given item.
+
+    :param item: item to fill out pds4 xml information with
+
+    :return: pds4 xml
+    """
+    # create env
+    env = Environment()
+
+    with importlib.resources.open_text(templates, "product-external-template.xml") as io:
+        template_text = io.read()
+        template = env.from_string(template_text)
+
+        item_title = item["assets"]["data"]["title"]
+
+        last_slash_i = item["assets"]["data"]["href"].rfind("/")
+        file = item["assets"]["data"]["href"][last_slash_i + 1:]
+
+        # fill out template params
+        lid = "urn:nasa:pds:geostac:external:" + item_title.lower()
+        title = item_title
+        modification_date = str(date.today())
+        lid_ref = item_title.lower()
+        bb_west = item["bbox"][0]
+        bb_south = item["bbox"][1]
+        bb_east = item["bbox"][2]
+        bb_north = item["bbox"][3]
+        file_name = file
+        lid_file = item_title.lower()
+        file_date = item["properties"]["datetime"][:10]
+        file_url = item["assets"]["data"]["href"]
+        # file_size not in api, but also not required by pds
+        encoding_standard = item["properties"]["pc:encoding"]
+
+        context = {
+            "lid": lid,
+            "title": title,
+            "modification_date": modification_date,
+            "lid_ref": lid_ref,
+            "bb_west": bb_west,
+            "bb_east": bb_east,
+            "bb_north": bb_north,
+            "bb_south": bb_south,
+            "file_name": file_name,
+            "lid_file": lid_file,
+            "file_date": file_date,
+            "file_url": file_url,
+            # file_size
+            "encoding_standard": encoding_standard
+        }
+
+        return template.render(context)
+
+
+def create_product_browse(item):
+    """Creates Product_Browse for given item.
+
+    :param item: item to fill out pds4 xml information with
+
+    :return: pds4 xml
+    """
+    # create env
+    env = Environment()
+
+    with importlib.resources.open_text(templates, "product-browse-template.xml") as io:
+        template_text = io.read()
+        template = env.from_string(template_text)
+
+        item_title = item["assets"]["data"]["title"]
+
+        last_slash_i = item["assets"]["thumbnail"]["href"].rfind("/")
+        file = item["assets"]["thumbnail"]["href"][last_slash_i + 1:]
+        data_type = item["assets"]["thumbnail"]["type"]
+        encoding_map = {"image/jpeg": "JPEG"}
+
+        # fill out template params
+        lid = "urn:nasa:pds:geostac:browse:" + item_title.lower() + "_browse"
+        title = item_title
+        lid_ref = item_title.lower() + "_browse"
+        file_name = file
+        file_date = item["properties"]["datetime"][:10]
+        file_url = item["assets"]["thumbnail"]["href"]
+        encoding_standard = encoding_map[data_type]
+
+        context = {
+            "lid": lid,
+            "title": title,
+            "lid_ref": lid_ref,
+            "file_name": file_name,
+            "file_date": file_date,
+            "file_url": file_url,
+            "encoding_standard": encoding_standard
+        }
+
+        return template.render(context)
+
+
+def parse_args():
+    """Parses arguments of command.
+
+    :return: args dictionary
+    """
+    # set up command line args
+    parser = argparse.ArgumentParser(description="Create and save pds4 xml labels created for Lola point clouds in GeoSTAC",
+                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+
+    parser.add_argument("-s",
+                        "--save-xml",
+                        action="store_true",
+                        help="True if you want the xml files to be saved",
+                        default=True)
+    parser.add_argument("-d",
+                        "--destination-directory",
+                        help="Directory to save pds4 xml files",
+                        default="lola_xml")
+
+    args = parser.parse_args()
+
+    return args
+
+
+def main():
+    """Main function of script."""
+    # parse args
+    args = parse_args()
+    dest = args.destination_directory
+    save_xml = args.save_xml
+
+    # get json from url
+    url = "https://stac.astrogeology.usgs.gov/api//collections/lunar_orbiter_laser_altimeter/items?limit=100000"
+    response = requests.get(url, timeout=30)
+    json_data = response.json()
+    features = json_data["features"]
+
+    for item in features:
+        external_pds4 = create_product_external(item)
+        browse_pds4 = create_product_browse(item)
+
+        if save_xml:
+            # create destination directoru if they don't exist
+            Path(dest + "/product_external").mkdir(parents=True, exist_ok=True)
+            Path(dest + "/product_browse").mkdir(parents=True, exist_ok=True)
+
+            ex_path = dest + "/product_external/" + item["assets"]["data"]["title"] + "_product_external.xml"
+            with open(ex_path, "w") as f:
+                f.write(external_pds4)
+
+            br_path = dest + "/product_browse/" + item["assets"]["data"]["title"] + "_product_browse.xml"
+            with open(br_path, "w") as f:
+                f.write(browse_pds4)
+
+
+if __name__ == "__main__":
+    main()

src/pds/registry/utils/geostac/templates/__init__.py

@@ -0,0 +1 @@
+"""GEOSTAC PDS4 XML Templates."""

src/pds/registry/utils/geostac/templates/product-browse-template.xml

@@ -0,0 +1,41 @@
+<?xml-model
+href="https://pds.nasa.gov/pds4/pds/v1/PDS4_PDS_1L00.sch" schematypens="http://purl.oclc.org/dsdl/schematron"
+?>
+<Product_Browse>
+  <Identification_Area>
+    <logical_identifier>
+      {{lid}}
+    </logical_identifier>
+    <version_id>1.0</version_id>
+    <title>
+      {{title}}
+    </title>
+    <information_model_version>1.21.0.0</information_model_version>
+    <product_class>Product_Browse</product_class>
+  </Identification_Area>
+  <Reference_List>
+    <Internal_Reference>
+      <!-->lid_reference is incorrect, used as filler right now</-->
+      <lid_reference>
+        {{lid_ref}}
+      </lid_reference>
+      <reference_type>browse_to_data</reference_type>
+      <comment>
+        This is a reference to LOLA RDR point observations as scraped from the NASA PDS and converted to LOLA/COPC format.
+      </comment>
+    </Internal_Reference>
+  </Reference_List>
+  <File_Area_Browse>
+    <File>
+      <file_name>{{file_name}}</file_name>
+      <local_identifier>BROWSE_FILE</local_identifier>
+      <creation_date_time>{{file_date}}</creation_date_time>
+      <file_URL>{{file_url}}</file_URL>
+    </File>
+    <Encoded_Image>
+      <local_identifier>BROWSE_IMAGE</local_identifier>
+      <offset unit="byte">0</offset>
+      <encoding_standard_id>{{encoding}}</encoding_standard_id>
+    </Encoded_Image>
+  </File_Area_Browse>
+</Product_Browse>

src/pds/registry/utils/geostac/templates/product-external-template.xml

@@ -0,0 +1,91 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="https://pds.nasa.gov/pds4/pds/v1/PDS4_PDS_1L00.sch" schematypens="http://purl.oclc.org/dsdl/schematron"?>
+<?xml-model href="https://pds.nasa.gov/pds4/cart/v1/PDS4_CART_1L00_1970.sch" schematypens="http://purl.oclc.org/dsdl/schematron"?>
+
+<Product_External
+  xmlns="http://pds.nasa.gov/pds4/pds/v1"
+  xmlns:pds="http://pds.nasa.gov/pds4/pds/v1"
+  xmlns:cart="http://pds.nasa.gov/pds4/cart/v1"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+
+  xsi:schemaLocation="http://pds.nasa.gov/pds4/pds/v1 https://pds.nasa.gov/pds4/pds/v1/PDS4_PDS_1L00.xsd
+  http://pds.nasa.gov/pds4/cart/v1 https://pds.nasa.gov/pds4/cart/v1/PDS4_CART_1L00_1970.xsd">
+
+  <Identification_Area>
+    <logical_identifier>{{lid}}</logical_identifier>
+    <version_id>1.0</version_id>
+    <title>{{title}}</title>
+    <information_model_version>1.21.0.0</information_model_version>
+    <product_class>Product_External</product_class>
+    <Modification_History>
+      <Modification_Detail>
+        <modification_date>{{modification_date}}</modification_date>
+        <version_id>1.0</version_id>
+        <description>
+          Derived release of the LOLA LIDAR shots in a stretched LAZ/COPC format for streaming services
+        </description>
+      </Modification_Detail>
+    </Modification_History>
+  </Identification_Area>
+  <Context_Area>
+    <Target_Identification>
+			<name>Moon</name>
+			<type>Satellite</type>
+		</Target_Identification>
+    <Discipline_Area>
+      <cart:Cartography>
+        <Local_Internal_Reference>
+          <local_identifier_reference>{{lid_ref}}</local_identifier_reference>
+          <local_reference_type>cartography_parameters_to_service</local_reference_type>
+        </Local_Internal_Reference>
+        <cart:Spatial_Domain>
+          <cart:Bounding_Coordinates>
+            <cart:west_bounding_coordinate unit="deg">{{bb_west}}</cart:west_bounding_coordinate>
+            <cart:east_bounding_coordinate unit="deg">{{bb_east}}</cart:east_bounding_coordinate>
+            <cart:north_bounding_coordinate unit="deg">{{bb_north}}</cart:north_bounding_coordinate>
+            <cart:south_bounding_coordinate unit="deg">{{bb_south}}</cart:south_bounding_coordinate>
+          </cart:Bounding_Coordinates>
+        </cart:Spatial_Domain>
+        <cart:Spatial_Reference_Information>
+          <cart:Horizontal_Coordinate_System_Definition>
+            <cart:Geographic>
+              <cart:geographic_description>
+                The actual shots in the LAZ/COPC file are in geocentric meters. To use in a lat/lon viewer,
+                the X,Y,Z data will need to be converted to Longitude,Latitude. The Z can be maintained as radius
+                or elevation using "radius - 1737400".
+              </cart:geographic_description>
+              <!-- <cart:latitude_resolution unit="deg">0.001</cart:latitude_resolution>
+              <cart:longitude_resolution unit="deg">0.001</cart:longitude_resolution> -->
+            </cart:Geographic>
+            <cart:Geodetic_Model>
+              <cart:latitude_type>Planetocentric</cart:latitude_type>
+              <cart:spheroid_name>MOON</cart:spheroid_name>
+              <cart:a_axis_radius unit="m">1737400</cart:a_axis_radius>
+              <cart:b_axis_radius unit="m">1737400</cart:b_axis_radius>
+              <cart:c_axis_radius unit="m">1737400</cart:c_axis_radius>
+              <cart:longitude_direction>Positive East</cart:longitude_direction>
+            </cart:Geodetic_Model>
+          </cart:Horizontal_Coordinate_System_Definition>
+        </cart:Spatial_Reference_Information>
+      </cart:Cartography>
+    </Discipline_Area>
+  </Context_Area>
+  <File_Area_External>
+    <File>
+      <file_name>{{file_name}}</file_name>
+      <local_identifier>{{lid_file}}</local_identifier>
+      <creation_date_time>{{file_date}}</creation_date_time>
+      <file_URL>{{file_url}}</file_URL>
+      <!-- <file_size unit="byte">{{file_size}}</file_size> -->
+      <comment>
+        A Cloud Optimized Point Cloud (COpCG) is a regular, but generally compressed, LAZ file,
+        aimed at being hosted on a HTTPS file server, with an internal organization that
+        enables more efficient workflows on the cloud.
+      </comment>
+    </File>
+    <Encoded_External>
+      <offset unit="byte">0</offset>
+      <encoding_standard_id>{{encoding_standard}}</encoding_standard_id>
+    </Encoded_External>
+  </File_Area_External>
+</Product_External>