From f1e90ed79f0a349155e49b3658d0a0d472f65ab1 Mon Sep 17 00:00:00 2001 From: Rob Emanuele Date: Thu, 14 Jan 2021 13:53:28 -0500 Subject: [PATCH 1/3] Modify asset HREFs based on catalog type. Previously only link href types (absolute vs relative) were modified as part of the `Catalog.save` process based on the catalog types: for ABSOLUTE_PUBLISHED, all link hrefs were made absolute, etc. The asset HREFs were not modified, which was counter to the expected result outlined in https://github.com/stac-utils/stactools/issues/31. This change causes asset HREFs to also be changed to absolute or relative based on the catalog type. Note that if the asset HREF cannot be made relative, e.g. the asset HREF points to an HTTP location and the item is being stored locally, then the HREF is left unchanged. --- docs/concepts.rst | 2 +- pystac/catalog.py | 2 ++ tests/test_writing.py | 17 ++++++++++++++++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 30d65a33c..035ee2e6d 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -101,7 +101,7 @@ Catalog Types The STAC `best practices document `_ lays out different catalog types, and how their links should be formatted. A brief description is below, but check out the document for the official take on these types: -Note that the catalog types do not dictate the asset HREF formats, only link formats. Asset HREFs in any catalog type can be relative or absolute; see the section on :ref:`rel vs abs asset` below. +The catalog types will also dictate the asset HREF formats. Asset HREFs in any catalog type can be relative or absolute may be absolute depending on their location; see the section on :ref:`rel vs abs asset` below. Self-Contained Catalogs diff --git a/pystac/catalog.py b/pystac/catalog.py index 3993c8fed..3215b0835 100644 --- a/pystac/catalog.py +++ b/pystac/catalog.py @@ -608,8 +608,10 @@ def save(self, catalog_type=None): # Ensure relative vs absolute if catalog_type == CatalogType.ABSOLUTE_PUBLISHED: self.make_all_links_absolute() + self.make_all_asset_hrefs_absolute() elif catalog_type in (CatalogType.SELF_CONTAINED, CatalogType.RELATIVE_PUBLISHED): self.make_all_links_relative() + self.make_all_asset_hrefs_relative() else: raise ValueError(f'catalog_type is not a CatalogType: "{catalog_type}"') diff --git a/tests/test_writing.py b/tests/test_writing.py index 172690355..abb67f092 100644 --- a/tests/test_writing.py +++ b/tests/test_writing.py @@ -3,7 +3,7 @@ from pystac import (STAC_IO, STACObject, Collection, CatalogType, LinkType) from pystac.serialization import (STACObjectType) -from pystac.utils import make_absolute_href +from pystac.utils import is_absolute_href, make_absolute_href, make_relative_href from pystac.validation import validate_dict from tests.utils import TestCases @@ -31,6 +31,19 @@ def validate_file(self, path, object_type): return validate_dict(d, object_type) def validate_link_types(self, root_href, catalog_type): + def validate_asset_href_type(item, item_href, link_type): + for asset in item.assets.values(): + if link_type == LinkType.ABSOLUTE: + self.assertTrue(is_absolute_href(asset.href)) + else: + is_valid = not is_absolute_href(asset.href) + if not is_valid: + # If the item href and asset href don't share + # the same root, the asset href must be absolute + rel_href = make_relative_href(asset.href, item_href) + self.assertEqual(asset.href, rel_href) + else: + self.assertTrue(is_valid) def validate_item_link_type(href, link_type, should_include_self): item_dict = STAC_IO.read_json(href) item = STACObject.from_file(href) @@ -38,6 +51,8 @@ def validate_item_link_type(href, link_type, should_include_self): if not link.rel == 'self': self.assertEqual(link.link_type, link_type) + validate_asset_href_type(item, href, link_type) + rels = set([link['rel'] for link in item_dict['links']]) self.assertEqual('self' in rels, should_include_self) From 7048001df72d9b6299e403d9f5dabd0706aa5082 Mon Sep 17 00:00:00 2001 From: Rob Emanuele Date: Thu, 14 Jan 2021 14:22:25 -0500 Subject: [PATCH 2/3] Add scripts/format; format code. --- pystac/validation/stac_validator.py | 12 ++++++++++-- scripts/format | 22 ++++++++++++++++++++++ tests/test_writing.py | 1 + 3 files changed, 33 insertions(+), 2 deletions(-) create mode 100755 scripts/format diff --git a/pystac/validation/stac_validator.py b/pystac/validation/stac_validator.py index ebc2442b7..08669a45d 100644 --- a/pystac/validation/stac_validator.py +++ b/pystac/validation/stac_validator.py @@ -34,7 +34,11 @@ def validate_core(self, stac_dict, stac_object_type, stac_version, href=None): pass @abstractmethod - def validate_extension(self, stac_dict, stac_object_type, stac_version, extension_id, + def validate_extension(self, + stac_dict, + stac_object_type, + stac_version, + extension_id, href=None): """Validate an extension stac object. @@ -173,7 +177,11 @@ def validate_core(self, stac_dict, stac_object_type, stac_version, href=None): stac_dict.get('id')) raise STACValidationError(msg, source=e) from e - def validate_extension(self, stac_dict, stac_object_type, stac_version, extension_id, + def validate_extension(self, + stac_dict, + stac_object_type, + stac_version, + extension_id, href=None): """Validate an extension stac object. diff --git a/scripts/format b/scripts/format new file mode 100755 index 000000000..a7e68fd4f --- /dev/null +++ b/scripts/format @@ -0,0 +1,22 @@ +#!/bin/bash + +set -e + +if [[ -n "${CI}" ]]; then + set -x +fi + +function usage() { + echo -n \ + "Usage: $(basename "$0") +Format code with yapf +" +} + +if [ "${BASH_SOURCE[0]}" = "${0}" ]; then + if [ "${1:-}" = "--help" ]; then + usage + else + yapf -ipr pystac tests + fi +fi diff --git a/tests/test_writing.py b/tests/test_writing.py index abb67f092..315be7d2a 100644 --- a/tests/test_writing.py +++ b/tests/test_writing.py @@ -44,6 +44,7 @@ def validate_asset_href_type(item, item_href, link_type): self.assertEqual(asset.href, rel_href) else: self.assertTrue(is_valid) + def validate_item_link_type(href, link_type, should_include_self): item_dict = STAC_IO.read_json(href) item = STACObject.from_file(href) From 2944349adfa54a84846d931a3cfc50af7ae699a7 Mon Sep 17 00:00:00 2001 From: Rob Emanuele Date: Thu, 14 Jan 2021 14:33:41 -0500 Subject: [PATCH 3/3] Upgrade yapf to 0.30.* --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 03874014e..03a4f079e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,6 +7,6 @@ sphinx-autobuild==0.7.1 sphinxcontrib-fulltoc==1.2.0 sphinxcontrib-napoleon==0.7 flake8==3.8.* -yapf==0.28.* +yapf==0.30.* nbsphinx==0.7.1 coverage==5.2.*