Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Configurable warnings #480

Merged
merged 43 commits into from
May 4, 2023
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
f8171c0
Xarray-style options
jsignell Apr 13, 2023
19cb420
Refine options and add mechanism to easily respond to events based on…
jsignell Apr 13, 2023
c0877f2
Fix types
jsignell Apr 14, 2023
2de758e
Fix message
jsignell Apr 14, 2023
40c5eaf
Use warnings rather than options
jsignell Apr 14, 2023
671d945
Expect warnings
jsignell Apr 14, 2023
1a63a2c
Add another warning and put back assert_conforms_to
jsignell Apr 14, 2023
9001e78
Rollback to passing tests
jsignell Apr 14, 2023
1ae1e0f
Apply suggestions from code review
jsignell Apr 17, 2023
20d427f
Get rid of ignore_conformance
jsignell Apr 18, 2023
9af577a
Let users control conformsTo
jsignell Apr 20, 2023
193b521
Fix up tests
jsignell Apr 20, 2023
e1661df
Merge branch 'main' into warnings
jsignell Apr 20, 2023
60cb2b8
Rewrite a cassette
jsignell Apr 20, 2023
7efd55f
Rewrite a cassette
jsignell Apr 20, 2023
618324f
More tests to demonstrate helper methods
jsignell Apr 20, 2023
eaa85ab
Tests for altering conforms_to via cli
jsignell Apr 21, 2023
85c4697
Add cassettes
jsignell Apr 21, 2023
e1fbd66
fix, tests: rewrite some cassettes, filter warning
gadomski Apr 21, 2023
d747b93
Merge branch 'main' into warnings
gadomski Apr 21, 2023
025dd80
Merge branch 'main' into warnings
jsignell Apr 24, 2023
5b84cf2
Put back in `ignore_conformance`
jsignell Apr 24, 2023
67a5983
Apply suggestions from code review
jsignell Apr 24, 2023
df9d698
Make warnings easier to use
jsignell Apr 24, 2023
38a2ba3
Add _supports_collections
jsignell Apr 24, 2023
0accb69
Better examples
jsignell Apr 24, 2023
de3eff6
Don't deprecate stac_io, add warnings for conformance
jsignell Apr 24, 2023
9e3cefd
Make sure stacklevel is always 2
jsignell Apr 24, 2023
1109d13
Try again to record the cassette
jsignell Apr 24, 2023
de34f4c
Docs and simplify a bit
jsignell Apr 24, 2023
c14ce2d
Merge branch 'main' into warnings
jsignell Apr 25, 2023
d3fb5ba
Merge branch 'main' into warnings
jsignell Apr 25, 2023
9804aad
Merge branch 'main' into warnings
gadomski Apr 27, 2023
98cc2a1
tests: rewrite cassette
gadomski Apr 27, 2023
718d23f
Start working on docs
jsignell Apr 27, 2023
b1aa915
Respond to PR comments
jsignell Apr 27, 2023
037368e
Finish first pass at docs
jsignell Apr 27, 2023
d0c9092
Merge branch 'main' into warnings
jsignell May 1, 2023
a34e592
Update docs/usage.rst
jsignell May 4, 2023
bfa8546
Don't capture warnings in logs
jsignell May 4, 2023
60a3ac6
Update changelog
jsignell May 4, 2023
3493978
Merge branch 'main' into warnings
jsignell May 4, 2023
8031adc
Merge branch 'main' into warnings
jsignell May 4, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 45 additions & 67 deletions pystac_client/client.py
jsignell marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from functools import lru_cache
from typing import TYPE_CHECKING, Any, Callable, Dict, Iterator, List, Optional, Union
import warnings

import pystac
import pystac.utils
Expand Down Expand Up @@ -27,6 +28,7 @@
SortbyLike,
)
from pystac_client.stac_api_io import StacApiIO
from pystac_client.warnings import FALLBACK_MSG, FallbackToPystac, MissingLink

if TYPE_CHECKING:
from pystac.item import Item as Item_Type
Expand Down Expand Up @@ -203,14 +205,14 @@ def from_file( # type: ignore

return client

def conforms_to(self, *conformance_classes: ConformanceClasses) -> bool:
return bool(self._stac_io and self._stac_io.conforms_to(*conformance_classes))
jsignell marked this conversation as resolved.
Show resolved Hide resolved

def _supports_collections(self) -> bool:
return self._conforms_to(ConformanceClasses.COLLECTIONS) or self._conforms_to(
ConformanceClasses.FEATURES
return self.conforms_to(
ConformanceClasses.COLLECTIONS, ConformanceClasses.FEATURES
)

def _conforms_to(self, conformance_class: ConformanceClasses) -> bool:
return self._stac_io.conforms_to(conformance_class) # type: ignore

@classmethod
def from_dict(
cls,
Expand All @@ -236,15 +238,19 @@ def from_dict(
return result

@lru_cache()
def get_collection(self, collection_id: str) -> Optional[Collection]:
def get_collection(
self, collection_id: str
) -> Optional[Union[Collection, CollectionClient]]:
"""Get a single collection from this Catalog/API

Args:
collection_id: The Collection ID to get

Returns:
CollectionClient: A STAC Collection
Union[Collection, CollectionClient]: A STAC Collection
"""
collection: Union[Collection, CollectionClient]

if self._supports_collections() and self._stac_io:
url = self._get_collections_href(collection_id)
collection = CollectionClient.from_dict(
Expand All @@ -255,21 +261,22 @@ def get_collection(self, collection_id: str) -> Optional[Collection]:
call_modifier(self.modifier, collection)
return collection
else:
for col in self.get_collections():
if col.id == collection_id:
call_modifier(self.modifier, col)
return col
warnings.warn(FALLBACK_MSG, category=FallbackToPystac)
for collection in super().get_collections():
if collection.id == collection_id:
call_modifier(self.modifier, collection)
return collection

return None

def get_collections(self) -> Iterator[Collection]:
def get_collections(self) -> Iterator[Union[Collection, CollectionClient]]:
"""Get Collections in this Catalog

Gets the collections from the /collections endpoint if supported,
otherwise fall back to Catalog behavior of following child links

Return:
Iterator[Collection]: Iterator over Collections in Catalog/API
Iterator[Union[Collection, CollectionClient]]: Collections in Catalog/API
"""
collection: Union[Collection, CollectionClient]

Expand All @@ -285,6 +292,7 @@ def get_collections(self) -> Iterator[Collection]:
call_modifier(self.modifier, collection)
yield collection
else:
warnings.warn(FALLBACK_MSG, category=FallbackToPystac)
for collection in super().get_collections():
call_modifier(self.modifier, collection)
yield collection
Expand All @@ -296,10 +304,11 @@ def get_items(self) -> Iterator["Item_Type"]:
Iterator[Item]:: Iterator of items whose parent is this
catalog.
"""
if self._conforms_to(ConformanceClasses.ITEM_SEARCH):
if self.conforms_to(ConformanceClasses.ITEM_SEARCH):
search = self.search()
yield from search.items()
else:
warnings.warn(FALLBACK_MSG, category=FallbackToPystac)
for item in super().get_items():
call_modifier(self.modifier, item)
yield item
Expand All @@ -313,13 +322,7 @@ def get_all_items(self) -> Iterator["Item_Type"]:
catalogs or collections connected to this catalog through
child links.
"""
if self._conforms_to(ConformanceClasses.ITEM_SEARCH):
# these are already modified
yield from self.get_items()
else:
for item in super().get_items():
call_modifier(self.modifier, item)
yield item
yield from self.get_items()

def search(
self,
Expand Down Expand Up @@ -438,26 +441,8 @@ def search(
or does not have a link with
a ``"rel"`` type of ``"search"``.
"""
if not self._conforms_to(ConformanceClasses.ITEM_SEARCH):
raise NotImplementedError(
"This catalog does not support search because it "
f'does not conform to "{ConformanceClasses.ITEM_SEARCH}"'
)
search_link = self.get_search_link()
if search_link:
if isinstance(search_link.target, str):
search_href = search_link.target
else:
raise NotImplementedError(
"Link with rel=search was an object rather than a URI"
)
else:
raise NotImplementedError(
"No link with rel=search could be found in this catalog"
)

return ItemSearch(
url=search_href,
url=self._get_search_href(),
method=method,
max_items=max_items,
stac_io=self._stac_io,
Expand Down Expand Up @@ -497,35 +482,28 @@ def get_search_link(self) -> Optional[pystac.Link]:
None,
)

def _get_search_href(self) -> str:
search_link = self.get_search_link()
href = self._get_href("search", search_link, "search")
return href

def _get_collections_href(self, collection_id: Optional[str] = None) -> str:
self_href = self.get_self_href()
if self_href is None:
data_link = self.get_single_link("data")
if data_link is None:
raise ValueError(
"cannot build a collections href without a self href or a data link"
)
else:
collections_href = data_link.href
data_link = self.get_single_link("data")
href = self._get_href("data", data_link, "collections")
if collection_id is None:
return href
else:
collections_href = f"{self_href.rstrip('/')}/collections"

if not pystac.utils.is_absolute_href(collections_href):
collections_href = self._make_absolute_href(collections_href)
return f"{href.rstrip('/')}/{collection_id}"

if collection_id is None:
return collections_href
def _get_href(self, rel: str, link: Optional[pystac.Link], endpoint: str) -> str:
if link and isinstance(link.href, str):
href = link.href
if not pystac.utils.is_absolute_href(href):
href = pystac.utils.make_absolute_href(href, self.self_href)
else:
return f"{collections_href.rstrip('/')}/{collection_id}"

def _make_absolute_href(self, href: str) -> str:
self_link = self.get_single_link("self")
if self_link is None:
raise ValueError("cannot build an absolute href without a self link")
elif not pystac.utils.is_absolute_href(self_link.href):
raise ValueError(
"cannot build an absolute href from "
f"a relative self link: {self_link.href}"
warnings.warn(
f"No link with {rel=} could be found in this catalog",
category=MissingLink,
)
else:
return pystac.utils.make_absolute_href(href, self_link.href)
href = f"{self.self_href.rstrip('/')}/{endpoint}"
jsignell marked this conversation as resolved.
Show resolved Hide resolved
return href
4 changes: 2 additions & 2 deletions pystac_client/collection_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ def get_item(self, id: str, recursive: bool = False) -> Optional["Item_Type"]:
else:
search_link = None
if (
stac_io.conforms_to(ConformanceClasses.FEATURES)
stac_io._conforms_to(ConformanceClasses.FEATURES)
and items_link is not None
):
url = f"{items_link.href}/{id}"
Expand All @@ -152,7 +152,7 @@ def get_item(self, id: str, recursive: bool = False) -> Optional["Item_Type"]:
raise err
assert isinstance(item, pystac.Item)
elif (
stac_io.conforms_to(ConformanceClasses.ITEM_SEARCH)
stac_io._conforms_to(ConformanceClasses.ITEM_SEARCH)
and search_link
and search_link.href
):
Expand Down
13 changes: 5 additions & 8 deletions pystac_client/item_search.py
Original file line number Diff line number Diff line change
Expand Up @@ -278,7 +278,7 @@ def __init__(
else:
self._stac_io = StacApiIO()

self._assert_conforms_to(ConformanceClasses.ITEM_SEARCH)
self._stac_io.conforms_to(ConformanceClasses.ITEM_SEARCH)

self._max_items = max_items
if self._max_items is not None and limit is not None:
Expand Down Expand Up @@ -308,9 +308,6 @@ def __init__(
k: v for k, v in params.items() if v is not None
}

def _assert_conforms_to(self, conformance_class: ConformanceClasses) -> None:
self._stac_io.assert_conforms_to(conformance_class)

def get_parameters(self) -> Dict[str, Any]:
if self.method == "POST":
return self._parameters
Expand Down Expand Up @@ -369,7 +366,7 @@ def _format_query(self, value: Optional[QueryLike]) -> Optional[Dict[str, Any]]:
if value is None:
return None

self._assert_conforms_to(ConformanceClasses.QUERY)
self._stac_io.conforms_to(ConformanceClasses.QUERY)

if isinstance(value, dict):
return value
Expand Down Expand Up @@ -418,7 +415,7 @@ def _format_filter(self, value: Optional[FilterLike]) -> Optional[FilterLike]:
if value is None:
return None

self._assert_conforms_to(ConformanceClasses.FILTER)
self._stac_io.conforms_to(ConformanceClasses.FILTER)

return value

Expand Down Expand Up @@ -562,7 +559,7 @@ def _format_sortby(self, value: Optional[SortbyLike]) -> Optional[Sortby]:
if value is None:
return None

self._assert_conforms_to(ConformanceClasses.SORT)
self._stac_io.conforms_to(ConformanceClasses.SORT)

if isinstance(value, str):
return [self._sortby_part_to_dict(part) for part in value.split(",")]
Expand Down Expand Up @@ -599,7 +596,7 @@ def _format_fields(self, value: Optional[FieldsLike]) -> Optional[Fields]:
if value is None:
return None

self._assert_conforms_to(ConformanceClasses.FIELDS)
self._stac_io.conforms_to(ConformanceClasses.FIELDS)

if isinstance(value, str):
return self._fields_to_dict(value.split(","))
Expand Down
22 changes: 18 additions & 4 deletions pystac_client/stac_api_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from copy import deepcopy
from typing import TYPE_CHECKING, Any, Callable, Dict, Iterator, List, Optional, Union
from urllib.parse import urlparse
import warnings

import pystac
from pystac.link import Link
Expand All @@ -20,6 +21,7 @@

from .conformance import CONFORMANCE_URIS, ConformanceClasses
from .exceptions import APIError
from .warnings import DoesNotConformTo

if TYPE_CHECKING:
from pystac.catalog import Catalog as Catalog_Type
Expand Down Expand Up @@ -263,7 +265,7 @@ def get_pages(
(link for link in page.get("links", []) if link["rel"] == "next"), None
)

def assert_conforms_to(self, conformance_class: ConformanceClasses) -> None:
jsignell marked this conversation as resolved.
Show resolved Hide resolved
def conforms_to(self, *conformance_classes: ConformanceClasses) -> bool:
"""Raises a :exc:`NotImplementedError` if the API does not publish the given
conformance class. This method only checks against the ``"conformsTo"``
property from the API landing page and does not make any additional
Expand All @@ -272,11 +274,23 @@ def assert_conforms_to(self, conformance_class: ConformanceClasses) -> None:
Args:
conformance_class: The ``ConformanceClasses`` key to check conformance
against.
Return:
bool: Indicates if the API conforms to the given spec or URI.

"""
if not self.conforms_to(conformance_class):
raise NotImplementedError(f"{conformance_class} not supported")
if any(map(self._conforms_to, conformance_classes)):
return True
else:
warnings.warn(
(
"Catalog does not conform to "
f"{', '.join(c.name for c in conformance_classes)}"
),
category=DoesNotConformTo,
)
jsignell marked this conversation as resolved.
Show resolved Hide resolved
return False

def conforms_to(self, conformance_class: ConformanceClasses) -> bool:
def _conforms_to(self, conformance_class: ConformanceClasses) -> bool:
"""Whether the API conforms to the given standard. This method only checks
against the ``"conformsTo"`` property from the API landing page and does not
make any additional calls to a ``/conformance`` endpoint even if the API
Expand Down
47 changes: 47 additions & 0 deletions pystac_client/warnings.py
jsignell marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
from contextlib import contextmanager
import warnings


FALLBACK_MSG = "Falling back to pystac. This might be slow."


class PystacClientWarning(UserWarning):
"""Base warning class"""

...


class DoesNotConformTo(PystacClientWarning):
"""Inform user when client does not conform to extension"""

...


class MissingLink(PystacClientWarning):
"""Inform user when link is properly implemented"""

...


class FallbackToPystac(PystacClientWarning):
"""Inform user when falling back to pystac implementation"""

...


@contextmanager
def strict(): # type: ignore
jsignell marked this conversation as resolved.
Show resolved Hide resolved
warnings.filterwarnings("error", category=PystacClientWarning)
try:
yield
finally:
warnings.resetwarnings()
gadomski marked this conversation as resolved.
Show resolved Hide resolved


@contextmanager
def lax(): # type: ignore
jsignell marked this conversation as resolved.
Show resolved Hide resolved
warnings.filterwarnings("ignore", category=PystacClientWarning)
try:
yield
finally:
warnings.resetwarnings()
Loading