Implement queryables through proxy to online json files (#44)

* Implement queryables through proxy to online json files

* lint

* Guard against 404

* Add logic for dynamic calculation of collection queryable intersection

* Add tests and appease mypy

* Update changelog

* Improve queryable test specificity

* Use cache and guard against errors

* Add test for /queryable 404 and use proper cache key

CHANGELOG.md

@@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Added support for /queryables endpoint [#44](https://github.com/microsoft/planetary-computer-apis/pull/44)
+
 ### Fixed
 
 
@@ -25,7 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 
 - Fixed renderconfigs for item tile links [#41](https://github.com/microsoft/planetary-computer-apis/pull/41)
-- Fixes hostname setting for URLs used in item links
+- Fixed hostname setting for URLs used in item links
 
 ## [2022.1.1]
 

pcstac/pcstac/cache.py

@@ -3,3 +3,7 @@
 # A TTL cache, only used for the (large) '/collections' endpoint
 # TTL set to 600 seconds == 10 minutes
 collections_endpoint_cache: TTLCache = TTLCache(maxsize=1, ttl=600)
+
+# A TTL cache, only used for computed all-collection queryables
+# TTL set to 21600 seconds == 6 hours
+queryables_endpoint_cache: TTLCache = TTLCache(maxsize=1, ttl=21600)

pcstac/pcstac/config.py

@@ -11,6 +11,8 @@
 )
 from stac_fastapi.extensions.core.filter.filter import FilterConformanceClasses
 
+from pcstac.filter import MSPCFiltersClient
+
 API_VERSION = "1.2"
 STAC_API_VERSION = "v1.0.0-beta.4"
 
@@ -29,12 +31,13 @@
     SortExtension(),
     FieldsExtension(),
     FilterExtension(
+        client=MSPCFiltersClient(),
         conformance_classes=[
             FilterConformanceClasses.FILTER,
             FilterConformanceClasses.ITEM_SEARCH_FILTER,
             FilterConformanceClasses.BASIC_CQL,
             FilterConformanceClasses.CQL_JSON,
-        ]
+        ],
     ),
     # stac_fastapi extensions
     TokenPaginationExtension(),

pcstac/pcstac/filter.py

@@ -0,0 +1,102 @@
+import asyncio
+import json
+from typing import Any, Dict, List, Optional, Set
+
+import requests
+from fastapi import HTTPException, Request
+from stac_fastapi.types.core import AsyncBaseFiltersClient
+
+from pcstac.cache import queryables_endpoint_cache
+
+
+class MSPCFiltersClient(AsyncBaseFiltersClient):
+    """Defines a pattern for implementing the STAC filter extension."""
+
+    queryable_url_template = (
+        "https://planetarycomputer.microsoft.com/stac/{cid}/queryables.json"
+    )
+
+    async def get_queryable_intersection(self, request: Request) -> dict:
+        """Generate json schema with intersecting properties of all collections.
+        When queryables are requested without specifying a collection (/queryable
+        from the root), a json schema encapsulating only the properties shared by all
+        collections should be returned. This function gathers all collection
+        queryables, calculates the intersection, and caches the results so that the
+        work can be saved for future queries.
+        """
+        pool = request.app.state.readpool
+
+        async with pool.acquire() as conn:
+            collections = await conn.fetchval(
+                """
+                SELECT * FROM all_collections();
+                """
+            )
+        collection_ids = [collection["id"] for collection in collections]
+        all_queryables = await asyncio.gather(
+            *[self.get_queryables(cid) for cid in collection_ids],
+            return_exceptions=True
+        )
+        all_queryables = [
+            queryable
+            for queryable in all_queryables
+            if not isinstance(queryable, Exception)
+        ]
+        all_properties: List[dict] = [
+            queryable["properties"] for queryable in all_queryables
+        ]
+        all_property_keys: List[Set[str]] = list(
+            map(lambda x: set(x.keys()), all_properties)
+        )
+        intersecting_props = {}
+        if len(all_property_keys) > 0:
+            property_name_intersection: List[str] = list(
+                set.intersection(*all_property_keys)
+            )
+            for name in property_name_intersection:
+                for idx, properties in enumerate(all_properties):
+                    if idx == 0:
+                        maybe_match = properties[name]
+                    else:
+                        if properties[name] != maybe_match:
+                            break
+                if maybe_match is not None:
+                    intersecting_props[name] = maybe_match
+
+        intersection_schema = {
+            "$schema": "http://json-schema.org/draft-07/schema#",
+            "$id": "https://example.org/queryables",
+            "type": "object",
+            "title": "",
+            "properties": intersecting_props,
+        }
+        return intersection_schema
+
+    async def get_queryables(
+        self, collection_id: Optional[str] = None, **kwargs: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        """Get the queryables available for the given collection_id.
+        If collection_id is None, returns the intersection of all
+        queryables over all collections.
+        This base implementation returns a blank queryable schema. This is not allowed
+        under OGC CQL but it is allowed by the STAC API Filter Extension
+        https://github.com/radiantearth/stac-api-spec/tree/master/fragments/filter#queryables
+        """
+        if not collection_id:
+            try:
+                queryable_resp = queryables_endpoint_cache["/queryables"]
+            except KeyError:
+                request = kwargs["request"]
+                if isinstance(request, Request):
+                    queryable_resp = await self.get_queryable_intersection(request)
+                    queryables_endpoint_cache["/queryables"] = queryable_resp
+        else:
+            r = requests.get(self.queryable_url_template.format(cid=collection_id))
+            if r.status_code == 404:
+                raise HTTPException(status_code=404)
+            elif r.status_code == 200:
+                try:
+                    queryable_resp = r.json()
+                except json.decoder.JSONDecodeError:
+                    raise HTTPException(status_code=404)
+        return queryable_resp

pcstac/tests/resources/test_queryables.py

@@ -0,0 +1,31 @@
+from typing import Callable
+
+import pytest
+
+
+@pytest.mark.asyncio
+async def test_queryables(app_client, load_test_data: Callable):
+    resp = await app_client.get("/queryables")
+    assert resp.status_code == 200
+    properties = resp.json()["properties"]
+    assert "id" in properties
+    assert "datetime" in properties
+    assert "naip:year" in properties
+    assert "naip:state" in properties
+
+
+@pytest.mark.asyncio
+async def test_collection_queryables(app_client, load_test_data: Callable):
+    resp = await app_client.get("/collections/naip/queryables")
+    assert resp.status_code == 200
+    properties = resp.json()["properties"]
+    assert "id" in properties
+    assert "datetime" in properties
+    assert "naip:year" in properties
+    assert "naip:state" in properties
+
+
+@pytest.mark.asyncio
+async def test_collection_queryables_404(app_client, load_test_data: Callable):
+    resp = await app_client.get("/collections/does-not-exist/queryables")
+    assert resp.status_code == 404