From 0e4fea3b17b509f01f83bf03cc523296c21b098c Mon Sep 17 00:00:00 2001 From: Daniel Wiesmann Date: Wed, 13 Jul 2022 16:40:10 +0100 Subject: [PATCH] Make item geometry and bbox nullable for sqlalchemy backend (#398) Closes #350 --- CHANGES.md | 1 + ...bf_make_item_geometry_and_bbox_nullable.py | 46 +++++ stac_fastapi/sqlalchemy/setup.py | 2 +- .../sqlalchemy/models/database.py | 6 +- .../stac_fastapi/sqlalchemy/serializers.py | 14 +- stac_fastapi/sqlalchemy/tests/api/test_api.py | 15 ++ .../tests/data/test_item_geometry_null.json | 169 ++++++++++++++++++ 7 files changed, 247 insertions(+), 6 deletions(-) create mode 100644 stac_fastapi/sqlalchemy/alembic/versions/7016c1bf3fbf_make_item_geometry_and_bbox_nullable.py create mode 100644 stac_fastapi/sqlalchemy/tests/data/test_item_geometry_null.json diff --git a/CHANGES.md b/CHANGES.md index f28d116c6..98c19189b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -19,6 +19,7 @@ ([#355](https://github.com/stac-utils/stac-fastapi/issues/355)) * docker-compose now runs uvicorn with hot-reloading enabled * Bump version of PGStac to 0.6.2 that includes support for hydrating results in the API backed ([#397](https://github.com/stac-utils/stac-fastapi/pull/397)) +* Make item geometry and bbox nullable in sqlalchemy backend. ([#398](https://github.com/stac-utils/stac-fastapi/pull/398)) ### Removed diff --git a/stac_fastapi/sqlalchemy/alembic/versions/7016c1bf3fbf_make_item_geometry_and_bbox_nullable.py b/stac_fastapi/sqlalchemy/alembic/versions/7016c1bf3fbf_make_item_geometry_and_bbox_nullable.py new file mode 100644 index 000000000..804361b02 --- /dev/null +++ b/stac_fastapi/sqlalchemy/alembic/versions/7016c1bf3fbf_make_item_geometry_and_bbox_nullable.py @@ -0,0 +1,46 @@ +"""Make item geometry and bbox nullable + +Revision ID: 7016c1bf3fbf +Revises: 5909bd10f2e6 +Create Date: 2022-04-28 10:40:06.856826 + +""" +from alembic import op + +# revision identifiers, used by Alembic. +revision = "7016c1bf3fbf" +down_revision = "5909bd10f2e6" +branch_labels = None +depends_on = None + + +def upgrade(): + op.alter_column( + schema="data", + table_name="items", + column_name="geometry", + nullable=True, + ) + op.alter_column( + schema="data", + table_name="items", + column_name="bbox", + nullable=True, + ) + + +def downgrade(): + # Downgrading will require the user to update or remove all null geometry + # cases from the DB, otherwise the downgrade migration will fail. + op.alter_column( + schema="data", + table_name="items", + column_name="geometry", + nullable=False, + ) + op.alter_column( + schema="data", + table_name="items", + column_name="bbox", + nullable=False, + ) diff --git a/stac_fastapi/sqlalchemy/setup.py b/stac_fastapi/sqlalchemy/setup.py index d6a606a3c..c7d6e98b6 100644 --- a/stac_fastapi/sqlalchemy/setup.py +++ b/stac_fastapi/sqlalchemy/setup.py @@ -8,7 +8,7 @@ install_requires = [ "attrs", "pydantic[dotenv]", - "stac_pydantic==2.0.*", + "stac_pydantic>=2.0.3", "stac-fastapi.types", "stac-fastapi.api", "stac-fastapi.extensions", diff --git a/stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/models/database.py b/stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/models/database.py index 73969bbe0..ed9d8cef0 100644 --- a/stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/models/database.py +++ b/stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/models/database.py @@ -64,8 +64,10 @@ class Item(BaseModel): # type:ignore id = sa.Column(sa.VARCHAR(1024), nullable=False, primary_key=True) stac_version = sa.Column(sa.VARCHAR(300)) stac_extensions = sa.Column(sa.ARRAY(sa.VARCHAR(300)), nullable=True) - geometry = sa.Column(GeojsonGeometry("GEOMETRY", srid=4326, spatial_index=True)) - bbox = sa.Column(sa.ARRAY(sa.NUMERIC), nullable=False) + geometry = sa.Column( + GeojsonGeometry("GEOMETRY", srid=4326, spatial_index=True), nullable=True + ) + bbox = sa.Column(sa.ARRAY(sa.NUMERIC), nullable=True) properties = sa.Column(JSONB) assets = sa.Column(JSONB) collection_id = sa.Column( diff --git a/stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/serializers.py b/stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/serializers.py index 93590378e..948d06e30 100644 --- a/stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/serializers.py +++ b/stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/serializers.py @@ -78,6 +78,10 @@ def db_to_stac(cls, db_model: database.Item, base_url: str) -> stac_types.Item: if isinstance(geometry, str): geometry = json.loads(geometry) + bbox = db_model.bbox + if bbox is not None: + bbox = [float(x) for x in db_model.bbox] + return stac_types.Item( type="Feature", stac_version=db_model.stac_version, @@ -85,7 +89,7 @@ def db_to_stac(cls, db_model: database.Item, base_url: str) -> stac_types.Item: id=db_model.id, collection=db_model.collection_id, geometry=geometry, - bbox=[float(x) for x in db_model.bbox], + bbox=bbox, properties=properties, links=item_links, assets=db_model.assets, @@ -111,13 +115,17 @@ def stac_to_db( stac_data["properties"]["created"] = now stac_data["properties"]["updated"] = now + geometry = stac_data["geometry"] + if geometry is not None: + geometry = json.dumps(geometry) + return database.Item( id=stac_data["id"], collection_id=stac_data["collection"], stac_version=stac_data["stac_version"], stac_extensions=stac_data.get("stac_extensions"), - geometry=json.dumps(stac_data["geometry"]), - bbox=stac_data["bbox"], + geometry=geometry, + bbox=stac_data.get("bbox"), properties=stac_data["properties"], assets=stac_data["assets"], **indexed_fields, diff --git a/stac_fastapi/sqlalchemy/tests/api/test_api.py b/stac_fastapi/sqlalchemy/tests/api/test_api.py index 0abd7cb00..1ee196923 100644 --- a/stac_fastapi/sqlalchemy/tests/api/test_api.py +++ b/stac_fastapi/sqlalchemy/tests/api/test_api.py @@ -92,6 +92,21 @@ def test_app_search_response_multipolygon( assert resp_json.get("features")[0]["geometry"]["type"] == "MultiPolygon" +def test_app_search_response_geometry_null( + load_test_data, app_client, postgres_transactions +): + item = load_test_data("test_item_geometry_null.json") + postgres_transactions.create_item(item, request=MockStarletteRequest) + + resp = app_client.get("/search", params={"collections": ["test-collection"]}) + assert resp.status_code == 200 + resp_json = resp.json() + + assert resp_json.get("type") == "FeatureCollection" + assert resp_json.get("features")[0]["geometry"] is None + assert resp_json.get("features")[0]["bbox"] is None + + def test_app_context_extension(load_test_data, app_client, postgres_transactions): item = load_test_data("test_item.json") postgres_transactions.create_item(item, request=MockStarletteRequest) diff --git a/stac_fastapi/sqlalchemy/tests/data/test_item_geometry_null.json b/stac_fastapi/sqlalchemy/tests/data/test_item_geometry_null.json new file mode 100644 index 000000000..27ef327a7 --- /dev/null +++ b/stac_fastapi/sqlalchemy/tests/data/test_item_geometry_null.json @@ -0,0 +1,169 @@ +{ + "type": "Feature", + "stac_version": "1.0.0", + "stac_extensions": [ + "https://landsat.usgs.gov/stac/landsat-ard-extension/v1.0.0/schema.json", + "https://stac-extensions.github.io/projection/v1.0.0/schema.json", + "https://stac-extensions.github.io/eo/v1.0.0/schema.json", + "https://stac-extensions.github.io/alternate-assets/v1.1.0/schema.json", + "https://stac-extensions.github.io/storage/v1.0.0/schema.json" + ], + "id": "LE07_CU_002012_20150101_20210502_02_BA", + "description": "Landsat Collection 2 Level-3 Burned Area Product", + "geometry": null, + "properties": { + "datetime": "2015-01-01T18:39:12.4885358Z", + "platform": "LANDSAT_7", + "instruments": [ + "ETM" + ], + "landsat:grid_horizontal": "02", + "landsat:grid_vertical": "12", + "landsat:grid_region": "CU", + "landsat:scene_count": 1, + "eo:cloud_cover": 0.0759, + "landsat:cloud_shadow_cover": 0.1394, + "landsat:snow_ice_cover": 0, + "landsat:fill": 95.4286, + "proj:epsg": null, + "proj:shape": [ + 5000, + 5000 + ], + "proj:transform": [ + 30, + 0, + -2265585, + 0, + -30, + 1514805 + ], + "created": "2022-02-08T20:07:38.885Z", + "updated": "2022-02-08T20:07:38.885Z" + }, + "assets": { + "index": { + "title": "HTML index page", + "type": "text/html", + "roles": [ + "metadata" + ], + "href": "https://landsatlook.usgs.gov/stac-browser/collection02/BA/2015/CU/002/012/LE07_CU_002012_20150101_20210502_02_BA/LE07_CU_002012_20150101_20210502_02" + }, + "bp": { + "title": "Burn Probability", + "description": "Collection 2 Level-3 Albers Burn Probability Burned Area", + "type": "image/vnd.stac.geotiff; cloud-optimized=true", + "roles": [ + "data" + ], + "href": "https://landsatlook.usgs.gov/level-3/collection02/BA/2015/CU/002/012/LE07_CU_002012_20150101_20210502_02_BA/LE07_CU_002012_20150101_20210502_02_BP.TIF", + "alternate": { + "s3": { + "storage:platform": "AWS", + "storage:requester_pays": true, + "href": "s3://usgs-landsat-level-3/collection02/BA/2015/CU/002/012/LE07_CU_002012_20150101_20210502_02_BA/LE07_CU_002012_20150101_20210502_02_BP.TIF" + } + } + }, + "bc": { + "title": "Burn Classification", + "description": "Collection 2 Level-3 Albers Burn Classification Burned Area", + "type": "image/vnd.stac.geotiff; cloud-optimized=true", + "roles": [ + "data" + ], + "href": "https://landsatlook.usgs.gov/level-3/collection02/BA/2015/CU/002/012/LE07_CU_002012_20150101_20210502_02_BA/LE07_CU_002012_20150101_20210502_02_BC.TIF", + "alternate": { + "s3": { + "storage:platform": "AWS", + "storage:requester_pays": true, + "href": "s3://usgs-landsat-level-3/collection02/BA/2015/CU/002/012/LE07_CU_002012_20150101_20210502_02_BA/LE07_CU_002012_20150101_20210502_02_BC.TIF" + } + } + }, + "quick_look": { + "title": "Quick Look File", + "description": "Collection 2 Level-3 Albers Quick Look File Burned Area", + "type": "image/png", + "roles": [ + "data" + ], + "href": "https://landsatlook.usgs.gov/level-3/collection02/BA/2015/CU/002/012/LE07_CU_002012_20150101_20210502_02_BA/LE07_CU_002012_20150101_20210502_02_QuickLook.png", + "alternate": { + "s3": { + "storage:platform": "AWS", + "storage:requester_pays": true, + "href": "s3://usgs-landsat-level-3/collection02/BA/2015/CU/002/012/LE07_CU_002012_20150101_20210502_02_BA/LE07_CU_002012_20150101_20210502_02_QuickLook.png" + } + } + }, + "xml": { + "title": "Extensible Metadata File", + "description": "Collection 2 Level-3 Albers Extensible Metadata File Burned Area", + "type": "application/xml", + "roles": [ + "metadata" + ], + "href": "https://landsatlook.usgs.gov/level-3/collection02/BA/2015/CU/002/012/LE07_CU_002012_20150101_20210502_02_BA/LE07_CU_002012_20150101_20210502_02.xml", + "alternate": { + "s3": { + "storage:platform": "AWS", + "storage:requester_pays": true, + "href": "s3://usgs-landsat-level-3/collection02/BA/2015/CU/002/012/LE07_CU_002012_20150101_20210502_02_BA/LE07_CU_002012_20150101_20210502_02.xml" + } + } + }, + "json": { + "title": "Extensible Metadata File (json)", + "description": "Collection 2 Level-3 Albers Extensible Metadata File (json) Burned Area", + "type": "application/json", + "roles": [ + "metadata" + ], + "href": "https://landsatlook.usgs.gov/level-3/collection02/BA/2015/CU/002/012/LE07_CU_002012_20150101_20210502_02_BA/LE07_CU_002012_20150101_20210502_02.json", + "alternate": { + "s3": { + "storage:platform": "AWS", + "storage:requester_pays": true, + "href": "s3://usgs-landsat-level-3/collection02/BA/2015/CU/002/012/LE07_CU_002012_20150101_20210502_02_BA/LE07_CU_002012_20150101_20210502_02.json" + } + } + } + }, + "links": [ + { + "rel": "self", + "href": "https://landsatlook.usgs.gov/stac-server/collections/landsat-c2l3-ba/items/LE07_CU_002012_20150101_20210502_02_BA" + }, + { + "rel": "derived_from", + "href": "https://landsatlook.usgs.gov/stac-server/collections/landsat-c2ard-sr/items/LE07_CU_002012_20150101_20210502_02_SR" + }, + { + "rel": "derived_from", + "href": "https://landsatlook.usgs.gov/stac-server/collections/landsat-c2ard-st/items/LE07_CU_002012_20150101_20210502_02_ST" + }, + { + "rel": "derived_from", + "href": "https://landsatlook.usgs.gov/stac-server/collections/landsat-c2ard-ta/items/LE07_CU_002012_20150101_20210502_02_TOA" + }, + { + "rel": "derived_from", + "href": "https://landsatlook.usgs.gov/stac-server/collections/landsat-c2ard-bt/items/LE07_CU_002012_20150101_20210502_02_BT" + }, + { + "rel": "parent", + "href": "https://landsatlook.usgs.gov/stac-server/collections/landsat-c2l3-ba" + }, + { + "rel": "collection", + "href": "https://landsatlook.usgs.gov/stac-server/collections/landsat-c2l3-ba" + }, + { + "rel": "root", + "href": "https://landsatlook.usgs.gov/stac-server/" + } + ], + "collection": "test-collection" + } \ No newline at end of file