From 48e0a2249c8b4331f56d3b3c35a1561c4dbf023d Mon Sep 17 00:00:00 2001 From: Anne Haley Date: Mon, 24 Jun 2024 19:23:04 +0000 Subject: [PATCH] Add module-based dataset loading --- sample_data/ingest_sample_data.py | 1 + .../use_cases/boston_floods/cities.json | 14 ---- .../use_cases/new_york_energy/charts.json | 1 + .../use_cases/new_york_energy/contexts.json | 19 +++++ .../use_cases/new_york_energy/datasets.json | 72 +++++++++++++++++++ .../use_cases/new_york_energy/nysdp.py | 55 ++++++++++++++ 6 files changed, 148 insertions(+), 14 deletions(-) delete mode 100644 sample_data/use_cases/boston_floods/cities.json create mode 100644 sample_data/use_cases/new_york_energy/charts.json create mode 100644 sample_data/use_cases/new_york_energy/contexts.json create mode 100644 sample_data/use_cases/new_york_energy/datasets.json create mode 100644 sample_data/use_cases/new_york_energy/nysdp.py diff --git a/sample_data/ingest_sample_data.py b/sample_data/ingest_sample_data.py index fa77a8ff..3292b6d4 100644 --- a/sample_data/ingest_sample_data.py +++ b/sample_data/ingest_sample_data.py @@ -1,4 +1,5 @@ from datetime import datetime +import importlib import json import os from pathlib import Path diff --git a/sample_data/use_cases/boston_floods/cities.json b/sample_data/use_cases/boston_floods/cities.json deleted file mode 100644 index b00ea6b4..00000000 --- a/sample_data/use_cases/boston_floods/cities.json +++ /dev/null @@ -1,14 +0,0 @@ -[ - { - "name": "Boston, MA", - "latitude": 42.4, - "longitude": -71.1, - "default_zoom": 11 - }, - { - "name": "Washington, DC", - "latitude": 38.9, - "longitude": -77.1, - "default_zoom": 11 - } -] diff --git a/sample_data/use_cases/new_york_energy/charts.json b/sample_data/use_cases/new_york_energy/charts.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/sample_data/use_cases/new_york_energy/charts.json @@ -0,0 +1 @@ +[] diff --git a/sample_data/use_cases/new_york_energy/contexts.json b/sample_data/use_cases/new_york_energy/contexts.json new file mode 100644 index 00000000..189f92fd --- /dev/null +++ b/sample_data/use_cases/new_york_energy/contexts.json @@ -0,0 +1,19 @@ +[ + { + "name": "NY Energy", + "default_map_center": [ + 43.5, + -75.5 + ], + "default_map_zoom": 8, + "datasets": [ + "National Grid Substations", + "National Grid DistAssetsOverview", + "National Grid Electrification_Data", + "National Grid EV_Load_Serving_Capacity", + "National Grid Hosting_Capacity_Data", + "National Grid LSRV", + "National Grid NY_SubT_SDP" + ] + } +] diff --git a/sample_data/use_cases/new_york_energy/datasets.json b/sample_data/use_cases/new_york_energy/datasets.json new file mode 100644 index 00000000..fc4d61d2 --- /dev/null +++ b/sample_data/use_cases/new_york_energy/datasets.json @@ -0,0 +1,72 @@ +[ + { + "name": "National Grid Substations", + "description": "Substation vector data", + "category": "energy", + "module": "nysdp", + "function": "create_vector_features", + "kwargs": { + "service_name": "Substations" + } + }, + { + "name": "National Grid DistAssetsOverview", + "description": "DistAssetsOverview vector data", + "category": "energy", + "module": "nysdp", + "function": "create_vector_features", + "kwargs": { + "service_name": "DistAssetsOverview" + } + }, + { + "name": "National Grid Electrification_Data", + "description": "Electrification_Data vector data", + "category": "energy", + "module": "nysdp", + "function": "create_vector_features", + "kwargs": { + "service_name": "Electrification_Data" + } + }, + { + "name": "National Grid EV_Load_Serving_Capacity", + "description": "EV_Load_Serving_Capacity vector data", + "category": "energy", + "module": "nysdp", + "function": "create_vector_features", + "kwargs": { + "service_name": "EV_Load_Serving_Capacity" + } + }, + { + "name": "National Grid Hosting_Capacity_Data", + "description": "Hosting_Capacity_Data vector data", + "category": "energy", + "module": "nysdp", + "function": "create_vector_features", + "kwargs": { + "service_name": "Hosting_Capacity_Data" + } + }, + { + "name": "National Grid LSRV", + "description": "LSRV vector data", + "category": "energy", + "module": "nysdp", + "function": "create_vector_features", + "kwargs": { + "service_name": "LSRV" + } + }, + { + "name": "National Grid NY_SubT_SDP", + "description": "NY_SubT_SDP vector data", + "category": "energy", + "module": "nysdp", + "function": "create_vector_features", + "kwargs": { + "service_name": "NY_SubT_SDP" + } + } +] diff --git a/sample_data/use_cases/new_york_energy/nysdp.py b/sample_data/use_cases/new_york_energy/nysdp.py new file mode 100644 index 00000000..2e56a84a --- /dev/null +++ b/sample_data/use_cases/new_york_energy/nysdp.py @@ -0,0 +1,55 @@ +import json +import geopandas +import requests + +from django.contrib.gis.geos import GEOSGeometry +from uvdat.core.models import VectorMapLayer, VectorFeature + + +NYDSP_URL = 'https://systemdataportal.nationalgrid.com/arcgis/rest/services/NYSDP' +RECORDS_PER_PAGE = 100 +SERVICE_SUFFIX = 'MapServer' +FORMAT_SUFFIX = 'f=pjson' +QUERY_CONTENT = f'where=1%3D1&returnGeometry=true&outFields=*&resultRecordCount={RECORDS_PER_PAGE}&f=geojson' + + +def fetch_vector_features(service_name=None): + feature_sets = {} + if service_name is None: + return feature_sets + service_url = f'{NYDSP_URL}/{service_name}/{SERVICE_SUFFIX}' + service_info = requests.get(f'{service_url}?{FORMAT_SUFFIX}').json() + for layer in service_info.get('layers', []): + feature_set = [] + layer_id = layer.get('id') + if layer_id is not None: + feature_page = None + result_offset = 0 + while feature_page is None or len(feature_page) == RECORDS_PER_PAGE: + query_response = requests.get( + f"{service_url}/{layer_id}/query?resultOffset={result_offset}&{QUERY_CONTENT}" + ).json() + feature_page = query_response.get('features', []) + feature_set += feature_page + result_offset += RECORDS_PER_PAGE + if len(feature_set): + feature_sets[layer_id] = feature_set + return feature_sets + + +def create_vector_features(dataset, service_name=None): + VectorMapLayer.objects.filter(dataset=dataset).delete() + + feature_sets = fetch_vector_features(service_name=service_name) + vector_features = [] + for index, feature_set in feature_sets.items(): + map_layer = VectorMapLayer.objects.create(dataset=dataset, index=index) + for feature in feature_set: + vector_features.append( + VectorFeature( + map_layer=map_layer, + geometry=GEOSGeometry(json.dumps(feature['geometry'])), + properties=feature['properties'], + ) + ) + VectorFeature.objects.bulk_create(vector_features)