From b1a7a673846e1e3e54c7206787fdeab23c3e1101 Mon Sep 17 00:00:00 2001 From: kls Date: Tue, 25 Jun 2024 20:04:26 -0500 Subject: [PATCH] Update: add functions --- geosyspy/geosys.py | 117 +++++++++++++----- geosyspy/services/map_product_service.py | 109 +++++++++------- .../master_data_management_service.py | 31 +++-- 3 files changed, 171 insertions(+), 86 deletions(-) diff --git a/geosyspy/geosys.py b/geosyspy/geosys.py index c4829ba..a096d42 100644 --- a/geosyspy/geosys.py +++ b/geosyspy/geosys.py @@ -1,19 +1,19 @@ """ Geosysoy class""" -from enum import Enum import io -import zipfile -from typing import List, Optional -from pathlib import Path import logging +import zipfile from datetime import datetime +from enum import Enum +from pathlib import Path +from typing import List, Optional import numpy as np import pandas as pd import rasterio +import retrying import xarray as xr from rasterio.io import MemoryFile -import retrying from geosyspy import image_reference from geosyspy.services.agriquest_service import AgriquestService @@ -24,25 +24,25 @@ from geosyspy.services.master_data_management_service import MasterDataManagementService from geosyspy.services.vegetation_time_series_service import VegetationTimeSeriesService from geosyspy.services.weather_service import WeatherService -from geosyspy.utils.geosys_platform_urls import GEOSYS_API_URLS, GIS_API_URLS from geosyspy.utils.constants import ( - Env, - Region, - WeatherTypeCollection, LR_SATELLITE_COLLECTION, - SatelliteImageryCollection, MR_SATELLITE_COLLECTION, - Harvest, AgriquestBlocks, AgriquestCommodityCode, AgriquestWeatherType, CropIdSeason, Emergence, + Env, + Harvest, + Region, + SatelliteImageryCollection, + WeatherTypeCollection, ZarcCycleType, ZarcSoilType, ) -from geosyspy.utils.http_client import HttpClient +from geosyspy.utils.geosys_platform_urls import GEOSYS_API_URLS, GIS_API_URLS from geosyspy.utils.helper import Helper +from geosyspy.utils.http_client import HttpClient class Geosys: @@ -200,21 +200,30 @@ def get_satellite_image_time_series( if not season_field_id: # extract seasonfield id from geometry season_field_id = ( - self.__master_data_management_service.extract_season_field_id(polygon) + self.__master_data_management_service.extract_season_field_id( + polygon + ) + ) + elif ( + not self.__master_data_management_service.check_season_field_exists( + season_field_id ) - elif not self.__master_data_management_service.check_season_field_exists( - season_field_id ): raise ValueError( f"Cannot access {season_field_id}. It is not existing or connected user doens't have access to it." ) - + return self.__vts_service.get_time_series_by_pixel( season_field_id, start_date, end_date, indicators[0] ) elif set(collections).issubset(set(MR_SATELLITE_COLLECTION)): return self.__get_images_as_dataset( - season_field_id, polygon, start_date, end_date, collections, indicators[0] + season_field_id, + polygon, + start_date, + end_date, + collections, + indicators[0], ) else: raise TypeError( @@ -237,7 +246,7 @@ def get_satellite_coverage_image_references( ], polygon: Optional[str] = None, season_field_id: Optional[str] = None, - coveragePercent: Optional[int] = 80 + coveragePercent: Optional[int] = 80, ) -> tuple: """Retrieves a list of images that covers a polygon on a specific date range. The return is a tuple: a dataframe with all the images covering the polygon, and @@ -262,7 +271,13 @@ def get_satellite_coverage_image_references( ) df = self.__map_product_service.get_satellite_coverage( - season_field_id, polygon, start_date, end_date, "", coveragePercent, collections + season_field_id, + polygon, + start_date, + end_date, + "", + coveragePercent, + collections, ) images_references = {} if df is not None: @@ -297,9 +312,27 @@ def download_image(self, polygon, image_id, indicator: str = "", path: str = "") self.logger.info("writing to %s", path) f.write(response_zipped_tiff.content) - def get_product(self, season_field_id, image_id, indicator, image= None): + def download_image_difference_map( + self, season_field_id, image_id_before, image_id_after + ): + """Downloads a satellite image locally resulting of the difference between 2 images + + Args: + season_field_id : season_field_id + image_id_before : the image reference from the satellite coverage before. + image_id_before : the image reference from the satellite coverage after. + """ + response = self.__map_product_service.get_zipped_tiff_difference_map( + season_field_id, image_id_before, image_id_after + ) + + return response + + def get_product(self, season_field_id, image_id, indicator, image=None): - response = self.__map_product_service.get_product(season_field_id, image_id, indicator, image) + response = self.__map_product_service.get_product( + season_field_id, image_id, indicator, image + ) return response @@ -311,7 +344,7 @@ def __get_images_as_dataset( end_date: datetime, collections: Optional[list[SatelliteImageryCollection]], indicator: str, - coveragePercent: int = 80 + coveragePercent: int = 80, ) -> "np.ndarray[np.Any , np.dtype[np.float64]]": """Returns all the 'sensors_list' images covering 'polygon' between 'start_date' and 'end_date' as a xarray dataset. @@ -349,7 +382,13 @@ def get_coordinates_by_pixel(raster): # and sorts them by resolution, from the highest to the lowest. # Keeps only the first image if two are found on the same date. df_coverage = self.__map_product_service.get_satellite_coverage( - season_field_id, polygon, start_date, end_date, indicator, coveragePercent, collections + season_field_id, + polygon, + start_date, + end_date, + indicator, + coveragePercent, + collections, ) # Return empty dataset if no coverage on the polygon between start_date, end_date @@ -606,12 +645,28 @@ def get_available_permissions(self): permissions: a string array containing all available permissions of the connected user """ # get crop code list - result = self.__master_data_management_service.get_permission_codes() + result = self.__master_data_management_service.get_profile("permissions") # build a string array with all available permission codes for the connected user return result["permissions"] + def get_user_area_conversion_rate(self): + """Returns the user's defined area's unit of measurement conversion's rate to square metres.""" + + # get crop code list + result = self.__master_data_management_service.get_profile( + "unitProfileUnitCategories" + ) + + conversion_rate = list( + filter( + lambda x: x["unitCategory"]["id"] == "FIELD_SURFACE", + result["unitProfileUnitCategories"], + ) + )[0]["unit"]["conversionRate"] + return conversion_rate + def get_sfid_from_geometry(self, geometry: str): """Retrieves every season field ID contained within the passed geometry. @@ -620,8 +675,12 @@ def get_sfid_from_geometry(self, geometry: str): Returns: ids: an array containing all the seasonfield ids """ - result = self.__master_data_management_service.retrieve_season_fields_in_polygon(geometry) - ids = [item['id'] for item in result.json()] + result = ( + self.__master_data_management_service.retrieve_season_fields_in_polygon( + geometry + ) + ) + ids = [item["id"] for item in result.json()] return ids def get_season_fields(self, season_field_ids: List[str]): @@ -632,7 +691,9 @@ def get_season_fields(self, season_field_ids: List[str]): Returns: result: an array containing all the seasonfield """ - result = self.__master_data_management_service.get_season_fields(season_field_ids) + result = self.__master_data_management_service.get_season_fields( + season_field_ids + ) return result ########################################### @@ -1218,7 +1279,7 @@ def get_zarc_analytics( ) return self.check_status_and_metrics(task_id, "ZARC", sf_unique_id) - def get_farm_info_from_location(self, latitude:str, longitude:str): + def get_farm_info_from_location(self, latitude: str, longitude: str): """get farm info from CAR layer Args: diff --git a/geosyspy/services/map_product_service.py b/geosyspy/services/map_product_service.py index ed77bbc..df74b8c 100644 --- a/geosyspy/services/map_product_service.py +++ b/geosyspy/services/map_product_service.py @@ -2,16 +2,17 @@ import logging from datetime import datetime -from urllib.parse import urljoin from typing import Optional -from requests import HTTPError +from urllib.parse import urljoin + import pandas as pd +from requests import HTTPError + from geosyspy.utils.constants import ( + PRIORITY_HEADERS, GeosysApiEndpoints, SatelliteImageryCollection, - PRIORITY_HEADERS, ) - from geosyspy.utils.http_client import HttpClient @@ -76,25 +77,13 @@ def get_satellite_coverage( fields = f"&$fields=coveragePercent,maps,image.id,image.sensor,image.availableBands,image.spatialResolution,image.date,seasonField.id" flm_url: str = urljoin( self.base_url, - GeosysApiEndpoints.FLM_CATALOG_IMAGERY_POST.value + parameters + fields + GeosysApiEndpoints.FLM_CATALOG_IMAGERY_POST.value + parameters + fields, ) if not season_field_id: - payload = { - "seasonFields": [ - { - "geometry": polygon - } - ] - } + payload = {"seasonFields": [{"geometry": polygon}]} else: - payload = { - "seasonFields": [ - { - "id": season_field_id - } - ] - } + payload = {"seasonFields": [{"id": season_field_id}]} response = self.http_client.post( flm_url, @@ -113,7 +102,7 @@ def get_satellite_coverage( "maps", "image.id", "image.availableBands", - "image.sensor", + "image.sensor", "image.spatialResolution", "image.date", "seasonField.id", @@ -122,37 +111,29 @@ def get_satellite_coverage( else: self.logger.info(response.status_code) - def get_zipped_tiff(self, field_id: str, - field_geometry: str, - image_id: str, - indicator: str): - + def get_zipped_tiff( + self, field_id: str, field_geometry: str, image_id: str, indicator: str + ): + if indicator != "" and indicator.upper() != "REFLECTANCE": parameters = f"/{indicator.upper()}/image.tiff.zip?resolution=Sensor" - download_tiff_url: str = urljoin(self.base_url, - GeosysApiEndpoints.FLM_BASE_REFERENCE_MAP_POST.value + parameters) + download_tiff_url: str = urljoin( + self.base_url, + GeosysApiEndpoints.FLM_BASE_REFERENCE_MAP_POST.value + parameters, + ) else: parameters = f"/TOC/image.tiff.zip?resolution=Sensor" - download_tiff_url: str = urljoin(self.base_url, GeosysApiEndpoints.FLM_REFLECTANCE_MAP.value + parameters) + download_tiff_url: str = urljoin( + self.base_url, GeosysApiEndpoints.FLM_REFLECTANCE_MAP.value + parameters + ) - if not field_id or field_id == '': + if not field_id or field_id == "": payload = { - "image": { - "id": image_id - }, - "seasonField": { - "geometry": field_geometry - } + "image": {"id": image_id}, + "seasonField": {"geometry": field_geometry}, } else: - payload = { - "image": { - "id": image_id - }, - "seasonField": { - "id": field_id - } - } + payload = {"image": {"id": image_id}, "seasonField": {"id": field_id}} response_zipped_tiff = self.http_client.post( download_tiff_url, @@ -166,7 +147,9 @@ def get_zipped_tiff(self, field_id: str, ) return response_zipped_tiff - def get_product(self, field_id: str, image_id: str, indicator: str, image: str = None): + def get_product( + self, field_id: str, image_id: str, indicator: str, image: str = None + ): """ Retrieves image product for a given season field and image reference from MP API. @@ -186,7 +169,8 @@ def get_product(self, field_id: str, image_id: str, indicator: str, image: str = get_product_url: str = urljoin( self.base_url, - GeosysApiEndpoints.FLM_BASE_REFERENCE_MAP.value.format(field_id) + parameters + GeosysApiEndpoints.FLM_BASE_REFERENCE_MAP.value.format(field_id) + + parameters, ) response_product = self.http_client.get( get_product_url, @@ -198,7 +182,7 @@ def get_product(self, field_id: str, image_id: str, indicator: str, image: str = "Unable to retrieve product. Server error: " + str(response_product.status_code) ) - + if image is not None: with open("output" + image, "wb") as file: file.write(response_product.content) @@ -206,3 +190,36 @@ def get_product(self, field_id: str, image_id: str, indicator: str, image: str = else: df = pd.json_normalize(response_product.json()) return df + + def get_zipped_tiff_difference_map( + self, field_id: str, image_id_before: str, image_id_after: str + ): + """ + Retrieves tiff resulting of a difference between 2 in-season images for a given season field from MP API. + + Args: + season_field_id (str): The identifier for the season field. + image_id_before (str): The image reference from the satellite coverage before. + image_id_after (str): The image reference from the satellite coverage after. + + Returns: + zipped tiff + """ + + parameters = f"/{image_id_before}/base-reference-map/DIFFERENCE_INSEASON_NDVI/difference-with/{image_id_after}/image.tiff.zip?$epsg-out=3857" + download_tiff_url: str = urljoin( + self.base_url, + GeosysApiEndpoints.FLM_BASE_REFERENCE_MAP.value.format(field_id) + + parameters, + ) + + response_zipped_tiff = self.http_client.get( + download_tiff_url, + {"X-Geosys-Task-Code": PRIORITY_HEADERS[self.priority_queue]}, + ) + if response_zipped_tiff.status_code != 200: + raise HTTPError( + "Unable to download tiff.zip file. Server error: " + + str(response_zipped_tiff.status_code) + ) + return response_zipped_tiff diff --git a/geosyspy/services/master_data_management_service.py b/geosyspy/services/master_data_management_service.py index 74ef860..2395d42 100644 --- a/geosyspy/services/master_data_management_service.py +++ b/geosyspy/services/master_data_management_service.py @@ -1,15 +1,14 @@ """ Mastaer data managenement service class""" import logging -from urllib.parse import urljoin -from typing import List from datetime import datetime +from typing import List, Optional +from urllib.parse import urljoin -from geosyspy.utils.constants import GeosysApiEndpoints, SEASON_FIELD_ID_REGEX +from geosyspy.utils.constants import SEASON_FIELD_ID_REGEX, GeosysApiEndpoints from geosyspy.utils.helper import Helper from geosyspy.utils.http_client import HttpClient - class MasterDataManagementService: def __init__(self, base_url: str, http_client: HttpClient): @@ -140,6 +139,7 @@ def check_season_field_exists(self, season_field_id: str) -> str: return True return False + def get_available_crops_code(self) -> List[str]: """Extracts the list of available crops for the connected user @@ -167,7 +167,7 @@ def get_available_crops_code(self) -> List[str]: f"Cannot handle HTTP response : {str(response.status_code)} : {str(response.json())}" ) - def get_permission_codes(self) -> List[str]: + def get_profile(self, fields: Optional[str] = None) -> List[str]: """Extracts the list of available permissions for the connected user Args: @@ -178,11 +178,18 @@ def get_permission_codes(self) -> List[str]: Raises: ValueError: The response status code is not as expected. """ - mdm_url: str = urljoin( - self.base_url, - GeosysApiEndpoints.MASTER_DATA_MANAGEMENT_ENDPOINT.value - + "/profile?$fields=permissions&$limit=none", - ) + if fields == None: + mdm_url: str = urljoin( + self.base_url, + GeosysApiEndpoints.MASTER_DATA_MANAGEMENT_ENDPOINT.value + + f"/profile", + ) + else: + mdm_url: str = urljoin( + self.base_url, + GeosysApiEndpoints.MASTER_DATA_MANAGEMENT_ENDPOINT.value + + f"/profile?$fields={fields}&$limit=none", + ) response = self.http_client.get(mdm_url) @@ -210,7 +217,7 @@ def get_season_fields(self, season_field_ids: str) -> List[object]: mdm_url: str = urljoin( self.base_url, GeosysApiEndpoints.MASTER_DATA_MANAGEMENT_ENDPOINT.value - +'/seasonfields?$fields=id,geometry,sowingDate,estimatedHarvestDate,Crop.Id' #add other fields if needed + +'/seasonfields?$fields=id,geometry,sowingDate,estimatedHarvestDate,Crop.Id,acreage' #add other fields if needed +'&Id=$in:' + '|'.join(season_field_ids) +'&$limit=none' ) @@ -223,4 +230,4 @@ def get_season_fields(self, season_field_ids: str) -> List[object]: return dict_response raise ValueError( f"Cannot handle HTTP response : {str(response.status_code)} : {str(dict_response)}" - ) + ) \ No newline at end of file