From 661c8f012104a94dee36191df39c24ceca4ccabc Mon Sep 17 00:00:00 2001 From: Frederique Date: Wed, 13 Dec 2023 14:36:40 +0100 Subject: [PATCH 1/4] The data was saved in the fiat folder where the model was read and not in the output. I moved the script to the write function to write it to the correct folder --- hydromt_fiat/fiat.py | 59 ++++++++++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/hydromt_fiat/fiat.py b/hydromt_fiat/fiat.py index f00fbe8a..c39e0174 100644 --- a/hydromt_fiat/fiat.py +++ b/hydromt_fiat/fiat.py @@ -71,6 +71,8 @@ def __init__( self.exposure = None self.vulnerability = None self.vf_ids_and_linking_df = pd.DataFrame() + self.additional_attributes_fn = "" # Path or paths to the additional attributes dataset(s) + self.building_footprint_fn = "" # Path to the building footprints dataset def setup_global_settings( self, @@ -769,33 +771,22 @@ def setup_aggregation_areas( Parameters ---------- exposure_gdf : gpd.GeoDataFrame - Exposure data to join the aggregation areas to as "Aggregation - Label: `label_names`". + Exposure data to join the aggregation areas to as `label_names`. aggregation_area_fn : Union[List[str], List[Path], str, Path] Path(s) to the aggregation area(s). attribute_names : Union[List[str], str] Name of the attribute(s) to join. label_names : Union[List[str], str] - Name of the label(s) to join. - + The name that the new attribute will get in the exposure data. """ exposure_gdf = self.exposure.get_full_gdf(self.exposure.exposure_db) self.exposure.exposure_db = join_exposure_aggregation_areas( exposure_gdf, aggregation_area_fn, attribute_names, label_names ) - - # Create additional attributes folder in root - additional_att_input = Path(self.root).joinpath("additional_attributes") - if not os.path.exists(additional_att_input): - os.makedirs(additional_att_input) - - if isinstance(aggregation_area_fn,list): - for file in aggregation_area_fn: - shutil.copy2(file, additional_att_input) - else: - shutil.copy2(aggregation_area_fn, additional_att_input) + # Set the additional_attributes_fn property to save the additional datasets + self.additional_attributes_fn = aggregation_area_fn def setup_building_footprint( self, @@ -822,15 +813,9 @@ def setup_building_footprint( building_footprint_fn, attribute_name, ) - # Create BF folder in Exposure - building_footprints_exp = Path(self.root).joinpath("exposure" , "building_footprints") - if not os.path.exists(building_footprints_exp): - os.makedirs(building_footprints_exp) - if isinstance(building_footprint_fn,list): - for file in building_footprint_fn: - shutil.copy2(file, building_footprints_exp) - else: - shutil.copy2(building_footprint_fn, building_footprints_exp) + + # Set the building_footprint_fn property to save the building footprints + self.building_footprint_fn = building_footprint_fn # Update functions def update_all(self): @@ -959,6 +944,32 @@ def write(self): self.write_geoms(fn="exposure/{name}.gpkg", driver="GPKG") if self._tables: self.write_tables() + if self.additional_attributes_fn: + folder = Path(self.root).joinpath("additional_attributes") + self.copy_datasets(self.additional_attributes_fn, folder) + if self.building_footprint_fn: + folder = Path(self.root).joinpath("exposure" , "building_footprints") + self.copy_datasets(self.building_footprint_fn, folder) + + def copy_datasets(self, data: Union[list, str, Path], folder: Union[Path, str]) -> None: + """Copies datasets to another folder + + Parameters + ---------- + data : Union[list, str, Path] + _description_ + folder : Union[Path, str] + _description_ + """ + # Create additional attributes folder in root + if not os.path.exists(folder): + os.makedirs(folder) + + if isinstance(data, list): + for file in data: + shutil.copy2(file, folder) + elif isinstance(data, Path) or isinstance(data, str): + shutil.copy2(data, folder) def write_tables(self) -> None: if len(self._tables) == 0: From 09183aff44abc51ae2e8762bfa20587b5e82d465 Mon Sep 17 00:00:00 2001 From: Frederique Date: Wed, 13 Dec 2023 14:41:43 +0100 Subject: [PATCH 2/4] Added additional assert for correctly saving the files --- tests/test_aggregation_areas.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_aggregation_areas.py b/tests/test_aggregation_areas.py index cf4d1c5e..4ccf6ccd 100644 --- a/tests/test_aggregation_areas.py +++ b/tests/test_aggregation_areas.py @@ -69,3 +69,14 @@ def test_aggregation_areas(case: ParameterSet | Sequence[object] | object): # Check if the vulnerability functions exist assert len(fm.vulnerability.functions) > 0 + + # Check if the additional_attributes folder exists + assert root.joinpath("additional_attributes").exists() + + # Check if the files are copied to the right folder + aggregation_area_fn = _cases[case]["configuration"]["setup_aggregation_areas"]["aggregation_area_fn"] + if isinstance(aggregation_area_fn, Path): + aggregation_area_fn = [aggregation_area_fn] + + for a in aggregation_area_fn: + assert root.joinpath("additional_attributes", a.name).exists() \ No newline at end of file From 1b002c468d573af139e6f187c8eeffb846d9f902 Mon Sep 17 00:00:00 2001 From: Frederique Date: Wed, 13 Dec 2023 15:21:00 +0100 Subject: [PATCH 3/4] aligning with floodadapt needs --- .../attribute_linking/NSI_attributes_to_FIAT.json | 2 +- hydromt_fiat/fiat.py | 7 ++++++- hydromt_fiat/workflows/equity_data.py | 11 ++++++++--- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/hydromt_fiat/data/attribute_linking/NSI_attributes_to_FIAT.json b/hydromt_fiat/data/attribute_linking/NSI_attributes_to_FIAT.json index 62f976b7..411b0e40 100644 --- a/hydromt_fiat/data/attribute_linking/NSI_attributes_to_FIAT.json +++ b/hydromt_fiat/data/attribute_linking/NSI_attributes_to_FIAT.json @@ -8,5 +8,5 @@ "Ground Floor Height": "found_ht", "Ground Elevation": "ground_elv", "geometry": "geometry", - "Aggregation Label: Census Block": "cbfips" + "Aggregation Label: Census Blockgroup": "cbfips" } \ No newline at end of file diff --git a/hydromt_fiat/fiat.py b/hydromt_fiat/fiat.py index c39e0174..b07cc114 100644 --- a/hydromt_fiat/fiat.py +++ b/hydromt_fiat/fiat.py @@ -757,9 +757,13 @@ def setup_equity_data( equity.match_geo_ID() equity.download_shp_geom(year_data, county_numbers) equity.merge_equity_data_shp() + equity.clean() self.set_tables(df=equity.equity_data_shp, name="equity_data") + # Update (if necessary) the aggregation label: census block + # Save the census block aggregation area data + def setup_aggregation_areas( self, aggregation_area_fn: Union[List[str], List[Path], str, Path], @@ -779,7 +783,8 @@ def setup_aggregation_areas( label_names : Union[List[str], str] The name that the new attribute will get in the exposure data. """ - + # TODO: add a census block option to automatically download and save + # the data exposure_gdf = self.exposure.get_full_gdf(self.exposure.exposure_db) self.exposure.exposure_db = join_exposure_aggregation_areas( exposure_gdf, aggregation_area_fn, attribute_names, label_names diff --git a/hydromt_fiat/workflows/equity_data.py b/hydromt_fiat/workflows/equity_data.py index 98cb5bcd..3d3eb078 100644 --- a/hydromt_fiat/workflows/equity_data.py +++ b/hydromt_fiat/workflows/equity_data.py @@ -177,7 +177,7 @@ def download_shp_geom(self, year_data: int, counties: List[str]): block_groups_shp = shp.dissolve(by=attrs, as_index=False) block_groups_shp = block_groups_shp[attrs + ["geometry"]] - # block_groups["Census_Bg"] = block_groups['TRACTCE' + code].astype(str) + "-block" + block_groups['BLKGRPCE' + code].astype(str) + block_groups_shp["GEOID_short"] = block_groups_shp['STATEFP' + code].astype(str) + block_groups_shp['COUNTYFP' + code].astype(str) + block_groups_shp['TRACTCE' + code].astype(str) + block_groups_shp['BLKGRPCE' + code].astype(str) block_groups_shp["GEO_ID"] = "1500000US" + block_groups_shp['STATEFP' + code].astype(str) + block_groups_shp['COUNTYFP' + code].astype(str) + block_groups_shp['TRACTCE' + code].astype(str) + block_groups_shp['BLKGRPCE' + code].astype(str) block_groups_list.append(block_groups_shp) @@ -191,8 +191,13 @@ def merge_equity_data_shp(self): # Delete the rows that do not have a geometry column self.equity_data_shp = self.equity_data_shp.loc[self.equity_data_shp["geometry"].notnull()] - #self.svi_data_shp.drop(columns=columns_to_drop, inplace=True) self.equity_data_shp = self.equity_data_shp.to_crs(epsg=4326) self.logger.info( "The geometry information was successfully added to the equity information" - ) \ No newline at end of file + ) + + aggregation_areas = self.block_groups[["GEOID_short", "geometry"]] + + def clean(self): + """Removes unnecessary columns""" + self.svi_data_shp = self.svi_data_shp[["GEOID_short", "TotalPopulationBG", "PerCapitaIncomeBG"]] \ No newline at end of file From 4bbfd757c59083a1ee399e3ab4585225909f289f Mon Sep 17 00:00:00 2001 From: Frederique Date: Thu, 14 Dec 2023 14:11:16 +0100 Subject: [PATCH 4/4] Added census block aggregation area in the exposure data when using setup_equity --- hydromt_fiat/fiat.py | 49 ++++++++++++++------- hydromt_fiat/workflows/aggregation_areas.py | 13 +++--- hydromt_fiat/workflows/equity_data.py | 17 +++++-- 3 files changed, 54 insertions(+), 25 deletions(-) diff --git a/hydromt_fiat/fiat.py b/hydromt_fiat/fiat.py index b07cc114..001bd1b2 100644 --- a/hydromt_fiat/fiat.py +++ b/hydromt_fiat/fiat.py @@ -10,7 +10,7 @@ import pandas as pd from hydromt.models.model_grid import GridModel from shapely.geometry import box -import os +import os import shutil from hydromt_fiat import DATADIR @@ -71,7 +71,9 @@ def __init__( self.exposure = None self.vulnerability = None self.vf_ids_and_linking_df = pd.DataFrame() - self.additional_attributes_fn = "" # Path or paths to the additional attributes dataset(s) + self.additional_attributes_fn = ( + "" # Path or paths to the additional attributes dataset(s) + ) self.building_footprint_fn = "" # Path to the building footprints dataset def setup_global_settings( @@ -105,7 +107,6 @@ def setup_output( output_dir: str = "output", output_csv_name: str = "output.csv", output_vector_name: Union[str, List[str]] = "spatial.gpkg", - ) -> None: """Setup Delft-FIAT output folder and files. @@ -429,7 +430,7 @@ def update_max_potential_damage( method=method, max_dist=max_dist, ) - + def update_ground_elevation( self, ground_elevation: Union[int, float, None, str, Path], @@ -505,7 +506,9 @@ def setup_hazard( # read maps and retrieve their attributes da_map_fn, da_name, da_type = read_maps(params, da_map_fn, idx) - da = self.data_catalog.get_rasterdataset(da_map_fn) # removed geom=self.region because it is not always there + da = self.data_catalog.get_rasterdataset( + da_map_fn + ) # removed geom=self.region because it is not always there # Convert to units of the exposure data if required if ( @@ -761,12 +764,21 @@ def setup_equity_data( self.set_tables(df=equity.equity_data_shp, name="equity_data") - # Update (if necessary) the aggregation label: census block # Save the census block aggregation area data + block_groups = equity.get_block_groups() + self.set_geoms(block_groups, "aggregation_areas/block_groups") + + # Update the aggregation label: census block + del self.exposure.exposure_db["Aggregation Label: Census Blockgroup"] + self.setup_aggregation_areas( + aggregation_area_fn=block_groups, + attribute_names="GEOID_short", + label_names="Aggregation Label: Census Blockgroup", + ) def setup_aggregation_areas( self, - aggregation_area_fn: Union[List[str], List[Path], str, Path], + aggregation_area_fn: Union[List[str], List[Path], List[gpd.GeoDataFrame], str, Path, gpd.GeoDataFrame], attribute_names: Union[List[str], str], label_names: Union[List[str], str], ): @@ -774,8 +786,6 @@ def setup_aggregation_areas( Parameters ---------- - exposure_gdf : gpd.GeoDataFrame - Exposure data to join the aggregation areas to as `label_names`. aggregation_area_fn : Union[List[str], List[Path], str, Path] Path(s) to the aggregation area(s). attribute_names : Union[List[str], str] @@ -783,15 +793,20 @@ def setup_aggregation_areas( label_names : Union[List[str], str] The name that the new attribute will get in the exposure data. """ - # TODO: add a census block option to automatically download and save - # the data exposure_gdf = self.exposure.get_full_gdf(self.exposure.exposure_db) self.exposure.exposure_db = join_exposure_aggregation_areas( exposure_gdf, aggregation_area_fn, attribute_names, label_names ) # Set the additional_attributes_fn property to save the additional datasets - self.additional_attributes_fn = aggregation_area_fn + if isinstance(aggregation_area_fn, list): + the_type = type(aggregation_area_fn[0]) + else: + the_type = type(aggregation_area_fn) + if the_type != gpd.GeoDataFrame: + # This copies data from one location to the root folder for the FIAT + # model, only use user-input data here (not the census blocks) + self.additional_attributes_fn = aggregation_area_fn def setup_building_footprint( self, @@ -818,7 +833,7 @@ def setup_building_footprint( building_footprint_fn, attribute_name, ) - + # Set the building_footprint_fn property to save the building footprints self.building_footprint_fn = building_footprint_fn @@ -953,10 +968,12 @@ def write(self): folder = Path(self.root).joinpath("additional_attributes") self.copy_datasets(self.additional_attributes_fn, folder) if self.building_footprint_fn: - folder = Path(self.root).joinpath("exposure" , "building_footprints") + folder = Path(self.root).joinpath("exposure", "building_footprints") self.copy_datasets(self.building_footprint_fn, folder) - def copy_datasets(self, data: Union[list, str, Path], folder: Union[Path, str]) -> None: + def copy_datasets( + self, data: Union[list, str, Path], folder: Union[Path, str] + ) -> None: """Copies datasets to another folder Parameters @@ -966,7 +983,7 @@ def copy_datasets(self, data: Union[list, str, Path], folder: Union[Path, str]) folder : Union[Path, str] _description_ """ - # Create additional attributes folder in root + # Create additional attributes folder in root if not os.path.exists(folder): os.makedirs(folder) diff --git a/hydromt_fiat/workflows/aggregation_areas.py b/hydromt_fiat/workflows/aggregation_areas.py index c0148a03..b9916942 100644 --- a/hydromt_fiat/workflows/aggregation_areas.py +++ b/hydromt_fiat/workflows/aggregation_areas.py @@ -12,7 +12,7 @@ def process_value(value): def join_exposure_aggregation_multiple_areas( exposure_gdf: gpd.GeoDataFrame, - aggregation_area_fn: Union[List[str], List[Path]], + aggregation_area_fn: Union[List[str], List[Path], List[gpd.GeoDataFrame]], attribute_names: List[str], label_names: List[str], ) -> gpd.GeoDataFrame: @@ -35,7 +35,10 @@ def join_exposure_aggregation_multiple_areas( _description_ """ for file_path, attribute_name, label_name in zip(aggregation_area_fn, attribute_names, label_names): - aggregation_gdf = gpd.read_file(file_path) + if isinstance(file_path, str) or isinstance(file_path, Path): + aggregation_gdf = gpd.read_file(file_path) + else: + aggregation_gdf = file_path ## check the projection of both gdf and if not match, reproject if exposure_gdf.crs != aggregation_gdf.crs: @@ -50,7 +53,7 @@ def join_exposure_aggregation_multiple_areas( exposure_gdf = gpd.sjoin( exposure_gdf, aggregation_gdf[["geometry", attribute_name]], - op="intersects", + predicate="intersects", how="left", ) @@ -69,7 +72,7 @@ def join_exposure_aggregation_multiple_areas( def join_exposure_aggregation_areas( exposure_gdf: gpd.GeoDataFrame, - aggregation_area_fn: Union[List[str], List[Path], str, Path], + aggregation_area_fn: Union[List[str], List[Path], List[gpd.GeoDataFrame], str, Path, gpd.GeoDataFrame], attribute_names: Union[List[str], str], label_names: Union[List[str], str], ) -> gpd.GeoDataFrame: @@ -86,7 +89,7 @@ def join_exposure_aggregation_areas( label_names : Union[List[str], str] Name of the label(s) to join. """ - if isinstance(aggregation_area_fn, str) or isinstance(aggregation_area_fn, Path): + if isinstance(aggregation_area_fn, str) or isinstance(aggregation_area_fn, Path) or isinstance(aggregation_area_fn, gpd.GeoDataFrame): aggregation_area_fn = [aggregation_area_fn] if isinstance(attribute_names, str): attribute_names = [attribute_names] diff --git a/hydromt_fiat/workflows/equity_data.py b/hydromt_fiat/workflows/equity_data.py index 3d3eb078..44120524 100644 --- a/hydromt_fiat/workflows/equity_data.py +++ b/hydromt_fiat/workflows/equity_data.py @@ -9,6 +9,7 @@ from zipfile import ZipFile from pathlib import Path from typing import List +import shutil class EquityData: @@ -25,7 +26,7 @@ def __init__(self, data_catalog: DataCatalog, logger: Logger, save_folder: str): self.pd_domain_scores_geo = pd.DataFrame() self.logger = logger - self.svi_data_shp = gpd.GeoDataFrame() + self.equity_data_shp = gpd.GeoDataFrame() self.block_groups = gpd.GeoDataFrame() def set_up_census_key(self, census_key: str): @@ -177,12 +178,19 @@ def download_shp_geom(self, year_data: int, counties: List[str]): block_groups_shp = shp.dissolve(by=attrs, as_index=False) block_groups_shp = block_groups_shp[attrs + ["geometry"]] - block_groups_shp["GEOID_short"] = block_groups_shp['STATEFP' + code].astype(str) + block_groups_shp['COUNTYFP' + code].astype(str) + block_groups_shp['TRACTCE' + code].astype(str) + block_groups_shp['BLKGRPCE' + code].astype(str) block_groups_shp["GEO_ID"] = "1500000US" + block_groups_shp['STATEFP' + code].astype(str) + block_groups_shp['COUNTYFP' + code].astype(str) + block_groups_shp['TRACTCE' + code].astype(str) + block_groups_shp['BLKGRPCE' + code].astype(str) + block_groups_shp["GEOID_short"] = block_groups_shp["GEO_ID"].str.split("US").str[1] block_groups_list.append(block_groups_shp) self.block_groups = gpd.GeoDataFrame(pd.concat(block_groups_list)) + # Delete the shapefile, that is not used anymore + shp_folder = Path(self.save_folder) / "shapefiles" + try: + shutil.rmtree(shp_folder) + except Exception as e: + self.logger.warning(f"Folder {shp_folder} cannot be removed: {e}") + def merge_equity_data_shp(self): """Merges the geometry data with the equity_data downloaded""" self.equity_data_shp = self.pd_domain_scores_geo.merge(self.block_groups[["GEO_ID", "geometry"]], on="GEO_ID", how="left") @@ -196,8 +204,9 @@ def merge_equity_data_shp(self): "The geometry information was successfully added to the equity information" ) - aggregation_areas = self.block_groups[["GEOID_short", "geometry"]] + def get_block_groups(self): + return self.block_groups[["GEOID_short", "geometry"]] def clean(self): """Removes unnecessary columns""" - self.svi_data_shp = self.svi_data_shp[["GEOID_short", "TotalPopulationBG", "PerCapitaIncomeBG"]] \ No newline at end of file + self.equity_data_shp = self.equity_data_shp[["GEOID_short", "TotalPopulationBG", "PerCapitaIncomeBG"]] \ No newline at end of file