diff --git a/examples/data/vulnerability_test_file_input.csv b/examples/data/vulnerability_test_file_input.csv new file mode 100644 index 00000000..c4ad5321 --- /dev/null +++ b/examples/data/vulnerability_test_file_input.csv @@ -0,0 +1,10 @@ +Name,Link,BldgDmgFnID,Occupancy,Source,Description +My vulnerability function 1,APT,105,RES1,FIA,"one floor, no basement, Structure, A-Zone" +My vulnerability function 2,APT2,105,RES1,FIA,"one floor, no basement, Structure, A-Zone" +My vulnerability function 3,APT3,105,RES1,FIA,"one floor, no basement, Structure, A-Zone" +My vulnerability function 4,HOUS,693,RES3A,BCAR - Jan 2011,"1to2 Stories, Elevated (Obstr)+6ft - no basement, Coastal A or V Zone" +My vulnerability function 5,SHOP,228,COM1,USACE - Galveston,"Drug Store, structure" +My vulnerability function 6,SHOP2,358,COM2,USACE - Galveston,"T.V. Repair, structure" +My vulnerability function 7,MALL,459,COM4,USACE - New Orleans,"Utility Company, structure, salt water, short duration" +My vulnerability function 8,MALL2,459,COM4,USACE - New Orleans,"Utility Company, structure, salt water, short duration" +My vulnerability function 9,OFFI,550,IND1,USACE - Galveston,"Fabrication Shop, structure" \ No newline at end of file diff --git a/examples/fiat_flood.ini b/examples/fiat_flood.ini index f8d09152..3efeb94a 100644 --- a/examples/fiat_flood.ini +++ b/examples/fiat_flood.ini @@ -1,8 +1,8 @@ [global] -artifact_data = True # use latest artifact data +artifact_data = False # use latest artifact data [setup_config] -case = baseline # name of case +case = test_exposure # name of case strategy = base # name of strategy (optional) scenario = base # name of scenario (optional) year = 2021 # case year (optional) @@ -16,36 +16,10 @@ risk_output = True # indicator that specifies whether a risk calculation map_output = True # indicator that specifies whether the result maps should be included in the output (default is True) [setup_hazard1] -map_fn = data/flood_hand/hand_050cm_rp02.tif # absolute or relative (with respect to the configuration.ini) path to the hazard file +map_fn = max_depth # absolute or relative (with respect to the configuration.ini) path to the hazard file map_type = water_depth # description of the hazard file type -rp = 2 # hazard return period in years, required for a risk calculation (optional) -crs = 3452 # coordinate reference system of the hazard file (optional) +rp = None # hazard return period in years, required for a risk calculation (optional) +crs = 4326 # coordinate reference system of the hazard file (optional) nodata = -9999 # value that is assigned as nodata (optional) var = None # hazard variable name in NetCDF input files (optional) chunks = 100 # chunk sizes along each dimension used to load the hazard file into a dask array (default is 'auto') (optional) - -[setup_hazard2] -map_fn = data/flood_hand/hand_150cm_rp50.tif # absolute or relative (with respect to the configuration.ini) path to the hazard file -map_type = water_depth # description of the hazard file type -rp = 50 # hazard return period in years, required for a risk calculation (optional) -crs = 3452 # coordinate reference system of the hazard file (optional) -nodata = -9999 # value that is assigned as nodata (optional) -var = None # hazard variable name in NetCDF input files (optional) -chunks = 100 # chunk sizes along each dimension used to load the hazard file into a dask array (default is 'auto') (optional) - -[setup_buildings_value] -bld_fn = guf_bld_2012 # name tag of or absolute or relative (with respect to the configuration file) path to the building footprint file (default is 'wsf_bld_2015') -pop_fn = ghs_pop_2015_54009 # name tag of or absolute or relative (with respect to the configuration file) path to the population count file (default is 'ghs_pop_2015') -chunks = 100 # chunk sizes along each dimension used to load the building footprint and population count files into a dask arrays (default is 'auto') (optional) -function_fn = None # absolute or relative (with respect to the configuration file or susceptibility_dp) path to the susceptibility file (default is the JCR continental susceptibilty function (https://publications.jrc.ec.europa.eu/repository/handle/JRC105688) related to the country parameter) (optional) -scale_factor = 1 # scaling factor of the exposure values (default is 1) (optional) -weight_factor = 1 # weight factor of the exposure values in the total damage and risk results (default is 1) (optional) - -# Alternatives to conduct a risk calculation (triggered by a multiple hazard input): -# [setup_hazard] -# map_fn = [hazard/RP_2.tif, hazard/RP_100.tif] # TODO: Not properly working yet, input is '[.., ..]' instead of ['..', '..'] -# rp = [2, 100] # TODO: Not properly working yet, input is not parseble. - -# Alternative to derive rp from filename: -# [setup_hazard1] -# map_fn = hazard/RP_*.tif \ No newline at end of file diff --git a/hydromt_fiat/data/AllDDF.xlsx b/hydromt_fiat/data/AllDDF.xlsx new file mode 100644 index 00000000..4fede5bf Binary files /dev/null and b/hydromt_fiat/data/AllDDF.xlsx differ diff --git a/hydromt_fiat/data/vulnerability_test_file_input.csv b/hydromt_fiat/data/vulnerability_test_file_input.csv new file mode 100644 index 00000000..27d64f7b --- /dev/null +++ b/hydromt_fiat/data/vulnerability_test_file_input.csv @@ -0,0 +1,12 @@ +Name,Link,ID,Occupancy,Source,Description +My vulnerability function 1,APT,105,RES1,FIA,"one floor, no basement, Structure, A-Zone" +My vulnerability function 2,APT2,105,RES1,FIA,"one floor, no basement, Structure, A-Zone" +My vulnerability function 3,APT3,105,RES1,FIA,"one floor, no basement, Structure, A-Zone" +My vulnerability function 4,HOUS,693,RES3A,BCAR - Jan 2011,"1to2 Stories, Elevated (Obstr)+6ft - no basement, Coastal A or V Zone" +My vulnerability function 5,SHOP,228,COM1,USACE - Galveston,"Drug Store, structure" +My vulnerability function 6,SHOP2,358,COM2,USACE - Galveston,"T.V. Repair, structure" +My vulnerability function 7,MALL,459,COM4,USACE - New Orleans,"Utility Company, structure, salt water, short duration" +My vulnerability function 8,MALL2,459,COM4,USACE - New Orleans,"Utility Company, structure, salt water, short duration" +My vulnerability function 9,OFFI,550,IND1,USACE - Galveston,"Fabrication Shop, structure" +My vulnerability function 10,HOUS2,699,RES3A,BCAR - Jan 2011,"1to2 Stories, Elevated (Obstr)+10ft - no basement, Coastal A or V Zone" +My vulnerability function 11,HOUS3,699,RES3A,BCAR - Jan 2011,"1to2 Stories, Elevated (Obstr)+10ft - no basement, Coastal A or V Zone" diff --git a/hydromt_fiat/fiat.py b/hydromt_fiat/fiat.py index 27c4b8b1..caad04b0 100644 --- a/hydromt_fiat/fiat.py +++ b/hydromt_fiat/fiat.py @@ -1,5 +1,6 @@ """Implement fiat model class""" +from hydromt_fiat.workflows.vulnerability import Vulnerability from hydromt.models.model_grid import GridModel from hydromt_fiat.workflows.exposure_vector import ExposureVector import logging @@ -10,11 +11,9 @@ from shapely.geometry import box from typing import Union -from shutil import copy from hydromt_fiat.workflows.social_vulnerability_index import SocialVulnerabilityIndex - from . import DATADIR __all__ = ["FiatModel"] @@ -100,18 +99,39 @@ def setup_exposure_vector( def setup_exposure_raster(self): NotImplemented - def setup_vulnerability(self): - NotImplemented + def setup_vulnerability( + self, + vulnerability_source: str, + vulnerability_identifiers_and_linking: str, + unit: str, + ) -> None: + """Setup the vulnerability curves from various possible inputs. + + Parameters + ---------- + vulnerability_source : str + The (relative) path or ID from the data catalog to the source of the vulnerability functions. + vulnerability_identifiers_and_linking : str + The (relative) path to the table that links the vulnerability functions and exposure categories. + unit : str + The unit of the vulnerability functions. + """ + vul = Vulnerability(self.data_catalog) + vul.get_vulnerability_functions_from_one_file( + vulnerability_source, vulnerability_identifiers_and_linking, unit + ) def setup_hazard(self, map_fn): NotImplemented - def setup_social_vulnerability_index(self, census_key: str, path:str, state_abbreviation:str): + def setup_social_vulnerability_index( + self, census_key: str, path: str, state_abbreviation: str + ): - #Create SVI object + # Create SVI object svi = SocialVulnerabilityIndex(self.data_catalog, self.config) - #Call functionalities of SVI + # Call functionalities of SVI svi.set_up_census_key(census_key) svi.variable_code_csv_to_pd_df(path) svi.set_up_download_codes() @@ -124,8 +144,8 @@ def setup_social_vulnerability_index(self, census_key: str, path:str, state_abbr svi.domain_scores() svi.composite_scores() -#TO DO: JOIN WITH GEOMETRIES. FOR MAPPING. -#this link can be used: https://github.com/datamade/census + # TO DO: JOIN WITH GEOMETRIES. FOR MAPPING. + # this link can be used: https://github.com/datamade/census def read(self): """Method to read the complete model schematization and configuration from file.""" diff --git a/hydromt_fiat/workflows/exposure_vector.py b/hydromt_fiat/workflows/exposure_vector.py index ec0924dc..b01c9d25 100644 --- a/hydromt_fiat/workflows/exposure_vector.py +++ b/hydromt_fiat/workflows/exposure_vector.py @@ -2,6 +2,8 @@ from hydromt_fiat.workflows.exposure import Exposure import geopandas as gpd import json +import geopandas as gpd +import json class ExposureVector(Exposure): @@ -65,6 +67,9 @@ def setup_from_single_source(self, source: str) -> None: nsi_fiat_translation[column_name] ] + exposure_1_df["Extraction Method"] = "centroid" + exposure_1_df["First Floor Elevation"] = exposure_1["FOUND_HT"] # TOCHANGE + else: NotImplemented diff --git a/hydromt_fiat/workflows/vulnerability.py b/hydromt_fiat/workflows/vulnerability.py index 63e8f09b..8bdb3c06 100644 --- a/hydromt_fiat/workflows/vulnerability.py +++ b/hydromt_fiat/workflows/vulnerability.py @@ -1,35 +1,194 @@ import pandas as pd -from pathlib import Path +from pandas.api.types import is_numeric_dtype +import numpy as np +from hydromt.data_catalog import DataCatalog +import logging + -### TO BE UPDATED ### class Vulnerability: - def __init__(self, datadir, config: dict): - self.datadir = datadir - self.config = config - - def get_vulnerability_function(self): - # TODO: @Luis: this function needs to be changed. Now it looks up a damage function - # according to country that is chosen but we want the user to choose their - # damage functions per category (res, com, etc.). So our function will be - # quite different. - """Return the susceptibility function id and the maximum damage number.""" - - # Read the global exposure configuration. - df_config = pd.read_excel( - Path(self.datadir).joinpath("global_configuration.xlsx"), - sheet_name="Buildings", + def __init__(self, data_catalog: DataCatalog): + self.data_catalog = data_catalog + + def get_vulnerability_functions_from_one_file( + self, + vulnerability_source: str, + vulnerability_identifiers_and_linking: str, + unit: str, + ): + + # Load the vulnerability functions from the source + df_source = self.data_catalog.get_dataframe(vulnerability_source) + + # USER INPUT, read vulnerability identifiers and linking csv generated via gui or manual input. + df_identifiers_linking = pd.read_csv(vulnerability_identifiers_and_linking) + + # TODO: add some checks to validate the df_source and df_identifiers_linking + + # Identify the unique combinations of values from the identifiers and linking data frame that will be used to select subsets of values from the source data frame. + # unique_combinations = df_identifiers_linking.groupby(['ID', 'Occupancy', 'Source', 'Description']).nunique().reset_index() + + identifier_columns = self.get_identifier_names(df_identifiers_linking) + df_source = self.add_full_identifier_column(df_source, identifier_columns) + df_identifiers_linking = self.add_full_identifier_column( + df_identifiers_linking, identifier_columns + ) + + combined_df = self.link_vfs_from_source(df_source, df_identifiers_linking) + + # Delete the column used to link the dataframes + del combined_df["full_identifier"] + + # Get the columns with damage fractions + fraction_cols = self.get_hazard_value_columns(combined_df) + + # Get the hazard values from the columns + hazard_values = self.get_hazard_values_from_columns(fraction_cols) + + # Get vulnerability factors and apply FIAT format. + vf_values_only = combined_df[fraction_cols].values.T + + # Check whether the vulnerability factors are fractions or percentages and + # convert into fractions if they are percentages + if vf_values_only.max() > 1: + vf_values_only = vf_values_only / 100 + + vf_names = df_identifiers_linking["Name"].values + v_dataframe = self.combine_vulnerability_functions( + hazard_values, vf_values_only, vf_names, unit + ) + + # Export the dataframe to a csv. + v_dataframe.to_csv( + "vulnerability_test_file_output.csv", index=False, header=False + ) + + @staticmethod + def get_identifier_names( + df: pd.DataFrame, to_remove: list = ["Name", "Link"] + ) -> list: + """_summary_ + + Parameters + ---------- + df : pd.DataFrame + _description_ + to_remove : list + _description_ + + Returns + ------- + list + _description_ + """ + # Check which columns are used as identifiers. These are all columns beside the 'to_remove' columns + _identifier_columns = list(df.columns) + for col in to_remove: + _identifier_columns.remove(col) + return _identifier_columns + + @staticmethod + def add_full_identifier_column( + df: pd.DataFrame, identifier_columns: list + ) -> pd.DataFrame: + # Create temporary columns of the identifier columns to be able to filter them + # with flexible user input + for c in identifier_columns: + if is_numeric_dtype(df[c]): + df[c] = df[c].astype(int).astype(str) + + df["full_identifier"] = df[identifier_columns].apply( + lambda x: x.str.cat(sep=""), axis=1 + ) + df["full_identifier"] = df["full_identifier"].str.replace(" ", "") + return df + + @staticmethod + def link_vfs_from_source( + vf_source: pd.DataFrame, vf_identifiers_linking: pd.DataFrame + ) -> pd.DataFrame: + # Initialize an empty list to hold the subsets + subsets = [] + + # Loop over the unique combinations of values + for i in range(len(vf_identifiers_linking)): + # Use the unique combination of values to select the corresponding subset of values + # from the first data frame using boolean indexing + subset = vf_source.loc[ + vf_source["full_identifier"] + == vf_identifiers_linking.loc[i, "full_identifier"] + ] + + # Check if the subset is empty + if subset.empty: + logging.warn( + f"No vulnerability curves found for unique combination {vf_identifiers_linking.loc[i]}" + ) + + # Append the subset of values to the list of subsets + subsets.append(subset) + + # Concatenate all of the necessary vulnerability curves info into a single data frame using pd.concat() + return pd.concat(subsets, ignore_index=True) + + @staticmethod + def get_hazard_value_columns(df: pd.DataFrame): + # Get the columns with damage fractions + def has_numbers(inputString): + return any(char.isdigit() for char in inputString) + + return [c for c in df.columns if has_numbers(c)] + + @staticmethod + def get_hazard_values_from_columns(columns: list) -> list: + # Get the hazard values from the column names + list_hazard_values = list( + map( + lambda sub: float("".join([ele for ele in sub if ele.isnumeric()])), + columns, + ) + ) + if list_hazard_values[0] != 0: + # The hazard values start with negative values + list_hazard_values[: list_hazard_values.index(0)] = [ + -x for x in list_hazard_values[: list_hazard_values.index(0)] + ] + return list_hazard_values + + @staticmethod + def combine_vulnerability_functions( + list_hazard_values: list, vf_values_only: np.array, vf_names: list, unit: str + ) -> pd.DataFrame: + """_summary_ + + Parameters + ---------- + list_hazard_values : list + _description_ + vf_values_only : np.array + _description_ + vf_names : list + _description_ + unit : str + _description_ + + Returns + ------- + pd.DataFrame + _description_ + """ + vf_water_depths_numbers = np.array(list_hazard_values) + vf_water_depths = np.array(vf_water_depths_numbers.reshape(-1, 1)) + vf_raw = np.hstack([vf_water_depths, vf_values_only]) + + vf_names_header = np.append("water depth", vf_names) + top_header_array = np.full( + (1, vf_names_header.shape[0]), fill_value="", dtype="