From 2651724f078b516626d82f656f29fe70cadfcf6c Mon Sep 17 00:00:00 2001 From: Frederique Date: Wed, 27 Sep 2023 16:16:50 +0200 Subject: [PATCH 1/2] Applied black, implemented the geom_names variable for ExposureVector, updated related tests --- docs/conf.py | 4 +- examples/data/hydromt_fiat_catalog_global.yml | 2 +- hydromt_fiat/api/exposure_vm.py | 4 +- .../data/hydromt_fiat_catalog_global.yml | 30 ++ .../JRC_damage_functions_linking.csv | 2 +- hydromt_fiat/data_apis/open_street_maps.py | 4 +- hydromt_fiat/fiat.py | 33 +- hydromt_fiat/validation.py | 34 +- hydromt_fiat/workflows/exposure_raster.py | 1 + hydromt_fiat/workflows/exposure_vector.py | 30 +- hydromt_fiat/workflows/hazard.py | 2 + .../workflows/social_vulnerability_index.py | 435 ++++++++++-------- hydromt_fiat/workflows/vulnerability.py | 3 +- tests/test_SVI_exposure.py | 3 +- tests/test_integration.py | 2 +- tests/test_integrations_hazard.py | 46 +- tests/test_raise_ground_floor_height.py | 6 +- tests/test_read_write.py | 55 +++ tests/test_reader.py | 8 +- tests/test_update_max_potential_damage.py | 6 +- tests/test_vulnerability.py | 13 +- ...nerability_exposure_add_to_data_catalog.py | 7 +- ...t_vulnerability_exposure_global_default.py | 2 +- tests/test_vulnerability_from_csvs.py | 6 +- 24 files changed, 468 insertions(+), 270 deletions(-) create mode 100644 hydromt_fiat/data/hydromt_fiat_catalog_global.yml rename hydromt_fiat/data/{damage_functions/flooding => vulnerability_linking}/JRC_damage_functions_linking.csv (67%) create mode 100644 tests/test_read_write.py diff --git a/docs/conf.py b/docs/conf.py index 7acb9d0f..9d493159 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -203,7 +203,7 @@ def write_nested_dropdown(name, data_cat, note="", categories=[]): "github_url": "https://github.com", # or your GitHub Enterprise interprise "github_user": "Deltares", "github_repo": "hydromt_fiat", - "github_version": "main", + "github_version": "main", "doc_path": "docs", "default_mode": "light", } @@ -324,4 +324,4 @@ def write_nested_dropdown(name, data_cat, note="", categories=[]): """ # Allow errors in notebooks -# nbsphinx_allow_errors = True \ No newline at end of file +# nbsphinx_allow_errors = True diff --git a/examples/data/hydromt_fiat_catalog_global.yml b/examples/data/hydromt_fiat_catalog_global.yml index 94317fd1..982f52df 100644 --- a/examples/data/hydromt_fiat_catalog_global.yml +++ b/examples/data/hydromt_fiat_catalog_global.yml @@ -11,7 +11,7 @@ jrc_vulnerability_curves: paper_doi: https://dx.doi.org/10.2760/16510 jrc_vulnerability_curves_linking: - path: damage_functions/flooding/JRC_damage_functions_linking.csv + path: vulnerability_linking/JRC_damage_functions_linking.csv data_type: DataFrame driver: csv meta: diff --git a/hydromt_fiat/api/exposure_vm.py b/hydromt_fiat/api/exposure_vm.py index da07bbc0..eff8315d 100644 --- a/hydromt_fiat/api/exposure_vm.py +++ b/hydromt_fiat/api/exposure_vm.py @@ -62,9 +62,7 @@ def set_asset_locations_source( self.exposure_model.occupancy_type = input_source self.exposure_model.max_potential_damage = input_source self.exposure_model.ground_floor_height = 1 # TODO: make flexible - self.exposure_model.unit = ( - Units.ft.value - ) # TODO: make flexible + self.exposure_model.unit = Units.ft.value # TODO: make flexible # Download NSI from the database region = self.data_catalog.get_geodataframe("area_of_interest") diff --git a/hydromt_fiat/data/hydromt_fiat_catalog_global.yml b/hydromt_fiat/data/hydromt_fiat_catalog_global.yml new file mode 100644 index 00000000..982f52df --- /dev/null +++ b/hydromt_fiat/data/hydromt_fiat_catalog_global.yml @@ -0,0 +1,30 @@ + +jrc_vulnerability_curves: + path: damage_functions/flooding/JRC_damage_functions.csv + data_type: DataFrame + driver: csv + meta: + category: vulnerability + description: JRC depth-damage functions for flooding, processed into a handy format for HydroMT-FIAT. + source_url: https://publications.jrc.ec.europa.eu/repository/handle/JRC105688 + paper_ref: Huizinga, J., De Moel, H. and Szewczyk, W., Global flood depth-damage functions - Methodology and the database with guidelines, EUR 28552 EN, Publications Office of the European Union, Luxembourg, 2017, ISBN 978-92-79-67781-6, doi:10.2760/16510, JRC105688. + paper_doi: https://dx.doi.org/10.2760/16510 + +jrc_vulnerability_curves_linking: + path: vulnerability_linking/JRC_damage_functions_linking.csv + data_type: DataFrame + driver: csv + meta: + category: vulnerability + description: Default linking table for the JRC damage functions (e.g., the residential damage function links to residential buildings). + +jrc_damage_values: + path: max_potential_damages/JRC_base_damage_values.csv + data_type: DataFrame + driver: csv + meta: + category: vulnerability + description: Base damage values from the JRC publicated Excel from the tab "MaxDamage-Data", processed into a handy format for HydroMT-FIAT. + source_url: https://publications.jrc.ec.europa.eu/repository/handle/JRC105688 + paper_ref: Huizinga, J., De Moel, H. and Szewczyk, W., Global flood depth-damage functions - Methodology and the database with guidelines, EUR 28552 EN, Publications Office of the European Union, Luxembourg, 2017, ISBN 978-92-79-67781-6, doi:10.2760/16510, JRC105688. + paper_doi: https://dx.doi.org/10.2760/16510 \ No newline at end of file diff --git a/hydromt_fiat/data/damage_functions/flooding/JRC_damage_functions_linking.csv b/hydromt_fiat/data/vulnerability_linking/JRC_damage_functions_linking.csv similarity index 67% rename from hydromt_fiat/data/damage_functions/flooding/JRC_damage_functions_linking.csv rename to hydromt_fiat/data/vulnerability_linking/JRC_damage_functions_linking.csv index 65e94202..6f711297 100644 --- a/hydromt_fiat/data/damage_functions/flooding/JRC_damage_functions_linking.csv +++ b/hydromt_fiat/data/vulnerability_linking/JRC_damage_functions_linking.csv @@ -1,4 +1,4 @@ -Name,Link,Damage Type,type +FIAT Damage Function Name,Exposure Link,Damage Type,type residential,residential,total,residential commercial,commercial,total,commercial industrial,industrial,total,industrial diff --git a/hydromt_fiat/data_apis/open_street_maps.py b/hydromt_fiat/data_apis/open_street_maps.py index a1d4347b..868e5170 100644 --- a/hydromt_fiat/data_apis/open_street_maps.py +++ b/hydromt_fiat/data_apis/open_street_maps.py @@ -27,9 +27,7 @@ def get_assets_from_osm(polygon: Polygon): def get_landuse_from_osm(polygon: Polygon): tags = {"landuse": True} # this is the tag we use to find the correct OSM data - landuse = ox.features.features_from_polygon( - polygon, tags - ) # then we query the data + landuse = ox.features.features_from_polygon(polygon, tags) # then we query the data if landuse.empty: logging.warning("No land use data found from OSM") diff --git a/hydromt_fiat/fiat.py b/hydromt_fiat/fiat.py index 3209fdc7..428172f6 100644 --- a/hydromt_fiat/fiat.py +++ b/hydromt_fiat/fiat.py @@ -306,7 +306,6 @@ def setup_exposure_vector( self.set_config("exposure.geom.crs", self.exposure.crs) self.set_config("exposure.geom.unit", unit) - def setup_exposure_raster(self): """Setup raster exposure data for Delft-FIAT. This function will be implemented at a later stage. @@ -574,14 +573,12 @@ def setup_social_vulnerability_index( # Link the SVI score to the exposure data exposure_data = self.exposure.get_full_gdf(self.exposure.exposure_db) exposure_data.sort_values("Object ID") - + if svi.svi_data_shp.crs != exposure_data.crs: - svi.svi_data_shp.to_crs(crs=exposure_data.crs, inplace = True) + svi.svi_data_shp.to_crs(crs=exposure_data.crs, inplace=True) - svi_exp_joined = gpd.sjoin( - exposure_data, svi.svi_data_shp, how="left" - ) - svi_exp_joined.drop(columns=['geometry'], inplace=True) + svi_exp_joined = gpd.sjoin(exposure_data, svi.svi_data_shp, how="left") + svi_exp_joined.drop(columns=["geometry"], inplace=True) svi_exp_joined = pd.DataFrame(svi_exp_joined) self.exposure.exposure_db = svi_exp_joined @@ -616,12 +613,16 @@ def update_tables(self): def update_geoms(self): # Update the exposure data geoms if self.exposure and "exposure" in self._tables: - for i, geom in enumerate(self.exposure.exposure_geoms): - file_suffix = i if i > 0 else "" - self.set_geoms(geom=geom, name=f"exposure{file_suffix}") + for i, geom_and_name in enumerate( + zip(self.exposure.exposure_geoms, self.exposure.geom_names) + ): + geom = geom_and_name[0] + name = geom_and_name[1] + + self.set_geoms(geom=geom, name=name) self.set_config( f"exposure.geom.file{str(i+1)}", - f"./exposure/exposure{file_suffix}.gpkg", + f"./exposure/{name}.gpkg", ) if not self.region.empty: @@ -632,7 +633,8 @@ def update_maps(self): # I/O def read(self): - """Method to read the complete model schematization and configuration from file.""" + """Method to read the complete model schematization and configuration from + file.""" self.logger.info(f"Reading model data from {self.root}") # Read the configuration file @@ -654,13 +656,6 @@ def _configread(self, fn): config = Config() return config.load_file(fn) - def check_path_exists(self, fn): - """TODO: decide to use this or another function (check_file_exist in py)""" - path = Path(fn) - self.logger.debug(f"Reading file {str(path.name)}") - if not fn.is_file(): - logging.warning(f"File {fn} does not exist!") - def read_tables(self): """Read the model tables for vulnerability and exposure data.""" if not self._write: diff --git a/hydromt_fiat/validation.py b/hydromt_fiat/validation.py index 4507e46f..e1c9b20d 100644 --- a/hydromt_fiat/validation.py +++ b/hydromt_fiat/validation.py @@ -1,5 +1,6 @@ from pathlib import Path + def check_dir_exist(dir, name=None): """ """ @@ -7,7 +8,8 @@ def check_dir_exist(dir, name=None): raise TypeError( f"The directory indicated by the '{name}' parameter does not exist." ) - + + def check_file_exist(root, param_lst, name=None): root = Path(root) param_lst = [Path(p) for p in param_lst] @@ -38,12 +40,13 @@ def check_file_exist(root, param_lst, name=None): else: assert isinstance(param_lst[param_idx], Path) except AssertionError: - raise TypeError( - f"The file indicated by the '{name}' parameter does not" - f" exist in the directory '{root}'." - ) - -#TODO: Improve this tool without calling model.get_congif(input_dir) + raise TypeError( + f"The file indicated by the '{name}' parameter does not" + f" exist in the directory '{root}'." + ) + + +# TODO: Improve this tool without calling model.get_congif(input_dir) # def check_file_exist(get_config, root, param_lst, name=None, input_dir=None): # root = Path(root) # param_lst = [Path(p) for p in param_lst] @@ -96,8 +99,8 @@ def check_file_exist(root, param_lst, name=None): # f"'{get_config(input_dir)}'." # ) -def check_uniqueness(map_name_lst): +def check_uniqueness(map_name_lst): def check_duplicates(lst): unique_elements = set() for element in lst: @@ -105,15 +108,14 @@ def check_duplicates(lst): return True # Found a duplicate unique_elements.add(element) return False # No duplicates found - + check = check_duplicates(map_name_lst) - if check: - raise ValueError( - f"The filenames of the hazard maps should be unique." - ) + if check: + raise ValueError(f"The filenames of the hazard maps should be unique.") + -#TODO: Improve this tool without calling model. Just checking the maps names +# TODO: Improve this tool without calling model. Just checking the maps names # def check_uniqueness(model, *args, file_type=None, filename=None): # """ """ @@ -136,6 +138,7 @@ def check_duplicates(lst): # ): # raise ValueError(f"Each model input layers must be unique.") + def check_param_type(param, name=None, types=None): """ """ @@ -155,6 +158,7 @@ def check_param_type(param, name=None, types=None): f"{type(i)} instead." ) + def get_param(param_lst, map_fn_lst, file_type, filename, i, param_name): """ """ @@ -166,4 +170,4 @@ def get_param(param_lst, map_fn_lst, file_type, filename, i, param_name): raise IndexError( f"Could not derive the {param_name} parameter for {file_type} " f"map: {filename}." - ) \ No newline at end of file + ) diff --git a/hydromt_fiat/workflows/exposure_raster.py b/hydromt_fiat/workflows/exposure_raster.py index a94cf611..9c4484dc 100644 --- a/hydromt_fiat/workflows/exposure_raster.py +++ b/hydromt_fiat/workflows/exposure_raster.py @@ -8,6 +8,7 @@ logger = logging.getLogger(__name__) + ### TO BE UPDATED ### class ExposureRaster(Exposure): def setup_buildings_value( diff --git a/hydromt_fiat/workflows/exposure_vector.py b/hydromt_fiat/workflows/exposure_vector.py index f90f6d63..6d976ce0 100644 --- a/hydromt_fiat/workflows/exposure_vector.py +++ b/hydromt_fiat/workflows/exposure_vector.py @@ -74,8 +74,8 @@ def __init__( ) self.exposure_db = pd.DataFrame() self.exposure_geoms = list() # A list of GeoDataFrames - self.source = gpd.GeoDataFrame() self.unit = unit + self._geom_names = list() # A list of (original) names of the geometry (files) def read_table(self, fn: Union[str, Path]): """Read the Delft-FIAT exposure data. @@ -102,6 +102,7 @@ def read_geoms(self, fn: Union[List[str], List[Path], str, Path]): fn = [fn] for f in fn: + self.set_geom_names(Path(f).stem) self.set_exposure_geoms(gpd.read_file(f, engine="pyogrio")) def setup_from_single_source( @@ -235,7 +236,28 @@ def setup_asset_locations(self, asset_locations: str) -> None: assets["Object ID"] = range(1, len(assets.index) + 1) # Set the asset locations to the geometry variable (self.exposure_geoms) + # and set the geom name self.set_exposure_geoms(assets) + self.set_geom_names("buildings") + + def set_geom_names(self, name: str) -> None: + """Append a name to the list of geometry names `geom_names`.""" + self.logger.info(f"Setting geometry name to {name}...") + self._geom_names.append(name) + + @property + def geom_names(self) -> List[str]: + """Returns a list with the geom names.""" + if len(self._geom_names) > 0 and len(self.exposure_geoms) > 0: + return self._geom_names + elif len(self._geom_names) == 0 and len(self.exposure_geoms) == 1: + return ["exposure"] + else: + self.logger.warning( + "No geometry names found, returning a list with the default names " + "'exposure_X'." + ) + return [f"exposure_{i}" for i in range(len(self.exposure_geoms))] def set_exposure_geoms(self, gdf: gpd.GeoDataFrame) -> None: """Append a GeoDataFrame to the exposure geometries `exposure_geoms`.""" @@ -856,7 +878,10 @@ def link_exposure_vulnerability( damage_types: Optional[List[str]] = ["Structure", "Content"], ): linking_dict = dict( - zip(exposure_linking_table["Exposure Link"], exposure_linking_table["FIAT Damage Function Name"]) + zip( + exposure_linking_table["Exposure Link"], + exposure_linking_table["FIAT Damage Function Name"], + ) ) # Find the column to link the exposure data to the vulnerability data @@ -1027,6 +1052,7 @@ def set_exposure_geoms_from_xy(self): }, crs=self.crs, ) + self.set_geom_names("exposure_points") self.set_exposure_geoms(exposure_geoms) def get_full_gdf( diff --git a/hydromt_fiat/workflows/hazard.py b/hydromt_fiat/workflows/hazard.py index 0f014aa3..b814c647 100644 --- a/hydromt_fiat/workflows/hazard.py +++ b/hydromt_fiat/workflows/hazard.py @@ -260,6 +260,7 @@ def read_floodmaps( return da_map_fn, da_name, da_type + def load_floodmaps( data_catalog: DataCatalog, region: gpd.GeoDataFrame, @@ -316,6 +317,7 @@ def load_floodmaps( return da + # def load_floodmaps( # data_catalog: DataCatalog, # region: gpd.GeoDataFrame, diff --git a/hydromt_fiat/workflows/social_vulnerability_index.py b/hydromt_fiat/workflows/social_vulnerability_index.py index ba9a0a8f..b0ca9e7d 100644 --- a/hydromt_fiat/workflows/social_vulnerability_index.py +++ b/hydromt_fiat/workflows/social_vulnerability_index.py @@ -1,16 +1,14 @@ -from census import Census #install in your environment using pip install Census -from us import states #install in your environment using pip install us +from census import Census # install in your environment using pip install Census +from us import states # install in your environment using pip install us from hydromt.data_catalog import DataCatalog from hydromt.log import setuplog -import logging +import logging import pandas as pd import numpy as np import geopandas as gpd - class SocialVulnerabilityIndex: - def __init__(self, data_catalog: DataCatalog = None, logger: logging.Logger = None): self.data_catalog = data_catalog self.census_key = Census @@ -33,11 +31,11 @@ def __init__(self, data_catalog: DataCatalog = None, logger: logging.Logger = No def read_dataset(self, path: str): """If you have your own dataset (e.g. already downloaded census data), you can use this to load it from a csv Make sure to only use the relevant functions in fiat then""" - self.pd_census_data = self.data_catalog.get_dataframe(path, sep = ";", header = 0) + self.pd_census_data = self.data_catalog.get_dataframe(path, sep=";", header=0) self.logger.info(f"You loaded your dataset from {path}") - def set_up_census_key(self, census_key:str): - """The Census key can be inputted in the ini file. + def set_up_census_key(self, census_key: str): + """The Census key can be inputted in the ini file. This is a unique key that every user needs to specify to download census data Parameters @@ -47,9 +45,11 @@ def set_up_census_key(self, census_key:str): """ self.census_key = Census(census_key) - self.logger.info(f"your census key {census_key} is used to download data from the Census website ") - - def variable_code_csv_to_pd_df(self, path:str): + self.logger.info( + f"your census key {census_key} is used to download data from the Census website " + ) + + def variable_code_csv_to_pd_df(self, path: str): """Loading the csv in and converting to pd_dataframe to be used in the other functions. Parameters @@ -57,7 +57,7 @@ def variable_code_csv_to_pd_df(self, path:str): path : specify the path where the census_vulnerability_data_codebook.csv is stored. The Excel/CSV specifies the names of the variables that need to be downloaded from the Census API - It contains additional information that can be called and used in the other functions. + It contains additional information that can be called and used in the other functions. """ self.codebook = self.data_catalog.get_dataframe(path) @@ -67,9 +67,9 @@ def set_up_download_codes(self): In this way the download_census_data function knows which variables to retrieve from the API. """ self.download_codes = list(self.codebook["Census_code_withE"]) - self.download_codes.append('NAME') - - def set_up_state_code(self, state_abbreviation:str): + self.download_codes.append("NAME") + + def set_up_state_code(self, state_abbreviation: str): """download census data for a state Parameters @@ -77,29 +77,32 @@ def set_up_state_code(self, state_abbreviation:str): state_abbreviation : str Abbreviation of the state for which you want to set up the census data download """ - state = [state_abbreviation] #read in the state abbreviation as specified in the ini file + state = [ + state_abbreviation + ] # read in the state abbreviation as specified in the ini file state_obj = getattr(states, state[0]) self.state_fips = state_obj.fips self.logger.info(f"The state abbreviation specified is: {state_abbreviation}") - + def download_census_data(self): """download the census data it is possible to also make the county, tract and blockgroup flexible so that a user could specify exactly what to download But: bear in mind, with social vulneraiblity we compare areas against each other, so Ideally you would have a large enough dataset (for statistical and validity purposes) """ download_census_codes = self.census_key.acs.state_county_blockgroup( - fields=self.download_codes, - state_fips=self.state_fips, - county_fips="*", - tract="*", - blockgroup="*" - ) + fields=self.download_codes, + state_fips=self.state_fips, + county_fips="*", + tract="*", + blockgroup="*", + ) self.pd_census_data = pd.DataFrame(download_census_codes) - self.logger.info("The census data was successfully downloaded using the specified variables in the codebook Excel") + self.logger.info( + "The census data was successfully downloaded using the specified variables in the codebook Excel" + ) return self.pd_census_data - - def rename_census_data(self, from_name:str, to_name:str): + def rename_census_data(self, from_name: str, to_name: str): """renaming the columns so that they have variable names instead of variable codes as their headers Parameters @@ -110,33 +113,39 @@ def rename_census_data(self, from_name:str, to_name:str): The name to replace with """ name_dict = dict(zip(self.codebook[from_name], self.codebook[to_name])) - self.pd_census_data = self.pd_census_data.rename(columns = name_dict) - + self.pd_census_data = self.pd_census_data.rename(columns=name_dict) def identify_no_data(self): - """This function identifies and replaces negative values in the pd_census_data DataFrame with NaN. - It then filters out rows that have no values greater than 0 in the columns (excluding specific columns) and updates the DataFrame accordingly. + """This function identifies and replaces negative values in the pd_census_data DataFrame with NaN. + It then filters out rows that have no values greater than 0 in the columns (excluding specific columns) and updates the DataFrame accordingly. The resulting DataFrame will contain only the rows where at least one value is greater than 0 in the selected columns, with negative values replaced by NaN. """ - self.columns_to_exclude = ["GEO_ID", "NAME", "state", "county", "tract", "block group", "Index"] + self.columns_to_exclude = [ + "GEO_ID", + "NAME", + "state", + "county", + "tract", + "block group", + "Index", + ] rows_to_keep = [] - + for column, series in self.pd_census_data.items(): if column in self.columns_to_exclude: continue for idx, value in series.items(): if value < 0: self.pd_census_data.loc[idx, column] = np.nan - + for index, row in self.pd_census_data.iterrows(): values_to_check = row.loc[~row.index.isin(self.columns_to_exclude)] if any(values_to_check > 0): rows_to_keep.append(index) - + self.pd_census_data = self.pd_census_data.loc[rows_to_keep] - - - def check_nan_variable_columns(self, column_to_check:str, indicator_column:str): + + def check_nan_variable_columns(self, column_to_check: str, indicator_column: str): """ Parameters ---------- @@ -151,7 +160,9 @@ def check_nan_variable_columns(self, column_to_check:str, indicator_column:str): When a census variable is dropped from the dataset because of too many nan-values (>5%) we print a logging message """ - nan_percentages = (self.pd_census_data.isna().sum() / len(self.pd_census_data)) * 100 + nan_percentages = ( + self.pd_census_data.isna().sum() / len(self.pd_census_data) + ) * 100 self.percentage = [] self.missing_values = {} @@ -171,14 +182,18 @@ def check_nan_variable_columns(self, column_to_check:str, indicator_column:str): self.fill_nan_columns(column) for value in self.codebook[column_to_check]: - if value not in self.pd_census_data.columns: - indicator_name = self.codebook.loc[self.codebook[column_to_check] == value, indicator_column].iloc[0] - self.missing_values[value] = indicator_name - self.logger.info(f"From indicator '{indicator_name}' the variable '{value}' is dropped because of too many missing values (exceeding the threshold of 5%)") - - def fill_nan_columns(self, nancolumn:str): + if value not in self.pd_census_data.columns: + indicator_name = self.codebook.loc[ + self.codebook[column_to_check] == value, indicator_column + ].iloc[0] + self.missing_values[value] = indicator_name + self.logger.info( + f"From indicator '{indicator_name}' the variable '{value}' is dropped because of too many missing values (exceeding the threshold of 5%)" + ) + + def fill_nan_columns(self, nancolumn: str): """This function is created to fill no data with the column mean - + Parameters ---------- nancolumn : str @@ -186,40 +201,41 @@ def fill_nan_columns(self, nancolumn:str): In the case of the census data: The column mean is the average value of all regions, for that specific variable """ - + column_average = self.pd_census_data[nancolumn].mean() self.pd_census_data[nancolumn].fillna(column_average, inplace=True) def check_zeroes_variable_rows(self): - #Here you count the zeroes in a row. - #This is under the assumption that if more than half of the data is zero, it indicates missing data. - #Note: in theory a region can be least vulnerable on all commponents and score zero everywhere. We assume however that it is unlikely that this is the case for more than 50% + # Here you count the zeroes in a row. + # This is under the assumption that if more than half of the data is zero, it indicates missing data. + # Note: in theory a region can be least vulnerable on all commponents and score zero everywhere. We assume however that it is unlikely that this is the case for more than 50% self.excluded_regions = [] num_columns = self.pd_census_data.shape[1] - if 'NAME' in self.pd_census_data.columns: + if "NAME" in self.pd_census_data.columns: num_columns = self.pd_census_data.shape[1] - 1 for index, row in self.pd_census_data.iterrows(): num_zeros = (row == 0).sum() percentage_zeros = num_zeros / num_columns - + if percentage_zeros > 0.5: self.pd_census_data = self.pd_census_data.drop(index) names_exluced_areas = row["NAME"] self.excluded_regions.append(names_exluced_areas) - - def create_indicator_groups(self, column_to_group:str, aggregation_name:str) -> dict: - """With this code we can group the different variables into their indicator group + def create_indicator_groups( + self, column_to_group: str, aggregation_name: str + ) -> dict: + """With this code we can group the different variables into their indicator group We can also group the indicators into the domains with this code - + Parameters ---------- column_to_group : str - This is the column which you want to group together into an aggregate group. + This is the column which you want to group together into an aggregate group. For example grouping all the Census variables into the indicators they belong to aggregation_name : str - This is the column by which you want to group. So for example: + This is the column by which you want to group. So for example: this column indicates the group name for the indicators to which variables belong. Returns @@ -231,8 +247,8 @@ def create_indicator_groups(self, column_to_group:str, aggregation_name:str) -> for indicator_code, group_df in self.codebook.groupby(aggregation_name): translation[indicator_code] = set(group_df[column_to_group].tolist()) return translation - - def processing_svi_data(self, translation_var_ind:dict): + + def processing_svi_data(self, translation_var_ind: dict): """ This is a function to pre-process the Census data. The function is quite specific to the census data! Check if it can be applied to other datasets! @@ -249,9 +265,9 @@ def processing_svi_data(self, translation_var_ind:dict): The preprocessing needs to be done to convert variables into indicators, for example by dividing a variable by a total * 100 to find a percentage. The variables with 'total' in their name, are used to do divisions for the creation of the percentages / shares. Therefore, this function below firstly finds for every indicator group if there is a variable with 'total' in the name. - If that is not the case, the indicator is not a percentage or share but a Median value. + If that is not the case, the indicator is not a percentage or share but a Median value. We can just keep the Median value and put it in the final dataframe (processed_census_data) - When there are more than 2 columns, we first need to sum all the data (except for the total). + When there are more than 2 columns, we first need to sum all the data (except for the total). Then we compute the percentage of that sum as a share of the whole (the total). For example to find the population under 5 years and over 65 years we need to add females and males and the different age groups and then take that as a percentage of the total population. If there are exactly two columns, then the one needs to be divided by the other. The other is always the total. Together these procedures make up the processed indicators. @@ -265,11 +281,13 @@ def processing_svi_data(self, translation_var_ind:dict): # Loop over the indicator_groups for key, values in translation_var_ind.items(): - updated_values = [value for value in values if value not in self.missing_values.keys()] + updated_values = [ + value for value in values if value not in self.missing_values.keys() + ] if len(updated_values) > 0: updated_translation_var_ind[key] = updated_values if key in self.columns_to_exclude: - continue + continue # Update translation_var_ind with the key-value pairs and remove the 'missing values' / the excluded values @@ -280,46 +298,55 @@ def processing_svi_data(self, translation_var_ind:dict): group_vars = self.pd_census_data[values] group_varslist.append(group_vars) else: - continue + continue # identify the columns that contain 'total' in the name. These are the columns that are used for division, to calculate percentages etc. - total_cols = [col for col in group_vars.columns if 'total' in col.lower()] - - #if there is no total column, the indicator is a single column (e.g. Median) and we just want to take that column - if len(total_cols) == 0: - self.processed_census_data[key] = group_vars.iloc[:,0] - - #if there are more than 2 columns we need to first sum the columns together and then divide over the total to get a percentage - elif len(group_vars.columns) > 2: + total_cols = [col for col in group_vars.columns if "total" in col.lower()] + + # if there is no total column, the indicator is a single column (e.g. Median) and we just want to take that column + if len(total_cols) == 0: + self.processed_census_data[key] = group_vars.iloc[:, 0] + + # if there are more than 2 columns we need to first sum the columns together and then divide over the total to get a percentage + elif len(group_vars.columns) > 2: # identify the columns that do not contain 'total' in the name - value_cols = [col for col in group_vars.columns if col not in total_cols] - #sum the values across the rows (axis = 1) for value columns + value_cols = [ + col for col in group_vars.columns if col not in total_cols + ] + # sum the values across the rows (axis = 1) for value columns sum_values = group_vars[value_cols].sum(axis=1) # divide the sum of values by the total column for this group total_col = total_cols[0] result = sum_values / group_vars[total_col] * 100 # add a new column to the result dataframe with the result and the key as the column header self.processed_census_data[key] = result - - #if the length is exactly 2, you do not need to sum but you do need to divide over the corresponding total column - elif len(group_vars.columns) == 2: + + # if the length is exactly 2, you do not need to sum but you do need to divide over the corresponding total column + elif len(group_vars.columns) == 2: total_col = total_cols[0] - value_cols = [col for col in group_vars.columns if col not in total_cols] - if 'capita' in value_cols[0]: #if 'capita' is in the column name, do not multiply by 100 - result = group_vars[value_cols].iloc[:,0] / group_vars[total_col] + value_cols = [ + col for col in group_vars.columns if col not in total_cols + ] + if ( + "capita" in value_cols[0] + ): # if 'capita' is in the column name, do not multiply by 100 + result = group_vars[value_cols].iloc[:, 0] / group_vars[total_col] else: - result = group_vars[value_cols].iloc[:,0] / group_vars[total_col] * 100 + result = ( + group_vars[value_cols].iloc[:, 0] / group_vars[total_col] * 100 + ) # add a new column to the result dataframe with the result and the key as the column header self.processed_census_data[key] = result self.processed_census_data["NAME"] = self.pd_census_data["NAME"] - self.logger.info("The specified variables are processed into their corresponding indicators, as specified in the Excel codebook") + self.logger.info( + "The specified variables are processed into their corresponding indicators, as specified in the Excel codebook" + ) def zscore(self, dataseries): return (dataseries - dataseries.mean()) / dataseries.std() - - def normalization_svi_data(self): + def normalization_svi_data(self): """This function normalizes the processed indicator dataset using zscores for the first time The normalization direction is specified in the input excel codebook The inverse or normal standardization is based in the indicator's relation with social vulnerability @@ -331,30 +358,39 @@ def normalization_svi_data(self): vulnerability_features = selected_data["Indicator_code"].tolist() direction = selected_data["zscore"].tolist() - self.svi_fiat['NAME'] = self.processed_census_data['NAME'] + self.svi_fiat["NAME"] = self.processed_census_data["NAME"] svi_fiat_names = [] for column, dir in zip(vulnerability_features, direction): - if column not in self.processed_census_data.columns: #If there are variables in the codebook but not in the data, this line ensures the model does not crash + if ( + column not in self.processed_census_data.columns + ): # If there are variables in the codebook but not in the data, this line ensures the model does not crash continue - if dir == 'normal': - self.svi_fiat[column + '_norm'] = self.zscore(self.processed_census_data[column]) - svi_fiat_names.append(column + '_norm') - elif dir == 'inverse': - self.svi_fiat[column + '_norm'] = self.zscore(self.processed_census_data[column]) * -1 - svi_fiat_names.append(column + '_norm') + if dir == "normal": + self.svi_fiat[column + "_norm"] = self.zscore( + self.processed_census_data[column] + ) + svi_fiat_names.append(column + "_norm") + elif dir == "inverse": + self.svi_fiat[column + "_norm"] = ( + self.zscore(self.processed_census_data[column]) * -1 + ) + svi_fiat_names.append(column + "_norm") else: - print('Direction is not provided') - self.logger.info("The data is standardized according to their relation specified in the codebook Excel (inverse or normal z-scores)") + print("Direction is not provided") + self.logger.info( + "The data is standardized according to their relation specified in the codebook Excel (inverse or normal z-scores)" + ) def domain_scores(self): """This function groups the indicators together in domains (called 'Category' in the input excel codebook) and calculates their domain score First, the domaingroups need to contain the right indicators, specific to the dataset It could be that an indiator is missing because of variable exclusion on the basis of nan-criteria >5% and there are no variables left to compute an indicator - In that case, the indicators are removed from domaingroups and the domain score is composed of the indicators that are there to make up a domain""" - + In that case, the indicators are removed from domaingroups and the domain score is composed of the indicators that are there to make up a domain + """ + self.pd_domain_scores = pd.DataFrame() - + # Get the column names from the processed dataframe columns_data = self.processed_census_data.columns.tolist() columns_data.remove("NAME") @@ -362,48 +398,83 @@ def domain_scores(self): self.missing_indicators = [] # Remove key-value pairs without a match -> you want to remove from the default match set (based in codebook) the variables that are no longer in your dataset - if any(value not in columns_data for values in domaingroups.values() for value in values): - self.missing_indicators.append([value for values in domaingroups.values() for value in values if value not in columns_data]) - # Create a new dictionary with the values that are present in columns_data - domaingroups = {key: [value for value in values if value in columns_data] for key, values in domaingroups.items()} - + if any( + value not in columns_data + for values in domaingroups.values() + for value in values + ): + self.missing_indicators.append( + [ + value + for values in domaingroups.values() + for value in values + if value not in columns_data + ] + ) + # Create a new dictionary with the values that are present in columns_data + domaingroups = { + key: [value for value in values if value in columns_data] + for key, values in domaingroups.items() + } + for key, values in domaingroups.items(): # Add _norm to all values to match with the normalization column names - values_norm = [v + '_norm' for v in values] - sum_variables = self.svi_fiat.filter(items=values_norm) #use the normalized dataset! - sum_values = sum_variables.sum(axis=1) / len(values_norm) #use the normalized dataset! + values_norm = [v + "_norm" for v in values] + sum_variables = self.svi_fiat.filter( + items=values_norm + ) # use the normalized dataset! + sum_values = sum_variables.sum(axis=1) / len( + values_norm + ) # use the normalized dataset! self.pd_domain_scores[key] = sum_values - + self.pd_domain_scores_z = self.zscore(self.pd_domain_scores) - #Now we add the names so we can later match these regions with their geometry + # Now we add the names so we can later match these regions with their geometry self.pd_domain_scores_z["NAME"] = self.processed_census_data["NAME"] - self.pd_domain_scores["NAME"] = self.processed_census_data["NAME"] - - self.logger.info("The indicators are grouped together into their respective domains and are standardized again using zscores") + self.pd_domain_scores["NAME"] = self.processed_census_data["NAME"] + + self.logger.info( + "The indicators are grouped together into their respective domains and are standardized again using zscores" + ) def composite_scores(self): - """create composite scores from the domain scores - """ + """create composite scores from the domain scores""" columns_to_sum = self.pd_domain_scores_z.columns.drop("NAME") - self.pd_domain_scores_z["composite_SVI"] = self.pd_domain_scores_z[columns_to_sum].sum(axis=1) - self.pd_domain_scores_z["composite_svi_z"] = self.zscore(self.pd_domain_scores_z["composite_SVI"]) - self.logger.info("Composite SVI scores are created based on the domain scores using equal weighting and are standardized again using zscores") - + self.pd_domain_scores_z["composite_SVI"] = self.pd_domain_scores_z[ + columns_to_sum + ].sum(axis=1) + self.pd_domain_scores_z["composite_svi_z"] = self.zscore( + self.pd_domain_scores_z["composite_SVI"] + ) + self.logger.info( + "Composite SVI scores are created based on the domain scores using equal weighting and are standardized again using zscores" + ) + def match_geo_ID(self): """Matches GEO_IDs for the regions that are still in the dataset""" self.pd_domain_scores_geo = self.pd_domain_scores_z.copy() - self.pd_domain_scores_geo['GEO_ID'] = None # Create a new column 'GEO_ID' with initial values set to None + self.pd_domain_scores_geo[ + "GEO_ID" + ] = None # Create a new column 'GEO_ID' with initial values set to None for index, value in enumerate(self.pd_domain_scores_geo["NAME"]): if value in self.pd_census_data["NAME"].values: - matching_row = self.pd_census_data.loc[self.pd_census_data["NAME"] == value] - geo_id = matching_row["GEO_ID"].values[0] # Assuming there's only one matching row, extract the GEO_ID value - self.pd_domain_scores_geo.at[index, 'GEO_ID'] = geo_id # Assign the GEO_ID value to the corresponding row in self.pd_domain_scores_geo - self.pd_domain_scores_geo['GEOID_short'] = self.pd_domain_scores_geo['GEO_ID'].str.split('US').str[1] - + matching_row = self.pd_census_data.loc[ + self.pd_census_data["NAME"] == value + ] + geo_id = matching_row["GEO_ID"].values[ + 0 + ] # Assuming there's only one matching row, extract the GEO_ID value + self.pd_domain_scores_geo.at[ + index, "GEO_ID" + ] = geo_id # Assign the GEO_ID value to the corresponding row in self.pd_domain_scores_geo + self.pd_domain_scores_geo["GEOID_short"] = ( + self.pd_domain_scores_geo["GEO_ID"].str.split("US").str[1] + ) + def load_shp_geom(self, path): - """loading geometry data from the user's specified shapefile. - The user can download the right block groups for their preferred state from: https://www.census.gov/cgi-bin/geo/shapefiles/index.php + """loading geometry data from the user's specified shapefile. + The user can download the right block groups for their preferred state from: https://www.census.gov/cgi-bin/geo/shapefiles/index.php Parameters ---------- @@ -411,78 +482,82 @@ def load_shp_geom(self, path): A shapefile containing block groups geometry: https://www.census.gov/cgi-bin/geo/shapefiles/index.php """ self.shape_data = self.data_catalog.get_geodataframe(path) - self.logger.info(f"Succesfully read in the shapefile from {path}") + self.logger.info(f"Succesfully read in the shapefile from {path}") def merge_svi_data_shp(self): - """Merges the geometry data with the social vulnerability index computed in the module - """ - self.svi_data_shp = self.pd_domain_scores_geo.merge(self.shape_data, left_on='GEOID_short', right_on='GEOID') + """Merges the geometry data with the social vulnerability index computed in the module""" + self.svi_data_shp = self.pd_domain_scores_geo.merge( + self.shape_data, left_on="GEOID_short", right_on="GEOID" + ) self.svi_data_shp = gpd.GeoDataFrame(self.svi_data_shp) - columns_to_drop = ['STATEFP','COUNTYFP','TRACTCE','BLKGRPCE', 'GEOID','NAMELSAD','MTFCC','FUNCSTAT','ALAND', 'AWATER','INTPTLAT','INTPTLON'] - self.svi_data_shp.drop(columns=columns_to_drop,inplace=True) + columns_to_drop = [ + "STATEFP", + "COUNTYFP", + "TRACTCE", + "BLKGRPCE", + "GEOID", + "NAMELSAD", + "MTFCC", + "FUNCSTAT", + "ALAND", + "AWATER", + "INTPTLAT", + "INTPTLON", + ] + self.svi_data_shp.drop(columns=columns_to_drop, inplace=True) self.svi_data_shp = self.svi_data_shp.to_crs(epsg=4326) - self.logger.info("The geometry information was successfully added to the social vulnerability information") - + self.logger.info( + "The geometry information was successfully added to the social vulnerability information" + ) ###OTHER -#import requests -#import json - -#This function does not work yet. Somehow the json data does not contain the acutal geometry data. - #svi.load_geometry_data("https://tigerweb.geo.census.gov/arcgis/rest/services/TIGERweb/tigerWMS_ACS2022/MapServer/8?f=pjson") - #def load_geometry_data(self, url:str): - - # # Specify the parameters for the request - # params = { - # "where": "1=1", - # "geometryType": "esriGeometryPolygon", - # "spatialRel": "esriSpatialRelIntersects", - # "outFields": "*", - # "returnGeometry": True, - # "f": "geojson" # Retrieve data as GeoJSON - # } - - # # Send the GET request - # response = requests.get(url, params=params) - # response2 = response.text - # # Load the JSON data - # json_data = json.loads(response2) - - # geometry_data = json_data['geometryField'] - - # # Extract the GEOID data - # geoid_data = [feature['attributes']['GEOID'] for feature in json_data['features']] - - # # Create a GeoPandas DataFrame - # df = gpd.GeoDataFrame(pd.DataFrame(geoid_data, columns=['GEOID'])) - # df['geometry'] = gpd.GeoSeries.from_wkt(geometry_data) - - # # Print the resulting GeoPandas DataFrame - # print(df) - - - - - #Testing the functions - #Removing some variables from the codebook - #I think in all cases you would need the total column, otherwise you would mess up the making of your indicators and thus final scores - #but you would not see that in the results as they will be standardized anyways - #the code works, so that does not really matter +# import requests +# import json - #lowering the nan-threshold to 3% - #it runs +# This function does not work yet. Somehow the json data does not contain the acutal geometry data. +# svi.load_geometry_data("https://tigerweb.geo.census.gov/arcgis/rest/services/TIGERweb/tigerWMS_ACS2022/MapServer/8?f=pjson") +# def load_geometry_data(self, url:str): - #changing the input data now also works. The code can still run if there are varaibles missing or if there is a large amount of Nans - #we now assume that we will still make the indciator if data is missing, as a default. In future versions we might want to chagne this for accuracy purposes - #We now printed the information about missing variables / indicators to lists that can be inspected so that informed decision can be made. +# # Specify the parameters for the request +# params = { +# "where": "1=1", +# "geometryType": "esriGeometryPolygon", +# "spatialRel": "esriSpatialRelIntersects", +# "outFields": "*", +# "returnGeometry": True, +# "f": "geojson" # Retrieve data as GeoJSON +# } +# # Send the GET request +# response = requests.get(url, params=params) +# response2 = response.text +# # Load the JSON data +# json_data = json.loads(response2) +# geometry_data = json_data['geometryField'] +# # Extract the GEOID data +# geoid_data = [feature['attributes']['GEOID'] for feature in json_data['features']] +# # Create a GeoPandas DataFrame +# df = gpd.GeoDataFrame(pd.DataFrame(geoid_data, columns=['GEOID'])) +# df['geometry'] = gpd.GeoSeries.from_wkt(geometry_data) +# # Print the resulting GeoPandas DataFrame +# print(df) +# Testing the functions +# Removing some variables from the codebook +# I think in all cases you would need the total column, otherwise you would mess up the making of your indicators and thus final scores +# but you would not see that in the results as they will be standardized anyways +# the code works, so that does not really matter +# lowering the nan-threshold to 3% +# it runs +# changing the input data now also works. The code can still run if there are varaibles missing or if there is a large amount of Nans +# we now assume that we will still make the indciator if data is missing, as a default. In future versions we might want to chagne this for accuracy purposes +# We now printed the information about missing variables / indicators to lists that can be inspected so that informed decision can be made. diff --git a/hydromt_fiat/workflows/vulnerability.py b/hydromt_fiat/workflows/vulnerability.py index 08055c8c..dc942a98 100644 --- a/hydromt_fiat/workflows/vulnerability.py +++ b/hydromt_fiat/workflows/vulnerability.py @@ -283,7 +283,8 @@ def get_vulnerability_functions_from_one_file( @staticmethod def get_identifier_names( - df: pd.DataFrame, to_remove: list = ["FIAT Damage Function Name", "Exposure Link", "Damage Type"] + df: pd.DataFrame, + to_remove: list = ["FIAT Damage Function Name", "Exposure Link", "Damage Type"], ) -> list: """_summary_ diff --git a/tests/test_SVI_exposure.py b/tests/test_SVI_exposure.py index f50d7d64..bfe40408 100644 --- a/tests/test_SVI_exposure.py +++ b/tests/test_SVI_exposure.py @@ -104,9 +104,8 @@ def test_SVI_exposure(case): fm.write() assert fm - + # Check if the exposure data exists assert root.joinpath("exposure", "exposure.gpkg").exists() assert root.joinpath("exposure", "exposure.csv").exists() assert root.joinpath("exposure", "region.gpkg").exists() - diff --git a/tests/test_integration.py b/tests/test_integration.py index 5c83164f..a500cb4b 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -159,4 +159,4 @@ def test_integration(case): assert root.joinpath("hazard").exists() # Check if the output data folder exists - assert root.joinpath("output").exists() \ No newline at end of file + assert root.joinpath("output").exists() diff --git a/tests/test_integrations_hazard.py b/tests/test_integrations_hazard.py index e0b18986..0b7de142 100644 --- a/tests/test_integrations_hazard.py +++ b/tests/test_integrations_hazard.py @@ -8,7 +8,7 @@ EXAMPLEDIR = Path( "P:/11207949-dhs-phaseii-floodadapt/Model-builder/Delft-FIAT/local_test_database" -) +) _cases = { "integration": { @@ -18,6 +18,7 @@ }, } + @pytest.mark.parametrize("case", list(_cases.keys())) def test_hazard(case): # Read model in examples folder. @@ -27,31 +28,33 @@ def test_hazard(case): # uncomment to test event analysis from geotiff file configuration = { - "setup_hazard": { - "map_fn": ["P:/11207949-dhs-phaseii-floodadapt/Model-builder/Delft-FIAT/local_test_database/data/Hazard/Current_prob_event_set_combined_doNothing_withSeaWall_RP=1_max_flood_depth.tif"], - "map_type": "water_depth", - "rp": None, - "crs": None, - "nodata": -99999, - "var": None, - "chunks": "auto", + "setup_hazard": { + "map_fn": [ + "P:/11207949-dhs-phaseii-floodadapt/Model-builder/Delft-FIAT/local_test_database/data/Hazard/Current_prob_event_set_combined_doNothing_withSeaWall_RP=1_max_flood_depth.tif" + ], + "map_type": "water_depth", + "rp": None, + "crs": None, + "nodata": -99999, + "var": None, + "chunks": "auto", "name_catalog": None, - "risk_output": False, + "risk_output": False, } } # uncomment to test risk analysis from geotiff file # configuration = { - # "setup_hazard": { - # "map_fn": ["P:/11207949-dhs-phaseii-floodadapt/Model-builder/Delft-FIAT/local_test_database/data/Hazard/Current_prob_event_set_combined_doNothing_withSeaWall_RP=1_max_flood_depth.tif", "P:/11207949-dhs-phaseii-floodadapt/Model-builder/Delft-FIAT/local_test_database/data/Hazard/Current_prob_event_set_combined_doNothing_withSeaWall_RP=2_max_flood_depth.tif", "P:/11207949-dhs-phaseii-floodadapt/Model-builder/Delft-FIAT/local_test_database/data/Hazard/Current_prob_event_set_combined_doNothing_withSeaWall_RP=5_max_flood_depth.tif"], - # "map_type": "water_depth", - # "rp": None, - # "crs": None, - # "nodata": -99999, - # "var": None, - # "chunks": "auto", + # "setup_hazard": { + # "map_fn": ["P:/11207949-dhs-phaseii-floodadapt/Model-builder/Delft-FIAT/local_test_database/data/Hazard/Current_prob_event_set_combined_doNothing_withSeaWall_RP=1_max_flood_depth.tif", "P:/11207949-dhs-phaseii-floodadapt/Model-builder/Delft-FIAT/local_test_database/data/Hazard/Current_prob_event_set_combined_doNothing_withSeaWall_RP=2_max_flood_depth.tif", "P:/11207949-dhs-phaseii-floodadapt/Model-builder/Delft-FIAT/local_test_database/data/Hazard/Current_prob_event_set_combined_doNothing_withSeaWall_RP=5_max_flood_depth.tif"], + # "map_type": "water_depth", + # "rp": None, + # "crs": None, + # "nodata": -99999, + # "var": None, + # "chunks": "auto", # "name_catalog": None, - # "risk_output": True, + # "risk_output": True, # } # } @@ -80,7 +83,7 @@ def test_hazard(case): # var = "zsmax" # configuration = { - # "setup_hazard": { + # "setup_hazard": { # "map_fn": map_fn, # absolute or relative (with respect to the configuration.ini) path to the hazard file # "map_type": "water_depth", # description of the hazard file type # "rp": None, # hazard return period in years, required for a risk calculation (optional) @@ -89,11 +92,10 @@ def test_hazard(case): # "var": var, # hazard variable name in NetCDF input files (optional) # "chunks": "auto", # chunk sizes along each dimension used to load the hazard file into a dask array (default is 'auto') (optional) # "name_catalog": None, - # "risk_output": risk_output, + # "risk_output": risk_output, # } # } - logger = setuplog("hydromt_fiat", log_level=10) data_catalog_yml = str(_cases[case]["data_catalogue"]) diff --git a/tests/test_raise_ground_floor_height.py b/tests/test_raise_ground_floor_height.py index ce572606..341bc0a3 100644 --- a/tests/test_raise_ground_floor_height.py +++ b/tests/test_raise_ground_floor_height.py @@ -5,7 +5,9 @@ import shutil import pandas as pd -EXAMPLEDIR = Path("P:/11207949-dhs-phaseii-floodadapt/Model-builder/Delft-FIAT/local_test_database") +EXAMPLEDIR = Path( + "P:/11207949-dhs-phaseii-floodadapt/Model-builder/Delft-FIAT/local_test_database" +) _cases = { "raise_ground_floor_height": { @@ -61,4 +63,4 @@ def test_raise_ground_floor_height(case): exposure_modified["Ground Floor Height"], exposure_modified["Ground Elevation"], ) - ) \ No newline at end of file + ) diff --git a/tests/test_read_write.py b/tests/test_read_write.py new file mode 100644 index 00000000..673b3863 --- /dev/null +++ b/tests/test_read_write.py @@ -0,0 +1,55 @@ +from hydromt_fiat.fiat import FiatModel +from hydromt.log import setuplog +from pathlib import Path +import pytest +import shutil + + +EXAMPLEDIR = Path( + "P:/11207949-dhs-phaseii-floodadapt/Model-builder/Delft-FIAT/local_test_database" +) + +_cases = { + "read_read_write_single_file": { + "dir": "test_read", + "new_root": EXAMPLEDIR / "test_read_write_single", + }, + "read_read_write_multiple_files": { + "dir": "test_read_multiple", + "new_root": EXAMPLEDIR / "test_read_write_multiple", + }, +} + + +@pytest.mark.parametrize("case", list(_cases.keys())) +def test_read_write(case): + # Read model in examples folder. + root = EXAMPLEDIR.joinpath(_cases[case]["dir"]) + logger = setuplog("hydromt_fiat", log_level=10) + + fm = FiatModel(root=root, mode="r", logger=logger) + fm.read() + + # Get the name of the spatial exposure data in the model that is read + exposure_files = [k for k in fm.config["exposure"]["geom"].keys() if "file" in k] + original_exposure_names = set( + [ + fm.config["exposure"]["geom"][f].split("/")[-1].split(".")[0] + for f in exposure_files + ] + ) + + # Remove the new root folder if it already exists + if _cases[case]["new_root"].exists(): + shutil.rmtree(_cases[case]["new_root"]) + + # Set the new root and write the model + fm.set_root(_cases[case]["new_root"]) + fm.write() + + # Check if the name of the spatial exposure data in the new_root is correct + new_exposure_names = set( + [f.stem for f in _cases[case]["new_root"].joinpath("exposure").glob("*.gpkg")] + ) + + assert original_exposure_names.issubset(new_exposure_names) diff --git a/tests/test_reader.py b/tests/test_reader.py index 30dbdb8c..f72e393f 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -13,20 +13,18 @@ _cases = { "read": { - "data_catalogue": EXAMPLEDIR / "fiat_catalog.yml", "dir": "test_read", }, } @pytest.mark.parametrize("case", list(_cases.keys())) -def test_read_fiat_config(case): +def test_read(case): # Read model in examples folder. root = EXAMPLEDIR.joinpath(_cases[case]["dir"]) logger = setuplog("hydromt_fiat", log_level=10) - data_catalog_yml = str(_cases[case]["data_catalogue"]) - fm = FiatModel(root=root, mode="r", data_libs=[data_catalog_yml], logger=logger) + fm = FiatModel(root=root, mode="r", logger=logger) fm.read() # Check if the exposure object exists @@ -39,4 +37,4 @@ def test_read_fiat_config(case): assert isinstance(fm.vulnerability, Vulnerability) # Check if the vulnerability functions exist - assert len(fm.vulnerability.functions) > 0 \ No newline at end of file + assert len(fm.vulnerability.functions) > 0 diff --git a/tests/test_update_max_potential_damage.py b/tests/test_update_max_potential_damage.py index e9302bcc..695aecdb 100644 --- a/tests/test_update_max_potential_damage.py +++ b/tests/test_update_max_potential_damage.py @@ -62,10 +62,12 @@ def test_update_max_potential_damage(case): # check if the max potential damage is updated updated_max_pot_damage.reset_index(inplace=True, drop=True) - updated_max_pot_damage["Object Name"] = updated_max_pot_damage["Object Name"].astype(int) + updated_max_pot_damage["Object Name"] = updated_max_pot_damage[ + "Object Name" + ].astype(int) pd.testing.assert_frame_equal( updated_max_pot_damage, exposure_modified, check_dtype=False, check_column_type=False, - ) \ No newline at end of file + ) diff --git a/tests/test_vulnerability.py b/tests/test_vulnerability.py index ab8e2165..c9af930e 100644 --- a/tests/test_vulnerability.py +++ b/tests/test_vulnerability.py @@ -43,14 +43,19 @@ def test_hazus_vulnerability_curves(case): fm.write() # Check if the vulnerability data exists - vulnerability_curves_path = root.joinpath("vulnerability", "vulnerability_curves.csv") + vulnerability_curves_path = root.joinpath( + "vulnerability", "vulnerability_curves.csv" + ) assert vulnerability_curves_path.exists() - # Check if the vulnerability data is correct by comparing it to data that is known + # Check if the vulnerability data is correct by comparing it to data that is known # to be correct vulnerability_curves = pd.read_csv(vulnerability_curves_path) correct_vulnerability_curves = pd.read_csv( - Path(__file__).parent / "data" / "correct" / "test_hazus_vulnerability_curves.csv" + Path(__file__).parent + / "data" + / "correct" + / "test_hazus_vulnerability_curves.csv" ) - assert vulnerability_curves.equals(correct_vulnerability_curves) \ No newline at end of file + assert vulnerability_curves.equals(correct_vulnerability_curves) diff --git a/tests/test_vulnerability_exposure_add_to_data_catalog.py b/tests/test_vulnerability_exposure_add_to_data_catalog.py index f10528f9..d1b364f8 100644 --- a/tests/test_vulnerability_exposure_add_to_data_catalog.py +++ b/tests/test_vulnerability_exposure_add_to_data_catalog.py @@ -62,7 +62,10 @@ }, } -@pytest.mark.skip(reason="Th functionalities to test with this test are not yet finalized") + +@pytest.mark.skip( + reason="Th functionalities to test with this test are not yet finalized" +) @pytest.mark.parametrize("case", list(_cases.keys())) def test_vulnerability_and_exposure_add_to_data_catalog(case): # Read model in examples folder. @@ -103,4 +106,4 @@ def test_vulnerability_and_exposure_add_to_data_catalog(case): assert root.joinpath("hazard").exists() # Check if the output data folder exists - assert root.joinpath("output").exists() \ No newline at end of file + assert root.joinpath("output").exists() diff --git a/tests/test_vulnerability_exposure_global_default.py b/tests/test_vulnerability_exposure_global_default.py index 64bb41ea..a8718a4f 100644 --- a/tests/test_vulnerability_exposure_global_default.py +++ b/tests/test_vulnerability_exposure_global_default.py @@ -96,4 +96,4 @@ def test_vulnerability_and_exposure_global_default(case): assert root.joinpath("hazard").exists() # Check if the output data folder exists - assert root.joinpath("output").exists() \ No newline at end of file + assert root.joinpath("output").exists() diff --git a/tests/test_vulnerability_from_csvs.py b/tests/test_vulnerability_from_csvs.py index 977c1d13..792698b1 100644 --- a/tests/test_vulnerability_from_csvs.py +++ b/tests/test_vulnerability_from_csvs.py @@ -38,10 +38,12 @@ def test_csvs(case): fm.write() # Check if the vulnerability data exists - vulnerability_curves_path = root.joinpath("vulnerability", "vulnerability_curves.csv") + vulnerability_curves_path = root.joinpath( + "vulnerability", "vulnerability_curves.csv" + ) assert vulnerability_curves_path.exists() - # Check if the vulnerability data is correct by comparing it to data that is known + # Check if the vulnerability data is correct by comparing it to data that is known # to be correct vulnerability_curves = pd.read_csv(vulnerability_curves_path) correct_vulnerability_curves = pd.read_csv( From e609e0a2ff1a17fcc0fa5bcc3b6b2e306838c839 Mon Sep 17 00:00:00 2001 From: Frederique Date: Wed, 27 Sep 2023 17:22:20 +0200 Subject: [PATCH 2/2] Updated tests according to changes issue #137 --- hydromt_fiat/fiat.py | 12 +--- hydromt_fiat/workflows/exposure_vector.py | 8 ++- .../test_hazus_vulnerability_curves.csv | 67 ++++++++++--------- tests/test_SVI_exposure.py | 11 +-- tests/test_hazard.py | 1 + tests/test_integration.py | 8 +-- tests/test_integrations_hazard.py | 1 + tests/test_vulnerability.py | 4 +- tests/test_vulnerability_exposure.py | 2 +- ...nerability_exposure_add_to_data_catalog.py | 2 +- ...t_vulnerability_exposure_global_default.py | 2 +- 11 files changed, 60 insertions(+), 58 deletions(-) diff --git a/hydromt_fiat/fiat.py b/hydromt_fiat/fiat.py index 428172f6..694c4486 100644 --- a/hydromt_fiat/fiat.py +++ b/hydromt_fiat/fiat.py @@ -271,13 +271,13 @@ def setup_exposure_vector( if asset_locations == occupancy_type == max_potential_damage: # The source for the asset locations, occupancy type and maximum potential # damage is the same, use one source to create the exposure data. - self.exposure.setup_from_single_source( + self.exposure.setup_buildings_from_single_source( asset_locations, ground_floor_height, extraction_method ) else: # The source for the asset locations, occupancy type and maximum potential # damage is different, use three sources to create the exposure data. - self.exposure.setup_from_multiple_sources( + self.exposure.setup_buildings_from_multiple_sources( asset_locations, occupancy_type, max_potential_damage, @@ -582,14 +582,6 @@ def setup_social_vulnerability_index( svi_exp_joined = pd.DataFrame(svi_exp_joined) self.exposure.exposure_db = svi_exp_joined - # exposure opnieuw opslaan in self._tables - - # TODO: geometries toevoegen aan de dataset met API - # we now use the shape download function by the census, the user needs to download their own shape data. They can download this from: https://www.census.gov/cgi-bin/geo/shapefiles/index.php - # #wfs python get request -> geometries - - # this link can be used: https://github.com/datamade/census - # Update functions def update_all(self): self.logger.info("Updating all data objects...") diff --git a/hydromt_fiat/workflows/exposure_vector.py b/hydromt_fiat/workflows/exposure_vector.py index 6d976ce0..6b9be6ec 100644 --- a/hydromt_fiat/workflows/exposure_vector.py +++ b/hydromt_fiat/workflows/exposure_vector.py @@ -105,7 +105,7 @@ def read_geoms(self, fn: Union[List[str], List[Path], str, Path]): self.set_geom_names(Path(f).stem) self.set_exposure_geoms(gpd.read_file(f, engine="pyogrio")) - def setup_from_single_source( + def setup_buildings_from_single_source( self, source: Union[str, Path], ground_floor_height: Union[int, float, str, Path, None], @@ -179,7 +179,10 @@ def setup_from_single_source( # Set the geoms from the X and Y coordinates self.set_exposure_geoms_from_xy() - def setup_from_multiple_sources( + # Set the name to the geom_names + self.set_geom_names("buildings") + + def setup_buildings_from_multiple_sources( self, asset_locations: Union[str, Path], occupancy_source: Union[str, Path], @@ -1052,7 +1055,6 @@ def set_exposure_geoms_from_xy(self): }, crs=self.crs, ) - self.set_geom_names("exposure_points") self.set_exposure_geoms(exposure_geoms) def get_full_gdf( diff --git a/tests/data/correct/test_hazus_vulnerability_curves.csv b/tests/data/correct/test_hazus_vulnerability_curves.csv index 81d37acf..6dcbb006 100644 --- a/tests/data/correct/test_hazus_vulnerability_curves.csv +++ b/tests/data/correct/test_hazus_vulnerability_curves.csv @@ -1,32 +1,37 @@ #UNIT=feet -#METHOD,max,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean -water depth [feet],AGR1,COM1,COM10,COM2,COM3,COM4,COM5,COM6,COM7,COM8,COM9,EDU1,GOV1,GOV2,IND1,IND2,IND3,IND4,IND5,IND6,REL1,RES1-1SNB,RES1-1SWB,RES1-2SNB,RES1-2SWB,RES1-3SNB,RES1-3SWB,RES2,RES3A,RES3B,RES3C,RES3D,RES3E,RES3F,RES4,RES5,RES6 --4.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.07,0.0,0.04,0.0,0.03,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 --3.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.07,0.0,0.04,0.0,0.03,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 --2.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.07,0.0,0.08,0.0,0.06,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 --1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.11,0.0,0.14,0.0,0.1,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 -0.0,0.0,0.01,0.0,0.0,0.0,0.02,0.0,0.0,0.02,0.01,0.0,0.0,0.0,0.0,0.01,0.01,0.0,0.0,0.0,0.0,0.0,0.18,0.17,0.11,0.19,0.05,0.12,0.11,0.12,0.12,0.12,0.12,0.12,0.12,0.0,0.0,0.0 -1.0,0.06,0.09,0.03,0.05,0.09,0.11,0.11,0.0,0.11,0.09,0.02,0.05,0.05,0.07,0.1,0.09,0.13,0.1,0.13,0.22,0.1,0.22,0.21,0.12,0.21,0.08,0.14,0.44,0.25,0.25,0.25,0.25,0.25,0.25,0.03,0.07,0.07 -2.0,0.11,0.14,0.05,0.08,0.12,0.16,0.11,0.0,0.12,0.11,0.04,0.07,0.08,0.1,0.12,0.14,0.14,0.14,0.14,0.31,0.11,0.25,0.29,0.14,0.26,0.12,0.2,0.63,0.5,0.5,0.5,0.5,0.5,0.5,0.05,0.1,0.1 -3.0,0.15,0.16,0.06,0.11,0.13,0.22,0.12,0.2,0.13,0.12,0.05,0.09,0.13,0.11,0.15,0.17,0.19,0.18,0.19,0.37,0.11,0.28,0.34,0.18,0.29,0.17,0.25,0.73,0.75,0.75,0.75,0.75,0.75,0.75,0.06,0.14,0.14 -4.0,0.19,0.18,0.07,0.13,0.16,0.28,0.13,0.25,0.14,0.14,0.05,0.09,0.14,0.12,0.19,0.22,0.22,0.22,0.22,0.43,0.12,0.3,0.38,0.2,0.34,0.19,0.31,0.78,1.0,1.0,1.0,1.0,1.0,1.0,0.07,0.15,0.15 -5.0,0.25,0.2,0.08,0.16,0.19,0.35,0.15,0.3,0.16,0.16,0.05,0.1,0.14,0.15,0.22,0.26,0.25,0.26,0.25,0.47,0.12,0.31,0.43,0.22,0.39,0.22,0.36,0.79,1.0,1.0,1.0,1.0,1.0,1.0,0.09,0.15,0.15 -6.0,0.3,0.23,0.1,0.19,0.22,0.38,0.17,0.35,0.17,0.18,0.06,0.11,0.15,0.17,0.26,0.3,0.28,0.34,0.28,0.5,0.13,0.4,0.5,0.24,0.44,0.24,0.38,0.81,1.0,1.0,1.0,1.0,1.0,1.0,0.12,0.16,0.16 -7.0,0.35,0.26,0.13,0.22,0.25,0.41,0.19,0.4,0.18,0.2,0.08,0.13,0.17,0.2,0.3,0.32,0.3,0.41,0.3,0.54,0.14,0.43,0.5,0.26,0.5,0.25,0.41,0.82,1.0,1.0,1.0,1.0,1.0,1.0,0.14,0.18,0.18 -8.0,0.41,0.3,0.17,0.25,0.28,0.44,0.22,0.43,0.2,0.22,0.1,0.15,0.19,0.23,0.35,0.35,0.33,0.42,0.33,0.57,0.14,0.43,0.54,0.3,0.55,0.3,0.44,0.83,1.0,1.0,1.0,1.0,1.0,1.0,0.18,0.2,0.2 -9.0,0.46,0.34,0.21,0.29,0.32,0.47,0.24,0.47,0.22,0.26,0.12,0.17,0.22,0.27,0.39,0.37,0.34,0.42,0.34,0.61,0.15,0.45,0.55,0.34,0.57,0.35,0.48,0.84,1.0,1.0,1.0,1.0,1.0,1.0,0.21,0.23,0.23 -10.0,0.51,0.38,0.25,0.32,0.35,0.5,0.28,0.5,0.24,0.29,0.15,0.2,0.26,0.31,0.42,0.39,0.36,0.45,0.36,0.63,0.17,0.46,0.55,0.38,0.59,0.38,0.5,0.85,1.0,1.0,1.0,1.0,1.0,1.0,0.26,0.26,0.26 -11.0,0.57,0.42,0.3,0.37,0.39,0.54,0.31,0.53,0.27,0.33,0.2,0.24,0.31,0.35,0.48,0.43,0.39,0.47,0.39,0.64,0.19,0.47,0.57,0.39,0.61,0.39,0.52,0.86,1.0,1.0,1.0,1.0,1.0,1.0,0.31,0.3,0.3 -12.0,0.63,0.47,0.35,0.41,0.43,0.57,0.34,0.55,0.3,0.37,0.24,0.28,0.37,0.4,0.5,0.46,0.4,0.49,0.4,0.65,0.24,0.47,0.58,0.4,0.63,0.4,0.54,0.88,1.0,1.0,1.0,1.0,1.0,1.0,0.36,0.34,0.34 -13.0,0.7,0.51,0.41,0.45,0.47,0.59,0.37,0.57,0.34,0.41,0.29,0.33,0.44,0.44,0.51,0.48,0.42,0.5,0.42,0.67,0.3,0.49,0.6,0.42,0.65,0.42,0.56,0.89,1.0,1.0,1.0,1.0,1.0,1.0,0.41,0.38,0.38 -14.0,0.75,0.55,0.47,0.49,0.5,0.62,0.4,0.6,0.37,0.45,0.35,0.39,0.51,0.48,0.53,0.5,0.42,0.5,0.42,0.68,0.38,0.5,0.62,0.43,0.66,0.43,0.57,0.9,1.0,1.0,1.0,1.0,1.0,1.0,0.46,0.42,0.42 -15.0,0.79,0.58,0.52,0.52,0.54,0.66,0.44,0.6,0.41,0.5,0.42,0.45,0.59,0.52,0.54,0.51,0.43,0.5,0.43,0.69,0.45,0.5,0.63,0.44,0.68,0.44,0.59,0.91,1.0,1.0,1.0,1.0,1.0,1.0,0.5,0.47,0.47 -16.0,0.82,0.61,0.58,0.55,0.57,0.68,0.48,0.6,0.44,0.53,0.49,0.52,0.65,0.56,0.55,0.54,0.43,0.5,0.43,0.7,0.52,0.5,0.65,0.45,0.69,0.45,0.61,0.92,1.0,1.0,1.0,1.0,1.0,1.0,0.54,0.52,0.52 -17.0,0.84,0.64,0.65,0.58,0.61,0.7,0.51,0.6,0.48,0.57,0.56,0.59,0.7,0.6,0.55,0.55,0.44,0.5,0.44,0.71,0.58,0.51,0.67,0.47,0.71,0.47,0.63,0.94,1.0,1.0,1.0,1.0,1.0,1.0,0.58,0.57,0.57 -18.0,0.87,0.67,0.71,0.61,0.64,0.72,0.55,0.6,0.51,0.6,0.62,0.64,0.74,0.64,0.56,0.57,0.44,0.5,0.44,0.72,0.64,0.51,0.69,0.48,0.72,0.48,0.65,0.95,1.0,1.0,1.0,1.0,1.0,1.0,0.62,0.62,0.62 -19.0,0.89,0.69,0.76,0.63,0.68,0.74,0.59,0.6,0.54,0.63,0.68,0.69,0.79,0.68,0.56,0.59,0.44,0.5,0.44,0.73,0.69,0.52,0.7,0.49,0.74,0.49,0.67,0.96,1.0,1.0,1.0,1.0,1.0,1.0,0.66,0.67,0.67 -20.0,0.9,0.71,0.81,0.66,0.71,0.76,0.63,0.6,0.56,0.66,0.74,0.74,0.83,0.72,0.57,0.6,0.44,0.5,0.44,0.74,0.74,0.52,0.72,0.5,0.75,0.5,0.69,0.97,1.0,1.0,1.0,1.0,1.0,1.0,0.7,0.72,0.72 -21.0,0.92,0.74,0.86,0.68,0.75,0.77,0.67,0.6,0.59,0.69,0.8,0.79,0.87,0.76,0.57,0.62,0.44,0.5,0.44,0.75,0.78,0.53,0.74,0.52,0.77,0.52,0.71,0.98,1.0,1.0,1.0,1.0,1.0,1.0,0.74,0.77,0.77 -22.0,0.93,0.76,0.91,0.7,0.78,0.78,0.71,0.6,0.61,0.73,0.86,0.84,0.91,0.8,0.57,0.63,0.45,0.5,0.45,0.76,0.82,0.53,0.76,0.53,0.79,0.53,0.73,0.99,1.0,1.0,1.0,1.0,1.0,1.0,0.78,0.82,0.82 -23.0,0.95,0.78,0.96,0.71,0.82,0.79,0.75,0.6,0.64,0.76,0.92,0.89,0.95,0.84,0.58,0.65,0.45,0.5,0.45,0.76,0.85,0.54,0.77,0.54,0.8,0.54,0.75,1.0,1.0,1.0,1.0,1.0,1.0,1.0,0.82,0.87,0.87 -24.0,0.96,0.8,1.0,0.73,0.85,0.8,0.79,0.6,0.66,0.78,0.98,0.94,0.98,0.88,0.58,0.66,0.45,0.5,0.45,0.77,0.88,0.54,0.79,0.56,0.82,0.56,0.77,1.0,1.0,1.0,1.0,1.0,1.0,1.0,0.86,0.92,0.92 +#METHOD,max,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean,mean +water depth [feet],AGR1,COM1,COM10,COM2,COM3,COM4,COM5,COM6,COM7,COM8,COM9,EDU1,EDU2,GOV1,GOV2,IND1,IND2,IND3,IND4,IND5,IND6,REL1,RES1-1SNB,RES1-1SWB,RES1-2SNB,RES1-2SWB,RES1-3SNB,RES1-3SWB,RES2,RES3A,RES3B,RES3C,RES3D,RES3E,RES3F,RES4,RES5,RES6 +-9.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +-8.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1e-06,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +-7.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.008,0.0,0.01,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +-6.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.021,0.0,0.023,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +-5.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.037,0.0,0.037,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +-4.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.057,0.0,0.052,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +-3.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.08,0.0,0.068,0.0,0.05,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +-2.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.105,0.0,0.084,0.0,0.07,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +-1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.024,0.132,0.01,0.101,0.0,0.08,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +0.0,0.06,0.02,0.0,0.03,0.004,0.0,0.0,0.0,0.0,0.13,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.02,0.15,0.02,0.0,0.0,0.081,0.16,0.05,0.119,0.06,0.14,0.03,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +1.0,0.2,0.26,0.11,0.16,0.164,0.2,0.5,0.146,0.28,0.45,0.04,0.217,0.27,0.3,0.08,0.15,0.193,0.2,0.2,0.2,0.2,0.293,0.133,0.189,0.087,0.138,0.11,0.18,0.27,0.217,0.217,0.217,0.217,0.217,0.217,0.161,0.15,0.38 +2.0,0.43,0.42,0.17,0.27,0.289,0.3,0.74,0.27,0.51,0.55,0.06,0.304,0.38,0.59,0.2,0.24,0.31,0.41,0.26,0.41,0.35,0.484,0.179,0.218,0.122,0.157,0.15,0.22,0.49,0.304,0.304,0.304,0.304,0.304,0.304,0.263,0.25,0.6 +3.0,0.58,0.56,0.2,0.36,0.409,0.4,0.83,0.37,0.6,0.64,0.08,0.39,0.53,0.74,0.38,0.34,0.423,0.51,0.31,0.51,0.47,0.6,0.22,0.247,0.155,0.177,0.19,0.25,0.64,0.39,0.39,0.39,0.39,0.39,0.39,0.341,0.4,0.73 +4.0,0.65,0.68,0.23,0.49,0.577,0.575,1.0,0.534,0.63,0.73,0.09,0.45,0.64,0.83,0.55,0.41,0.523,0.62,0.37,0.62,0.56,0.693,0.257,0.274,0.185,0.198,0.23,0.29,0.7,0.45,0.45,0.45,0.45,0.45,0.45,0.397,0.5,0.81 +5.0,0.66,0.78,0.25,0.57,0.633,0.7,1.0,0.7,0.67,0.77,0.1,0.479,0.68,0.9,0.7,0.47,0.607,0.67,0.4,0.67,0.59,0.764,0.288,0.3,0.213,0.22,0.26,0.31,0.76,0.479,0.479,0.479,0.479,0.479,0.479,0.487,0.58,0.88 +6.0,0.66,0.83,0.29,0.63,0.707,0.8,1.0,0.791,0.71,0.8,0.12,0.519,0.7,1.0,0.81,0.52,0.72,0.71,0.44,0.71,0.66,0.814,0.315,0.324,0.239,0.243,0.29,0.34,0.78,0.519,0.519,0.519,0.519,0.519,0.519,0.524,0.65,0.94 +7.0,0.67,0.85,0.35,0.69,0.793,0.838,1.0,0.856,0.72,0.82,0.17,0.557,0.72,1.0,0.89,0.57,0.821,0.73,0.48,0.73,0.69,0.884,0.338,0.345,0.263,0.267,0.32,0.36,0.79,0.557,0.557,0.557,0.557,0.557,0.557,0.584,0.78,1.0 +8.0,0.7,0.87,0.42,0.72,0.843,1.0,1.0,0.925,0.74,0.83,0.22,0.593,0.75,1.0,0.98,0.6,0.907,0.76,0.53,0.76,0.71,0.943,0.357,0.363,0.284,0.291,0.35,0.39,0.81,0.593,0.593,0.593,0.593,0.593,0.593,0.613,0.9,1.0 +9.0,0.75,0.88,0.51,0.76,0.871,1.0,1.0,0.956,0.77,0.85,0.3,0.606,0.79,1.0,1.0,0.63,0.943,0.78,0.56,0.78,0.72,0.971,0.372,0.377,0.303,0.317,0.41,0.44,0.83,0.606,0.606,0.606,0.606,0.606,0.606,0.631,0.9,1.0 +10.0,0.76,0.89,0.63,0.8,0.871,1.0,1.0,0.963,0.81,0.87,0.41,0.634,0.83,1.0,1.0,0.64,0.95,0.79,0.57,0.79,0.78,0.971,0.384,0.386,0.32,0.344,0.43,0.46,0.83,0.634,0.634,0.634,0.634,0.634,0.634,0.649,0.92,1.0 +11.0,0.76,0.9,0.77,0.82,0.871,1.0,1.0,0.963,0.86,0.89,0.57,0.634,0.88,1.0,1.0,0.66,0.95,0.82,0.6,0.82,0.79,0.971,0.392,0.391,0.334,0.372,0.45,0.47,0.83,0.634,0.634,0.634,0.634,0.634,0.634,0.649,0.92,1.0 +12.0,0.76,0.91,0.93,0.84,0.871,1.0,1.0,0.963,0.92,0.9,0.66,0.634,0.94,1.0,1.0,0.68,0.95,0.83,0.62,0.83,0.8,0.971,0.397,0.391,0.347,0.4,0.48,0.5,0.83,0.634,0.634,0.634,0.634,0.634,0.634,0.649,0.92,1.0 +13.0,0.77,0.92,1.0,0.86,0.871,1.0,1.0,0.963,0.94,0.91,0.73,0.634,1.0,1.0,1.0,0.69,0.95,0.84,0.63,0.84,0.8,0.971,0.4,0.391,0.356,0.43,0.5,0.52,0.83,0.634,0.634,0.634,0.634,0.634,0.634,0.649,0.92,1.0 +14.0,0.77,0.92,1.0,0.87,0.871,1.0,1.0,0.963,0.97,0.92,0.79,0.634,1.0,1.0,1.0,0.72,0.95,0.86,0.63,0.86,0.81,0.971,0.4,0.391,0.364,0.461,0.52,0.53,0.83,0.634,0.634,0.634,0.634,0.634,0.634,0.649,0.92,1.0 +15.0,0.77,0.92,1.0,0.87,0.871,1.0,1.0,0.963,0.99,0.93,0.84,0.634,1.0,1.0,1.0,0.73,0.95,0.87,0.63,0.87,0.81,0.971,0.4,0.391,0.369,0.493,0.54,0.55,0.83,0.634,0.634,0.634,0.634,0.634,0.634,0.649,0.92,1.0 +16.0,0.78,0.93,1.0,0.88,0.871,1.0,1.0,0.963,1.0,0.94,0.9,0.634,1.0,1.0,1.0,0.73,0.95,0.87,0.64,0.87,0.81,0.971,0.4,0.391,0.372,0.526,0.56,0.57,0.83,0.634,0.634,0.634,0.634,0.634,0.634,0.649,0.92,1.0 +17.0,0.78,0.93,1.0,0.89,0.871,1.0,1.0,0.963,1.0,0.95,0.97,0.634,1.0,1.0,1.0,0.73,0.95,0.88,0.65,0.88,0.82,0.971,0.4,0.391,0.372,0.526,0.58,0.58,0.83,0.634,0.634,0.634,0.634,0.634,0.634,0.649,0.92,1.0 +18.0,0.78,0.94,1.0,0.89,0.871,1.0,1.0,0.963,1.0,0.96,0.98,0.634,1.0,1.0,1.0,0.74,0.95,0.88,0.65,0.88,0.82,0.971,0.4,0.391,0.372,0.526,0.6,0.6,0.83,0.634,0.634,0.634,0.634,0.634,0.634,0.649,0.92,1.0 +19.0,0.79,0.94,1.0,0.89,0.871,1.0,1.0,0.963,1.0,0.96,1.0,0.634,1.0,1.0,1.0,0.74,0.95,0.88,0.65,0.88,0.82,0.971,0.4,0.391,0.372,0.526,0.6,0.6,0.83,0.634,0.634,0.634,0.634,0.634,0.634,0.649,0.92,1.0 +20.0,0.79,0.94,1.0,0.89,0.871,1.0,1.0,0.963,1.0,0.96,1.0,0.634,1.0,1.0,1.0,0.74,0.95,0.88,0.65,0.88,0.83,0.971,0.4,0.391,0.372,0.526,0.6,0.6,0.83,0.634,0.634,0.634,0.634,0.634,0.634,0.649,0.92,1.0 +21.0,0.79,0.94,1.0,0.89,0.871,1.0,1.0,0.963,1.0,0.96,1.0,0.634,1.0,1.0,1.0,0.74,0.95,0.88,0.65,0.88,0.83,0.971,0.4,0.391,0.372,0.526,0.6,0.6,0.83,0.634,0.634,0.634,0.634,0.634,0.634,0.649,0.92,1.0 +22.0,0.79,0.94,1.0,0.89,0.871,1.0,1.0,0.963,1.0,0.96,1.0,0.634,1.0,1.0,1.0,0.75,0.95,0.88,0.65,0.88,0.83,0.971,0.4,0.391,0.372,0.526,0.6,0.6,0.83,0.634,0.634,0.634,0.634,0.634,0.634,0.649,0.92,1.0 +23.0,0.8,0.94,1.0,0.89,0.871,1.0,1.0,0.963,1.0,0.96,1.0,0.634,1.0,1.0,1.0,0.75,0.95,0.88,0.65,0.88,0.83,0.971,0.4,0.391,0.372,0.526,0.6,0.6,0.83,0.634,0.634,0.634,0.634,0.634,0.634,0.649,0.92,1.0 +24.0,0.8,0.94,1.0,0.89,0.871,1.0,1.0,0.963,1.0,0.96,1.0,0.634,1.0,1.0,1.0,0.75,0.95,0.88,0.65,0.88,0.83,0.971,0.4,0.391,0.372,0.526,0.6,0.6,0.83,0.634,0.634,0.634,0.634,0.634,0.634,0.649,0.92,1.0 diff --git a/tests/test_SVI_exposure.py b/tests/test_SVI_exposure.py index bfe40408..3128ee7e 100644 --- a/tests/test_SVI_exposure.py +++ b/tests/test_SVI_exposure.py @@ -100,12 +100,15 @@ def test_SVI_exposure(case): fm.data_catalog.from_dict(to_add) fm.build(region={"geom": region}, opt=_cases[case]["configuration"]) - fm.write() - assert fm - # Check if the exposure data exists - assert root.joinpath("exposure", "exposure.gpkg").exists() + assert root.joinpath("exposure", "buildings.gpkg").exists() assert root.joinpath("exposure", "exposure.csv").exists() assert root.joinpath("exposure", "region.gpkg").exists() + + # Check if the social vulnerability data exists + assert root.joinpath("exposure", "social_vulnerability_scores.csv").exists() + + # Check if the vulnerability data exists + assert root.joinpath("vulnerability", "vulnerability_curves.csv").exists() \ No newline at end of file diff --git a/tests/test_hazard.py b/tests/test_hazard.py index b5b942cb..52c280c1 100644 --- a/tests/test_hazard.py +++ b/tests/test_hazard.py @@ -18,6 +18,7 @@ } +@pytest.mark.skip(reason="This test will be removed") @pytest.mark.parametrize("case", list(_cases.keys())) def test_hazard(case): # Read model in examples folder. diff --git a/tests/test_integration.py b/tests/test_integration.py index a500cb4b..9b61a7f5 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -45,7 +45,7 @@ }, "setup_vulnerability": { "vulnerability_fn": "default_vulnerability_curves", - "vulnerability_identifiers_and_linking_fn": "default_vulnerability_curves", + "vulnerability_identifiers_and_linking_fn": "default_hazus_iwr_linking", "functions_mean": "default", "functions_max": ["AGR1"], "unit": "feet", @@ -68,7 +68,6 @@ "nodata": -99999, # value that is assigned as nodata (optional) "var": None, # hazard variable name in NetCDF input files (optional) "chunks": "auto", # chunk sizes along each dimension used to load the hazard file into a dask array (default is 'auto') (optional) - "name_catalog": "flood_maps", "risk_output": False, }, }, @@ -86,7 +85,7 @@ }, "setup_vulnerability": { "vulnerability_fn": "default_vulnerability_curves", - "vulnerability_identifiers_and_linking_fn": "default_vulnerability_curves", + "vulnerability_identifiers_and_linking_fn": "default_hazus_iwr_linking", "functions_mean": "default", "functions_max": ["AGR1"], "unit": "feet", @@ -123,7 +122,6 @@ "nodata": -99999, # value that is assigned as nodata (optional) "var": None, # hazard variable name in NetCDF input files (optional) "chunks": "auto", # chunk sizes along each dimension used to load the hazard file into a dask array (default is 'auto') (optional) - "name_catalog": "flood_maps", "risk_output": True, }, }, @@ -148,7 +146,7 @@ def test_integration(case): fm.write() # Check if the exposure data exists - assert root.joinpath("exposure", "exposure.gpkg").exists() + assert root.joinpath("exposure", "buildings.gpkg").exists() assert root.joinpath("exposure", "exposure.csv").exists() assert root.joinpath("exposure", "region.gpkg").exists() diff --git a/tests/test_integrations_hazard.py b/tests/test_integrations_hazard.py index 0b7de142..53af79f6 100644 --- a/tests/test_integrations_hazard.py +++ b/tests/test_integrations_hazard.py @@ -19,6 +19,7 @@ } +@pytest.mark.skip(reason="Hazard functions not yet finalized") @pytest.mark.parametrize("case", list(_cases.keys())) def test_hazard(case): # Read model in examples folder. diff --git a/tests/test_vulnerability.py b/tests/test_vulnerability.py index c9af930e..45f4ce3d 100644 --- a/tests/test_vulnerability.py +++ b/tests/test_vulnerability.py @@ -16,8 +16,8 @@ "dir": "test_hazus_vulnerability_curves", "configuration": { "setup_vulnerability": { - "vulnerability_fn": "hazus_vulnerability_curves", - "vulnerability_identifiers_and_linking_fn": ".\\examples\\data\\vulnerability_test_file_input.csv", + "vulnerability_fn": "default_vulnerability_curves", + "vulnerability_identifiers_and_linking_fn": "default_hazus_iwr_linking", "functions_mean": "default", "functions_max": ["AGR1"], "unit": "feet", diff --git a/tests/test_vulnerability_exposure.py b/tests/test_vulnerability_exposure.py index 25b4be40..dbeb376c 100644 --- a/tests/test_vulnerability_exposure.py +++ b/tests/test_vulnerability_exposure.py @@ -122,7 +122,7 @@ def test_vulnerability_exposure_NSI(case): fm.write() # Check if the exposure data exists - assert root.joinpath("exposure", "exposure.gpkg").exists() + assert root.joinpath("exposure", "buildings.gpkg").exists() assert root.joinpath("exposure", "exposure.csv").exists() assert root.joinpath("exposure", "region.gpkg").exists() diff --git a/tests/test_vulnerability_exposure_add_to_data_catalog.py b/tests/test_vulnerability_exposure_add_to_data_catalog.py index d1b364f8..3d61e903 100644 --- a/tests/test_vulnerability_exposure_add_to_data_catalog.py +++ b/tests/test_vulnerability_exposure_add_to_data_catalog.py @@ -95,7 +95,7 @@ def test_vulnerability_and_exposure_add_to_data_catalog(case): fm.write() # Check if the exposure data exists - assert root.joinpath("exposure", "exposure.gpkg").exists() + assert root.joinpath("exposure", "buildings.gpkg").exists() assert root.joinpath("exposure", "exposure.csv").exists() assert root.joinpath("exposure", "region.gpkg").exists() diff --git a/tests/test_vulnerability_exposure_global_default.py b/tests/test_vulnerability_exposure_global_default.py index a8718a4f..2ca2b9ee 100644 --- a/tests/test_vulnerability_exposure_global_default.py +++ b/tests/test_vulnerability_exposure_global_default.py @@ -85,7 +85,7 @@ def test_vulnerability_and_exposure_global_default(case): fm.write() # Check if the exposure data exists - assert root.joinpath("exposure", "exposure.gpkg").exists() + assert root.joinpath("exposure", "buildings.gpkg").exists() assert root.joinpath("exposure", "exposure.csv").exists() assert root.joinpath("exposure", "region.gpkg").exists()