Configuration based TUDelft calculation (#353)

* Update config documentation.
* Tidying and test improvements; fix precision-related bug in applying SOP.
* Fix damage function.
* Changes to standard of protection and use Sequence in place of List/Iterable.
* Fix ruff exclusion of unused imports.
---------

Signed-off-by: Joe Moorhouse <5102656+joemoorhouse@users.noreply.github.com>

docs/vulnerability_functions/inundation_jrc/raw.csv

@@ -35,15 +35,15 @@ Transport,3,1,,1,1,,,1,,,,0,,,
 Transport,4,1,,1,1,,,1,,,,0,,,
 Transport,5,1,,1,1,,,1,,,,0,,,
 Transport,6,1,,1,1,,,1,,,,0,,,
-Infrastructure  roads,0,0,,,0,,,0,,,,0,,,
-Infrastructure  roads,0.5,0.25,,,0.214436906,,,0.232218453,,,,0.069410857,,,
-Infrastructure  roads,1,0.42,,,0.37275441,,,0.396377205,,,,0.073943937,,,
-Infrastructure  roads,1.5,0.55,,,0.603934871,,,0.576967436,,,,0.205814323,,,
-Infrastructure  roads,2,0.65,,,0.709659091,,,0.679829545,,,,0.270555844,,,
-Infrastructure  roads,3,0.8,,,0.808409091,,,0.804204545,,,,0.221245379,,,
-Infrastructure  roads,4,0.9,,,0.887159091,,,0.893579545,,,,0.133514877,,,
-Infrastructure  roads,5,1,,,0.96875,,,0.984375,,,,0.0625,,,
-Infrastructure  roads,6,1,,,1,,,1,,,,0,,,
+Infrastructure roads,0,0,,,0,,,0,,,,0,,,
+Infrastructure roads,0.5,0.25,,,0.214436906,,,0.232218453,,,,0.069410857,,,
+Infrastructure roads,1,0.42,,,0.37275441,,,0.396377205,,,,0.073943937,,,
+Infrastructure roads,1.5,0.55,,,0.603934871,,,0.576967436,,,,0.205814323,,,
+Infrastructure roads,2,0.65,,,0.709659091,,,0.679829545,,,,0.270555844,,,
+Infrastructure roads,3,0.8,,,0.808409091,,,0.804204545,,,,0.221245379,,,
+Infrastructure roads,4,0.9,,,0.887159091,,,0.893579545,,,,0.133514877,,,
+Infrastructure roads,5,1,,,0.96875,,,0.984375,,,,0.0625,,,
+Infrastructure roads,6,1,,,1,,,1,,,,0,,,
 Agriculture,0,0,0.018575388,,0,0,,0,,,,0,0,,
 Agriculture,0.5,0.3,0.267797668,,0.135,0.242873563,,0.236417808,,,,0.188944436,0.095973845,,
 Agriculture,1,0.55,0.473677377,,0.37,0.47183908,,0.466379114,,,,0.351069794,0.17508562,,

pyproject.toml

@@ -206,7 +206,6 @@ ignore = [
     # Ignore this rules so that precommit passes. Uncomment to start fixing them
     "B006", "B008", "B904", "B012", "B024",
     "D",
-    "F401",
 ]
 
 [tool.ruff.lint.extend-per-file-ignores]

src/physrisk/data/pregenerated_hazard_model.py

@@ -1,7 +1,7 @@
 import concurrent.futures
 from collections import defaultdict
 import logging
-from typing import Dict, List, Mapping, MutableMapping, Optional, Type
+from typing import Dict, List, Mapping, MutableMapping, Optional, Sequence, Type
 
 import numpy as np
 
@@ -30,8 +30,8 @@ def __init__(
     ):
         self.hazard_data_providers = hazard_data_providers
 
-    def get_hazard_events(  # noqa: C901
-        self, requests: List[HazardDataRequest]
+    def get_hazard_data(  # noqa: C901
+        self, requests: Sequence[HazardDataRequest]
     ) -> Mapping[HazardDataRequest, HazardDataResponse]:
         # A note on concurrency.
         # The requests will be batched up with batches accessing the same data set
@@ -45,10 +45,10 @@ def get_hazard_events(  # noqa: C901
         # for now we do 2; 1 might be preferred if the number of threads needed to download
         # data in parallel becomes large (probably not, given use of async in Zarr).
 
-        return self._get_hazard_events(requests)
+        return self._get_hazard_data(requests)
 
-    def _get_hazard_events(  # noqa: C901
-        self, requests: List[HazardDataRequest]
+    def _get_hazard_data(  # noqa: C901
+        self, requests: Sequence[HazardDataRequest]
     ) -> Mapping[HazardDataRequest, HazardDataResponse]:
         batches = defaultdict(list)
         for request in requests:
@@ -58,13 +58,13 @@ def _get_hazard_events(  # noqa: C901
         # can change max_workers=1 for debugging
         with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
             futures = [
-                executor.submit(self._get_hazard_events_batch, batches[key], responses)
+                executor.submit(self._get_hazard_data_batch, batches[key], responses)
                 for key in batches.keys()
             ]
             concurrent.futures.wait(futures)
         return responses
 
-    def _get_hazard_events_batch(
+    def _get_hazard_data_batch(
         self,
         batch: List[HazardDataRequest],
         responses: MutableMapping[HazardDataRequest, HazardDataResponse],
@@ -113,7 +113,10 @@ def _get_hazard_events_batch(
                             np.array([0]),
                         )
                     responses[req] = HazardEventDataResponse(
-                        valid_periods, valid_intensities, units, path
+                        valid_periods,
+                        valid_intensities.astype(dtype="float64"),
+                        units,
+                        path,
                     )
             else:  # type: ignore
                 parameters, defns, units, path = hazard_data_provider.get_data(
@@ -128,7 +131,10 @@ def _get_hazard_events_batch(
                 for i, req in enumerate(batch):
                     valid = ~np.isnan(parameters[i, :])
                     responses[req] = HazardParameterDataResponse(
-                        parameters[i, :][valid], defns[valid], units, path
+                        parameters[i, :][valid].astype(dtype="float64"),
+                        defns[valid],
+                        units,
+                        path,
                     )
         except Exception as err:
             # e.g. the requested data is unavailable

src/physrisk/data/static/vulnerability/vulnerability_config.csv

@@ -0,0 +1,19 @@
+hazard_class,asset_class,asset_identifier,indicator_id,indicator_units,impact_id,impact_units,curve_type,points_x,points_y,points_z,cap_of_points_x,cap_of_points_y,activation_of_points_x
+"CoastalInundation,PluvialInundation,RiverineInundation",RealEstateAsset,"type=Buildings/Residential,location=Europe",flood_depth,metres,damage,,indicator/piecewise_linear,"[0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 5.0, 6.0]","[0.0, 0.25, 0.4, 0.5, 0.6, 0.75, 0.85, 0.95, 1.0]",,,,
+"CoastalInundation,PluvialInundation,RiverineInundation",RealEstateAsset,"type=Buildings/Residential,location=North America",flood_depth,metres,damage,,indicator/piecewise_linear,"[0.0, 0.01, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 5.0, 6.0]","[0.0, 0.201804543, 0.443269857, 0.582754693, 0.682521912, 0.783957148, 0.854348922, 0.923670101, 0.958522773, 1.0]",,,,
+"CoastalInundation,PluvialInundation,RiverineInundation",RealEstateAsset,"type=Buildings/Residential,location=South America",flood_depth,metres,damage,,indicator/piecewise_linear,"[0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 5.0, 6.0]","[0.0, 0.490885951, 0.711294067, 0.842026011, 0.949369096, 0.983636977, 1.0, 1.0, 1.0]",,,,
+"CoastalInundation,PluvialInundation,RiverineInundation",RealEstateAsset,"type=Buildings/Residential,location=Asia",flood_depth,metres,damage,,indicator/piecewise_linear,"[0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 5.0, 6.0]","[0.0, 0.326556502, 0.494050324, 0.616572124, 0.720711764, 0.869528213, 0.931487084, 0.983604148, 1.0]",,,,
+"CoastalInundation,PluvialInundation,RiverineInundation",RealEstateAsset,"type=Buildings/Residential,location=Africa",flood_depth,metres,damage,,indicator/piecewise_linear,"[0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 5.0, 6.0]","[0.0, 0.219925401, 0.378226846, 0.530589082, 0.635636733, 0.81693978, 0.903434688, 0.957152173, 1.0]",,,,
+"CoastalInundation,PluvialInundation,RiverineInundation",RealEstateAsset,"type=Buildings/Residential,location=Oceania",flood_depth,metres,damage,,indicator/piecewise_linear,"[0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 5.0, 6.0]","[0.0, 0.475418119, 0.640393124, 0.714614662, 0.787726348, 0.928779884, 0.967381853, 0.982795444, 1.0]",,,,
+"CoastalInundation,PluvialInundation,RiverineInundation",RealEstateAsset,"type=Buildings/Commercial,location=Europe",flood_depth,metres,damage,,indicator/piecewise_linear,"[0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 5.0, 6.0]","[0.0, 0.15, 0.3, 0.45, 0.55, 0.75, 0.9, 1.0, 1.0]",,,,
+"CoastalInundation,PluvialInundation,RiverineInundation",RealEstateAsset,"type=Buildings/Commercial,location=North America",flood_depth,metres,damage,,indicator/piecewise_linear,"[0.0, 0.01, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 5.0, 6.0]","[0.0, 0.018404908, 0.239263804, 0.374233129, 0.466257669, 0.552147239, 0.687116564, 0.82208589, 0.90797546, 1.0]",,,,
+"CoastalInundation,PluvialInundation,RiverineInundation",RealEstateAsset,"type=Buildings/Commercial,location=South America",flood_depth,metres,damage,,indicator/piecewise_linear,"[0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 5.0, 6.0]","[0.0, 0.611477587, 0.839531094, 0.923588457, 0.991972477, 1.0, 1.0, 1.0, 1.0]",,,,
+"CoastalInundation,PluvialInundation,RiverineInundation",RealEstateAsset,"type=Buildings/Commercial,location=Asia",flood_depth,metres,damage,,indicator/piecewise_linear,"[0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 5.0, 6.0]","[0.0, 0.376789623, 0.537681619, 0.659336684, 0.762845232, 0.883348656, 0.941854895, 0.98075938, 1.0]",,,,
+"CoastalInundation,PluvialInundation,RiverineInundation",RealEstateAsset,"type=Buildings/Commercial,location=Oceania",flood_depth,metres,damage,,indicator/piecewise_linear,"[0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 5.0, 6.0]","[0.0, 0.238953575, 0.481199682, 0.673795091, 0.864583333, 1.0, 1.0, 1.0, 1.0]",,,,
+"CoastalInundation,PluvialInundation,RiverineInundation",RealEstateAsset,"type=Buildings/Commercial,location=Global",flood_depth,metres,damage,,indicator/piecewise_linear,"[0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 5.0, 6.0]","[0.0, 0.323296918, 0.506529105, 0.63459558, 0.744309656, 0.864093044, 0.932788157, 0.977746968, 1.0]",,,,
+"CoastalInundation,PluvialInundation,RiverineInundation",RealEstateAsset,"type=Buildings/Industrial,location=Europe",flood_depth,metres,damage,,indicator/piecewise_linear,"[0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 5.0, 6.0]","[0.0, 0.15, 0.27, 0.4, 0.52, 0.7, 0.85, 1.0, 1.0]",,,,
+"CoastalInundation,PluvialInundation,RiverineInundation",RealEstateAsset,"type=Buildings/Industrial,location=North America",flood_depth,metres,damage,,indicator/piecewise_linear,"[0.0, 0.01, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 5.0, 6.0]","[0.0, 0.025714286, 0.322857143, 0.511428571, 0.637142857, 0.74, 0.86, 0.937142857, 0.98, 1.0]",,,,
+"CoastalInundation,PluvialInundation,RiverineInundation",RealEstateAsset,"type=Buildings/Industrial,location=South America",flood_depth,metres,damage,,indicator/piecewise_linear,"[0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 5.0, 6.0]","[0.0, 0.6670194, 0.888712522, 0.946737213, 1.0, 1.0, 1.0, 1.0, 1.0]",,,,
+"CoastalInundation,PluvialInundation,RiverineInundation",RealEstateAsset,"type=Buildings/Industrial,location=Asia",flood_depth,metres,damage,,indicator/piecewise_linear,"[0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 5.0, 6.0]","[0.0, 0.283181524, 0.481615653, 0.629218894, 0.717240588, 0.85667503, 0.908577004, 0.955327463, 1.0]",,,,
+"CoastalInundation,PluvialInundation,RiverineInundation",RealEstateAsset,"type=Buildings/Industrial,location=Africa",flood_depth,metres,damage,,indicator/piecewise_linear,"[0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 5.0, 6.0]","[0.0, 0.062682043, 0.247196046, 0.403329984, 0.494488633, 0.684652389, 0.918589786, 1.0, 1.0]",,,,
+"CoastalInundation,PluvialInundation,RiverineInundation",RealEstateAsset,"type=Buildings/Industrial,location=Global",flood_depth,metres,damage,,indicator/piecewise_linear,"[0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 5.0, 6.0]","[0.0, 0.297148022, 0.479790559, 0.60328579, 0.694345844, 0.820265484, 0.922861929, 0.987065493, 1.0]",,,,

src/physrisk/data/static/vulnerability/vulnerability_config_README.md

@@ -0,0 +1,5 @@
+# Vulnerability configuration
+
+Vulnerability configuration is generated by a test: `test_generate_all_config`.
+
+This test writes to a temporary file `candidate_vulnerability_config.csv` that can be copied into `vulnerability.config.csv`.

src/physrisk/data/zarr_reader.py

@@ -1,6 +1,6 @@
 import os
 from pathlib import PurePosixPath
-from typing import Any, Callable, MutableMapping, Optional, Sequence, Union
+from typing import Callable, MutableMapping, Optional, Sequence, Union
 
 import numpy as np
 import s3fs

src/physrisk/hazard_models/hazard_model_factory.py

@@ -1,5 +1,5 @@
 from collections import defaultdict
-from typing import Dict, List, Mapping, MutableMapping, Optional
+from typing import Dict, List, Mapping, MutableMapping, Optional, Sequence
 
 from physrisk.data.hazard_data_provider import SourcePath
 from physrisk.data.pregenerated_hazard_model import ZarrHazardModel
@@ -65,7 +65,7 @@ def __init__(
         self.max_jba_requests = provider_max_requests.get("jba", 0)
         self.jba_hazard_model = (
             JBAHazardModel(cache_store, credentials, max_requests=self.max_jba_requests)
-            if not self.credentials.jba_api_disabled() and self.max_jba_requests > 0
+            if not self.credentials.jba_api_disabled() and self.max_jba_requests >= 0
             else None
         )
         self.zarr_hazard_model = ZarrHazardModel(
@@ -76,27 +76,27 @@ def __init__(
         )
 
     def hazard_model(self, type):
-        if (
-            not self.credentials.jba_api_disabled()
-            and self.max_jba_requests > 0
-            and (type == RiverineInundation or type == PluvialInundation)
+        if self.jba_hazard_model is not None and (
+            type == RiverineInundation or type == PluvialInundation
         ):
             return self.jba_hazard_model
         else:
             return self.zarr_hazard_model
 
-    def get_hazard_events(
-        self, requests: List[HazardDataRequest]
+    def get_hazard_data(
+        self, requests: Sequence[HazardDataRequest]
     ) -> Mapping[HazardDataRequest, HazardDataResponse]:
-        requests_by_model = defaultdict(list)
+        requests_by_model: Dict[HazardModel, List[HazardDataRequest]] = defaultdict(
+            list
+        )
 
         for request in requests:
             requests_by_model[self.hazard_model(request.hazard_type)].append(request)
 
         responses: Dict[HazardDataRequest, HazardDataResponse] = {}
 
         for model, reqs in requests_by_model.items():
-            events_reponses = model.get_hazard_events(reqs)
+            events_reponses = model.get_hazard_data(reqs)
             responses.update(events_reponses)
 
         return responses

src/physrisk/hazard_models/jba_hazard_model.py

@@ -85,7 +85,7 @@ def __init__(
         self.lock = Lock()
         self.max_requests = max_requests
 
-    def check_requests(self, requests: List[HazardDataRequest]):
+    def check_requests(self, requests: Sequence[HazardDataRequest]):
         if any(
             r
             for r in requests
@@ -96,8 +96,8 @@ def check_requests(self, requests: List[HazardDataRequest]):
         ):
             raise ValueError("invalid request")
 
-    def get_hazard_events(
-        self, requests: List[HazardDataRequest]
+    def get_hazard_data(
+        self, requests: Sequence[HazardDataRequest]
     ) -> Mapping[HazardDataRequest, HazardDataResponse]:
         # noqa:C90
 

src/physrisk/kernel/__init__.py

@@ -1,6 +0,0 @@
-from .assets import Asset, PowerGeneratingAsset
-from .curve import ExceedanceCurve
-from .hazard_event_distrib import HazardEventDistrib
-from .hazards import Drought, Hazard, RiverineInundation
-from .vulnerability_distrib import VulnerabilityDistrib
-from .vulnerability_model import VulnerabilityModelAcuteBase

src/physrisk/kernel/curve.py

@@ -18,7 +18,7 @@ def add_x_value_to_curve(x, curve_x, curve_y):
         curve_x = np.insert(curve_x, i, x)
     elif x == curve_x[i]:
         # point already exists; nothing to do
-        return curve_x, curve_y
+        return curve_x, curve_y, i
     elif i == 0:
         curve_y = np.insert(curve_y, 0, curve_y[0])  # flat extrapolation
         curve_x = np.insert(curve_x, 0, x)
@@ -30,7 +30,7 @@ def add_x_value_to_curve(x, curve_x, curve_y):
         curve_y = np.insert(curve_y, i, prob)
         curve_x = np.insert(curve_x, i, x)
 
-    return curve_x, curve_y
+    return curve_x, curve_y, i
 
 
 def to_exceedance_curve(bin_edges, probs):
@@ -148,7 +148,7 @@ def add_value_point(self, value):
         """Add a point to the curve with specified value and exceedance
         probability determined from existing curve by linear interpolation.
         """
-        values, probs = add_x_value_to_curve(value, self.values, self.probs)
+        values, probs, _ = add_x_value_to_curve(value, self.values, self.probs)
         return ExceedanceCurve(probs, values)
 
     def get_value(self, prob):

src/physrisk/kernel/exposure.py

@@ -3,7 +3,7 @@
 from abc import abstractmethod
 from dataclasses import dataclass
 from enum import Enum
-from typing import Dict, Iterable, List, Tuple
+from typing import Dict, List, Sequence, Tuple
 
 import numpy as np
 
@@ -55,7 +55,7 @@ class AssetExposureResult:
 class ExposureMeasure(DataRequester):
     @abstractmethod
     def get_exposures(
-        self, asset: Asset, data_responses: Iterable[HazardDataResponse]
+        self, asset: Asset, data_responses: Sequence[HazardDataResponse]
     ) -> Dict[type, Tuple[Category, float, str]]: ...
 
 
@@ -65,7 +65,7 @@ def __init__(self):
 
     def get_data_requests(
         self, asset: Asset, *, scenario: str, year: int
-    ) -> Iterable[HazardDataRequest]:
+    ) -> Sequence[HazardDataRequest]:
         return [
             HazardDataRequest(
                 hazard_type,
@@ -82,7 +82,7 @@ def get_data_requests(
             for (hazard_type, indicator_id) in self.exposure_bins.keys()
         ]
 
-    def get_exposures(self, asset: Asset, data_responses: Iterable[HazardDataResponse]):
+    def get_exposures(self, asset: Asset, data_responses: Sequence[HazardDataResponse]):
         result: Dict[type, Tuple[Category, float, str]] = {}
         for (k, v), resp in zip(self.exposure_bins.items(), data_responses):
             if isinstance(resp, HazardParameterDataResponse):
@@ -161,7 +161,7 @@ def get_exposure_bins(self):
         )
         return categories
 
-    def bounds_to_lookup(self, bounds: Iterable[Bounds]):
+    def bounds_to_lookup(self, bounds: Sequence[Bounds]):
         lower_bounds = np.array([b.lower for b in bounds])
         categories = np.array([b.category for b in bounds])
         return (lower_bounds, categories)

src/physrisk/kernel/hazard_model.py

@@ -1,7 +1,7 @@
 import sys
 from abc import ABC, abstractmethod
 from collections import defaultdict
-from typing import Dict, List, Mapping, Optional, Protocol, Tuple, Type
+from typing import Dict, Mapping, Optional, Protocol, Sequence, Tuple, Type
 
 import numpy as np
 
@@ -146,16 +146,29 @@ def hazard_model(
 
 
 class HazardModel(ABC):
-    """Hazard event model. The model accepts a set of EventDataRequests and returns the corresponding
-    EventDataResponses."""
+    """Hazard model. The model accepts a set of HazardDataRequests and returns the corresponding
+    HazardDataResponses."""
 
     @abstractmethod
-    def get_hazard_events(
-        self, requests: List[HazardDataRequest]
+    def get_hazard_data(
+        self, requests: Sequence[HazardDataRequest]
     ) -> Mapping[HazardDataRequest, HazardDataResponse]:
-        """Process the hazard data requests and return responses."""
+        """Process the hazard indicator data requests and return responses.
+
+        Args:
+            requests (Sequence[HazardDataRequest]): Hazard indicator data requests.
+
+        Returns:
+            Mapping[HazardDataRequest, HazardDataResponse]: Responses for all hazard indicator data requests.
+        """
         ...
 
+    def get_hazard_events(
+        self, requests: Sequence[HazardDataRequest]
+    ) -> Mapping[HazardDataRequest, HazardDataResponse]:
+        """Deprecated: this has been renamed to get_hazard_data."""
+        return self.get_hazard_data(requests)
+
 
 class DataSource(Protocol):
     def __call__(
@@ -169,8 +182,8 @@ class CompositeHazardModel(HazardModel):
     def __init__(self, hazard_models: Dict[type, HazardModel]):
         self.hazard_models = hazard_models
 
-    def get_hazard_events(
-        self, requests: List[HazardDataRequest]
+    def get_hazard_data(
+        self, requests: Sequence[HazardDataRequest]
     ) -> Mapping[HazardDataRequest, HazardDataResponse]:
         requests_by_event_type = defaultdict(list)
 
@@ -179,7 +192,7 @@ def get_hazard_events(
 
         responses: Dict[HazardDataRequest, HazardDataResponse] = {}
         for event_type, reqs in requests_by_event_type.items():
-            events_reponses = self.hazard_models[event_type].get_hazard_events(reqs)
+            events_reponses = self.hazard_models[event_type].get_hazard_data(reqs)
             responses.update(events_reponses)
 
         return responses