From 23df0c898a009296cfac9e2c2ad141c6b476b03f Mon Sep 17 00:00:00 2001 From: Austin Orr Date: Thu, 7 Apr 2022 14:01:09 -0700 Subject: [PATCH 1/2] tolerant task failure and auto calculating imp & perv ro depths if missing --- .../api/api_v1/models/land_surface_models.py | 2 +- nereid/nereid/src/land_surface/loading.py | 7 ++ nereid/nereid/src/land_surface/tasks.py | 103 ++++++++++-------- nereid/nereid/src/treatment_facility/tasks.py | 36 +++--- nereid/nereid/src/treatment_site/tasks.py | 103 +++++++++--------- .../test_land_surface/test_loading.py | 18 ++- 6 files changed, 154 insertions(+), 115 deletions(-) diff --git a/nereid/nereid/api/api_v1/models/land_surface_models.py b/nereid/nereid/api/api_v1/models/land_surface_models.py index d3a58ecf..d1b6f53a 100644 --- a/nereid/nereid/api/api_v1/models/land_surface_models.py +++ b/nereid/nereid/api/api_v1/models/land_surface_models.py @@ -102,7 +102,7 @@ class LandSurfaceDetails(LandSurfaceBase): class LandSurfaceResults(BaseModel): - summary: List[LandSurfaceSummary] + summary: Optional[List[LandSurfaceSummary]] = None details: Optional[List[LandSurfaceDetails]] = None errors: Optional[List[str]] = None diff --git a/nereid/nereid/src/land_surface/loading.py b/nereid/nereid/src/land_surface/loading.py index a202fbc5..1a89e42e 100644 --- a/nereid/nereid/src/land_surface/loading.py +++ b/nereid/nereid/src/land_surface/loading.py @@ -32,6 +32,13 @@ def detailed_volume_loading_results(df: pandas.DataFrame) -> pandas.DataFrame: df = df.loc[df["area_acres"] > 0] + if "precip_depth_inches" in df: + if "imp_ro_depth_inches" not in df: # pragma: no branch + df["imp_ro_depth_inches"] = df["precip_depth_inches"] * df["imp_ro_coeff"] + + if "perv_ro_depth_inches" not in df: # pragma: no branch + df["perv_ro_depth_inches"] = df["precip_depth_inches"] * df["perv_ro_coeff"] + # method chaining with 'df.assign' looks better, but it's much less memory efficient df["imp_pct"] = 100 * df["imp_area_acres"] / df["area_acres"] df["perv_area_acres"] = df["area_acres"] - df["imp_area_acres"] diff --git a/nereid/nereid/src/land_surface/tasks.py b/nereid/nereid/src/land_surface/tasks.py index eb4d8a94..799a0bde 100644 --- a/nereid/nereid/src/land_surface/tasks.py +++ b/nereid/nereid/src/land_surface/tasks.py @@ -22,56 +22,63 @@ def land_surface_loading( aggregate the load to each node_id, and are always returned. """ - response: Dict[str, Any] = {} - response["errors"] = [] + response: Dict[str, Any] = {"errors": []} land_surface_list = land_surfaces.get("land_surfaces") or [] - if land_surface_list: # pragma: no branch - df = pandas.DataFrame(land_surface_list) - df["imp_pct"] = 100 * df["imp_area_acres"] / df["area_acres"] - - df, messages = parse_configuration_logic( - df=df, - config_section="api_recognize", - config_object="land_surfaces", - context=context, - ) - - # TODO: add validator function to ensure config & request are complete. - - if len(messages) > 0: - response["errors"].extend(messages) - - wet_weather_parameters = init_wq_parameters("land_surface_emc_table", context) - dry_weather_parameters = init_wq_parameters( - "dry_weather_land_surface_emc_table", context - ) - - seasons = ( - context.get("project_reference_data", {}) - .get("dry_weather_flow_table", {}) - .get("seasons", {}) - ) - - detailed_results = detailed_loading_results( - df, - wet_weather_parameters, - dry_weather_parameters, - seasons, - ) - summary_results = summary_loading_results( - detailed_results, - wet_weather_parameters, - dry_weather_parameters, - season_names=seasons.keys(), - ) - - response["summary"] = summary_results.fillna(0).to_dict(orient="records") - - if details: - response["details"] = detailed_results.fillna(0).to_dict(orient="records") - else: # pragma: no cover - response["errors"].append("ERROR: no land surface input data provided.") + try: + if land_surface_list: # pragma: no branch + df = pandas.DataFrame(land_surface_list) + df["imp_pct"] = 100 * df["imp_area_acres"] / df["area_acres"] + + df, messages = parse_configuration_logic( + df=df, + config_section="api_recognize", + config_object="land_surfaces", + context=context, + ) + + # TODO: add validator function to ensure config & request are complete. + + if len(messages) > 0: + response["errors"].extend(messages) + + wet_weather_parameters = init_wq_parameters( + "land_surface_emc_table", context + ) + dry_weather_parameters = init_wq_parameters( + "dry_weather_land_surface_emc_table", context + ) + + seasons = ( + context.get("project_reference_data", {}) + .get("dry_weather_flow_table", {}) + .get("seasons", {}) + ) + + detailed_results = detailed_loading_results( + df, + wet_weather_parameters, + dry_weather_parameters, + seasons, + ) + summary_results = summary_loading_results( + detailed_results, + wet_weather_parameters, + dry_weather_parameters, + season_names=seasons.keys(), + ) + + response["summary"] = summary_results.fillna(0).to_dict(orient="records") + + if details: + response["details"] = detailed_results.fillna(0).to_dict( + orient="records" + ) + else: # pragma: no cover + response["errors"].append("ERROR: no land surface input data provided.") + + except Exception as e: # pragma: no cover + response["errors"].append(str(e)) return response diff --git a/nereid/nereid/src/treatment_facility/tasks.py b/nereid/nereid/src/treatment_facility/tasks.py index 4e1e3402..ea4fd29c 100644 --- a/nereid/nereid/src/treatment_facility/tasks.py +++ b/nereid/nereid/src/treatment_facility/tasks.py @@ -17,26 +17,30 @@ def initialize_treatment_facilities( context: Dict[str, Any], ) -> Dict[str, Any]: - treatment_facility_list = treatment_facilities.get("treatment_facilities") or [] - if not pre_validated: - treatment_facility_list = validate_treatment_facility_models( - treatment_facility_list, context - ) + response: Dict[str, Any] = {"errors": []} - df, messages = parse_configuration_logic( - df=pandas.DataFrame(treatment_facility_list), - config_section="api_recognize", - config_object="treatment_facility", - context=context, - ) + try: + treatment_facility_list = treatment_facilities.get("treatment_facilities") or [] + if not pre_validated: + treatment_facility_list = validate_treatment_facility_models( + treatment_facility_list, context + ) + + df, messages = parse_configuration_logic( + df=pandas.DataFrame(treatment_facility_list), + config_section="api_recognize", + config_object="treatment_facility", + context=context, + ) - treatment_facility_nodes = build_treatment_facility_nodes(df=df) + treatment_facility_nodes = build_treatment_facility_nodes(df=df) - response: Dict[str, Any] = {"errors": []} + if len(messages) > 0: + response["errors"] = messages - if len(messages) > 0: - response["errors"] = messages + response["treatment_facilities"] = treatment_facility_nodes - response["treatment_facilities"] = treatment_facility_nodes + except Exception as e: # pragma: no cover + response["errors"].append(str(e)) return response diff --git a/nereid/nereid/src/treatment_site/tasks.py b/nereid/nereid/src/treatment_site/tasks.py index 097c95f2..cc5ac57f 100644 --- a/nereid/nereid/src/treatment_site/tasks.py +++ b/nereid/nereid/src/treatment_site/tasks.py @@ -9,58 +9,63 @@ def initialize_treatment_sites( response: Dict[str, Any] = {"errors": []} - sites = treatment_sites.get("treatment_sites") or [] - - # tmnt_map is connects the facility name with the treatment - # key for the influent-> effluent concentration transformation - # e.g. {"bioretention": "Biofiltration"} - tmnt_map = { - k: dct["tmnt_performance_facility_type"] - for k, dct in ( - context["api_recognize"] - .get("treatment_facility", {}) - .get("facility_type", {}) - .items() - ) - } - - tmnt_sites = [] - - if sites: - - _df = pandas.DataFrame(sites) - - remainder_data = [] - for node, g in _df.groupby("node_id"): - remainder = 100 - g["area_pct"].sum() - remainder_data.append( - { - "node_id": node, - "area_pct": remainder, - "facility_type": "no_treatment", - "eliminate_all_dry_weather_flow_override": False, - } + try: + + sites = treatment_sites.get("treatment_sites") or [] + + # tmnt_map is connects the facility name with the treatment + # key for the influent-> effluent concentration transformation + # e.g. {"bioretention": "Biofiltration"} + tmnt_map = { + k: dct["tmnt_performance_facility_type"] + for k, dct in ( + context["api_recognize"] + .get("treatment_facility", {}) + .get("facility_type", {}) + .items() ) + } + + tmnt_sites = [] + + if sites: + + _df = pandas.DataFrame(sites) - df = ( - pandas.concat([_df, pandas.DataFrame(remainder_data)]) - .fillna(0) - .assign( - tmnt_performance_facility_type=lambda df: df["facility_type"].replace( - tmnt_map + remainder_data = [] + for node, g in _df.groupby("node_id"): + remainder = 100 - g["area_pct"].sum() + remainder_data.append( + { + "node_id": node, + "area_pct": remainder, + "facility_type": "no_treatment", + "eliminate_all_dry_weather_flow_override": False, + } + ) + + df = ( + pandas.concat([_df, pandas.DataFrame(remainder_data)]) + .fillna(0) + .assign( + tmnt_performance_facility_type=lambda df: df[ + "facility_type" + ].replace(tmnt_map) ) ) - ) - - tmnt_sites = [ - { - "node_id": key, - "treatment_facilities": g.to_dict("records"), - "node_type": "site_based", - } - for key, g in df.groupby("node_id") - ] - - response["treatment_sites"] = tmnt_sites + + tmnt_sites = [ + { + "node_id": key, + "treatment_facilities": g.to_dict("records"), + "node_type": "site_based", + } + for key, g in df.groupby("node_id") + ] + + response["treatment_sites"] = tmnt_sites + + except Exception as e: # pragma: no cover + response["errors"].append(str(e)) return response diff --git a/nereid/nereid/tests/test_src/test_land_surface/test_loading.py b/nereid/nereid/tests/test_src/test_land_surface/test_loading.py index a4cc6e2a..1b120cc9 100644 --- a/nereid/nereid/tests/test_src/test_land_surface/test_loading.py +++ b/nereid/nereid/tests/test_src/test_land_surface/test_loading.py @@ -77,7 +77,23 @@ def test_detailed_land_surface_loading_results( dry_weather_parameters, seasons, ) - assert t["area_acres"].sum() == land_surfaces_df["area_acres"].sum() + + land_surfaces_df_reduced = land_surfaces_df.drop( + columns=["imp_ro_depth_inches", "perv_ro_depth_inches"] + ).assign(precip_depth_inches=12) + + t_precip = detailed_loading_results( + land_surfaces_df_reduced, + wet_weather_parameters, + dry_weather_parameters, + seasons, + ) + + assert ( + t["area_acres"].sum() + == land_surfaces_df["area_acres"].sum() + == t_precip["area_acres"].sum() + ) assert len(t) == len(land_surfaces_list) if not "no_joins" in key and not "no_params" in key: assert any(["conc" in c for c in t.columns]) From ce8f266959ff99c4af99ece67aac3fa0ca4482cd Mon Sep 17 00:00:00 2001 From: Austin Orr Date: Thu, 7 Apr 2022 14:29:27 -0700 Subject: [PATCH 2/2] bump dependency versions --- nereid/requirements.txt | 14 +++++++------- nereid/requirements_tests.txt | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/nereid/requirements.txt b/nereid/requirements.txt index 3780b6df..4e44aa3e 100644 --- a/nereid/requirements.txt +++ b/nereid/requirements.txt @@ -1,15 +1,15 @@ aiofiles==0.8.0 -celery==5.2.3 -fastapi==0.75.0 +celery==5.2.6 +fastapi==0.75.1 graphviz==0.19.1 -jinja2==3.0.3 +jinja2==3.1.1 matplotlib==3.5.1 networkx==2.7.1 orjson==3.6.7 -pandas==1.4.1 -pint==0.18 +pandas==1.4.2 +pint==0.19.1 pydot==1.4.2 -python-dotenv==0.19.2 +python-dotenv==0.20.0 pyyaml==6.0 -redis==4.1.4 +redis==4.2.2 scipy==1.8.0 \ No newline at end of file diff --git a/nereid/requirements_tests.txt b/nereid/requirements_tests.txt index 8a52abab..709d382f 100644 --- a/nereid/requirements_tests.txt +++ b/nereid/requirements_tests.txt @@ -1,7 +1,7 @@ -black==22.1.0 +black==22.3.0 codecov==2.1.12 coverage==6.3.2 isort==5.10.1 -mypy==0.941 -pytest==7.1.0 +mypy==0.942 +pytest==7.1.1 requests==2.27.1 \ No newline at end of file