diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 57516073..ee2a37ef 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,7 +8,7 @@ on: pull_request: workflow_dispatch: schedule: - # every other sunday at noon. + # twice per month - cron: "0 10 1,15 * *" jobs: diff --git a/nereid/nereid/api/api_v1/async_utils.py b/nereid/nereid/api/api_v1/async_utils.py index 5ec82621..cb798394 100644 --- a/nereid/nereid/api/api_v1/async_utils.py +++ b/nereid/nereid/api/api_v1/async_utils.py @@ -1,9 +1,9 @@ -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, Union from celery import Task from celery.exceptions import TimeoutError from celery.result import AsyncResult -from fastapi import APIRouter +from fastapi import Request from nereid.core.config import settings @@ -21,33 +21,37 @@ def wait_a_sec_and_see_if_we_can_return_some_data( def run_task( + request: Request, task: Task, - router: APIRouter, - get_route: str, + get_route: str = "get_task", force_foreground: Optional[bool] = False, + timeout: float = 0.2, ) -> Dict[str, Any]: if force_foreground or settings.FORCE_FOREGROUND: # pragma: no cover - response = dict(data=task(), task_id="foreground", result_route="foreground") + task_ret: Union[bytes, str] = task() + response = { + "data": task_ret, + "task_id": "foreground", + "result_route": "foreground", + } else: - response = standard_json_response(task.apply_async(), router, get_route) + response = standard_json_response( + request, task.apply_async(), get_route=get_route, timeout=timeout + ) return response def standard_json_response( + request: Request, task: AsyncResult, - router: APIRouter, - get_route: str, + get_route: str = "get_task", timeout: float = 0.2, - api_version: str = settings.API_LATEST, ) -> Dict[str, Any]: - router_path = router.url_path_for(get_route, task_id=task.id) - - result_route = f"{api_version}{router_path}" - _ = wait_a_sec_and_see_if_we_can_return_some_data(task, timeout=timeout) + result_route = str(request.url_for(get_route, task_id=task.id)) response = dict(task_id=task.task_id, status=task.status, result_route=result_route) diff --git a/nereid/nereid/api/api_v1/endpoints_async/land_surface_loading.py b/nereid/nereid/api/api_v1/endpoints_async/land_surface_loading.py index 08950e32..9505bf96 100644 --- a/nereid/nereid/api/api_v1/endpoints_async/land_surface_loading.py +++ b/nereid/nereid/api/api_v1/endpoints_async/land_surface_loading.py @@ -1,6 +1,6 @@ from typing import Any, Dict -from fastapi import APIRouter, Body, Depends +from fastapi import APIRouter, Body, Depends, Request from fastapi.responses import ORJSONResponse import nereid.bg_worker as bg @@ -21,6 +21,7 @@ response_class=ORJSONResponse, ) async def calculate_loading( + request: Request, land_surfaces: LandSurfaces = Body(...), details: bool = False, context: dict = Depends(get_valid_context), @@ -32,9 +33,7 @@ async def calculate_loading( land_surfaces=land_surfaces_req, details=details, context=context ) - return run_task( - task=task, router=router, get_route="get_land_surface_loading_result" - ) + return run_task(request, task, "get_land_surface_loading_result") @router.get( @@ -43,6 +42,9 @@ async def calculate_loading( response_model=LandSurfaceResponse, response_class=ORJSONResponse, ) -async def get_land_surface_loading_result(task_id: str) -> Dict[str, Any]: +async def get_land_surface_loading_result( + request: Request, + task_id: str, +) -> Dict[str, Any]: task = bg.land_surface_loading.AsyncResult(task_id, app=router) - return standard_json_response(task, router, "get_land_surface_loading_result") + return standard_json_response(request, task, "get_land_surface_loading_result") diff --git a/nereid/nereid/api/api_v1/endpoints_async/network.py b/nereid/nereid/api/api_v1/endpoints_async/network.py index d9c3cb0f..98ecfdb6 100644 --- a/nereid/nereid/api/api_v1/endpoints_async/network.py +++ b/nereid/nereid/api/api_v1/endpoints_async/network.py @@ -23,11 +23,12 @@ response_class=ORJSONResponse, ) async def validate_network( - graph: network_models.Graph = Body(..., examples=network_models.GraphExamples) + request: Request, + graph: network_models.Graph = Body(..., examples=network_models.GraphExamples), ) -> Dict[str, Any]: task = bg.validate_network.s(graph=graph.dict(by_alias=True)) - return run_task(task=task, router=router, get_route="get_validate_network_result") + return run_task(request, task, "get_validate_network_result") @router.get( @@ -36,10 +37,10 @@ async def validate_network( response_model=network_models.NetworkValidationResponse, response_class=ORJSONResponse, ) -async def get_validate_network_result(task_id: str) -> Dict[str, Any]: +async def get_validate_network_result(request: Request, task_id: str) -> Dict[str, Any]: task = bg.validate_network.AsyncResult(task_id, app=router) - return standard_json_response(task, router, "get_validate_network_result") + return standard_json_response(request, task, "get_validate_network_result") @router.post( @@ -49,12 +50,13 @@ async def get_validate_network_result(task_id: str) -> Dict[str, Any]: response_class=ORJSONResponse, ) async def subgraph_network( + request: Request, subgraph_req: network_models.SubgraphRequest = Body(...), ) -> Dict[str, Any]: task = bg.network_subgraphs.s(**subgraph_req.dict(by_alias=True)) - return run_task(task=task, router=router, get_route="get_subgraph_network_result") + return run_task(request, task, "get_subgraph_network_result") @router.get( @@ -63,10 +65,10 @@ async def subgraph_network( response_model=network_models.SubgraphResponse, response_class=ORJSONResponse, ) -async def get_subgraph_network_result(task_id: str) -> Dict[str, Any]: +async def get_subgraph_network_result(request: Request, task_id: str) -> Dict[str, Any]: task = bg.network_subgraphs.AsyncResult(task_id, app=router) - return standard_json_response(task, router, "get_subgraph_network_result") + return standard_json_response(request, task, "get_subgraph_network_result") @router.get( @@ -125,6 +127,7 @@ async def get_subgraph_network_as_img( response_class=ORJSONResponse, ) async def network_solution_sequence( + request: Request, graph: network_models.Graph = Body(..., examples=network_models.GraphExamples), min_branch_size: int = Query(4), ) -> Dict[str, Any]: @@ -133,7 +136,7 @@ async def network_solution_sequence( graph=graph.dict(by_alias=True), min_branch_size=min_branch_size ) - return run_task(task=task, router=router, get_route="get_network_solution_sequence") + return run_task(request, task, "get_network_solution_sequence") @router.get( @@ -142,10 +145,12 @@ async def network_solution_sequence( response_model=network_models.SolutionSequenceResponse, response_class=ORJSONResponse, ) -async def get_network_solution_sequence(task_id: str) -> Dict[str, Any]: +async def get_network_solution_sequence( + request: Request, task_id: str +) -> Dict[str, Any]: task = bg.solution_sequence.AsyncResult(task_id, app=router) - return standard_json_response(task, router, "get_network_solution_sequence") + return standard_json_response(request, task, "get_network_solution_sequence") @router.get( diff --git a/nereid/nereid/api/api_v1/endpoints_async/reference_data.py b/nereid/nereid/api/api_v1/endpoints_async/reference_data.py index 4f64df58..64eeb2bf 100644 --- a/nereid/nereid/api/api_v1/endpoints_async/reference_data.py +++ b/nereid/nereid/api/api_v1/endpoints_async/reference_data.py @@ -1,7 +1,7 @@ import base64 from io import BytesIO from pathlib import Path -from typing import Any, Dict, Optional, Union +from typing import Any, Callable, Dict, Optional, Union from fastapi import APIRouter, Depends, HTTPException from fastapi.requests import Request @@ -45,10 +45,11 @@ async def get_reference_data_json( state, region = context["state"], context["region"] if filepath.is_file(): + filedata: Union[Dict[str, Any], str] = "" + loader: Callable[[Union[Path, str]], Union[Dict[str, Any], str]] = load_file if "json" in filepath.suffix.lower(): - filedata: Dict[str, Any] = load_json(filepath) - else: - filedata: str = load_file(filepath) # type: ignore + loader = load_json + filedata = loader(filepath) else: detail = f"state '{state}', region '{region}', or filename '{filename}' not found. {filepath}" @@ -62,12 +63,7 @@ async def get_reference_data_json( return response -@router.get( - "/reference_data/nomograph", - tags=["reference_data"], - # response_model=ReferenceDataResponse, - # response_class=ORJSONResponse, -) +@router.get("/reference_data/nomograph", tags=["reference_data"]) async def get_nomograph( request: Request, context: dict = Depends(get_valid_context), diff --git a/nereid/nereid/api/api_v1/endpoints_async/tasks.py b/nereid/nereid/api/api_v1/endpoints_async/tasks.py index 85ed580f..04923487 100644 --- a/nereid/nereid/api/api_v1/endpoints_async/tasks.py +++ b/nereid/nereid/api/api_v1/endpoints_async/tasks.py @@ -1,6 +1,6 @@ from typing import Any, Dict -from fastapi import APIRouter +from fastapi import APIRouter, Request from fastapi.responses import ORJSONResponse from nereid.api.api_v1.async_utils import standard_json_response @@ -11,6 +11,6 @@ @router.get("/{task_id}", response_model=JSONAPIResponse) -async def get_task(task_id: str) -> Dict[str, Any]: +async def get_task(request: Request, task_id: str) -> Dict[str, Any]: task = celery_app.AsyncResult(task_id) - return standard_json_response(task, router=router, get_route="get_task") + return standard_json_response(request, task) diff --git a/nereid/nereid/api/api_v1/endpoints_async/treatment_facilities.py b/nereid/nereid/api/api_v1/endpoints_async/treatment_facilities.py index 89bf1729..c9207490 100644 --- a/nereid/nereid/api/api_v1/endpoints_async/treatment_facilities.py +++ b/nereid/nereid/api/api_v1/endpoints_async/treatment_facilities.py @@ -1,6 +1,6 @@ from typing import Any, Dict, Tuple, Union -from fastapi import APIRouter, Body, Depends +from fastapi import APIRouter, Body, Depends, Request from fastapi.responses import ORJSONResponse import nereid.bg_worker as bg @@ -40,9 +40,10 @@ def validate_facility_request( response_class=ORJSONResponse, ) async def initialize_treatment_facility_parameters( + request: Request, tmnt_facility_req: Tuple[TreatmentFacilities, Dict[str, Any]] = Depends( validate_facility_request - ) + ), ) -> Dict[str, Any]: treatment_facilities, context = tmnt_facility_req @@ -52,9 +53,7 @@ async def initialize_treatment_facility_parameters( pre_validated=True, context=context, ) - return run_task( - task=task, router=router, get_route="get_treatment_facility_parameters" - ) + return run_task(request, task, "get_treatment_facility_parameters") @router.get( @@ -63,6 +62,8 @@ async def initialize_treatment_facility_parameters( response_model=TreatmentFacilitiesResponse, response_class=ORJSONResponse, ) -async def get_treatment_facility_parameters(task_id: str) -> Dict[str, Any]: +async def get_treatment_facility_parameters( + request: Request, task_id: str +) -> Dict[str, Any]: task = bg.initialize_treatment_facilities.AsyncResult(task_id, app=router) - return standard_json_response(task, router, "get_treatment_facility_parameters") + return standard_json_response(request, task, "get_treatment_facility_parameters") diff --git a/nereid/nereid/api/api_v1/endpoints_async/watershed.py b/nereid/nereid/api/api_v1/endpoints_async/watershed.py index 34d10be0..9548a7aa 100644 --- a/nereid/nereid/api/api_v1/endpoints_async/watershed.py +++ b/nereid/nereid/api/api_v1/endpoints_async/watershed.py @@ -1,6 +1,6 @@ from typing import Any, Dict, Tuple -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, Request from fastapi.responses import ORJSONResponse import nereid.bg_worker as bg @@ -39,6 +39,7 @@ def validate_watershed_request( response_class=ORJSONResponse, ) async def post_solve_watershed( + request: Request, watershed_pkg: Tuple[Dict[str, Any], Dict[str, Any]] = Depends( validate_watershed_request ), @@ -48,7 +49,7 @@ async def post_solve_watershed( task = bg.solve_watershed.s( watershed=watershed, treatment_pre_validated=True, context=context ) - return run_task(task=task, router=router, get_route="get_watershed_result") + return run_task(request, task, "get_watershed_result") @router.get( @@ -57,6 +58,6 @@ async def post_solve_watershed( response_model=WatershedResponse, response_class=ORJSONResponse, ) -async def get_watershed_result(task_id: str) -> Dict[str, Any]: +async def get_watershed_result(request: Request, task_id: str) -> Dict[str, Any]: task = bg.solve_watershed.AsyncResult(task_id, app=router) - return standard_json_response(task, router, "get_watershed_result") + return standard_json_response(request, task, "get_watershed_result") diff --git a/nereid/nereid/api/api_v1/endpoints_sync/reference_data.py b/nereid/nereid/api/api_v1/endpoints_sync/reference_data.py index 69a8d401..a3aafeab 100644 --- a/nereid/nereid/api/api_v1/endpoints_sync/reference_data.py +++ b/nereid/nereid/api/api_v1/endpoints_sync/reference_data.py @@ -1,7 +1,7 @@ import base64 from io import BytesIO from pathlib import Path -from typing import Any, Dict, Optional, Union +from typing import Any, Callable, Dict, Optional, Union from fastapi import APIRouter, Depends, HTTPException from fastapi.requests import Request @@ -17,8 +17,7 @@ @router.get("/reference_data_file", tags=["reference_data"]) async def get_reference_data_file( - context: dict = Depends(get_valid_context), - filename: str = "", + context: dict = Depends(get_valid_context), filename: str = "" ) -> FileResponse: filepath = Path(context.get("data_path", "")) / filename @@ -39,21 +38,23 @@ async def get_reference_data_file( response_class=ORJSONResponse, ) async def get_reference_data_json( - context: dict = Depends(get_valid_context), - filename: str = "", + context: dict = Depends(get_valid_context), filename: str = "" ) -> Dict[str, Any]: filepath = Path(context.get("data_path", "")) / filename state, region = context["state"], context["region"] if filepath.is_file(): + filedata: Union[Dict[str, Any], str] = "" + loader: Callable[[Union[Path, str]], Union[Dict[str, Any], str]] = load_file if "json" in filepath.suffix.lower(): - filedata: Dict[str, Any] = load_json(filepath) - else: - filedata: str = load_file(filepath) # type: ignore + loader = load_json + filedata = loader(filepath) else: - detail = f"state '{state}', region '{region}', or filename '{filename}' not found. {filepath}" + detail = ( + f"state '{state}', region '{region}', or filename '{filename}' not found." + ) raise HTTPException(status_code=400, detail=detail) response = dict( @@ -64,12 +65,7 @@ async def get_reference_data_json( return response -@router.get( - "/reference_data/nomograph", - tags=["reference_data"], - # response_model=ReferenceDataResponse, - # response_class=ORJSONResponse, -) +@router.get("/reference_data/nomograph", tags=["reference_data"]) async def get_nomograph( request: Request, context: dict = Depends(get_valid_context), diff --git a/nereid/nereid/api/api_v1/utils.py b/nereid/nereid/api/api_v1/utils.py index 5fe4dd64..267f90e8 100644 --- a/nereid/nereid/api/api_v1/utils.py +++ b/nereid/nereid/api/api_v1/utils.py @@ -1,6 +1,6 @@ from typing import Any, Dict -from fastapi import HTTPException +from fastapi import HTTPException, Request from fastapi.templating import Jinja2Templates from nereid.core.config import nereid_path @@ -9,10 +9,17 @@ templates = Jinja2Templates(directory=f"{nereid_path}/static/templates") -def get_valid_context(state: str = "state", region: str = "region") -> Dict[str, Any]: - context = get_request_context(state, region) +def get_valid_context( + request: Request, + state: str = "state", + region: str = "region", +) -> Dict[str, Any]: + """This will redirect the context data directory according to the application instantiation.""" + datadir = request.app._settings.DATA_DIRECTORY + context: Dict[str, Any] = request.app._settings.APP_CONTEXT + + context = get_request_context(state, region, datadir=datadir, context=context) isvalid, msg = validate_request_context(context) if not isvalid: raise HTTPException(status_code=400, detail=msg) - return context diff --git a/nereid/nereid/core/config.py b/nereid/nereid/core/config.py index 857e03e4..26e3dc17 100644 --- a/nereid/nereid/core/config.py +++ b/nereid/nereid/core/config.py @@ -22,6 +22,7 @@ class Settings(BaseSettings): } ) VERSION: str = nereid.__version__ + DATA_DIRECTORY: Optional[str] = None FORCE_FOREGROUND: bool = False ENABLE_ASYNC_ROUTES: bool = False diff --git a/nereid/nereid/core/context.py b/nereid/nereid/core/context.py index b0c38f6a..be621ac4 100644 --- a/nereid/nereid/core/context.py +++ b/nereid/nereid/core/context.py @@ -1,6 +1,6 @@ from copy import deepcopy from pathlib import Path -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional, Tuple, Union from nereid.core.config import settings from nereid.core.io import load_cfg @@ -50,8 +50,8 @@ def validate_request_context(context: Dict[str, Any]) -> Tuple[bool, str]: def get_request_context( state: str = "state", region: str = "region", - datadir: Optional[str] = None, - context: Optional[dict] = None, + datadir: Optional[Union[str, Path]] = None, + context: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: if context is None: @@ -64,10 +64,11 @@ def get_request_context( default_path = Path(__file__).parent.parent / "data" basepath = Path(context.get("data_path", default_path)) + p: str = context.get("project_data_directory", "") if (state == "state") or (region == "region"): - datadir = basepath / context.get("default_data_directory", "") - else: - datadir = basepath / context.get("project_data_directory", "") + p = context.get("default_data_directory", "") + + datadir = basepath / p data_path = Path(datadir) / state / region diff --git a/nereid/nereid/core/io.py b/nereid/nereid/core/io.py index 937a0a1b..12ef4dac 100644 --- a/nereid/nereid/core/io.py +++ b/nereid/nereid/core/io.py @@ -3,6 +3,7 @@ from pathlib import Path from typing import Any, Dict, List, Tuple, Union +import numpy import orjson as json import pandas import yaml @@ -45,14 +46,14 @@ def load_json(filepath: PathType) -> Dict[str, Any]: return contents -def load_ref_data(tablename: str, context: dict) -> pandas.DataFrame: +def load_ref_data(tablename: str, context: dict) -> Tuple[pandas.DataFrame, List[str]]: data_path = Path(context["data_path"]) table_context = context.get("project_reference_data", {}).get(tablename, {}) filepath = data_path / table_context["file"] - ref_table = pandas.read_json(load_file(filepath), orient="table") + ref_table = pandas.read_json(load_file(filepath), orient="table", typ="frame") df, msg = parse_configuration_logic( df=ref_table, @@ -81,7 +82,7 @@ def parse_expand_fields( for ix, col in enumerate(cols): df[col] = df[field].str.split(sep).str[ix] - except: + except Exception: messages.append( f"unable to expand fields in {config_section}:{config_object} " f"for instructions {f}" @@ -105,10 +106,12 @@ def parse_collapse_fields( sep = f.get("sep", "_") cols = f.get("fields", []) - df[field] = df[cols[0]].astype(str) - for col in cols[1:]: - df[field] = df[field] + sep + df[col].astype(str) - except: + if field is not None: # pragma: no branch + df[field] = df[cols[0]].astype(str) + for col in cols[1:]: + df[field] = df[field] + sep + df[col].astype(str) + + except Exception: messages.append( f"unable to expand fields in {config_section}:{config_object} " f"for instructions {f}" @@ -210,7 +213,7 @@ def parse_remaps( how = remap["how"] mapping = remap["mapping"] - fillna = remap.get("fillna", pandas.NA) + fillna = remap.get("fillna", numpy.nan) try: if how == "addend": @@ -238,7 +241,7 @@ def parse_remaps( f"in {config_section}:{config_object}." ) - except: + except Exception: messages.append( f"ERROR: unable to apply mapping '{remap}' in " f"{config_section}:{config_object}." diff --git a/nereid/nereid/core/utils.py b/nereid/nereid/core/utils.py index 23eb32bc..b6d3c403 100644 --- a/nereid/nereid/core/utils.py +++ b/nereid/nereid/core/utils.py @@ -1,7 +1,8 @@ import importlib.resources as pkg_resources -from typing import Any, Dict +from typing import Any, Dict, Union import numpy +import pandas from pydantic import BaseModel, ValidationError @@ -50,7 +51,9 @@ def safe_divide(x: float, y: float) -> float: return x / y -def safe_array_divide(x: numpy.ndarray, y: numpy.ndarray) -> numpy.ndarray: +def safe_array_divide( + x: Union[numpy.ndarray, pandas.Series], y: Union[numpy.ndarray, pandas.Series] +) -> numpy.ndarray: return numpy.divide(x, y, out=numpy.zeros_like(x), where=y != 0) diff --git a/nereid/nereid/factory.py b/nereid/nereid/factory.py index c8239d20..cf8b10a0 100644 --- a/nereid/nereid/factory.py +++ b/nereid/nereid/factory.py @@ -1,6 +1,6 @@ from typing import Any, Dict, Optional -from fastapi import FastAPI +from fastapi import Depends, FastAPI from fastapi.middleware.cors import CORSMiddleware from fastapi.openapi.docs import get_redoc_html, get_swagger_ui_html from fastapi.staticfiles import StaticFiles @@ -10,13 +10,29 @@ from nereid.core.config import nereid_path, settings -def create_app(**settings_override: Optional[Dict[str, Any]]) -> FastAPI: +def create_app( + *, + settings_override: Optional[Dict[str, Any]] = None, + app_kwargs: Optional[Dict[str, Any]] = None +) -> FastAPI: _settings = settings.copy(deep=True) - _settings.update(settings_override) + if settings_override is not None: # pragma: no branch + _settings.update(settings_override) + + kwargs = {} + if app_kwargs is not None: # pragma: no cover + kwargs = app_kwargs + + _docs_url: Optional[str] = kwargs.pop("docs_url", None) + _redoc_url: Optional[str] = kwargs.pop("redoc_url", None) app = FastAPI( - title="nereid", version=_settings.VERSION, docs_url=None, redoc_url=None + title="nereid", + version=_settings.VERSION, + docs_url=_docs_url, + redoc_url=_redoc_url, + **kwargs ) setattr(app, "_settings", _settings) @@ -47,29 +63,32 @@ def create_app(**settings_override: Optional[Dict[str, Any]]) -> FastAPI: allow_headers=["*"], ) - @app.get("/docs", include_in_schema=False) - async def custom_swagger_ui_html(): - return get_swagger_ui_html( - openapi_url=str(app.openapi_url), - title=app.title + " - Swagger UI", - oauth2_redirect_url=app.swagger_ui_oauth2_redirect_url, - swagger_js_url="/static/swagger-ui-bundle.js", - swagger_css_url="/static/swagger-ui.css", - swagger_favicon_url="/static/logo/trident_neptune_logo.ico", - ) - - @app.get("/redoc", include_in_schema=False) - async def redoc_html(): - return get_redoc_html( - openapi_url=str(app.openapi_url), - title=app.title + " - ReDoc", - redoc_js_url="/static/redoc.standalone.js", - redoc_favicon_url="/static/logo/trident_neptune_logo.ico", - ) + if app.docs_url is None: # pragma: no branch + + @app.get("/docs", include_in_schema=False) + async def custom_swagger_ui_html(): + return get_swagger_ui_html( + openapi_url=str(app.openapi_url), + title=app.title + " - Swagger UI", + oauth2_redirect_url=app.swagger_ui_oauth2_redirect_url, + swagger_js_url="/static/swagger-ui-bundle.js", + swagger_css_url="/static/swagger-ui.css", + swagger_favicon_url="/static/logo/trident_neptune_logo.ico", + ) + + if app.redoc_url is None: # pragma: no branch + + @app.get("/redoc", include_in_schema=False) + async def redoc_html(): + return get_redoc_html( + openapi_url=str(app.openapi_url), + title=app.title + " - ReDoc", + redoc_js_url="/static/redoc.standalone.js", + redoc_favicon_url="/static/logo/trident_neptune_logo.ico", + ) @app.get("/config") - async def check_config(state="state", region="region"): - context = get_valid_context(state, region) + async def check_config(context=Depends(get_valid_context)): return context return app diff --git a/nereid/nereid/src/land_surface/loading.py b/nereid/nereid/src/land_surface/loading.py index 560e25ec..a202fbc5 100644 --- a/nereid/nereid/src/land_surface/loading.py +++ b/nereid/nereid/src/land_surface/loading.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Optional +from typing import Dict, Iterable, Optional import pandas @@ -54,7 +54,7 @@ def detailed_volume_loading_results(df: pandas.DataFrame) -> pandas.DataFrame: def detailed_dry_weather_volume_loading_results( - df: pandas.DataFrame, seasons: Dict[str, Optional[List[str]]] + df: pandas.DataFrame, seasons: Dict[str, Optional[Iterable[str]]] ) -> pandas.DataFrame: """This function aggregates the dry weather flowrate (dwf) by season according to the config file spec. @@ -83,9 +83,9 @@ def detailed_dry_weather_volume_loading_results( def detailed_pollutant_loading_results( df: pandas.DataFrame, - wet_weather_parameters: List[Dict[str, str]], - dry_weather_parameters: List[Dict[str, str]], - season_names: List[str], + wet_weather_parameters: Iterable[Dict[str, str]], + dry_weather_parameters: Iterable[Dict[str, str]], + season_names: Iterable[str], ) -> pandas.DataFrame: """convert the washoff concentration to load for both wet and dry seasons. @@ -125,9 +125,9 @@ def detailed_pollutant_loading_results( def detailed_loading_results( land_surfaces_df: pandas.DataFrame, - wet_weather_parameters: List[Dict[str, str]], - dry_weather_parameters: List[Dict[str, str]], - seasons: Dict[str, List[str]], + wet_weather_parameters: Iterable[Dict[str, str]], + dry_weather_parameters: Iterable[Dict[str, str]], + seasons: Dict[str, Iterable[str]], ) -> pandas.DataFrame: results = ( @@ -147,9 +147,9 @@ def detailed_loading_results( def summary_loading_results( detailed_results: pandas.DataFrame, - wet_weather_parameters: List[Dict[str, str]], - dry_weather_parameters: List[Dict[str, str]], - season_names: List[str], + wet_weather_parameters: Iterable[Dict[str, str]], + dry_weather_parameters: Iterable[Dict[str, str]], + season_names: Iterable[str], ) -> pandas.DataFrame: groupby_cols = ["node_id"] @@ -188,7 +188,7 @@ def summary_loading_results( } df = ( - detailed_results.reindex(columns=groupby_cols + output_columns_summable) + detailed_results.loc[:, groupby_cols + output_columns_summable] .groupby(groupby_cols) .agg(agg_dict) ) @@ -209,8 +209,7 @@ def summary_loading_results( factor: float = float(param["load_to_conc_factor"]) df[conc_col] = ( - safe_array_divide(df[load_col].values, df["runoff_volume_cuft"].values) - * factor + safe_array_divide(df[load_col], df["runoff_volume_cuft"]) * factor ) for season in season_names: @@ -222,8 +221,6 @@ def summary_loading_results( load_col = season + "_" + param["load_col"] factor = float(param["load_to_conc_factor"]) - df[conc_col] = ( - safe_array_divide(df[load_col].values, df[dw_vol_col].values) * factor - ) + df[conc_col] = safe_array_divide(df[load_col], df[dw_vol_col]) * factor return df diff --git a/nereid/nereid/src/network/algorithms.py b/nereid/nereid/src/network/algorithms.py index 3e17b548..d5c56830 100644 --- a/nereid/nereid/src/network/algorithms.py +++ b/nereid/nereid/src/network/algorithms.py @@ -79,6 +79,7 @@ def find_leafy_branch_larger_than_size(g: nx.DiGraph, size: int = 1) -> nx.DiGra us.add(node) if len(us) >= size: return g.subgraph(us) + return nx.Digraph() # pragma: no cover def sequential_subgraph_nodes(g: nx.DiGraph, size: int) -> List[List[Union[str, int]]]: diff --git a/nereid/nereid/src/network/render.py b/nereid/nereid/src/network/render.py index 89f6926c..f12d9f7b 100644 --- a/nereid/nereid/src/network/render.py +++ b/nereid/nereid/src/network/render.py @@ -1,11 +1,11 @@ from functools import lru_cache from io import BytesIO from itertools import cycle -from typing import IO, Dict, List, Optional, Tuple, Union +from typing import IO, Any, Dict, List, Optional, Tuple, Union -import matplotlib as mpl import networkx as nx import orjson as json +from matplotlib import axes, cm, figure from matplotlib import pyplot as plt @@ -14,9 +14,11 @@ def _cached_layout( edge_json: str, prog: str ) -> Dict[Union[str, int], Tuple[float, float]]: g = nx.from_edgelist(json.loads(edge_json), create_using=nx.MultiDiGraph) - layout: Dict[Union[str, int], Tuple[float, float]] = nx.nx_pydot.pydot_layout( - g, prog=prog - ) + layout: Optional[ + Dict[Union[str, int], Tuple[float, float]] + ] = nx.nx_pydot.pydot_layout(g, prog=prog) + if layout is None: # pragma: no cover + layout = {} return layout @@ -29,7 +31,7 @@ def cached_layout( def get_figure_width_height_from_graph_layout( - layout_dict: Dict[Union[int, str], Tuple[float, float]], + layout_dict: Dict[Union[str, int], Tuple[float, float]], npi: Optional[float] = None, min_width: float = 1.0, min_height: float = 1.0, @@ -61,15 +63,15 @@ def get_figure_width_height_from_graph_layout( def render_subgraphs( - g: nx.DiGraph, + g: nx.Graph, request_nodes: list, subgraph_nodes: list, layout: Optional[Dict[Union[str, int], Tuple[float, float]]] = None, npi: Optional[float] = None, - node_size: float = 200, - ax: Optional[mpl.axes.Axes] = None, + node_size: int = 200, + ax: Optional[axes.Axes] = None, fig_kwargs: Optional[Dict] = None, -) -> mpl.figure.Figure: +) -> figure.Figure: if fig_kwargs is None: # pragma: no branch fig_kwargs = {} @@ -135,7 +137,7 @@ def render_subgraphs( def render_solution_sequence( G: nx.DiGraph, solution_sequence: List[List[List]], - ax: Optional[mpl.axes.Axes] = None, + ax: Optional[axes.Axes] = None, layout: Optional[Dict[Union[str, int], Tuple[float, float]]] = None, npi: Optional[float] = None, min_marker_size: float = 40.0, @@ -144,7 +146,7 @@ def render_solution_sequence( marker_cycle_str: str = "^ovs>", nx_draw_kwargs: Optional[Dict] = None, fig_kwargs: Optional[Dict] = None, -) -> mpl.figure.Figure: +) -> figure.Figure: if layout is None: # pragma: no branch layout = cached_layout(G, prog="dot") if fig_kwargs is None: # pragma: no branch @@ -160,7 +162,7 @@ def render_solution_sequence( fig, ax = plt.subplots(**fig_kwargs) marker_cycle = cycle(marker_cycle_str) - cmap = mpl.cm.get_cmap(cmap_str) + cmap = cm.get_cmap(cmap_str) for k, series_graphs in enumerate(solution_sequence): node_shape = next(marker_cycle) @@ -201,7 +203,7 @@ def render_solution_sequence( return ax.get_figure() -def fig_to_image(fig: mpl.figure.Figure, **kwargs: dict) -> IO: +def fig_to_image(fig: figure.Figure, **kwargs: Any) -> IO: _kwargs = dict(bbox_inches="tight", format="svg", dpi=300) _kwargs.update(kwargs) diff --git a/nereid/nereid/src/network/tasks.py b/nereid/nereid/src/network/tasks.py index 8f415279..b7d6dd77 100644 --- a/nereid/nereid/src/network/tasks.py +++ b/nereid/nereid/src/network/tasks.py @@ -1,4 +1,4 @@ -from typing import IO, Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Union import networkx as nx @@ -29,9 +29,9 @@ def validate_network(graph: Dict) -> Dict[str, Union[bool, List]]: """ _graph = thin_graph_dict(graph) - G = graph_factory(_graph) + g = graph_factory(_graph) - isvalid = validate.is_valid(G) + isvalid = validate.is_valid(g) result: Dict[str, Union[bool, List]] = {"isvalid": isvalid} @@ -40,7 +40,7 @@ def validate_network(graph: Dict) -> Dict[str, Union[bool, List]]: else: _keys = ["node_cycles", "edge_cycles", "multiple_out_edges", "duplicate_edges"] - for key, value in zip(_keys, validate.validate_network(G)): + for key, value in zip(_keys, validate.validate_network(g)): result[key] = value return result @@ -54,9 +54,9 @@ def network_subgraphs( node_ids = [node["id"] for node in nodes] - G = graph_factory(_graph) - subset = get_subset(G, node_ids) - sub_g = G.subgraph(subset) + g = nx.DiGraph(graph_factory(_graph)) + subset = get_subset(g, node_ids) + sub_g = g.subgraph(subset) subgraph_nodes = [ {"nodes": [{"id": n} for n in nodes]} @@ -93,9 +93,9 @@ def solution_sequence( _graph = thin_graph_dict(graph) # strip unneeded metadata - G = graph_factory(_graph) + g = nx.DiGraph(graph_factory(_graph)) - _sequence = parallel_sequential_subgraph_nodes(G, min_branch_size) + _sequence = parallel_sequential_subgraph_nodes(g, min_branch_size) sequence = { "parallel": [ @@ -117,7 +117,7 @@ def render_solution_sequence_svg( _graph = thin_graph_dict(task_result["graph"]) # strip unneeded metadata - g = graph_factory(_graph) + g = nx.DiGraph(graph_factory(_graph)) _sequence = task_result["solution_sequence"] diff --git a/nereid/nereid/src/network/utils.py b/nereid/nereid/src/network/utils.py index 2a57791c..239938bd 100644 --- a/nereid/nereid/src/network/utils.py +++ b/nereid/nereid/src/network/utils.py @@ -1,10 +1,12 @@ import copy -from typing import Any, Collection, Dict +from typing import Any, Collection, Dict, Union import networkx as nx +GraphType = Union[nx.MultiGraph, nx.Graph, nx.MultiDiGraph, nx.DiGraph] -def graph_factory(graph: Dict[str, Any]) -> nx.Graph: + +def graph_factory(graph: Dict[str, Any]) -> GraphType: """ Parameters ---------- @@ -28,41 +30,47 @@ def graph_factory(graph: Dict[str, Any]) -> nx.Graph: # multi graphs so we can identify which edges are duplicated. if is_multigraph: if is_directed: - g = nx.MultiDiGraph() + cls = nx.MultiDiGraph else: - g = nx.MultiGraph() + cls = nx.MultiGraph # this is the most tolerant type of graph elif is_directed: - g = nx.DiGraph() + cls = nx.DiGraph else: - g = nx.Graph() # for testing purposes + cls = nx.Graph # for testing purposes if edges: - if g.is_multigraph(): - g = nx.from_edgelist( - [ - ( - d.get("source"), - d.get("target"), - d.get("key", None), - d.get("metadata", {}), - ) - for d in edges - ], - create_using=g, + if is_multigraph: + g = cls( + nx.from_edgelist( + [ + ( + d.get("source"), + d.get("target"), + d.get("key", None), + d.get("metadata", {}), + ) + for d in edges + ], + create_using=cls(), + ) ) else: - g = nx.from_edgelist( - [ - (d.get("source"), d.get("target"), d.get("metadata", {})) - for d in edges - ], - create_using=g, + g = cls( + nx.from_edgelist( + [ + (d.get("source"), d.get("target"), d.get("metadata", {})) + for d in edges + ], + create_using=cls(), + ) ) + else: + g = cls() if nodes: g.add_nodes_from([(n.get("id"), n.get("metadata", {})) for n in nodes]) @@ -91,7 +99,7 @@ def thin_graph_dict(graph_dict: Dict[str, Any]) -> Dict[str, Any]: return result -def nxGraph_to_dict(g: nx.Graph) -> Dict[str, Any]: +def nxGraph_to_dict(g: GraphType) -> Dict[str, Any]: """Convert a networkx garph object into a dictionary suitable for serialization. @@ -139,7 +147,7 @@ def nxGraph_to_dict(g: nx.Graph) -> Dict[str, Any]: return result -def clean_graph_dict(g: nx.Graph) -> Dict[str, Any]: +def clean_graph_dict(g: GraphType) -> Dict[str, Any]: """ Converts a graph to a dictionary, ensuring all node labels are converted to strings @@ -147,7 +155,7 @@ def clean_graph_dict(g: nx.Graph) -> Dict[str, Any]: return nxGraph_to_dict(nx.relabel_nodes(g, lambda x: str(x))) -def sum_node_attr(g: nx.Graph, nodes: Collection, attr: str) -> float: +def sum_node_attr(g: GraphType, nodes: Collection, attr: str) -> float: """Returns sum of one attribute for node's upstream nodes Parameters diff --git a/nereid/nereid/src/network/validate.py b/nereid/nereid/src/network/validate.py index 812a4e8f..21242982 100644 --- a/nereid/nereid/src/network/validate.py +++ b/nereid/nereid/src/network/validate.py @@ -5,31 +5,38 @@ import networkx as nx from nereid.src.network.algorithms import find_cycle +from nereid.src.network.utils import GraphType def validate_network( - G: nx.Graph, **kwargs: dict -) -> Tuple[List[List[str]], List[List[str]], List[List[str]], List[List[str]]]: + G: GraphType, **kwargs: dict +) -> Tuple[List[List], List[List[str]], List[List[str]], List[List[str]]]: """Checks if there is a cycle, and prints a helpful message if there is. """ _partial_sort: Callable = partial(sorted, key=lambda x: str(x)) # force cycles to be ordered so that we can test against them - node_cycles: List[List[str]] = list(map(_partial_sort, nx.simple_cycles(G))) + node_cycles: List[List] = list(map(_partial_sort, nx.simple_cycles(G))) edge_cycles = [list(map(str, _)) for _ in find_cycle(G, **kwargs)] - multiple_outs = [[str(k), str(v)] for k, v in G.out_degree() if v > 1] + multiple_outs = [ + [str(node), str(deg)] + for node, deg in nx.MultiDiGraph(G).out_degree() + if deg > 1 + ] - duplicate_edges: List[List[str]] = [] + duplicate_edges: List[List] = [] if len(G.edges()) != len(set(G.edges())): - duplicate_edges = [[s, t] for s, t, k in G.edges(keys=True) if k > 0] + duplicate_edges = [ + [s, t] for s, t, *k in nx.MultiGraph(G).edges(keys=True) if k[0] + ] return node_cycles, edge_cycles, multiple_outs, duplicate_edges -def is_valid(G: nx.Graph) -> bool: +def is_valid(G: GraphType) -> bool: try: # catch cycles deque(nx.topological_sort(G), maxlen=0) @@ -38,11 +45,11 @@ def is_valid(G: nx.Graph) -> bool: try: # catch multiple out connections - assert all((v <= 1 for k, v in G.out_degree())) + assert all((v <= 1 for k, v in nx.DiGraph(G).out_degree())) # catch assert len(G.edges()) == len(set(G.edges())) - except: + except Exception: return False return True diff --git a/nereid/nereid/src/nomograph/interpolators.py b/nereid/nereid/src/nomograph/interpolators.py index cd1ff4be..db8a0663 100644 --- a/nereid/nereid/src/nomograph/interpolators.py +++ b/nereid/nereid/src/nomograph/interpolators.py @@ -87,7 +87,7 @@ def __init__( x: Union[Sequence[float], pandas.Series, numpy.ndarray], t: Union[Sequence[float], pandas.Series, numpy.ndarray], y: Union[Sequence[float], pandas.Series, numpy.ndarray], - interp_kwargs: Dict[str, Any] = None, + interp_kwargs: Optional[Dict[str, Any]] = None, ) -> None: """This class manages 2D interpolations of stormwater treatment facility performance across the size, drawdown time, and long term capture. diff --git a/nereid/nereid/src/tmnt_performance/tmnt.py b/nereid/nereid/src/tmnt_performance/tmnt.py index 1535c171..b3868229 100644 --- a/nereid/nereid/src/tmnt_performance/tmnt.py +++ b/nereid/nereid/src/tmnt_performance/tmnt.py @@ -1,5 +1,5 @@ from functools import partial -from typing import Callable, Dict, Mapping, Optional, Tuple +from typing import Any, Callable, Dict, Mapping, Optional import numpy import pandas @@ -71,7 +71,7 @@ def effluent_conc( def build_effluent_function_map( df: pandas.DataFrame, facility_column: str, pollutant_column: str -) -> Mapping[Tuple[str, str], Callable]: +) -> Mapping[Any, Callable]: # this is close to what we want, but it has a lot of nans. _facility_dict = df.set_index([facility_column, pollutant_column]).to_dict("index") diff --git a/nereid/nereid/src/watershed/solve_watershed.py b/nereid/nereid/src/watershed/solve_watershed.py index 776d0218..f5ebab58 100644 --- a/nereid/nereid/src/watershed/solve_watershed.py +++ b/nereid/nereid/src/watershed/solve_watershed.py @@ -4,7 +4,7 @@ from nereid.core.utils import dictlist_to_dict from nereid.src.land_surface.tasks import land_surface_loading -from nereid.src.network.utils import graph_factory +from nereid.src.network.utils import GraphType, graph_factory from nereid.src.network.validate import is_valid, validate_network from nereid.src.nomograph.nomo import load_nomograph_mapping from nereid.src.tmnt_performance.tasks import effluent_function_map @@ -32,7 +32,7 @@ def initialize_graph( watershed: Dict[str, Any], treatment_pre_validated: bool, context: Dict[str, Any], -) -> Tuple[nx.DiGraph, List[str]]: +) -> Tuple[GraphType, List[str]]: errors: List[str] = [] diff --git a/nereid/nereid/src/watershed/wet_weather_loading.py b/nereid/nereid/src/watershed/wet_weather_loading.py index 16e0a6dc..852124aa 100644 --- a/nereid/nereid/src/watershed/wet_weather_loading.py +++ b/nereid/nereid/src/watershed/wet_weather_loading.py @@ -4,10 +4,7 @@ from nereid.core.utils import safe_divide from nereid.src.network.utils import sum_node_attr -from nereid.src.watershed.design_functions import ( - design_intensity_inhr, - design_volume_cuft, -) +from nereid.src.watershed.design_functions import design_volume_cuft from nereid.src.watershed.loading import compute_pollutant_load_reduction @@ -130,8 +127,11 @@ def accumulate_wet_weather_loading( g, predecessors, "design_volume_cuft_cumul" ) - data["design_volume_cuft_cumul"] = ( - data["design_volume_cuft_direct"] + data["design_volume_cuft_upstream"] + # land surface nodes don't have a design depth, so we have to recalc this + # if there is one set for this node, and it has to include the whole effective + # area upstream. Patched 2022-03-14. + data["design_volume_cuft_cumul"] = design_volume_cuft( + data.get("design_storm_depth_inches", 0.0), data["eff_area_acres_cumul"] ) # during storm detention doesn't exist for the 'current' node diff --git a/nereid/nereid/tests/__init__.py b/nereid/nereid/tests/__init__.py index dae366cb..41fc0f66 100644 --- a/nereid/nereid/tests/__init__.py +++ b/nereid/nereid/tests/__init__.py @@ -1,4 +1,4 @@ -from nereid.core.config import settings +from nereid.core.config import nereid_path def test(*args): # pragma: no cover @@ -7,6 +7,6 @@ def test(*args): # pragma: no cover except ImportError: raise ImportError("`pytest` is required to run the test suite") - options = [str(settings._nereid_path)] + options = [str(nereid_path)] options.extend(list(args)) return pytest.main(options) diff --git a/nereid/nereid/tests/test_api/conftest.py b/nereid/nereid/tests/test_api/conftest.py index 42fb623d..3ee9d5f7 100644 --- a/nereid/nereid/tests/test_api/conftest.py +++ b/nereid/nereid/tests/test_api/conftest.py @@ -17,7 +17,7 @@ def client(async_mode): mode = "none" if async_mode: mode = "replace" - app = create_app(ASYNC_MODE=mode) + app = create_app(settings_override=dict(ASYNC_MODE=mode)) with TestClient(app) as client: yield client diff --git a/nereid/nereid/tests/test_api/test_land_surface_loading.py b/nereid/nereid/tests/test_api/test_land_surface_loading.py index 2b1b1128..8f9ea3d9 100644 --- a/nereid/nereid/tests/test_api/test_land_surface_loading.py +++ b/nereid/nereid/tests/test_api/test_land_surface_loading.py @@ -1,3 +1,5 @@ +import json + import pytest from nereid.api.api_v1.models import land_surface_models @@ -36,17 +38,21 @@ def test_get_land_surface_loading( result_route = prjson["result_route"] get_response = client.get(result_route) - assert get_response.status_code == 200 + assert get_response.status_code == 200, get_response.content grjson = get_response.json() - assert land_surface_models.LandSurfaceResponse(**prjson) - assert grjson["task_id"] == prjson["task_id"] - assert grjson["result_route"] == prjson["result_route"] - assert grjson["status"].lower() != "failure" - - if grjson["status"].lower() == "success": # pragma: no branch - assert len(grjson["data"]["summary"]) <= n_nodes - if details == "true": - assert len(grjson["data"]["details"]) == n_rows - else: - assert grjson["data"]["details"] is None + try: + assert land_surface_models.LandSurfaceResponse(**prjson) + assert grjson["task_id"] == prjson["task_id"] + assert grjson["result_route"] == prjson["result_route"] + assert grjson["status"].lower() != "failure" + + if grjson["status"].lower() == "success": # pragma: no branch + assert len(grjson["data"]["summary"]) <= n_nodes + if details == "true": + assert len(grjson["data"]["details"]) == n_rows + else: + assert grjson["data"]["details"] is None + except AssertionError: # pragma: no cover + print(json.dumps(grjson, indent=2)) + raise diff --git a/nereid/nereid/tests/test_api/test_reference_data.py b/nereid/nereid/tests/test_api/test_reference_data.py index 506fbd09..21c162e0 100644 --- a/nereid/nereid/tests/test_api/test_reference_data.py +++ b/nereid/nereid/tests/test_api/test_reference_data.py @@ -1,7 +1,10 @@ +import matplotlib import pytest from nereid.core.config import settings +matplotlib.use("agg") + @pytest.mark.parametrize( "query, isvalid", diff --git a/nereid/nereid/tests/test_api/test_utils.py b/nereid/nereid/tests/test_api/test_utils.py deleted file mode 100644 index 6e2b5b8f..00000000 --- a/nereid/nereid/tests/test_api/test_utils.py +++ /dev/null @@ -1,28 +0,0 @@ -import pytest -from fastapi import HTTPException - -# import nereid.bg_worker as bg -from nereid.api.api_v1 import utils - - -@pytest.mark.parametrize( - "state, region, raises, exp", - [("state", "region", False, {"test": True}), ("wa", "sea", True, {})], -) -def test_get_valid_context(state, region, raises, exp): - if raises: - pytest.raises(HTTPException, utils.get_valid_context, state, region) - else: - req_context = utils.get_valid_context(state, region) - assert all([req_context[k] == v for k, v in exp.items()]) - - -# def test_run_task_by_name(subgraph_request_dict): - -# graph, nodes = subgraph_request_dict["graph"], subgraph_request_dict["nodes"] - -# task = bg.background_network_subgraphs.s(graph=graph, nodes=nodes) - -# return utils.run_task( -# task=task, router=r"¯\_(ツ)_/¯", get_route=r"¯\_(ツ)_/¯", force_foreground=True -# ) diff --git a/nereid/nereid/tests/test_core/test_io.py b/nereid/nereid/tests/test_core/test_io.py index 93df6630..3e6070e3 100644 --- a/nereid/nereid/tests/test_core/test_io.py +++ b/nereid/nereid/tests/test_core/test_io.py @@ -54,8 +54,9 @@ def test_io_load_multiple_cfgs(): def test_load_ref_data(contexts, table, key): context = contexts[key] - ref_table = io.load_ref_data(table, context) - assert len(ref_table) > 1 + ref_table, msg = io.load_ref_data(table, context) + assert len(ref_table) > 1, ref_table + assert all(["error" not in m.lower() for m in msg]), msg @pytest.mark.parametrize("n_rows", [10]) diff --git a/nereid/nereid/tests/test_src/test_land_surface/test_loading.py b/nereid/nereid/tests/test_src/test_land_surface/test_loading.py index 519099ea..a4cc6e2a 100644 --- a/nereid/nereid/tests/test_src/test_land_surface/test_loading.py +++ b/nereid/nereid/tests/test_src/test_land_surface/test_loading.py @@ -139,7 +139,7 @@ def test_detailed_land_surface_pollutant_loading_results( size = 4 runoff_volume_cuft = numpy.random.randint(0, 4, size) - FC_conc = numpy.random.randint(0, 1e5, size) # mpn/100ml + FC_conc = numpy.random.randint(0, 100_000, size) # mpn/100ml TCu_conc = numpy.random.randint(0, 1000, size) # ug/l TSS_conc = numpy.random.randint(0, 1000, size) # mg/l diff --git a/nereid/nereid/tests/test_src/test_network/conftest.py b/nereid/nereid/tests/test_src/test_network/conftest.py index 0395a9b9..5fc8dfac 100644 --- a/nereid/nereid/tests/test_src/test_network/conftest.py +++ b/nereid/nereid/tests/test_src/test_network/conftest.py @@ -1,6 +1,7 @@ import copy import itertools import string +from typing import Any, Dict, List import networkx as nx import pytest @@ -81,9 +82,14 @@ def _construct_graph_dicts(): for dct in g1["edges"]: dct["metadata"]["key"] = 0 - g2 = copy.deepcopy(g1) - - g2["nodes"] = [{"id": "A"}, {"id": "B"}, {"id": "C"}, {"id": "D"}] + g2: Dict[str, Any] = copy.deepcopy(g1) + _nodes: List[Dict[str, Any]] = [ + {"id": "A"}, + {"id": "B"}, + {"id": "C"}, + {"id": "D"}, + ] + g2["nodes"] = _nodes for i, (dct, l) in enumerate(zip(g2["nodes"], string.ascii_lowercase)): dct["metadata"] = {} diff --git a/nereid/nereid/tests/test_src/test_watershed/test_tasks.py b/nereid/nereid/tests/test_src/test_watershed/test_tasks.py index 977d08d6..60597a2b 100644 --- a/nereid/nereid/tests/test_src/test_watershed/test_tasks.py +++ b/nereid/nereid/tests/test_src/test_watershed/test_tasks.py @@ -106,6 +106,18 @@ def test_solve_watershed_with_treatment( # check that treatment happened assert outfall_results[load_type] > 0 + nested_bmps = [ + data + for data in response_dict["results"] + if (data["eff_area_acres_direct"] < data["eff_area_acres_cumul"]) + and "facility" in data.get("node_type", "") + ] + + for data in nested_bmps: + assert ( + data["design_volume_cuft_direct"] < data["design_volume_cuft_cumul"] + ), data.get("node_type", "") + def test_stable_watershed_stable_subgraph_solutions( contexts, watershed_requests, watershed_test_case @@ -129,7 +141,7 @@ def test_stable_watershed_stable_subgraph_solutions( ] } - g = graph_factory(watershed_request["graph"]) + g = nx.DiGraph(graph_factory(watershed_request["graph"])) # this subgraph is empty, has no data. subg = nx.DiGraph(g.subgraph(get_subset(g, nodes=dirty_nodes)).edges) @@ -148,6 +160,18 @@ def test_stable_watershed_stable_subgraph_solutions( check_subgraph_response_equal(subgraph_results, results) + nested_bmps = [ + data + for data in subgraph_results + if (data["eff_area_acres_direct"] < data["eff_area_acres_cumul"]) + and "facility" in data.get("node_type", "") + ] + + for data in nested_bmps: + assert ( + data["design_volume_cuft_direct"] < data["design_volume_cuft_cumul"] + ), data.get("node_type", "") + @pytest.mark.parametrize( "facility_type, eliminates_wet_volume, treats_wet_volume, captures_dwf, mitigates_peak_flow", diff --git a/nereid/requirements.txt b/nereid/requirements.txt index 158f5788..3780b6df 100644 --- a/nereid/requirements.txt +++ b/nereid/requirements.txt @@ -1,10 +1,10 @@ aiofiles==0.8.0 celery==5.2.3 -fastapi==0.74.1 +fastapi==0.75.0 graphviz==0.19.1 jinja2==3.0.3 matplotlib==3.5.1 -networkx==2.6.3 +networkx==2.7.1 orjson==3.6.7 pandas==1.4.1 pint==0.18 diff --git a/nereid/requirements_tests.txt b/nereid/requirements_tests.txt index 9f17505f..8a52abab 100644 --- a/nereid/requirements_tests.txt +++ b/nereid/requirements_tests.txt @@ -2,6 +2,6 @@ black==22.1.0 codecov==2.1.12 coverage==6.3.2 isort==5.10.1 -mypy==0.931 -pytest==7.0.1 +mypy==0.941 +pytest==7.1.0 requests==2.27.1 \ No newline at end of file diff --git a/setup.py b/setup.py index 52655f12..045b6dfe 100644 --- a/setup.py +++ b/setup.py @@ -2,11 +2,19 @@ from setuptools import setup + +def search(substr: str, content: str): + found = re.search(substr, content) + if found: + return found.group(1) + return "" + + with open("nereid/nereid/__init__.py", encoding="utf8") as f: content = f.read() - version = re.search(r'__version__ = "(.*?)"', content).group(1) - author = re.search(r'__author__ = "(.*?)"', content).group(1) - author_email = re.search(r'__email__ = "(.*?)"', content).group(1) + version = search(r'__version__ = "(.*?)"', content) + author = search(r'__author__ = "(.*?)"', content) + author_email = search(r'__email__ = "(.*?)"', content) setup(version=version, author=author, author_email=author_email)