diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..68649df2 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,73 @@ +name: Build and Push Image + +on: + push: + branches: + - develop + - master + workflow_dispatch: + +jobs: + build-and-push: + strategy: + fail-fast: false + matrix: + image: [nereid, redis, celeryworker] + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Prepare + id: prep + run: | + DOCKER_IMAGE=${{ secrets.ACR_SERVER }}/nereid/${{ matrix.image }} + VERSION=edge + if [[ $GITHUB_REF == refs/tags/* ]]; then + VERSION=${GITHUB_REF#refs/tags/v} + fi + if [ "${{ github.event_name }}" = "schedule" ]; then + VERSION=nightly + fi + TAGS="${DOCKER_IMAGE}:${VERSION}" + if [[ $VERSION =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then + TAGS="$TAGS,${DOCKER_IMAGE}:latest" + fi + echo "tags: $TAGS" + echo ::set-output name=tags::${TAGS} + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + + - name: Inspect builder + run: | + echo "Name: ${{ steps.buildx.outputs.name }}" + echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}" + echo "Status: ${{ steps.buildx.outputs.status }}" + echo "Flags: ${{ steps.buildx.outputs.flags }}" + echo "Platforms: ${{ steps.buildx.outputs.platforms }}" + + - name: Login to Azure + if: github.event_name != 'pull_request' + uses: docker/login-action@v1 + with: + registry: ${{ secrets.ACR_SERVER }} + username: ${{ secrets.ACR_USERNAME }} + password: ${{ secrets.ACR_PASSWORD }} + + - name: Build and Push + id: docker_build + uses: docker/build-push-action@v2 + with: + builder: ${{ steps.buildx.outputs.name }} + context: ./nereid + file: ./nereid/Dockerfile.multi + target: ${{ matrix.image }} + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.prep.outputs.tags }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4f3b53e5..079334d4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,9 +8,9 @@ on: pull_request: workflow_dispatch: schedule: - # every other sunday at noon. - - cron: '0 12 1-7,15-21,29-31 * 0' - + # every 2 weeks. + - cron: "0 0 */14 * *" + jobs: python-test: runs-on: ubuntu-latest @@ -77,7 +77,7 @@ jobs: - name: build stack run: | docker --version - bash ./scripts/build_deploy.sh + bash ./scripts/build_dev.sh docker-compose up -d nereid-tests - name: run tests run: docker-compose exec -T nereid-tests coverage run -m pytest -xv diff --git a/docker-compose.deploy.images.yml b/docker-compose.deploy.images.yml index b08ecf7a..22fd93ab 100644 --- a/docker-compose.deploy.images.yml +++ b/docker-compose.deploy.images.yml @@ -1,12 +1,12 @@ -version: '3.7' +version: "3.7" services: nereid: - image: sitkacontainers.azurecr.io/ocstormwatertools/nereid_nereid:${NEREID_IMAGE_TAG:-latest} + image: ${ACR_SERVER}/nereid/nereid:${NEREID_IMAGE_TAG:-latest} celeryworker: - image: sitkacontainers.azurecr.io/ocstormwatertools/nereid_celeryworker:${NEREID_IMAGE_TAG:-latest} + image: ${ACR_SERVER}/nereid/celeryworker:${NEREID_IMAGE_TAG:-latest} nereid-tests: - image: sitkacontainers.azurecr.io/ocstormwatertools/nereid_nereid-tests:${NEREID_IMAGE_TAG:-latest} + image: ${ACR_SERVER}/nereid/nereid-tests:${NEREID_IMAGE_TAG:-latest} redis: - image: sitkacontainers.azurecr.io/ocstormwatertools/nereid_redis:${NEREID_IMAGE_TAG:-latest} + image: ${ACR_SERVER}/nereid/redis:${NEREID_IMAGE_TAG:-latest} flower: - image: sitkacontainers.azurecr.io/ocstormwatertools/nereid_flower:${NEREID_IMAGE_TAG:-latest} + image: ${ACR_SERVER}/nereid/flower:${NEREID_IMAGE_TAG:-latest} diff --git a/make.bat b/make.bat index ea37c6fd..7dfcb31f 100644 --- a/make.bat +++ b/make.bat @@ -12,6 +12,7 @@ if /i %1 == cover-src goto :cover-src if /i %1 == dev-server goto :dev-server if /i %1 == restart goto :restart if /i %1 == lint goto :lint +if /i %1 == login goto :login :help echo Commands: @@ -31,6 +32,10 @@ goto :eof set COMPOSE_DOCKER_CLI_BUILD=1 +:login +bash scripts/az-login.sh +goto :eof + :test call make clean call make restart @@ -59,7 +64,7 @@ goto :eof :coverage call make clean call make restart -docker compose exec nereid-tests coverage run -m pytest -x +docker compose exec nereid-tests coverage run --branch -m pytest -x docker compose exec nereid-tests coverage report -m goto :eof diff --git a/nereid/nereid/__init__.py b/nereid/nereid/__init__.py index e697a370..c61bbf49 100644 --- a/nereid/nereid/__init__.py +++ b/nereid/nereid/__init__.py @@ -1,3 +1,3 @@ -__version__ = "0.4.3" +__version__ = "0.5.0" __author__ = "Austin Orr" __email__ = "aorr@geosyntec.com" diff --git a/nereid/nereid/api/api_v1/models/treatment_facility_models.py b/nereid/nereid/api/api_v1/models/treatment_facility_models.py index ba116932..fb8181f1 100644 --- a/nereid/nereid/api/api_v1/models/treatment_facility_models.py +++ b/nereid/nereid/api/api_v1/models/treatment_facility_models.py @@ -9,9 +9,24 @@ class FacilityBase(BaseModel): node_id: str facility_type: str - ref_data_key: str - design_storm_depth_inches: float = Field(..., gt=0) - eliminate_all_dry_weather_flow_override: bool = False + ref_data_key: str = Field( + ..., + description=( + """This attribute is used to determine which nomographs +to reference in order to compute the long-term volume +capture performance of the facility.""" + ), + ) + design_storm_depth_inches: float = Field( + ..., gt=0, description="""85th percentile design storm depth in inches""" + ) + eliminate_all_dry_weather_flow_override: bool = Field( + False, + description=( + """Whether to override the dr weather flow capture calculation +and set the performance to 'fully elimates all dry weather flow'. (default=False)""" + ), + ) class NTFacility(FacilityBase): @@ -77,12 +92,25 @@ class RetentionFacility(OnlineFaciltyBase): _constructor: str = "retention_facility_constructor" +class RetentionFacilityHSG(OnlineFaciltyBase): + total_volume_cuft: float + area_sqft: float + hsg: str + _constructor: str = "retention_facility_constructor" + + class DryWellFacility(OnlineFaciltyBase): total_volume_cuft: float treatment_rate_cfs: float _constructor: str = "dry_well_facility_constructor" +class DryWellFacilityFlowOrVolume(OnlineFaciltyBase): + total_volume_cuft: float + treatment_rate_cfs: float + _constructor: str = "dry_well_facility_flow_or_volume_constructor" + + class BioInfFacility(OnlineFaciltyBase): total_volume_cuft: float retention_volume_cuft: float @@ -154,10 +182,12 @@ class PermPoolFacility(OnlineFaciltyBase): BioInfFacility, FlowAndRetFacility, RetentionFacility, + RetentionFacilityHSG, TmntFacility, TmntFacilityWithRetentionOverride, CisternFacility, DryWellFacility, + DryWellFacilityFlowOrVolume, DryWeatherTreatmentLowFlowFacility, DryWeatherDiversionLowFlowFacility, LowFlowFacility, @@ -171,10 +201,12 @@ class PermPoolFacility(OnlineFaciltyBase): BioInfFacility, FlowAndRetFacility, RetentionFacility, + RetentionFacilityHSG, TmntFacility, TmntFacilityWithRetentionOverride, CisternFacility, DryWellFacility, + DryWellFacilityFlowOrVolume, DryWeatherTreatmentLowFlowFacility, DryWeatherDiversionLowFlowFacility, LowFlowFacility, diff --git a/nereid/nereid/core/config.py b/nereid/nereid/core/config.py index 98b4344e..cb468443 100644 --- a/nereid/nereid/core/config.py +++ b/nereid/nereid/core/config.py @@ -7,6 +7,9 @@ import nereid from nereid.core.io import load_cfg +with pkg_resources.path("nereid", "__init__.py") as file: + nereid_path = file.parent + class Settings(BaseSettings): API_V1_STR: str = "/api/v1" diff --git a/nereid/nereid/core/utils.py b/nereid/nereid/core/utils.py index 4b021c9e..21a56804 100644 --- a/nereid/nereid/core/utils.py +++ b/nereid/nereid/core/utils.py @@ -112,7 +112,7 @@ class Config: except ValidationError as e: unvalidated_data["errors"] = "ERROR: " + str(e) + " \n" model = fallback - valid = model(**unvalidated_data) + valid = model.construct(**unvalidated_data) return valid diff --git a/nereid/nereid/data/default_data/state/region/config.yml b/nereid/nereid/data/default_data/state/region/config.yml index c45946f1..1c325795 100644 --- a/nereid/nereid/data/default_data/state/region/config.yml +++ b/nereid/nereid/data/default_data/state/region/config.yml @@ -121,6 +121,16 @@ api_recognize: validation_fallback: NTFacility tmnt_performance_facility_type: ¯\_(ツ)_/¯ # wq improvement via retention only + permeable_pavement: + validator: RetentionFacility + validation_fallback: NTFacility + tmnt_performance_facility_type: ¯\_(ツ)_/¯ # wq improvement via retention only + + permeable_pavement_hsg: + validator: RetentionFacilityHSG + validation_fallback: NTFacility + tmnt_performance_facility_type: ¯\_(ツ)_/¯ # wq improvement via retention only + bioretention: validator: BioInfFacility validation_fallback: NTFacility @@ -156,6 +166,11 @@ api_recognize: validation_fallback: NTFacility tmnt_performance_facility_type: ¯\_(ツ)_/¯ + dry_well_flow_or_volume: + validator: DryWellFacilityFlowOrVolume + validation_fallback: NTFacility + tmnt_performance_facility_type: ¯\_(ツ)_/¯ + cistern: validator: CisternFacility validation_fallback: NTFacility diff --git a/nereid/nereid/main.py b/nereid/nereid/main.py index cfe96335..737ac172 100644 --- a/nereid/nereid/main.py +++ b/nereid/nereid/main.py @@ -7,10 +7,10 @@ from nereid.api.api_v1.api import api_router from nereid.api.api_v1.utils import get_valid_context from nereid.core.cache import redis_cache -from nereid.core.config import settings +from nereid.core.config import nereid_path, settings app = FastAPI(title="nereid", version=nereid.__version__, docs_url=None, redoc_url=None) -app.mount("/static", StaticFiles(directory="nereid/static"), name="static") +app.mount("/static", StaticFiles(directory=str(nereid_path / "static")), name="static") @app.get("/docs", include_in_schema=False) diff --git a/nereid/nereid/src/treatment_facility/constructors.py b/nereid/nereid/src/treatment_facility/constructors.py index e16ad639..6afcebf0 100644 --- a/nereid/nereid/src/treatment_facility/constructors.py +++ b/nereid/nereid/src/treatment_facility/constructors.py @@ -82,6 +82,26 @@ def dry_well_facility_constructor( return result + @staticmethod + def dry_well_facility_flow_or_volume_constructor( + *, total_volume_cuft: float, treatment_rate_cfs: float, **kwargs: dict + ) -> Dict[str, Any]: + + retention_volume_cuft = total_volume_cuft + retention_ddt_hr = safe_divide(total_volume_cuft, treatment_rate_cfs * 3600) + + result = dict( + retention_volume_cuft=retention_volume_cuft, + retention_ddt_hr=retention_ddt_hr, + # We need to override this because dry wells don't perform treatment + # in either wet weather or dry weather, only retention/volume reduction. + # ini_treatment_rate_cfs=treatment_rate_cfs, + retention_rate_cfs=treatment_rate_cfs, + node_type="dry_well_facility", + ) + + return result + @staticmethod def bioinfiltration_facility_constructor( *, diff --git a/nereid/nereid/src/watershed/solve_watershed.py b/nereid/nereid/src/watershed/solve_watershed.py index afbf285c..8559ac48 100644 --- a/nereid/nereid/src/watershed/solve_watershed.py +++ b/nereid/nereid/src/watershed/solve_watershed.py @@ -219,7 +219,9 @@ def solve_node( ) elif "facility" in node_type: - if any([_type in node_type for _type in ["volume_based", "flow_based",]]): + if any( + [_type in node_type for _type in ["volume_based", "flow_based", "dry_well"]] + ): compute_volume_capture_with_nomograph(data, nomograph_map) compute_wet_weather_volume_discharge(data) compute_wet_weather_load_reduction( diff --git a/nereid/nereid/src/watershed/treatment_facility_capture.py b/nereid/nereid/src/watershed/treatment_facility_capture.py index 99eee345..91621f24 100644 --- a/nereid/nereid/src/watershed/treatment_facility_capture.py +++ b/nereid/nereid/src/watershed/treatment_facility_capture.py @@ -1,6 +1,7 @@ from typing import Any, Callable, Dict, List, Mapping from nereid.core.utils import safe_divide +from nereid.src.watershed.design_functions import design_intensity_inhr def compute_volume_capture_with_nomograph( @@ -17,7 +18,7 @@ def compute_volume_capture_with_nomograph( Parameters ---------- data : dict - information about the node, inclding treatment facility sizing and inflow + information about the node, including treatment facility sizing and inflow characteristics nomograph_map : mapping this mapping uses the nomograph data filepath as the key to return a 2d nomograph @@ -83,6 +84,9 @@ def compute_volume_capture_with_nomograph( elif "flow_based_facility" in node_type: data = compute_flow_based_facility(data, flow_nomo, volume_nomo) + elif "dry_well_facility" in node_type: + data = compute_dry_well_facility(data, flow_nomo, volume_nomo) + else: # pragma: no cover # this should be impossible to reach since this function is called within the # solve_watershed_loading function @@ -383,10 +387,11 @@ def compute_flow_based_facility( msg = f"overriding tributary_area_tc_min from '{tc}' to 5 minutes." data["node_warnings"].append(msg) - captured_fraction = float( - flow_nomo(intensity=data.get("design_intensity_inhr", 0.0), tc=tc) or 0.0 + intensity = data["design_intensity_inhr"] = design_intensity_inhr( + data.get("treatment_rate_cfs", 0.0), data["eff_area_acres_cumul"] ) + captured_fraction = float(flow_nomo(intensity=intensity, tc=tc) or 0.0) size_fraction = safe_divide( data.get("retention_volume_cuft", 0.0), data["design_volume_cuft_cumul"] ) @@ -408,6 +413,64 @@ def compute_flow_based_facility( return data +def compute_dry_well_facility( + data: Dict[str, Any], flow_nomo: Callable, volume_nomo: Callable +) -> Dict[str, Any]: + """best of flow-based and volume based nomographs for bmp volume and treatment rate. + + the fate of all of the treatment for a drywell is _always_ retention. + + Parameters + ---------- + data : dict + all the current node's information. this will be treatment facilily size + information and characteristics of incoming upstream flow. + *_nomo : thinly wrapped 2D CloughTocher Interpolators + Reference: `nereid.src.nomograph.nomo` + + """ + + tc = data.get("tributary_area_tc_min") + if tc is None or tc < 5: + tc = 5 + msg = f"overriding tributary_area_tc_min from '{tc}' to 5 minutes." + data["node_warnings"].append(msg) + + # check flow nomo + intensity = data["design_intensity_inhr"] = design_intensity_inhr( + data.get("retention_rate_cfs", 0.0), data["eff_area_acres_cumul"] + ) + + flow_based_captured_fraction = float(flow_nomo(intensity=intensity, tc=tc) or 0.0) + + # check volume nomo + size_fraction = safe_divide( + data.get("retention_volume_cuft", 0.0), data["design_volume_cuft_cumul"] + ) + ret_ddt_hr = data.get("retention_ddt_hr", 0.0) + + volume_based_captured_fraction = float( + volume_nomo(size=size_fraction, ddt=ret_ddt_hr) or 0.0 + ) + + # check which is best capture + solution_type = "flow based" + if volume_based_captured_fraction > flow_based_captured_fraction: + solution_type = "volume based" + + captured_pct = ( + max(flow_based_captured_fraction, volume_based_captured_fraction) * 100 + ) + + # for dry wells, all capture is retention. + data["retained_pct"] = captured_pct + data["captured_pct"] = captured_pct + data["treated_pct"] = 0.0 + data["_nomograph_solution_status"] = f"successful; dry well {solution_type}" + + return data + + def compute_peak_flow_reduction( data: Dict[str, Any], peak_nomo: Callable ) -> Dict[str, Any]: diff --git a/nereid/nereid/src/watershed/wet_weather_loading.py b/nereid/nereid/src/watershed/wet_weather_loading.py index b701e257..240e2904 100644 --- a/nereid/nereid/src/watershed/wet_weather_loading.py +++ b/nereid/nereid/src/watershed/wet_weather_loading.py @@ -121,11 +121,6 @@ def accumulate_wet_weather_loading( data.get("retention_volume_cuft", 0.0) + data["retention_volume_cuft_upstream"] ) - # calculate design intensity - data["design_intensity_inhr"] = design_intensity_inhr( - data.get("treatment_rate_cfs", 0.0), data["eff_area_acres_cumul"] - ) - # accumulate design volume data["design_volume_cuft_direct"] = design_volume_cuft( data.get("design_storm_depth_inches", 0.0), data["eff_area_acres_direct"] diff --git a/nereid/nereid/tests/test_src/test_watershed/test_solve_watershed.py b/nereid/nereid/tests/test_src/test_watershed/test_solve_watershed.py index 61ce2e26..9d641013 100644 --- a/nereid/nereid/tests/test_src/test_watershed/test_solve_watershed.py +++ b/nereid/nereid/tests/test_src/test_watershed/test_solve_watershed.py @@ -294,7 +294,28 @@ def test_solve_watershed_stable_with_subsets( "flow_nomograph": "nomographs/100_LAGUNABEACH_flow_nomo.csv", "retention_volume_cuft": 2000.0, "retention_ddt_hr": 0.22222, - "node_type": "volume_based_facility_retention_only", + "node_type": "volume_based_facility", + }, + { + "facility_type": "dry_well", + "design_storm_depth_inches": 0.85, + "is_online": True, + "tributary_area_tc_min": 0.0, # bad tc + "total_volume_cuft": 6000.0, + "treatment_rate_cfs": 0.5, + "constructor": "dry_well_facility_constructor", + "validation_fallback": "NTFacility", + "validator": "DryWellFacility", + "tmnt_performance_facility_type": "¯\\_(ツ)_/¯", + "valid_model": "DryWellFacility", + "subbasin": "10101000", + "rain_gauge": "100_LAGUNABEACH", + "et_zone": "Zone4", + "volume_nomograph": "nomographs/100_LAGUNABEACH_volume_nomo.csv", + "flow_nomograph": "nomographs/100_LAGUNABEACH_flow_nomo.csv", + "retention_volume_cuft": 4000.0, + "retention_ddt_hr": 0.22222, + "node_type": "dry_well_facility", }, { "facility_type": "dry_weather_diversion", diff --git a/nereid/requirements.txt b/nereid/requirements.txt index b15ad04e..f3fb0e1c 100644 --- a/nereid/requirements.txt +++ b/nereid/requirements.txt @@ -1,15 +1,15 @@ -scipy==1.7.0 -pandas==1.3.0 -networkx==2.5.1 +scipy==1.7.1 +pandas==1.3.2 +networkx==2.6.2 pydot==1.4.2 -graphviz==0.16 -matplotlib==3.4.2 -fastapi==0.65.3 +graphviz==0.17 +matplotlib==3.4.3 +fastapi==0.68.1 aiofiles==0.7.0 celery==5.1.2 jinja2==3.0.1 redis==3.5.3 -orjson==3.5.4 +orjson==3.6.3 pyyaml==5.4.1 pint==0.17 -python-dotenv==0.18.0 +python-dotenv==0.19.0 diff --git a/nereid/requirements_tests.txt b/nereid/requirements_tests.txt index 97156a71..d05cbd19 100644 --- a/nereid/requirements_tests.txt +++ b/nereid/requirements_tests.txt @@ -1,7 +1,7 @@ -pytest==6.2.4 +pytest==6.2.5 coverage==5.5 -codecov==2.1.11 -requests==2.25.1 +codecov==2.1.12 +requests==2.26.0 mypy==0.910 black==19.10b0 isort==5.9.1 diff --git a/scripts/az-login.sh b/scripts/az-login.sh new file mode 100644 index 00000000..d9d50ac5 --- /dev/null +++ b/scripts/az-login.sh @@ -0,0 +1,5 @@ +#! /usr/bin/env sh + +set -e +source .env +az acr login --name $AZURE_CONTAINER_REGISTRY \ No newline at end of file diff --git a/scripts/build_deploy.sh b/scripts/build_deploy.sh index 293f1d86..1b1c0cf4 100644 --- a/scripts/build_deploy.sh +++ b/scripts/build_deploy.sh @@ -1,5 +1,6 @@ set -e export COMPOSE_FILE=docker-stack.yml +export COMPOSE_DOCKER_CLI_BUILD=1 docker-compose \ -f docker-compose.shared.depends.yml \ diff --git a/setup.cfg b/setup.cfg index 785835c6..9f575e10 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,9 +19,39 @@ classifiers = [options] packages = find: package_dir = - =nereid + = nereid include_package_data = true python_requires = >= 3.6 +install_requires = + python-dotenv>=0.14,<0.19 + scipy>=1.5,<1.8 + pandas>=1.1,<1.4 + networkx>=2.5,<2.6 + pydot>=1.4,<1.5 + graphviz + matplotlib>=3.3,<3.5 + fastapi>=0.65.3,<0.66 + aiofiles>=0.6,<0.8 + celery>=5.0,<5.2 + jinja2>=2.11,<3.1 + redis>=3.5,<3.6 + orjson>=3.4,<3.6 + pyyaml>=5.4.1,<5.5 + pint>=0.16,<0.18 + +[options.packages.find] +where = nereid + +[options.package_data] +nereid = + core/* + data/* + data/default_data/* + data/default_data/state/region/* + data/default_data/state/region/nomographs/* + static/* + static/logo/* + tests/test_data/* [isort] diff --git a/setup.py b/setup.py index d7750edd..52655f12 100644 --- a/setup.py +++ b/setup.py @@ -9,25 +9,4 @@ author_email = re.search(r'__email__ = "(.*?)"', content).group(1) -setup( - version=version, - author=author, - author_email=author_email, - install_requires=[ - "python-dotenv>=0.14,<0.19", - "scipy>=1.5,<1.8", - "pandas>=1.1,<1.4", - "networkx>=2.5,<2.6", - "pydot>=1.4,<1.5", - "graphviz", - "matplotlib>=3.3,<3.5", - "fastapi>=0.65.3,<0.66", - "aiofiles>=0.6,<0.8", - "celery>=5.0,<5.2", - "jinja2>=2.11,<3.1", - "redis>=3.5,<3.6", - "orjson>=3.4,<3.6", - "pyyaml>=5.4.1,<5.5", - "pint>=0.16,<0.18", - ], -) +setup(version=version, author=author, author_email=author_email)