diff --git a/.dockerignore b/.dockerignore index 3eeac5f5..6c33d2f4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,4 +2,4 @@ **/.pytest_cache **/.mypy_cache **/.coverage -**/.env +**/.env \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..b0c15221 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,76 @@ +name: Build and Push Image + +on: + push: + branches: + - develop + - master + workflow_dispatch: + +jobs: + build-and-push: + strategy: + fail-fast: false + matrix: + targets: [nereid, redis, celeryworker] + runs-on: ubuntu-latest + env: + DOCKER_BUILDKIT: 1 + COMPOSE_DOCKER_CLI_BUILD: 1 + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Prepare + id: prep + run: | + DOCKER_IMAGE=${{ secrets.ACR_SERVER }}/nereid/${{ matrix.image }} + VERSION=edge + if [[ $GITHUB_REF == refs/tags/* ]]; then + VERSION=${GITHUB_REF#refs/tags/v} + fi + if [ "${{ github.event_name }}" = "schedule" ]; then + VERSION=nightly + fi + TAGS="${DOCKER_IMAGE}:${VERSION}" + if [[ $VERSION =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then + TAGS="$TAGS,${DOCKER_IMAGE}:latest" + fi + echo "tags: $TAGS" + echo ::set-output name=tags::${TAGS} + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + + - name: Inspect builder + run: | + echo "Name: ${{ steps.buildx.outputs.name }}" + echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}" + echo "Status: ${{ steps.buildx.outputs.status }}" + echo "Flags: ${{ steps.buildx.outputs.flags }}" + echo "Platforms: ${{ steps.buildx.outputs.platforms }}" + + - name: Login to Azure + if: github.event_name != 'pull_request' + uses: docker/login-action@v1 + with: + registry: ${{ secrets.ACR_SERVER }} + username: ${{ secrets.ACR_USERNAME }} + password: ${{ secrets.ACR_PASSWORD }} + + - name: Build and Push + id: docker_build + uses: docker/build-push-action@v2 + with: + builder: ${{ steps.buildx.outputs.name }} + context: ./nereid + file: ./nereid/Dockerfile.multi + target: ${{ matrix.image }} + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.prep.outputs.tags }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4f3b53e5..b51fa259 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,8 +9,8 @@ on: workflow_dispatch: schedule: # every other sunday at noon. - - cron: '0 12 1-7,15-21,29-31 * 0' - + - cron: "0 12 1-7,15-21,29-31 * 0" + jobs: python-test: runs-on: ubuntu-latest @@ -20,7 +20,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.9 - name: Install Deps run: pip install -r nereid/requirements.txt -r nereid/requirements_tests.txt - name: Run Tests @@ -36,11 +36,11 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.9 - name: Install nereid as library run: | - pip install . - pip install -r nereid/requirements_tests.txt + pip install . + pip install pytest - name: Run Tests run: pytest nereid/nereid/tests/test_src -xv @@ -64,7 +64,7 @@ jobs: - name: Run Edge Tests continue-on-error: true run: | - pytest nereid/nereid/tests/test_src -xv + pytest nereid/nereid/tests -xv - name: Exit run: exit 0 @@ -77,10 +77,12 @@ jobs: - name: build stack run: | docker --version - bash ./scripts/build_deploy.sh + bash ./scripts/build_dev.sh docker-compose up -d nereid-tests - name: run tests - run: docker-compose exec -T nereid-tests coverage run -m pytest -xv + run: | + docker compose exec -T nereid-tests coverage run --branch -m pytest nereid/tests -xs + docker compose exec -T nereid-tests coverage run -a --branch -m pytest nereid/tests/test_api -xs --async - name: coverage run: | docker-compose exec -T nereid-tests coverage report -mi diff --git a/docker-compose.deploy.images.yml b/docker-compose.deploy.images.yml index b08ecf7a..22fd93ab 100644 --- a/docker-compose.deploy.images.yml +++ b/docker-compose.deploy.images.yml @@ -1,12 +1,12 @@ -version: '3.7' +version: "3.7" services: nereid: - image: sitkacontainers.azurecr.io/ocstormwatertools/nereid_nereid:${NEREID_IMAGE_TAG:-latest} + image: ${ACR_SERVER}/nereid/nereid:${NEREID_IMAGE_TAG:-latest} celeryworker: - image: sitkacontainers.azurecr.io/ocstormwatertools/nereid_celeryworker:${NEREID_IMAGE_TAG:-latest} + image: ${ACR_SERVER}/nereid/celeryworker:${NEREID_IMAGE_TAG:-latest} nereid-tests: - image: sitkacontainers.azurecr.io/ocstormwatertools/nereid_nereid-tests:${NEREID_IMAGE_TAG:-latest} + image: ${ACR_SERVER}/nereid/nereid-tests:${NEREID_IMAGE_TAG:-latest} redis: - image: sitkacontainers.azurecr.io/ocstormwatertools/nereid_redis:${NEREID_IMAGE_TAG:-latest} + image: ${ACR_SERVER}/nereid/redis:${NEREID_IMAGE_TAG:-latest} flower: - image: sitkacontainers.azurecr.io/ocstormwatertools/nereid_flower:${NEREID_IMAGE_TAG:-latest} + image: ${ACR_SERVER}/nereid/flower:${NEREID_IMAGE_TAG:-latest} diff --git a/docker-compose.shared.env.yml b/docker-compose.shared.env.yml index b80590cf..1d3f8e15 100644 --- a/docker-compose.shared.env.yml +++ b/docker-compose.shared.env.yml @@ -1,6 +1,8 @@ version: '3.7' services: nereid: + environment: + - NEREID_ASYNC_MODE=replace env_file: - env.env nereid-tests: diff --git a/docker-compose.shared.ports.yml b/docker-compose.shared.ports.yml index f65abe14..d244e5b9 100644 --- a/docker-compose.shared.ports.yml +++ b/docker-compose.shared.ports.yml @@ -1,8 +1,8 @@ version: '3.7' services: redis: - ports: - - 6379:6379 + expose: + - 6379 nereid: ports: - '80:80' diff --git a/env.env b/env.env index 152c62b8..c1ba64d7 100644 --- a/env.env +++ b/env.env @@ -1,4 +1,2 @@ -__CELERY_BROKER_URL=redis://guest@redis:6379/0 -__CELERY_RESULT_BACKEND=redis://guest@redis:6379/0 CELERY_BROKER_URL=redis://redis:6379/0 -CELERY_RESULT_BACKEND=redis://redis:6379/0 +CELERY_RESULT_BACKEND=redis://redis:6379/0 \ No newline at end of file diff --git a/make.bat b/make.bat index ea37c6fd..c5a7567b 100644 --- a/make.bat +++ b/make.bat @@ -8,10 +8,14 @@ if /i %1 == up goto :up if /i %1 == down goto :down if /i %1 == typecheck goto :typecheck if /i %1 == coverage goto :coverage +if /i %1 == coverage-sync goto :coverage-sync +if /i %1 == coverage-async goto :coverage-async +if /i %1 == coverage-all goto :coverage-all if /i %1 == cover-src goto :cover-src if /i %1 == dev-server goto :dev-server if /i %1 == restart goto :restart if /i %1 == lint goto :lint +if /i %1 == login goto :login :help echo Commands: @@ -31,10 +35,15 @@ goto :eof set COMPOSE_DOCKER_CLI_BUILD=1 +:login +bash scripts/az-login.sh +goto :eof + :test call make clean call make restart -docker compose exec nereid-tests pytest %2 %3 %4 %5 %6 +for /f "tokens=1,* delims= " %%a in ("%*") do set ALL_BUT_FIRST=%%b +docker compose exec nereid-tests pytest %ALL_BUT_FIRST% goto :eof :typecheck @@ -59,7 +68,29 @@ goto :eof :coverage call make clean call make restart -docker compose exec nereid-tests coverage run -m pytest -x +docker compose exec nereid-tests coverage run --branch -m pytest nereid/tests -xs +docker compose exec nereid-tests coverage report -m +goto :eof + +:coverage-sync +call make clean +call make restart +docker compose exec nereid-tests coverage run --branch -m pytest nereid/tests -xs +docker compose exec nereid-tests coverage report -m --omit=*test*,*bg_*,*celery_* +goto :eof + +:coverage-async +call make clean +call make restart +docker compose exec nereid-tests coverage run --branch -m pytest nereid/tests -xs --async +docker compose exec nereid-tests coverage report -m --omit=*test*,*_sync* +goto :eof + +:coverage-all +call make clean +call make restart +docker compose exec nereid-tests coverage run --branch -m pytest nereid/tests -xs +docker compose exec nereid-tests coverage run -a --branch -m pytest nereid/tests/test_api -xs --async docker compose exec nereid-tests coverage report -m goto :eof @@ -85,3 +116,8 @@ goto :eof :lint bash scripts/lint.sh goto :eof + +rem docker build --target nereid_install -t nereid_install -f nereid/Dockerfile.multi . +rem docker run -it -v %cd%:/nereid -p 8088:80 --expose 80 nereid_install bash +rem pip install -e .[app] +rem uvicorn nereid.main:app --port 80 --host 0.0.0.0 --reload \ No newline at end of file diff --git a/nereid/.coveragerc b/nereid/.coveragerc index a1fd6e1c..a4009a00 100644 --- a/nereid/.coveragerc +++ b/nereid/.coveragerc @@ -6,4 +6,4 @@ branch = True [report] include = nereid/*.py ignore_errors = True -omit = nereid/data/* +omit = nereid/data/*,*main.py diff --git a/nereid/Dockerfile.multi b/nereid/Dockerfile.multi index e3ecf50d..6cca7b0b 100644 --- a/nereid/Dockerfile.multi +++ b/nereid/Dockerfile.multi @@ -5,82 +5,111 @@ # time to build and is considerably more complex for scipy and pandas. -FROM redis:6.0.8-alpine3.12 as redis +FROM python:3.9-bullseye as nereid_install +RUN apt-get update -y \ + && apt-get install -y --no-install-recommends graphviz \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /nereid +CMD ["bash", "-c", "while true; do sleep 1; done"] + + +FROM redis:6.2.6-alpine3.15 as redis COPY redis.conf /redis.conf CMD ["redis-server", "/redis.conf"] -FROM python:3.8.6-alpine3.12 as flower -RUN apk add --no-cache ca-certificates && update-ca-certificates -RUN pip install --no-cache-dir redis==3.5.3 flower==0.9.5 celery==4.4.7 +FROM python:3.9-alpine3.15 as flower +RUN apk add --no-cache ca-certificates tzdata && update-ca-certificates +RUN pip install --no-cache-dir redis==4.1.0 flower==1.0.0 celery==5.2.3 ENV PYTHONUNBUFFERED=1 PYTHONHASHSEED=random PYTHONDONTWRITEBYTECODE=1 +ENV FLOWER_DATA_DIR=/data +ENV PYTHONPATH=${FLOWER_DATA_DIR} +WORKDIR $FLOWER_DATA_DIR +# Add a user with an explicit UID/GID and create necessary directories +ENV IMG_USER=flower +RUN set -eux; \ + addgroup -g 1000 ${IMG_USER}; \ + adduser -u 1000 -G ${IMG_USER} ${IMG_USER} -D; \ + mkdir -p "$FLOWER_DATA_DIR"; \ + chown ${IMG_USER}:${IMG_USER} "$FLOWER_DATA_DIR" +USER ${IMG_USER} +VOLUME $FLOWER_DATA_DIR +# Default port EXPOSE 5555 -USER nobody -ENTRYPOINT ["flower"] +CMD ["celery", "flower"] -FROM python:3.8.6-buster as builder +FROM python:3.9.9-bullseye as builder COPY requirements.txt /requirements.txt COPY requirements_tests.txt /requirements_tests.txt COPY requirements_server.txt /requirements_server.txt RUN mkdir /core && \ - pip wheel \ - --wheel-dir=/core \ - -r /requirements.txt + pip wheel \ + --wheel-dir=/core \ + -r /requirements.txt RUN mkdir /tsts && \ - pip wheel \ - --wheel-dir=/tsts \ - -r /requirements_tests.txt + pip wheel \ + --wheel-dir=/tsts \ + -r /requirements_tests.txt RUN mkdir /serve && \ - pip wheel \ - --wheel-dir=/serve \ - -r /requirements_server.txt + pip wheel \ + --wheel-dir=/serve \ + -r /requirements_server.txt -FROM python:3.8.6-slim-buster as core-runtime +FROM python:3.9.9-slim-bullseye as core-runtime RUN apt-get update -y \ - && apt-get install -y --no-install-recommends graphviz \ - && rm -rf /var/lib/apt/lists/* + && apt-get install -y --no-install-recommends graphviz \ + && rm -rf /var/lib/apt/lists/* WORKDIR /nereid ENV PYTHONPATH=/nereid -ENV PATH=/root/.local/bin:$PATH +ENV PATH=/opt/venv/bin:$PATH -FROM python:3.8.6-slim-buster as core-env +FROM python:3.9.9-slim-bullseye as core-env COPY --from=builder /core /core COPY requirements.txt /requirements.txt +RUN python -m venv /opt/venv +# Make sure we use the virtualenv: +ENV PATH=/opt/venv/bin:$PATH RUN pip install \ - --user \ - --no-index \ - --no-cache-dir \ - --find-links=/core \ - -r /requirements.txt \ - && rm -rf /core/* + --no-index \ + --no-cache-dir \ + --find-links=/core \ + -r /requirements.txt \ + && rm -rf /core/* FROM core-runtime as celeryworker -ENV C_FORCE_ROOT=1 -COPY --from=core-env /root/.local /root/.local -COPY ./scripts/run-worker.sh /run-worker.sh -RUN chmod +x /run-worker.sh +# Add a user with an explicit UID/GID and create necessary directories +ENV IMG_USER=celeryworker +RUN addgroup --gid 1000 ${IMG_USER} \ + && adduser --system --disabled-password --uid 1000 --gid 1000 ${IMG_USER} \ + && chown -R ${IMG_USER}:${IMG_USER} /nereid +USER ${IMG_USER} +COPY --from=core-env --chown=${IMG_USER} /opt/venv /opt/venv +COPY --chown=${IMG_USER} ./scripts/run-worker.sh /run-worker.sh +RUN chmod gu+x /run-worker.sh CMD ["bash", "/run-worker.sh"] -COPY ./nereid /nereid/nereid +COPY --chown=${IMG_USER} ./nereid /nereid/nereid FROM core-env as server-env COPY requirements_server.txt /requirements_server.txt COPY --from=builder /serve /serve +RUN python -m venv /opt/venv +# Make sure we use the virtualenv: +ENV PATH=/opt/venv/bin:$PATH RUN pip install \ - --user \ - --no-index \ - --no-cache-dir \ - --find-links=/serve \ - -r /requirements_server.txt \ - && rm -rf /serve/* + --no-index \ + --no-cache-dir \ + --find-links=/serve \ + -r /requirements_server.txt \ + && rm -rf /serve/* FROM core-runtime as nereid -COPY --from=server-env /root/.local /root/.local +COPY --from=server-env /opt/venv /opt/venv COPY gunicorn_conf.py /gunicorn_conf.py COPY ./scripts/start.sh /start.sh RUN chmod +x /start.sh @@ -94,30 +123,33 @@ COPY ./nereid /nereid/nereid FROM core-env as test-env COPY requirements_tests.txt /requirements_tests.txt COPY --from=builder /tsts /tsts +RUN python -m venv /opt/venv +# Make sure we use the virtualenv: +ENV PATH=/opt/venv/bin:$PATH RUN pip install \ - --user \ - --no-index \ - --no-cache-dir \ - --find-links=/tsts \ - -r /requirements_tests.txt \ - && rm -rf /tsts/* + --no-index \ + --no-cache-dir \ + --find-links=/tsts \ + -r /requirements_tests.txt \ + && rm -rf /tsts/* FROM core-runtime as nereid-tests -COPY --from=test-env /root/.local /root/.local +COPY --from=test-env /opt/venv /opt/venv COPY .coveragerc /nereid/.coveragerc +COPY conftest.py /nereid/conftest.py ## This will make the container wait, doing nothing, but alive CMD ["bash", "-c", "while true; do sleep 1; done"] EXPOSE 8888 COPY ./nereid /nereid/nereid -FROM python:3.8-buster as nereid-edge +FROM python:3.9-bullseye as nereid-edge RUN apt-get update && apt-get install -y graphviz COPY requirements.txt /requirements.txt COPY requirements_tests.txt /requirements_tests.txt RUN awk -F"==" '{print $1}' /requirements.txt /requirements_tests.txt \ - > /requirements_edge.txt + > /requirements_edge.txt RUN cat requirements_edge.txt RUN pip install -r /requirements_edge.txt COPY ./nereid /nereid/nereid diff --git a/nereid/conftest.py b/nereid/conftest.py new file mode 100644 index 00000000..9e82f199 --- /dev/null +++ b/nereid/conftest.py @@ -0,0 +1,2 @@ +def pytest_addoption(parser): + parser.addoption("--async", action="store_true", default=False) diff --git a/nereid/nereid/__init__.py b/nereid/nereid/__init__.py index e697a370..eefefe9b 100644 --- a/nereid/nereid/__init__.py +++ b/nereid/nereid/__init__.py @@ -1,3 +1,3 @@ -__version__ = "0.4.3" +__version__ = "0.5.0rc2" __author__ = "Austin Orr" __email__ = "aorr@geosyntec.com" diff --git a/nereid/nereid/api/api_v1/__init__.py b/nereid/nereid/api/api_v1/__init__.py index e69de29b..8b137891 100644 --- a/nereid/nereid/api/api_v1/__init__.py +++ b/nereid/nereid/api/api_v1/__init__.py @@ -0,0 +1 @@ + diff --git a/nereid/nereid/api/api_v1/api.py b/nereid/nereid/api/api_v1/api.py deleted file mode 100644 index 4927ad19..00000000 --- a/nereid/nereid/api/api_v1/api.py +++ /dev/null @@ -1,20 +0,0 @@ -from fastapi import APIRouter - -from nereid.api.api_v1.endpoints.land_surface_loading import ( - router as land_surface_loading_router, -) -from nereid.api.api_v1.endpoints.network import router as network_router -from nereid.api.api_v1.endpoints.reference_data import router as reference_data_router -from nereid.api.api_v1.endpoints.treatment_facilities import ( - router as treatment_facilities_router, -) -from nereid.api.api_v1.endpoints.treatment_sites import router as treatment_sites_router -from nereid.api.api_v1.endpoints.watershed import router as watershed_router - -api_router = APIRouter() -api_router.include_router(network_router) -api_router.include_router(reference_data_router) -api_router.include_router(land_surface_loading_router) -api_router.include_router(treatment_facilities_router) -api_router.include_router(treatment_sites_router) -api_router.include_router(watershed_router) diff --git a/nereid/nereid/api/api_v1/async_utils.py b/nereid/nereid/api/api_v1/async_utils.py new file mode 100644 index 00000000..5ec82621 --- /dev/null +++ b/nereid/nereid/api/api_v1/async_utils.py @@ -0,0 +1,57 @@ +from typing import Any, Dict, Optional + +from celery import Task +from celery.exceptions import TimeoutError +from celery.result import AsyncResult +from fastapi import APIRouter + +from nereid.core.config import settings + + +def wait_a_sec_and_see_if_we_can_return_some_data( + task: AsyncResult, timeout: float = 0.2 +) -> Optional[Dict[str, Any]]: + result = None + try: + result = task.get(timeout=timeout) + except TimeoutError: + pass + + return result + + +def run_task( + task: Task, + router: APIRouter, + get_route: str, + force_foreground: Optional[bool] = False, +) -> Dict[str, Any]: + + if force_foreground or settings.FORCE_FOREGROUND: # pragma: no cover + response = dict(data=task(), task_id="foreground", result_route="foreground") + + else: + response = standard_json_response(task.apply_async(), router, get_route) + + return response + + +def standard_json_response( + task: AsyncResult, + router: APIRouter, + get_route: str, + timeout: float = 0.2, + api_version: str = settings.API_LATEST, +) -> Dict[str, Any]: + router_path = router.url_path_for(get_route, task_id=task.id) + + result_route = f"{api_version}{router_path}" + + _ = wait_a_sec_and_see_if_we_can_return_some_data(task, timeout=timeout) + + response = dict(task_id=task.task_id, status=task.status, result_route=result_route) + + if task.successful(): + response["data"] = task.result + + return response diff --git a/nereid/nereid/api/api_v1/endpoints/__init__.py b/nereid/nereid/api/api_v1/endpoints/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/nereid/nereid/api/api_v1/endpoints/land_surface_loading.py b/nereid/nereid/api/api_v1/endpoints/land_surface_loading.py deleted file mode 100644 index 40a4a0e9..00000000 --- a/nereid/nereid/api/api_v1/endpoints/land_surface_loading.py +++ /dev/null @@ -1,83 +0,0 @@ -from typing import Any, Dict - -from fastapi import APIRouter, Body, Depends -from fastapi.responses import ORJSONResponse - -import nereid.bg_worker as bg -from nereid.api.api_v1.models.land_surface_models import ( - LandSurfaceResponse, - LandSurfaces, -) -from nereid.api.api_v1.utils import get_valid_context, run_task, standard_json_response - -router = APIRouter() - - -@router.post( - "/land_surface/loading", - tags=["land_surface", "loading"], - response_model=LandSurfaceResponse, - response_class=ORJSONResponse, -) -async def calculate_loading( - land_surfaces: LandSurfaces = Body( - ..., - example={ - "land_surfaces": [ - { - "node_id": "1", - "surface_key": "10101100-RESMF-A-5", - "area_acres": 1.834347898661638, - "imp_area_acres": 1.430224547955745, - }, - { - "node_id": "0", - "surface_key": "10101100-OSDEV-A-0", - "area_acres": 4.458327528535912, - "imp_area_acres": 0.4457209193544626, - }, - { - "node_id": "0", - "surface_key": "10101000-IND-A-10", - "area_acres": 3.337086111390218, - "imp_area_acres": 0.47675887386582366, - }, - { - "node_id": "0", - "surface_key": "10101100-COMM-C-0", - "area_acres": 0.5641157902710026, - "imp_area_acres": 0.40729090799199347, - }, - { - "node_id": "1", - "surface_key": "10101200-TRANS-C-5", - "area_acres": 0.007787658410143283, - "imp_area_acres": 0.007727004694355631, - }, - ] - }, - ), - details: bool = False, - context: dict = Depends(get_valid_context), -) -> Dict[str, Any]: - - land_surfaces_req = land_surfaces.dict(by_alias=True) - - task = bg.background_land_surface_loading.s( - land_surfaces=land_surfaces_req, details=details, context=context - ) - - return run_task( - task=task, router=router, get_route="get_land_surface_loading_result" - ) - - -@router.get( - "/land_surface/loading/{task_id}", - tags=["land_surface", "loading"], - response_model=LandSurfaceResponse, - response_class=ORJSONResponse, -) -async def get_land_surface_loading_result(task_id: str) -> Dict[str, Any]: - task = bg.background_land_surface_loading.AsyncResult(task_id, app=router) - return standard_json_response(task, router, "get_land_surface_loading_result") diff --git a/nereid/nereid/api/api_v1/endpoints_async/__init__.py b/nereid/nereid/api/api_v1/endpoints_async/__init__.py new file mode 100644 index 00000000..e7057f54 --- /dev/null +++ b/nereid/nereid/api/api_v1/endpoints_async/__init__.py @@ -0,0 +1,26 @@ +from fastapi import APIRouter + +from nereid.api.api_v1.endpoints_async.land_surface_loading import ( + router as land_surface_loading_router, +) +from nereid.api.api_v1.endpoints_async.network import router as network_router +from nereid.api.api_v1.endpoints_async.reference_data import ( + router as reference_data_router, +) +from nereid.api.api_v1.endpoints_async.tasks import router as task_router +from nereid.api.api_v1.endpoints_async.treatment_facilities import ( + router as treatment_facilities_router, +) +from nereid.api.api_v1.endpoints_async.treatment_sites import ( + router as treatment_sites_router, +) +from nereid.api.api_v1.endpoints_async.watershed import router as watershed_router + +async_router = APIRouter() +async_router.include_router(network_router) +async_router.include_router(reference_data_router) +async_router.include_router(land_surface_loading_router) +async_router.include_router(treatment_facilities_router) +async_router.include_router(treatment_sites_router) +async_router.include_router(watershed_router) +async_router.include_router(task_router) diff --git a/nereid/nereid/api/api_v1/endpoints_async/land_surface_loading.py b/nereid/nereid/api/api_v1/endpoints_async/land_surface_loading.py new file mode 100644 index 00000000..08950e32 --- /dev/null +++ b/nereid/nereid/api/api_v1/endpoints_async/land_surface_loading.py @@ -0,0 +1,48 @@ +from typing import Any, Dict + +from fastapi import APIRouter, Body, Depends +from fastapi.responses import ORJSONResponse + +import nereid.bg_worker as bg +from nereid.api.api_v1.async_utils import run_task, standard_json_response +from nereid.api.api_v1.models.land_surface_models import ( + LandSurfaceResponse, + LandSurfaces, +) +from nereid.api.api_v1.utils import get_valid_context + +router = APIRouter() + + +@router.post( + "/land_surface/loading", + tags=["land_surface", "loading"], + response_model=LandSurfaceResponse, + response_class=ORJSONResponse, +) +async def calculate_loading( + land_surfaces: LandSurfaces = Body(...), + details: bool = False, + context: dict = Depends(get_valid_context), +) -> Dict[str, Any]: + + land_surfaces_req = land_surfaces.dict(by_alias=True) + + task = bg.land_surface_loading.s( + land_surfaces=land_surfaces_req, details=details, context=context + ) + + return run_task( + task=task, router=router, get_route="get_land_surface_loading_result" + ) + + +@router.get( + "/land_surface/loading/{task_id}", + tags=["land_surface", "loading"], + response_model=LandSurfaceResponse, + response_class=ORJSONResponse, +) +async def get_land_surface_loading_result(task_id: str) -> Dict[str, Any]: + task = bg.land_surface_loading.AsyncResult(task_id, app=router) + return standard_json_response(task, router, "get_land_surface_loading_result") diff --git a/nereid/nereid/api/api_v1/endpoints/network.py b/nereid/nereid/api/api_v1/endpoints_async/network.py similarity index 53% rename from nereid/nereid/api/api_v1/endpoints/network.py rename to nereid/nereid/api/api_v1/endpoints_async/network.py index 221157ea..d9c3cb0f 100644 --- a/nereid/nereid/api/api_v1/endpoints/network.py +++ b/nereid/nereid/api/api_v1/endpoints_async/network.py @@ -1,23 +1,20 @@ -from typing import Any, Dict, List, Union +from typing import Any, Dict, Union from fastapi import APIRouter, Body, HTTPException, Query -from fastapi.encoders import jsonable_encoder from fastapi.requests import Request from fastapi.responses import ORJSONResponse -from fastapi.templating import Jinja2Templates import nereid.bg_worker as bg -from nereid.api.api_v1.models import network_models -from nereid.api.api_v1.utils import ( +from nereid.api.api_v1.async_utils import ( run_task, standard_json_response, wait_a_sec_and_see_if_we_can_return_some_data, ) +from nereid.api.api_v1.models import network_models +from nereid.api.api_v1.utils import templates router = APIRouter() -templates = Jinja2Templates(directory="nereid/api/templates") - @router.post( "/network/validate", @@ -26,17 +23,10 @@ response_class=ORJSONResponse, ) async def validate_network( - graph: network_models.Graph = Body( - ..., - example={ - "directed": True, - "nodes": [{"id": "A"}, {"id": "B"}], - "edges": [{"source": "A", "target": "B"}], - }, - ) + graph: network_models.Graph = Body(..., examples=network_models.GraphExamples) ) -> Dict[str, Any]: - task = bg.background_validate_network.s(graph=graph.dict(by_alias=True)) + task = bg.validate_network.s(graph=graph.dict(by_alias=True)) return run_task(task=task, router=router, get_route="get_validate_network_result") @@ -48,7 +38,7 @@ async def validate_network( ) async def get_validate_network_result(task_id: str) -> Dict[str, Any]: - task = bg.background_validate_network.AsyncResult(task_id, app=router) + task = bg.validate_network.AsyncResult(task_id, app=router) return standard_json_response(task, router, "get_validate_network_result") @@ -59,48 +49,10 @@ async def get_validate_network_result(task_id: str) -> Dict[str, Any]: response_class=ORJSONResponse, ) async def subgraph_network( - subgraph_req: network_models.SubgraphRequest = Body( - ..., - example={ - "graph": { - "directed": True, - "edges": [ - {"source": "3", "target": "1"}, - {"source": "5", "target": "3"}, - {"source": "7", "target": "1"}, - {"source": "9", "target": "1"}, - {"source": "11", "target": "1"}, - {"source": "13", "target": "3"}, - {"source": "15", "target": "9"}, - {"source": "17", "target": "7"}, - {"source": "19", "target": "17"}, - {"source": "21", "target": "15"}, - {"source": "23", "target": "1"}, - {"source": "25", "target": "5"}, - {"source": "27", "target": "11"}, - {"source": "29", "target": "7"}, - {"source": "31", "target": "11"}, - {"source": "33", "target": "25"}, - {"source": "35", "target": "23"}, - {"source": "4", "target": "2"}, - {"source": "6", "target": "2"}, - {"source": "8", "target": "6"}, - {"source": "10", "target": "2"}, - {"source": "12", "target": "2"}, - {"source": "14", "target": "2"}, - {"source": "16", "target": "12"}, - {"source": "18", "target": "12"}, - {"source": "20", "target": "8"}, - {"source": "22", "target": "6"}, - {"source": "24", "target": "12"}, - ], - }, - "nodes": [{"id": "3"}, {"id": "29"}, {"id": "18"}], - }, - ), + subgraph_req: network_models.SubgraphRequest = Body(...), ) -> Dict[str, Any]: - task = bg.background_network_subgraphs.s(**subgraph_req.dict(by_alias=True)) + task = bg.network_subgraphs.s(**subgraph_req.dict(by_alias=True)) return run_task(task=task, router=router, get_route="get_subgraph_network_result") @@ -113,7 +65,7 @@ async def subgraph_network( ) async def get_subgraph_network_result(task_id: str) -> Dict[str, Any]: - task = bg.background_network_subgraphs.AsyncResult(task_id, app=router) + task = bg.network_subgraphs.AsyncResult(task_id, app=router) return standard_json_response(task, router, "get_subgraph_network_result") @@ -130,7 +82,7 @@ async def get_subgraph_network_as_img( npi: float = Query(4.0), ) -> Union[Dict[str, Any], Any]: - task = bg.background_network_subgraphs.AsyncResult(task_id, app=router) + task = bg.network_subgraphs.AsyncResult(task_id, app=router) response = dict(task_id=task.task_id, status=task.status) if task.successful(): # pragma: no branch @@ -140,11 +92,9 @@ async def get_subgraph_network_as_img( render_task_id = task.task_id + f"-{media_type}-{npi}" if media_type == "svg": - render_task = bg.background_render_subgraph_svg.AsyncResult( - render_task_id, app=router - ) + render_task = bg.render_subgraph_svg.AsyncResult(render_task_id, app=router) if render_task.status.lower() != "started": # pragma: no branch - render_task = bg.background_render_subgraph_svg.apply_async( + render_task = bg.render_subgraph_svg.apply_async( args=(result, npi), task_id=render_task_id ) _ = wait_a_sec_and_see_if_we_can_return_some_data( @@ -175,46 +125,11 @@ async def get_subgraph_network_as_img( response_class=ORJSONResponse, ) async def network_solution_sequence( - graph: network_models.Graph = Body( - ..., - example={ - "directed": True, - "edges": [ - {"source": "3", "target": "1"}, - {"source": "5", "target": "3"}, - {"source": "7", "target": "1"}, - {"source": "9", "target": "1"}, - {"source": "11", "target": "1"}, - {"source": "13", "target": "3"}, - {"source": "15", "target": "9"}, - {"source": "17", "target": "7"}, - {"source": "19", "target": "17"}, - {"source": "21", "target": "15"}, - {"source": "23", "target": "1"}, - {"source": "25", "target": "5"}, - {"source": "27", "target": "11"}, - {"source": "29", "target": "7"}, - {"source": "31", "target": "11"}, - {"source": "33", "target": "25"}, - {"source": "35", "target": "23"}, - {"source": "4", "target": "2"}, - {"source": "6", "target": "2"}, - {"source": "8", "target": "6"}, - {"source": "10", "target": "2"}, - {"source": "12", "target": "2"}, - {"source": "14", "target": "2"}, - {"source": "16", "target": "12"}, - {"source": "18", "target": "12"}, - {"source": "20", "target": "8"}, - {"source": "22", "target": "6"}, - {"source": "24", "target": "12"}, - ], - }, - ), + graph: network_models.Graph = Body(..., examples=network_models.GraphExamples), min_branch_size: int = Query(4), ) -> Dict[str, Any]: - task = bg.background_solution_sequence.s( + task = bg.solution_sequence.s( graph=graph.dict(by_alias=True), min_branch_size=min_branch_size ) @@ -229,7 +144,7 @@ async def network_solution_sequence( ) async def get_network_solution_sequence(task_id: str) -> Dict[str, Any]: - task = bg.background_solution_sequence.AsyncResult(task_id, app=router) + task = bg.solution_sequence.AsyncResult(task_id, app=router) return standard_json_response(task, router, "get_network_solution_sequence") @@ -246,7 +161,7 @@ async def get_network_solution_sequence_as_img( npi: float = Query(4.0), ) -> Union[Dict[str, Any], Any]: - task = bg.background_solution_sequence.AsyncResult(task_id, app=router) + task = bg.solution_sequence.AsyncResult(task_id, app=router) response = dict(task_id=task.task_id, status=task.status) if task.successful(): # pragma: no branch @@ -257,11 +172,11 @@ async def get_network_solution_sequence_as_img( if media_type == "svg": - render_task = bg.background_render_solution_sequence_svg.AsyncResult( + render_task = bg.render_solution_sequence_svg.AsyncResult( render_task_id, app=router ) if render_task.status.lower() != "started": # pragma: no branch - render_task = bg.background_render_solution_sequence_svg.apply_async( + render_task = bg.render_solution_sequence_svg.apply_async( args=(result, npi), task_id=render_task_id ) _ = wait_a_sec_and_see_if_we_can_return_some_data( diff --git a/nereid/nereid/api/api_v1/endpoints/reference_data.py b/nereid/nereid/api/api_v1/endpoints_async/reference_data.py similarity index 94% rename from nereid/nereid/api/api_v1/endpoints/reference_data.py rename to nereid/nereid/api/api_v1/endpoints_async/reference_data.py index 1ca437e0..4f64df58 100644 --- a/nereid/nereid/api/api_v1/endpoints/reference_data.py +++ b/nereid/nereid/api/api_v1/endpoints_async/reference_data.py @@ -6,17 +6,14 @@ from fastapi import APIRouter, Depends, HTTPException from fastapi.requests import Request from fastapi.responses import FileResponse, ORJSONResponse -from fastapi.templating import Jinja2Templates from nereid.api.api_v1.models.reference_models import ReferenceDataResponse -from nereid.api.api_v1.utils import get_valid_context +from nereid.api.api_v1.utils import get_valid_context, templates from nereid.core.io import load_file, load_json, load_ref_data from nereid.src.nomograph.nomo import load_nomograph_mapping router = APIRouter() -templates = Jinja2Templates(directory="nereid/api/templates") - @router.get("/reference_data_file", tags=["reference_data"]) async def get_reference_data_file( @@ -108,7 +105,9 @@ async def get_nomograph( @router.get( - "/reference_data/{table}", tags=["reference_data"], response_class=ORJSONResponse, + "/reference_data/{table}", + tags=["reference_data"], + response_class=ORJSONResponse, ) async def get_reference_data_table( table: str, context: dict = Depends(get_valid_context) diff --git a/nereid/nereid/api/api_v1/endpoints_async/tasks.py b/nereid/nereid/api/api_v1/endpoints_async/tasks.py new file mode 100644 index 00000000..85ed580f --- /dev/null +++ b/nereid/nereid/api/api_v1/endpoints_async/tasks.py @@ -0,0 +1,16 @@ +from typing import Any, Dict + +from fastapi import APIRouter +from fastapi.responses import ORJSONResponse + +from nereid.api.api_v1.async_utils import standard_json_response +from nereid.api.api_v1.models.response_models import JSONAPIResponse +from nereid.core.celery_app import celery_app + +router = APIRouter(prefix="/task", default_response_class=ORJSONResponse) + + +@router.get("/{task_id}", response_model=JSONAPIResponse) +async def get_task(task_id: str) -> Dict[str, Any]: + task = celery_app.AsyncResult(task_id) + return standard_json_response(task, router=router, get_route="get_task") diff --git a/nereid/nereid/api/api_v1/endpoints/treatment_facilities.py b/nereid/nereid/api/api_v1/endpoints_async/treatment_facilities.py similarity index 88% rename from nereid/nereid/api/api_v1/endpoints/treatment_facilities.py rename to nereid/nereid/api/api_v1/endpoints_async/treatment_facilities.py index 31c7bc22..89bf1729 100644 --- a/nereid/nereid/api/api_v1/endpoints/treatment_facilities.py +++ b/nereid/nereid/api/api_v1/endpoints_async/treatment_facilities.py @@ -4,13 +4,14 @@ from fastapi.responses import ORJSONResponse import nereid.bg_worker as bg +from nereid.api.api_v1.async_utils import run_task, standard_json_response from nereid.api.api_v1.models.treatment_facility_models import ( TreatmentFacilities, TreatmentFacilitiesResponse, TreatmentFacilitiesStrict, validate_treatment_facility_models, ) -from nereid.api.api_v1.utils import get_valid_context, run_task, standard_json_response +from nereid.api.api_v1.utils import get_valid_context router = APIRouter() @@ -46,7 +47,7 @@ async def initialize_treatment_facility_parameters( treatment_facilities, context = tmnt_facility_req - task = bg.background_initialize_treatment_facilities.s( + task = bg.initialize_treatment_facilities.s( treatment_facilities=treatment_facilities.dict(), pre_validated=True, context=context, @@ -63,7 +64,5 @@ async def initialize_treatment_facility_parameters( response_class=ORJSONResponse, ) async def get_treatment_facility_parameters(task_id: str) -> Dict[str, Any]: - task = bg.background_initialize_treatment_facilities.AsyncResult( - task_id, app=router - ) + task = bg.initialize_treatment_facilities.AsyncResult(task_id, app=router) return standard_json_response(task, router, "get_treatment_facility_parameters") diff --git a/nereid/nereid/api/api_v1/endpoints/treatment_sites.py b/nereid/nereid/api/api_v1/endpoints_async/treatment_sites.py similarity index 100% rename from nereid/nereid/api/api_v1/endpoints/treatment_sites.py rename to nereid/nereid/api/api_v1/endpoints_async/treatment_sites.py diff --git a/nereid/nereid/api/api_v1/endpoints/watershed.py b/nereid/nereid/api/api_v1/endpoints_async/watershed.py similarity index 84% rename from nereid/nereid/api/api_v1/endpoints/watershed.py rename to nereid/nereid/api/api_v1/endpoints_async/watershed.py index 8dd01f3a..34d10be0 100644 --- a/nereid/nereid/api/api_v1/endpoints/watershed.py +++ b/nereid/nereid/api/api_v1/endpoints_async/watershed.py @@ -4,17 +4,19 @@ from fastapi.responses import ORJSONResponse import nereid.bg_worker as bg +from nereid.api.api_v1.async_utils import run_task, standard_json_response from nereid.api.api_v1.models.treatment_facility_models import ( validate_treatment_facility_models, ) from nereid.api.api_v1.models.watershed_models import Watershed, WatershedResponse -from nereid.api.api_v1.utils import get_valid_context, run_task, standard_json_response +from nereid.api.api_v1.utils import get_valid_context router = APIRouter() def validate_watershed_request( - watershed_req: Watershed, context: dict = Depends(get_valid_context), + watershed_req: Watershed, + context: dict = Depends(get_valid_context), ) -> Tuple[Dict[str, Any], Dict[str, Any]]: watershed: Dict[str, Any] = watershed_req.dict(by_alias=True) @@ -43,7 +45,7 @@ async def post_solve_watershed( ) -> Dict[str, Any]: watershed, context = watershed_pkg - task = bg.background_solve_watershed.s( + task = bg.solve_watershed.s( watershed=watershed, treatment_pre_validated=True, context=context ) return run_task(task=task, router=router, get_route="get_watershed_result") @@ -56,5 +58,5 @@ async def post_solve_watershed( response_class=ORJSONResponse, ) async def get_watershed_result(task_id: str) -> Dict[str, Any]: - task = bg.background_solve_watershed.AsyncResult(task_id, app=router) + task = bg.solve_watershed.AsyncResult(task_id, app=router) return standard_json_response(task, router, "get_watershed_result") diff --git a/nereid/nereid/api/api_v1/endpoints_sync/__init__.py b/nereid/nereid/api/api_v1/endpoints_sync/__init__.py new file mode 100644 index 00000000..e25e7435 --- /dev/null +++ b/nereid/nereid/api/api_v1/endpoints_sync/__init__.py @@ -0,0 +1,24 @@ +from fastapi.routing import APIRouter + +from nereid.api.api_v1.endpoints_sync.land_surface_loading import ( + router as land_surface_loading_router, +) +from nereid.api.api_v1.endpoints_sync.network import router as network_router +from nereid.api.api_v1.endpoints_sync.reference_data import ( + router as reference_data_router, +) +from nereid.api.api_v1.endpoints_sync.treatment_facilities import ( + router as treatment_facilities_router, +) +from nereid.api.api_v1.endpoints_sync.treatment_sites import ( + router as treatment_sites_router, +) +from nereid.api.api_v1.endpoints_sync.watershed import router as watershed_router + +sync_router = APIRouter() +sync_router.include_router(network_router) +sync_router.include_router(reference_data_router) +sync_router.include_router(land_surface_loading_router) +sync_router.include_router(treatment_facilities_router) +sync_router.include_router(treatment_sites_router) +sync_router.include_router(watershed_router) diff --git a/nereid/nereid/api/api_v1/endpoints_sync/land_surface_loading.py b/nereid/nereid/api/api_v1/endpoints_sync/land_surface_loading.py new file mode 100644 index 00000000..48a658ac --- /dev/null +++ b/nereid/nereid/api/api_v1/endpoints_sync/land_surface_loading.py @@ -0,0 +1,34 @@ +from typing import Any, Dict + +from fastapi import APIRouter, Body, Depends +from fastapi.responses import ORJSONResponse + +from nereid.api.api_v1.models.land_surface_models import ( + LandSurfaceResponse, + LandSurfaces, +) +from nereid.api.api_v1.utils import get_valid_context +from nereid.src import tasks + +router = APIRouter() + + +@router.post( + "/land_surface/loading", + tags=["land_surface", "loading"], + response_model=LandSurfaceResponse, + response_class=ORJSONResponse, +) +async def calculate_loading( + land_surfaces: LandSurfaces = Body(...), + details: bool = False, + context: dict = Depends(get_valid_context), +) -> Dict[str, Any]: + + land_surfaces_req = land_surfaces.dict(by_alias=True) + + data = tasks.land_surface_loading( + land_surfaces=land_surfaces_req, details=details, context=context + ) + + return {"data": data} diff --git a/nereid/nereid/api/api_v1/endpoints_sync/network.py b/nereid/nereid/api/api_v1/endpoints_sync/network.py new file mode 100644 index 00000000..2e3e3acd --- /dev/null +++ b/nereid/nereid/api/api_v1/endpoints_sync/network.py @@ -0,0 +1,53 @@ +from typing import Any, Dict + +from fastapi import APIRouter, Body, Query +from fastapi.responses import ORJSONResponse + +from nereid.api.api_v1.models import network_models +from nereid.src import tasks + +router = APIRouter() + + +@router.post( + "/network/validate", + tags=["network", "validate"], + response_model=network_models.NetworkValidationResponse, + response_class=ORJSONResponse, +) +async def validate_network( + graph: network_models.Graph = Body(..., examples=network_models.GraphExamples), +) -> Dict[str, Any]: + g: Dict[str, Any] = graph.dict(by_alias=True) + data = tasks.validate_network(graph=g) + return {"data": data} + + +@router.post( + "/network/subgraph", + tags=["network", "subgraph"], + response_model=network_models.SubgraphResponse, + response_class=ORJSONResponse, +) +async def subgraph_network( + subgraph_req: network_models.SubgraphRequest = Body(...), +) -> Dict[str, Any]: + + kwargs = subgraph_req.dict(by_alias=True) + data = tasks.network_subgraphs(**kwargs) + return {"data": data} + + +@router.post( + "/network/solution_sequence", + tags=["network", "sequence"], + response_model=network_models.SolutionSequenceResponse, + response_class=ORJSONResponse, +) +async def network_solution_sequence( + graph: network_models.Graph = Body(..., examples=network_models.GraphExamples), + min_branch_size: int = Query(4), +) -> Dict[str, Any]: + g = graph.dict(by_alias=True) + data = tasks.solution_sequence(graph=g, min_branch_size=min_branch_size) + return {"data": data} diff --git a/nereid/nereid/api/api_v1/endpoints_sync/reference_data.py b/nereid/nereid/api/api_v1/endpoints_sync/reference_data.py new file mode 100644 index 00000000..69a8d401 --- /dev/null +++ b/nereid/nereid/api/api_v1/endpoints_sync/reference_data.py @@ -0,0 +1,132 @@ +import base64 +from io import BytesIO +from pathlib import Path +from typing import Any, Dict, Optional, Union + +from fastapi import APIRouter, Depends, HTTPException +from fastapi.requests import Request +from fastapi.responses import FileResponse, ORJSONResponse + +from nereid.api.api_v1.models.reference_models import ReferenceDataResponse +from nereid.api.api_v1.utils import get_valid_context, templates +from nereid.core.io import load_file, load_json, load_ref_data +from nereid.src.nomograph.nomo import load_nomograph_mapping + +router = APIRouter() + + +@router.get("/reference_data_file", tags=["reference_data"]) +async def get_reference_data_file( + context: dict = Depends(get_valid_context), + filename: str = "", +) -> FileResponse: + + filepath = Path(context.get("data_path", "")) / filename + state, region = context["state"], context["region"] + + if filepath.is_file(): + return FileResponse(filepath) + + else: + detail = f"state '{state}', region '{region}', or filename '{filename}' not found. {filepath}" + raise HTTPException(status_code=400, detail=detail) + + +@router.get( + "/reference_data", + tags=["reference_data"], + response_model=ReferenceDataResponse, + response_class=ORJSONResponse, +) +async def get_reference_data_json( + context: dict = Depends(get_valid_context), + filename: str = "", +) -> Dict[str, Any]: + + filepath = Path(context.get("data_path", "")) / filename + state, region = context["state"], context["region"] + + if filepath.is_file(): + if "json" in filepath.suffix.lower(): + filedata: Dict[str, Any] = load_json(filepath) + else: + filedata: str = load_file(filepath) # type: ignore + + else: + detail = f"state '{state}', region '{region}', or filename '{filename}' not found. {filepath}" + raise HTTPException(status_code=400, detail=detail) + + response = dict( + status="SUCCESS", + data=dict(state=state, region=region, file=filename, filedata=filedata), + ) + + return response + + +@router.get( + "/reference_data/nomograph", + tags=["reference_data"], + # response_model=ReferenceDataResponse, + # response_class=ORJSONResponse, +) +async def get_nomograph( + request: Request, + context: dict = Depends(get_valid_context), + filename: str = "", + type: Optional[str] = None, +) -> Union[Dict[str, Any], Any]: + mapping = load_nomograph_mapping(context) + state, region = context["state"], context["region"] + nomo = mapping.get(filename) or None + if nomo: + if type == "surface": + fig = nomo.surface_plot().get_figure() # type: ignore + f = BytesIO() + fig.savefig(f, bbox_inches="tight", format="png", dpi=300) + f.seek(0) + encoded = base64.b64encode(f.getvalue()).decode("utf-8") + img: str = f"" + + else: + fig = nomo.plot().get_figure() # type: ignore + f = BytesIO() + fig.savefig(f, bbox_inches="tight", format="svg") + f.seek(0) + img = f.read().decode() + + return templates.TemplateResponse( + "display_svg.html", {"request": request, "svg": img} + ) + + else: + detail = ( + f"state '{state}', region '{region}', or filename '{filename}' not found." + ) + raise HTTPException(status_code=400, detail=detail) + + +@router.get( + "/reference_data/{table}", + tags=["reference_data"], + response_class=ORJSONResponse, +) +async def get_reference_data_table( + table: str, context: dict = Depends(get_valid_context) +) -> Dict[str, Any]: + + state, region = context["state"], context["region"] + tables = context.get("project_reference_data", {}).keys() + if table in tables: + df, msg = load_ref_data(table, context) + data = df.to_dict(orient="records") + + else: + detail = f"No such table. Options are {tables}" + raise HTTPException(status_code=400, detail=detail) + + response = dict( + status="SUCCESS", + data=dict(state=state, region=region, table=table, data=data, msg=msg), + ) + return response diff --git a/nereid/nereid/api/api_v1/endpoints_sync/treatment_facilities.py b/nereid/nereid/api/api_v1/endpoints_sync/treatment_facilities.py new file mode 100644 index 00000000..0de15082 --- /dev/null +++ b/nereid/nereid/api/api_v1/endpoints_sync/treatment_facilities.py @@ -0,0 +1,54 @@ +from typing import Any, Dict, Tuple, Union + +from fastapi import APIRouter, Body, Depends +from fastapi.responses import ORJSONResponse + +from nereid.api.api_v1.models.treatment_facility_models import ( + TreatmentFacilities, + TreatmentFacilitiesResponse, + TreatmentFacilitiesStrict, + validate_treatment_facility_models, +) +from nereid.api.api_v1.utils import get_valid_context +from nereid.src import tasks + +router = APIRouter() + + +def validate_facility_request( + treatment_facilities: Union[TreatmentFacilities, TreatmentFacilitiesStrict] = Body( + ... + ), + context: dict = Depends(get_valid_context), +) -> Tuple[TreatmentFacilities, Dict[str, Any]]: + + unvalidated_data = treatment_facilities.dict()["treatment_facilities"] + + valid_models = validate_treatment_facility_models(unvalidated_data, context) + + return ( + TreatmentFacilities.construct(treatment_facilities=valid_models), + context, + ) + + +@router.post( + "/treatment_facility/validate", + tags=["treatment_facility", "validate"], + response_model=TreatmentFacilitiesResponse, + response_class=ORJSONResponse, +) +async def initialize_treatment_facility_parameters( + tmnt_facility_req: Tuple[TreatmentFacilities, Dict[str, Any]] = Depends( + validate_facility_request + ), +) -> Dict[str, Any]: + + treatment_facilities, context = tmnt_facility_req + + data = tasks.initialize_treatment_facilities( + treatment_facilities=treatment_facilities.dict(), + pre_validated=True, + context=context, + ) + return {"data": data} diff --git a/nereid/nereid/api/api_v1/endpoints_sync/treatment_sites.py b/nereid/nereid/api/api_v1/endpoints_sync/treatment_sites.py new file mode 100644 index 00000000..d0fb00d4 --- /dev/null +++ b/nereid/nereid/api/api_v1/endpoints_sync/treatment_sites.py @@ -0,0 +1,28 @@ +from typing import Any, Dict + +from fastapi import APIRouter, Body, Depends +from fastapi.responses import ORJSONResponse + +from nereid.api.api_v1.models.treatment_site_models import ( + TreatmentSiteResponse, + TreatmentSites, +) +from nereid.api.api_v1.utils import get_valid_context +from nereid.src import tasks + +router = APIRouter() + + +@router.post( + "/treatment_site/validate", + tags=["treatment_site", "validate"], + response_model=TreatmentSiteResponse, + response_class=ORJSONResponse, +) +async def initialize_treatment_site( + treatment_sites: TreatmentSites = Body(...), + context: dict = Depends(get_valid_context), +) -> Dict[str, Any]: + data = tasks.initialize_treatment_sites(treatment_sites.dict(), context=context) + + return {"data": data} diff --git a/nereid/nereid/api/api_v1/endpoints_sync/watershed.py b/nereid/nereid/api/api_v1/endpoints_sync/watershed.py new file mode 100644 index 00000000..f4b6c4ca --- /dev/null +++ b/nereid/nereid/api/api_v1/endpoints_sync/watershed.py @@ -0,0 +1,50 @@ +from typing import Any, Dict, Tuple + +from fastapi import APIRouter, Depends +from fastapi.responses import ORJSONResponse + +from nereid.api.api_v1.models.treatment_facility_models import ( + validate_treatment_facility_models, +) +from nereid.api.api_v1.models.watershed_models import Watershed, WatershedResponse +from nereid.api.api_v1.utils import get_valid_context +from nereid.src import tasks + +router = APIRouter() + + +def validate_watershed_request( + watershed_req: Watershed, + context: dict = Depends(get_valid_context), +) -> Tuple[Dict[str, Any], Dict[str, Any]]: + + watershed: Dict[str, Any] = watershed_req.dict(by_alias=True) + + unvalidated_treatment_facilities = watershed.get("treatment_facilities") + if unvalidated_treatment_facilities is not None: + valid_models = validate_treatment_facility_models( + unvalidated_treatment_facilities, context + ) + + watershed["treatment_facilities"] = valid_models + + return watershed, context + + +@router.post( + "/watershed/solve", + tags=["watershed", "main"], + response_model=WatershedResponse, + response_class=ORJSONResponse, +) +async def post_solve_watershed( + watershed_pkg: Tuple[Dict[str, Any], Dict[str, Any]] = Depends( + validate_watershed_request + ), +) -> Dict[str, Any]: + watershed, context = watershed_pkg + + data = tasks.solve_watershed( + watershed=watershed, treatment_pre_validated=True, context=context + ) + return {"data": data} diff --git a/nereid/nereid/api/api_v1/models/land_surface_models.py b/nereid/nereid/api/api_v1/models/land_surface_models.py index 829ef4b6..d3a58ecf 100644 --- a/nereid/nereid/api/api_v1/models/land_surface_models.py +++ b/nereid/nereid/api/api_v1/models/land_surface_models.py @@ -10,13 +10,51 @@ class LandSurface(BaseModel): node_id: str surface_key: str = Field(..., example="104506-RESSFH-B-5") - area_acres: float - imp_area_acres: float + area_acres: float = Field(..., gt=0.0) + imp_area_acres: float = Field(..., ge=0.0) class LandSurfaces(BaseModel): land_surfaces: List[LandSurface] + class Config: + schema_extra = { + "example": { + "land_surfaces": [ + { + "node_id": "1", + "surface_key": "10101100-RESMF-A-5", + "area_acres": 1.834347898661638, + "imp_area_acres": 1.430224547955745, + }, + { + "node_id": "0", + "surface_key": "10101100-OSDEV-A-0", + "area_acres": 4.458327528535912, + "imp_area_acres": 0.4457209193544626, + }, + { + "node_id": "0", + "surface_key": "10101000-IND-A-10", + "area_acres": 3.337086111390218, + "imp_area_acres": 0.47675887386582366, + }, + { + "node_id": "0", + "surface_key": "10101100-COMM-C-0", + "area_acres": 0.5641157902710026, + "imp_area_acres": 0.40729090799199347, + }, + { + "node_id": "1", + "surface_key": "10101200-TRANS-C-5", + "area_acres": 0.007787658410143283, + "imp_area_acres": 0.007727004694355631, + }, + ] + } + } + ## Land Surface Response Models @@ -33,8 +71,8 @@ class LandSurfaceSummary(LandSurfaceBase): perv_area_acres: float imp_ro_volume_cuft: float perv_ro_volume_cuft: float - runoff_volume_cuft: float - eff_area_acres: float + runoff_volume_cuft: float # required for watershed solution + eff_area_acres: float # required for watershed solution land_surfaces_count: float imp_pct: float ro_coeff: float diff --git a/nereid/nereid/api/api_v1/models/network_models.py b/nereid/nereid/api/api_v1/models/network_models.py index 0f5888bb..c7dc7405 100644 --- a/nereid/nereid/api/api_v1/models/network_models.py +++ b/nereid/nereid/api/api_v1/models/network_models.py @@ -18,6 +18,97 @@ class Edge(BaseModel): metadata: Optional[dict] = {} +GraphExamples = { + "simple": { + "summary": "A normal simple graph", + "description": "This should work correctly.", + "value": { + "directed": True, + "nodes": [{"id": "A"}, {"id": "B"}], + "edges": [{"source": "A", "target": "B"}], + }, + }, + "simple edgelist": { + "summary": "Bare minimum graph definition.", + "description": "Graph will be both `directed=True` and `multigraph=True` by default.", + "value": { + "edges": [{"source": "A", "target": "B"}], + }, + }, + "duplicate edge": { + "summary": "Graph with duplicate edge.", + "description": "This graph will error because of the duplicate edge.", + "value": { + "nodes": [{"id": "A"}, {"id": "B"}], + "edges": [ + {"source": "A", "target": "B"}, + {"source": "A", "target": "B"}, + ], + }, + }, + "multiple out edges": { + "summary": "Graph with multiple out edges.", + "description": "This graph will error because of A points to both B and C.", + "value": { + "nodes": [{"id": "A"}, {"id": "B"}, {"id": "C"}], + "edges": [ + {"source": "A", "target": "B"}, + {"source": "A", "target": "C"}, + ], + }, + }, + "cycle": { + "summary": "Graph with a cycle.", + "description": "This graph will error because there is a cycle.", + "value": { + "nodes": [{"id": "A"}, {"id": "B"}, {"id": "C"}], + "edges": [ + {"source": "A", "target": "B"}, + {"source": "B", "target": "C"}, + {"source": "C", "target": "A"}, + ], + }, + }, + "complex": { + "summary": "A complex example", + "description": "A complex graph that works correctly.", + "value": { + "directed": True, + "edges": [ + {"source": "3", "target": "1"}, + {"source": "5", "target": "3"}, + {"source": "7", "target": "1"}, + {"source": "9", "target": "1"}, + {"source": "11", "target": "1"}, + {"source": "13", "target": "3"}, + {"source": "15", "target": "9"}, + {"source": "17", "target": "7"}, + {"source": "19", "target": "17"}, + {"source": "21", "target": "15"}, + {"source": "23", "target": "1"}, + {"source": "25", "target": "5"}, + {"source": "27", "target": "11"}, + {"source": "29", "target": "7"}, + {"source": "31", "target": "11"}, + {"source": "33", "target": "25"}, + {"source": "35", "target": "23"}, + {"source": "4", "target": "2"}, + {"source": "6", "target": "2"}, + {"source": "8", "target": "6"}, + {"source": "10", "target": "2"}, + {"source": "12", "target": "2"}, + {"source": "14", "target": "2"}, + {"source": "16", "target": "12"}, + {"source": "18", "target": "12"}, + {"source": "20", "target": "8"}, + {"source": "22", "target": "6"}, + {"source": "24", "target": "12"}, + ], + }, + }, +} + + class Graph(BaseModel): edges: List[Edge] nodes: Optional[List[Node]] @@ -40,6 +131,14 @@ class SubgraphRequest(BaseModel): graph: Graph nodes: List[Node] + class Config: + schema_extra = { + "example": { + "graph": GraphExamples["complex"]["value"], + "nodes": [{"id": "3"}, {"id": "29"}, {"id": "18"}], + }, + } + class SeriesSequence(BaseModel): series: List[Nodes] diff --git a/nereid/nereid/api/api_v1/models/treatment_facility_models.py b/nereid/nereid/api/api_v1/models/treatment_facility_models.py index ba116932..fb8181f1 100644 --- a/nereid/nereid/api/api_v1/models/treatment_facility_models.py +++ b/nereid/nereid/api/api_v1/models/treatment_facility_models.py @@ -9,9 +9,24 @@ class FacilityBase(BaseModel): node_id: str facility_type: str - ref_data_key: str - design_storm_depth_inches: float = Field(..., gt=0) - eliminate_all_dry_weather_flow_override: bool = False + ref_data_key: str = Field( + ..., + description=( + """This attribute is used to determine which nomographs +to reference in order to compute the long-term volume +capture performance of the facility.""" + ), + ) + design_storm_depth_inches: float = Field( + ..., gt=0, description="""85th percentile design storm depth in inches""" + ) + eliminate_all_dry_weather_flow_override: bool = Field( + False, + description=( + """Whether to override the dr weather flow capture calculation +and set the performance to 'fully elimates all dry weather flow'. (default=False)""" + ), + ) class NTFacility(FacilityBase): @@ -77,12 +92,25 @@ class RetentionFacility(OnlineFaciltyBase): _constructor: str = "retention_facility_constructor" +class RetentionFacilityHSG(OnlineFaciltyBase): + total_volume_cuft: float + area_sqft: float + hsg: str + _constructor: str = "retention_facility_constructor" + + class DryWellFacility(OnlineFaciltyBase): total_volume_cuft: float treatment_rate_cfs: float _constructor: str = "dry_well_facility_constructor" +class DryWellFacilityFlowOrVolume(OnlineFaciltyBase): + total_volume_cuft: float + treatment_rate_cfs: float + _constructor: str = "dry_well_facility_flow_or_volume_constructor" + + class BioInfFacility(OnlineFaciltyBase): total_volume_cuft: float retention_volume_cuft: float @@ -154,10 +182,12 @@ class PermPoolFacility(OnlineFaciltyBase): BioInfFacility, FlowAndRetFacility, RetentionFacility, + RetentionFacilityHSG, TmntFacility, TmntFacilityWithRetentionOverride, CisternFacility, DryWellFacility, + DryWellFacilityFlowOrVolume, DryWeatherTreatmentLowFlowFacility, DryWeatherDiversionLowFlowFacility, LowFlowFacility, @@ -171,10 +201,12 @@ class PermPoolFacility(OnlineFaciltyBase): BioInfFacility, FlowAndRetFacility, RetentionFacility, + RetentionFacilityHSG, TmntFacility, TmntFacilityWithRetentionOverride, CisternFacility, DryWellFacility, + DryWellFacilityFlowOrVolume, DryWeatherTreatmentLowFlowFacility, DryWeatherDiversionLowFlowFacility, LowFlowFacility, diff --git a/nereid/nereid/api/api_v1/models/treatment_site_models.py b/nereid/nereid/api/api_v1/models/treatment_site_models.py index 7ec24257..e018033b 100644 --- a/nereid/nereid/api/api_v1/models/treatment_site_models.py +++ b/nereid/nereid/api/api_v1/models/treatment_site_models.py @@ -19,6 +19,42 @@ class TreatmentSite(BaseModel): class TreatmentSites(BaseModel): treatment_sites: List[TreatmentSite] + class Config: + schema_extra = { + "example": { + "treatment_sites": [ + { + "node_id": "WQMP-1a-tmnt", + "facility_type": "bioretention", + "area_pct": 75, + "captured_pct": 80, + "retained_pct": 10, + }, + { + "node_id": "WQMP-1a-tmnt", + "facility_type": "nt", + "area_pct": 25, + "captured_pct": 0, + "retained_pct": 0, + }, + { + "node_id": "WQMP-1b-tmnt", + "facility_type": "bioretention", + "area_pct": 75, + "captured_pct": 50, + "retained_pct": 10, + }, + { + "node_id": "WQMP-1b-tmnt", + "facility_type": "nt", + "area_pct": 25, + "captured_pct": 0, + "retained_pct": 0, + }, + ] + } + } + ## Treatment Site Response Models diff --git a/nereid/nereid/api/api_v1/utils.py b/nereid/nereid/api/api_v1/utils.py index a96656c5..5fe4dd64 100644 --- a/nereid/nereid/api/api_v1/utils.py +++ b/nereid/nereid/api/api_v1/utils.py @@ -1,77 +1,18 @@ -from typing import Any, Dict, Optional +from typing import Any, Dict -from celery import Task -from celery.exceptions import TimeoutError -from celery.result import AsyncResult -from fastapi import APIRouter, HTTPException +from fastapi import HTTPException +from fastapi.templating import Jinja2Templates -import nereid.bg_worker as bg -from nereid.core import utils -from nereid.core.config import settings +from nereid.core.config import nereid_path +from nereid.core.context import get_request_context, validate_request_context - -def wait_a_sec_and_see_if_we_can_return_some_data( - task: AsyncResult, timeout: float = 0.2 -) -> Optional[Dict[str, Any]]: - result = None - try: - result = task.get(timeout=timeout) - except TimeoutError: - pass - - return result - - -def run_task( - task: Task, - router: APIRouter, - get_route: str, - force_foreground: Optional[bool] = False, -) -> Dict[str, Any]: - - if force_foreground or settings.FORCE_FOREGROUND: - response = dict(data=task(), task_id="foreground", result_route="foreground") - - else: - response = standard_json_response(task.apply_async(), router, get_route) - - return response - - -def standard_json_response( - task: AsyncResult, - router: APIRouter, - get_route: str, - timeout: float = 0.2, - api_version: str = settings.API_LATEST, -) -> Dict[str, Any]: - router_path = router.url_path_for(get_route, task_id=task.id) - - result_route = f"{api_version}{router_path}" - - _ = wait_a_sec_and_see_if_we_can_return_some_data(task, timeout=timeout) - - response = dict(task_id=task.task_id, status=task.status, result_route=result_route) - - if task.successful(): - response["data"] = task.result - - return response +templates = Jinja2Templates(directory=f"{nereid_path}/static/templates") def get_valid_context(state: str = "state", region: str = "region") -> Dict[str, Any]: - context = utils.get_request_context(state, region) - isvalid, msg = utils.validate_request_context(context) + context = get_request_context(state, region) + isvalid, msg = validate_request_context(context) if not isvalid: raise HTTPException(status_code=400, detail=msg) - if not settings.FORCE_FOREGROUND: # pragma: no branch - - task = bg.background_validate_request_context.s(context=context).apply_async() - isvalid, msg = task.get() - if not isvalid: # pragma: no cover - raise HTTPException( - status_code=400, detail="Error in celery worker: " + msg - ) - return context diff --git a/nereid/nereid/bg_worker.py b/nereid/nereid/bg_worker.py index f83d0fdd..f2dd0eb2 100644 --- a/nereid/nereid/bg_worker.py +++ b/nereid/nereid/bg_worker.py @@ -1,17 +1,7 @@ import logging from nereid.core.celery_app import celery_app -from nereid.core.utils import validate_request_context -from nereid.src.land_surface.tasks import land_surface_loading -from nereid.src.network.tasks import ( - network_subgraphs, - render_solution_sequence_svg, - render_subgraph_svg, - solution_sequence, - validate_network, -) -from nereid.src.treatment_facility.tasks import initialize_treatment_facilities -from nereid.src.watershed.tasks import solve_watershed +from nereid.src import tasks logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -24,53 +14,42 @@ def background_ping(): # pragma: no cover @celery_app.task(acks_late=True, track_started=True) -def background_validate_request_context(context): - return validate_request_context(context) # pragma: no cover +def validate_network(graph): # pragma: no cover + return tasks.validate_network(graph=graph) @celery_app.task(acks_late=True, track_started=True) -def background_validate_network(graph): - return validate_network(graph=graph) # pragma: no cover +def network_subgraphs(graph, nodes): # pragma: no cover + return tasks.network_subgraphs(graph=graph, nodes=nodes) @celery_app.task(acks_late=True, track_started=True) -def background_network_subgraphs(graph, nodes): - return network_subgraphs(graph=graph, nodes=nodes) # pragma: no cover +def render_subgraph_svg(task_result, npi): # pragma: no cover + return tasks.render_subgraph_svg(task_result=task_result, npi=npi).decode() @celery_app.task(acks_late=True, track_started=True) -def background_render_subgraph_svg(task_result, npi): - return render_subgraph_svg( # pragma: no cover - task_result=task_result, npi=npi - ).decode() +def solution_sequence(graph, min_branch_size): # pragma: no cover + return tasks.solution_sequence(graph=graph, min_branch_size=min_branch_size) @celery_app.task(acks_late=True, track_started=True) -def background_solution_sequence(graph, min_branch_size): - return solution_sequence( # pragma: no cover - graph=graph, min_branch_size=min_branch_size - ) - - -@celery_app.task(acks_late=True, track_started=True) -def background_render_solution_sequence_svg(task_result, npi): - return render_solution_sequence_svg( # pragma: no cover - task_result=task_result, npi=npi - ).decode() +def render_solution_sequence_svg(task_result, npi): # pragma: no cover + return tasks.render_solution_sequence_svg(task_result=task_result, npi=npi).decode() @celery_app.task(acks_late=True, track_started=True) -def background_land_surface_loading(land_surfaces, details, context): - return land_surface_loading( # pragma: no cover +def land_surface_loading(land_surfaces, details, context): # pragma: no cover + return tasks.land_surface_loading( land_surfaces=land_surfaces, details=details, context=context ) @celery_app.task(acks_late=True, track_started=True) -def background_initialize_treatment_facilities( +def initialize_treatment_facilities( treatment_facilities, pre_validated, context -): - return initialize_treatment_facilities( # pragma: no cover +): # pragma: no cover + return tasks.initialize_treatment_facilities( treatment_facilities=treatment_facilities, pre_validated=pre_validated, context=context, @@ -78,8 +57,8 @@ def background_initialize_treatment_facilities( @celery_app.task(acks_late=True, track_started=True) -def background_solve_watershed(watershed, treatment_pre_validated, context): - return solve_watershed( # pragma: no cover +def solve_watershed(watershed, treatment_pre_validated, context): # pragma: no cover + return tasks.solve_watershed( watershed=watershed, treatment_pre_validated=treatment_pre_validated, context=context, diff --git a/nereid/nereid/core/cache.py b/nereid/nereid/core/cache.py deleted file mode 100644 index 87394572..00000000 --- a/nereid/nereid/core/cache.py +++ /dev/null @@ -1,80 +0,0 @@ -import functools -import hashlib -import logging - -import redis - -logger = logging.getLogger(__name__) - - -redis_cache = redis.Redis(host="redis", port=6379, db=9) - -try: # pragma: no cover - # It's ok if redis isn't up, we'll fall back to an lru_cache if we can only - # use the main process. If redis is available, let's flush the cache to start - # fresh. - if redis_cache.ping(): - redis_cache.flushdb() - logger.debug("flushed redis function cache") -except redis.ConnectionError: # pragma: no cover - pass - - -def rcache(**rkwargs): - ex = rkwargs.pop("ex", 3600 * 24) - - def _rcache(obj): - cache = redis_cache - - @functools.wraps(obj) - def memoizer(*args, **kwargs): - sorted_kwargs = {k: kwargs[k] for k in sorted(kwargs.keys())} - - # hashing the key may not be necessary, but it keeps the server-side filepaths hidden - key = hashlib.sha1( - (str(args) + str(sorted_kwargs)).encode("utf-8") - ).hexdigest() - - if cache.get(key) is None: - logger.debug(f"redis cache miss {key}") - - cache.set(key, obj(*args, **kwargs), ex=ex) - - else: - logger.debug(f"redis hit cache {key}") - - return cache.get(key) - - return memoizer - - return _rcache - - -def no_cache(**rkwargs): # pragma: no cover - def _rcache(obj): - @functools.wraps(obj) - def memoizer(*args, **kwargs): - return obj(*args, **kwargs) - - return memoizer - - return _rcache - - -def get_cache_decorator(): - """fetch a cache decorator for functions. If redis is up, - use that, else use no_cache. - - The point of the no_cache fallback is to make development easier. - In production and even in CI this should use the redis cache. - """ - try: - if redis_cache.ping(): - return rcache - else: # pragma: no cover - return no_cache - except redis.ConnectionError: # pragma: no cover - return no_cache - - -cache_decorator = get_cache_decorator() diff --git a/nereid/nereid/core/config.py b/nereid/nereid/core/config.py index 98b4344e..857e03e4 100644 --- a/nereid/nereid/core/config.py +++ b/nereid/nereid/core/config.py @@ -1,17 +1,19 @@ -import importlib.resources as pkg_resources -from pathlib import Path from typing import Any, Dict, List, Literal, Optional, Union from pydantic import AnyHttpUrl, BaseSettings, validator import nereid from nereid.core.io import load_cfg +from nereid.core.utils import get_nereid_path + +nereid_path = get_nereid_path() class Settings(BaseSettings): + API_V1_STR: str = "/api/v1" API_LATEST: str = API_V1_STR - APP_CONTEXT: Dict[str, Any] = load_cfg(Path(__file__).parent / "base_config.yml") + APP_CONTEXT: Dict[str, Any] = load_cfg(nereid_path / "core" / "base_config.yml") APP_CONTEXT.update( { "version": nereid.__version__, @@ -19,8 +21,12 @@ class Settings(BaseSettings): "contact": nereid.__email__, } ) + VERSION: str = nereid.__version__ FORCE_FOREGROUND: bool = False + ENABLE_ASYNC_ROUTES: bool = False + ASYNC_ROUTE_PREFIX: str = "/async" + ASYNC_MODE: Literal["none", "add", "replace"] = "none" # ALLOW_CORS_ORIGINS is a JSON-formatted list of origins # e.g: '["http://localhost", "http://localhost:4200", "http://localhost:3000", \ @@ -38,11 +44,7 @@ def assemble_cors_origins(cls, v: Union[str, List[str]]) -> Union[List[str], str class Config: # pragma: no cover env_prefix = "NEREID_" - try: - with pkg_resources.path("nereid", ".env") as p: - env_file = p - except FileNotFoundError: - pass + env_file = ".env" extra = "allow" def update(self, other: dict) -> None: # pragma: no cover diff --git a/nereid/nereid/core/context.py b/nereid/nereid/core/context.py new file mode 100644 index 00000000..b0c38f6a --- /dev/null +++ b/nereid/nereid/core/context.py @@ -0,0 +1,82 @@ +from copy import deepcopy +from pathlib import Path +from typing import Any, Dict, Optional, Tuple + +from nereid.core.config import settings +from nereid.core.io import load_cfg + + +def validate_request_context(context: Dict[str, Any]) -> Tuple[bool, str]: + + dp = context.get("data_path") + state = context["state"] + region = context["region"] + + if not dp: + message = f"No configuration exists for the requested state: '{state}' and/or region: '{region}'." + return False, message + + data_path = Path(dp) + + if not data_path.exists(): + message = f"Requested path to reference data:{data_path} does not exist." + return False, message + + if not context.get("project_reference_data"): + return False, "Configuration has no 'project_reference_data' section" + + for tablename, attrs in context.get("project_reference_data", {}).items(): + try: + filename = attrs.get("file") + if filename: + + filepath = data_path / filename + if not filepath.exists(): + message = ( + f"Requested path to reference file: {filepath} does not exist." + ) + return False, message + + except Exception as e: + message = ( + f"Error in section '{tablename}' with entries: '{attrs}' " + f"Exception: {e}" + ) + return False, message + + return True, "valid" + + +def get_request_context( + state: str = "state", + region: str = "region", + datadir: Optional[str] = None, + context: Optional[dict] = None, +) -> Dict[str, Any]: + + if context is None: + context = settings.APP_CONTEXT + + context["state"] = state + context["region"] = region + + if datadir is None: + default_path = Path(__file__).parent.parent / "data" + basepath = Path(context.get("data_path", default_path)) + + if (state == "state") or (region == "region"): + datadir = basepath / context.get("default_data_directory", "") + else: + datadir = basepath / context.get("project_data_directory", "") + + data_path = Path(datadir) / state / region + + if not data_path.exists(): + return context + + request_context = deepcopy(context) + + request_context.update(load_cfg(data_path / "config.yml")) + request_context["data_path"] = str(data_path) + + return request_context diff --git a/nereid/nereid/core/io.py b/nereid/nereid/core/io.py index 3325bcb0..937a0a1b 100644 --- a/nereid/nereid/core/io.py +++ b/nereid/nereid/core/io.py @@ -1,4 +1,5 @@ from copy import deepcopy +from functools import lru_cache from pathlib import Path from typing import Any, Dict, List, Tuple, Union @@ -6,20 +7,19 @@ import pandas import yaml -from nereid.core.cache import cache_decorator - PathType = Union[Path, str] -@cache_decorator(ex=3600 * 24) # expires in 24 hours -def _load_file(filepath: PathType) -> str: +@lru_cache +def _load_file(filepath: PathType) -> bytes: + """returns bytes for redis cache compatability""" fp = Path(filepath) - return fp.read_text(encoding="utf-8") + return fp.read_bytes() def load_file(filepath: PathType) -> str: """wrapper to ensure the cache is called with an absolute path""" - contents: str = _load_file(Path(filepath).resolve()) + contents: str = _load_file(Path(filepath).resolve()).decode() return contents @@ -293,5 +293,4 @@ def parse_configuration_logic( df, msg = parse_collapse_fields( df, params, config_section, config_object, context, msg ) - return df, msg diff --git a/nereid/nereid/core/utils.py b/nereid/nereid/core/utils.py index 4b021c9e..23eb32bc 100644 --- a/nereid/nereid/core/utils.py +++ b/nereid/nereid/core/utils.py @@ -1,87 +1,13 @@ -from copy import deepcopy -from pathlib import Path -from typing import Any, Dict, Optional, Tuple +import importlib.resources as pkg_resources +from typing import Any, Dict +import numpy from pydantic import BaseModel, ValidationError -from nereid.core.config import settings -from nereid.core.io import load_cfg - -def validate_request_context(context: Dict[str, Any]) -> Tuple[bool, str]: - - dp = context.get("data_path") - state = context["state"] - region = context["region"] - - if not dp: - message = f"No configuration exists for the requested state: '{state}' and/or region: '{region}'." - return False, message - - data_path = Path(dp) - - if not data_path.exists(): - message = f"Requested path to reference data:{data_path} does not exist." - return False, message - - if not context.get("project_reference_data"): - return False, "Configuration has no 'project_reference_data' section" - - for tablename, attrs in context.get("project_reference_data", {}).items(): - try: - filename = attrs.get("file") - if filename: - - filepath = data_path / filename - if not filepath.exists(): - message = ( - f"Requested path to reference file: {filepath} does not exist." - ) - return False, message - - except Exception as e: - message = ( - f"Error in section '{tablename}' with entries: '{attrs}' " - f"Exception: {e}" - ) - return False, message - - return True, "valid" - - -def get_request_context( - state: str = "state", - region: str = "region", - datadir: Optional[str] = None, - context: Optional[dict] = None, -) -> Dict[str, Any]: - - if context is None: - context = settings.APP_CONTEXT - - context["state"] = state - context["region"] = region - - if datadir is None: - default_path = Path(__file__).parent.parent / "data" - basepath = Path(context.get("data_path", default_path)) - - if (state == "state") or (region == "region"): - datadir = basepath / context.get("default_data_directory", "") - else: - datadir = basepath / context.get("project_data_directory", "") - - data_path = Path(datadir) / state / region - - if not data_path.exists(): - return context - - request_context = deepcopy(context) - - request_context.update(load_cfg(data_path / "config.yml")) - request_context["data_path"] = str(data_path) - - return request_context +def get_nereid_path(): + with pkg_resources.path("nereid", "__init__.py") as file: + return file.parent def validate_with_discriminator( @@ -112,28 +38,33 @@ class Config: except ValidationError as e: unvalidated_data["errors"] = "ERROR: " + str(e) + " \n" model = fallback - valid = model(**unvalidated_data) + valid = model.construct(**unvalidated_data) return valid def safe_divide(x: float, y: float) -> float: - """This returns zero if the denominator is zero - """ + """This returns zero if the denominator is zero""" if y == 0.0: return 0.0 return x / y +def safe_array_divide(x: numpy.ndarray, y: numpy.ndarray) -> numpy.ndarray: + return numpy.divide(x, y, out=numpy.zeros_like(x), where=y != 0) + + def dictlist_to_dict(dictlist, key): """turn a list of dicts with a common key into a dict. values of the key should be unique within the dictlist Example ------- - >>>dict_list = [{"id":"a"}, {"id":"b"}] - >>>dictlist_to_dict(dict_list, "id") + ```python + dict_list = [{"id":"a"}, {"id":"b"}] + dictlist_to_dict(dict_list, "id") {'a': {'id': 'a'}, 'b': {'id': 'b'}} + ``` """ result = {} diff --git a/nereid/nereid/data/default_data/state/region/config.yml b/nereid/nereid/data/default_data/state/region/config.yml index c45946f1..1c325795 100644 --- a/nereid/nereid/data/default_data/state/region/config.yml +++ b/nereid/nereid/data/default_data/state/region/config.yml @@ -121,6 +121,16 @@ api_recognize: validation_fallback: NTFacility tmnt_performance_facility_type: ¯\_(ツ)_/¯ # wq improvement via retention only + permeable_pavement: + validator: RetentionFacility + validation_fallback: NTFacility + tmnt_performance_facility_type: ¯\_(ツ)_/¯ # wq improvement via retention only + + permeable_pavement_hsg: + validator: RetentionFacilityHSG + validation_fallback: NTFacility + tmnt_performance_facility_type: ¯\_(ツ)_/¯ # wq improvement via retention only + bioretention: validator: BioInfFacility validation_fallback: NTFacility @@ -156,6 +166,11 @@ api_recognize: validation_fallback: NTFacility tmnt_performance_facility_type: ¯\_(ツ)_/¯ + dry_well_flow_or_volume: + validator: DryWellFacilityFlowOrVolume + validation_fallback: NTFacility + tmnt_performance_facility_type: ¯\_(ツ)_/¯ + cistern: validator: CisternFacility validation_fallback: NTFacility diff --git a/nereid/nereid/factory.py b/nereid/nereid/factory.py new file mode 100644 index 00000000..c8239d20 --- /dev/null +++ b/nereid/nereid/factory.py @@ -0,0 +1,75 @@ +from typing import Any, Dict, Optional + +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware +from fastapi.openapi.docs import get_redoc_html, get_swagger_ui_html +from fastapi.staticfiles import StaticFiles + +from nereid.api.api_v1.endpoints_sync import sync_router +from nereid.api.api_v1.utils import get_valid_context +from nereid.core.config import nereid_path, settings + + +def create_app(**settings_override: Optional[Dict[str, Any]]) -> FastAPI: + + _settings = settings.copy(deep=True) + _settings.update(settings_override) + + app = FastAPI( + title="nereid", version=_settings.VERSION, docs_url=None, redoc_url=None + ) + setattr(app, "_settings", _settings) + + static_path = nereid_path / "static" + app.mount("/static", StaticFiles(directory=str(static_path)), name="static") + + if _settings.ASYNC_MODE == "replace": # pragma: no cover + from nereid.api.api_v1.endpoints_async import async_router + + app.include_router(async_router, prefix=_settings.API_V1_STR, tags=["async"]) + else: + app.include_router(sync_router, prefix=_settings.API_V1_STR) + if _settings.ASYNC_MODE == "add": # pragma: no cover + from nereid.api.api_v1.endpoints_async import async_router + + app.include_router( + async_router, + prefix=_settings.API_V1_STR + _settings.ASYNC_ROUTE_PREFIX, + tags=["async"], + ) + + app.add_middleware( + CORSMiddleware, + allow_origins=_settings.ALLOW_CORS_ORIGINS, + allow_origin_regex=_settings.ALLOW_CORS_ORIGIN_REGEX, + allow_credentials=False, + allow_methods=["GET", "OPTIONS", "POST"], + allow_headers=["*"], + ) + + @app.get("/docs", include_in_schema=False) + async def custom_swagger_ui_html(): + return get_swagger_ui_html( + openapi_url=str(app.openapi_url), + title=app.title + " - Swagger UI", + oauth2_redirect_url=app.swagger_ui_oauth2_redirect_url, + swagger_js_url="/static/swagger-ui-bundle.js", + swagger_css_url="/static/swagger-ui.css", + swagger_favicon_url="/static/logo/trident_neptune_logo.ico", + ) + + @app.get("/redoc", include_in_schema=False) + async def redoc_html(): + return get_redoc_html( + openapi_url=str(app.openapi_url), + title=app.title + " - ReDoc", + redoc_js_url="/static/redoc.standalone.js", + redoc_favicon_url="/static/logo/trident_neptune_logo.ico", + ) + + @app.get("/config") + async def check_config(state="state", region="region"): + context = get_valid_context(state, region) + return context + + return app diff --git a/nereid/nereid/main.py b/nereid/nereid/main.py index cfe96335..10f48b84 100644 --- a/nereid/nereid/main.py +++ b/nereid/nereid/main.py @@ -1,63 +1,3 @@ -from fastapi import FastAPI -from fastapi.middleware.cors import CORSMiddleware -from fastapi.openapi.docs import get_redoc_html, get_swagger_ui_html -from fastapi.staticfiles import StaticFiles +from nereid.factory import create_app -import nereid -from nereid.api.api_v1.api import api_router -from nereid.api.api_v1.utils import get_valid_context -from nereid.core.cache import redis_cache -from nereid.core.config import settings - -app = FastAPI(title="nereid", version=nereid.__version__, docs_url=None, redoc_url=None) -app.mount("/static", StaticFiles(directory="nereid/static"), name="static") - - -@app.get("/docs", include_in_schema=False) -async def custom_swagger_ui_html(): - return get_swagger_ui_html( - openapi_url=str(app.openapi_url), - title=app.title + " - Swagger UI", - oauth2_redirect_url=app.swagger_ui_oauth2_redirect_url, - swagger_js_url="/static/swagger-ui-bundle.js", - swagger_css_url="/static/swagger-ui.css", - swagger_favicon_url="/static/logo/trident_neptune_logo.ico", - ) - - -@app.get("/redoc", include_in_schema=False) -async def redoc_html(): - return get_redoc_html( - openapi_url=str(app.openapi_url), - title=app.title + " - ReDoc", - redoc_js_url="/static/redoc.standalone.js", - redoc_favicon_url="/static/logo/trident_neptune_logo.ico", - ) - - -@app.get("/config", include_in_schema=False) -async def check_config(state="state", region="region"): - - try: # pragma: no cover - # if redis is available, let's flush the cache to start - # fresh. - if redis_cache.ping(): - redis_cache.flushdb() - except: # pragma: no cover - pass - - context = get_valid_context(state, region) - - return context - - -app.include_router(api_router, prefix=settings.API_V1_STR) - -app.add_middleware( - CORSMiddleware, - allow_origins=settings.ALLOW_CORS_ORIGINS, - allow_origin_regex=settings.ALLOW_CORS_ORIGIN_REGEX, - allow_credentials=False, - allow_methods=["GET", "OPTIONS", "POST"], - allow_headers=["*"], -) +app = create_app() diff --git a/nereid/nereid/src/land_surface/loading.py b/nereid/nereid/src/land_surface/loading.py index 7c9e857b..560e25ec 100644 --- a/nereid/nereid/src/land_surface/loading.py +++ b/nereid/nereid/src/land_surface/loading.py @@ -2,6 +2,8 @@ import pandas +from nereid.core.utils import safe_array_divide + def clean_land_surface_dataframe(df: pandas.DataFrame) -> pandas.DataFrame: """this function cleans up the imperviousness passed by the client and uses @@ -15,7 +17,7 @@ def clean_land_surface_dataframe(df: pandas.DataFrame) -> pandas.DataFrame: return df -def detailed_volume_loading_results(df: pandas.DataFrame,) -> pandas.DataFrame: +def detailed_volume_loading_results(df: pandas.DataFrame) -> pandas.DataFrame: """ df must contain: area_acres: float @@ -28,6 +30,8 @@ def detailed_volume_loading_results(df: pandas.DataFrame,) -> pandas.DataFrame: """ + df = df.loc[df["area_acres"] > 0] + # method chaining with 'df.assign' looks better, but it's much less memory efficient df["imp_pct"] = 100 * df["imp_area_acres"] / df["area_acres"] df["perv_area_acres"] = df["area_acres"] - df["imp_area_acres"] @@ -202,9 +206,12 @@ def summary_loading_results( for param in wet_weather_parameters: conc_col = param["conc_col"] load_col = param["load_col"] - factor = param["load_to_conc_factor"] + factor: float = float(param["load_to_conc_factor"]) - df[conc_col] = (df[load_col] / df["runoff_volume_cuft"]) * factor + df[conc_col] = ( + safe_array_divide(df[load_col].values, df["runoff_volume_cuft"].values) + * factor + ) for season in season_names: dw_vol_col = f"{season}_dry_weather_flow_cuft" @@ -213,8 +220,10 @@ def summary_loading_results( for param in dry_weather_parameters: conc_col = season + "_" + param["conc_col"] load_col = season + "_" + param["load_col"] - factor = param["load_to_conc_factor"] + factor = float(param["load_to_conc_factor"]) - df[conc_col] = (df[load_col] / df[dw_vol_col]).fillna(0) * factor + df[conc_col] = ( + safe_array_divide(df[load_col].values, df[dw_vol_col].values) * factor + ) return df diff --git a/nereid/nereid/src/land_surface/tasks.py b/nereid/nereid/src/land_surface/tasks.py index f511ccdf..eb4d8a94 100644 --- a/nereid/nereid/src/land_surface/tasks.py +++ b/nereid/nereid/src/land_surface/tasks.py @@ -55,7 +55,10 @@ def land_surface_loading( ) detailed_results = detailed_loading_results( - df, wet_weather_parameters, dry_weather_parameters, seasons, + df, + wet_weather_parameters, + dry_weather_parameters, + seasons, ) summary_results = summary_loading_results( detailed_results, diff --git a/nereid/nereid/src/network/tasks.py b/nereid/nereid/src/network/tasks.py index aba9c5b9..8f415279 100644 --- a/nereid/nereid/src/network/tasks.py +++ b/nereid/nereid/src/network/tasks.py @@ -2,7 +2,6 @@ import networkx as nx -from nereid.core.cache import cache_decorator from nereid.src.network import validate from nereid.src.network.algorithms import get_subset, parallel_sequential_subgraph_nodes from nereid.src.network.render import ( @@ -71,8 +70,7 @@ def network_subgraphs( return result -@cache_decorator(ex=3600 * 24) # expires in 24 hours -def render_subgraph_svg(task_result: dict, npi: Optional[float] = None) -> IO: +def render_subgraph_svg(task_result: dict, npi: Optional[float] = None) -> bytes: g = graph_factory(task_result["graph"]) @@ -84,7 +82,7 @@ def render_subgraph_svg(task_result: dict, npi: Optional[float] = None) -> IO: ) svg_bin = fig_to_image(fig) - svg: IO = svg_bin.read() + svg: bytes = svg_bin.read() return svg @@ -113,8 +111,9 @@ def solution_sequence( return result -@cache_decorator(ex=3600 * 24) # expires in 24 hours -def render_solution_sequence_svg(task_result: dict, npi: Optional[float] = None) -> IO: +def render_solution_sequence_svg( + task_result: dict, npi: Optional[float] = None +) -> bytes: _graph = thin_graph_dict(task_result["graph"]) # strip unneeded metadata @@ -130,6 +129,6 @@ def render_solution_sequence_svg(task_result: dict, npi: Optional[float] = None) fig = render_solution_sequence(g, solution_sequence=solution_sequence, npi=npi) svg_bin = fig_to_image(fig) - svg: IO = svg_bin.read() + svg: bytes = svg_bin.read() return svg diff --git a/nereid/nereid/src/network/utils.py b/nereid/nereid/src/network/utils.py index df0ca57a..2a57791c 100644 --- a/nereid/nereid/src/network/utils.py +++ b/nereid/nereid/src/network/utils.py @@ -140,7 +140,7 @@ def nxGraph_to_dict(g: nx.Graph) -> Dict[str, Any]: def clean_graph_dict(g: nx.Graph) -> Dict[str, Any]: - """ + """ Converts a graph to a dictionary, ensuring all node labels are converted to strings """ diff --git a/nereid/nereid/src/nomograph/interpolators.py b/nereid/nereid/src/nomograph/interpolators.py index 5eecbe59..cd1ff4be 100644 --- a/nereid/nereid/src/nomograph/interpolators.py +++ b/nereid/nereid/src/nomograph/interpolators.py @@ -244,10 +244,12 @@ def __call__( y = numpy.clip(y, numpy.nanmin(self.y_data), numpy.nanmax(self.y_data)) if numpy.iterable(t) and numpy.iterable(y): - if len(t) == len(y): + t_iter = numpy.array(t) + y_iter = numpy.array(y) + if t_iter.size == y_iter.size: res: List[float] = [] - for _y, _t in zip(y, t): + for _y, _t in zip(y_iter, t_iter): guess, _ = self.get_x( at_y=_y, t=_t, atol=atol, max_iters=max_iters diff --git a/nereid/nereid/src/tasks.py b/nereid/nereid/src/tasks.py new file mode 100644 index 00000000..6770a3f5 --- /dev/null +++ b/nereid/nereid/src/tasks.py @@ -0,0 +1,15 @@ +from nereid.src.land_surface.tasks import land_surface_loading as land_surface_loading +from nereid.src.network.tasks import network_subgraphs as network_subgraphs +from nereid.src.network.tasks import ( + render_solution_sequence_svg as render_solution_sequence_svg, +) +from nereid.src.network.tasks import render_subgraph_svg as render_subgraph_svg +from nereid.src.network.tasks import solution_sequence as solution_sequence +from nereid.src.network.tasks import validate_network as validate_network +from nereid.src.treatment_facility.tasks import ( + initialize_treatment_facilities as initialize_treatment_facilities, +) +from nereid.src.treatment_site.tasks import ( + initialize_treatment_sites as initialize_treatment_sites, +) +from nereid.src.watershed.tasks import solve_watershed as solve_watershed diff --git a/nereid/nereid/src/tmnt_performance/tmnt.py b/nereid/nereid/src/tmnt_performance/tmnt.py index 988d6847..1535c171 100644 --- a/nereid/nereid/src/tmnt_performance/tmnt.py +++ b/nereid/nereid/src/tmnt_performance/tmnt.py @@ -61,7 +61,7 @@ def effluent_conc( _C = C * numpy.log(inf_conc) if any([A, B, C, D, E]): - eff = numpy.nansum([A, B * inf_conc, _C, e1, D * (inf_conc ** E) * e2]) + eff = numpy.nansum([A, B * inf_conc, _C, e1, D * (inf_conc**E) * e2]) result = float(numpy.nanmax([dl, numpy.nanmin([eff, inf_conc])])) result *= conversion_factor_from_to(from_unit=unit, to_unit=inf_unit) diff --git a/nereid/nereid/src/treatment_facility/constructors.py b/nereid/nereid/src/treatment_facility/constructors.py index e16ad639..c9c965b2 100644 --- a/nereid/nereid/src/treatment_facility/constructors.py +++ b/nereid/nereid/src/treatment_facility/constructors.py @@ -82,6 +82,26 @@ def dry_well_facility_constructor( return result + @staticmethod + def dry_well_facility_flow_or_volume_constructor( + *, total_volume_cuft: float, treatment_rate_cfs: float, **kwargs: dict + ) -> Dict[str, Any]: + + retention_volume_cuft = total_volume_cuft + retention_ddt_hr = safe_divide(total_volume_cuft, treatment_rate_cfs * 3600) + + result = dict( + retention_volume_cuft=retention_volume_cuft, + retention_ddt_hr=retention_ddt_hr, + # We need to override this because dry wells don't perform treatment + # in either wet weather or dry weather, only retention/volume reduction. + # ini_treatment_rate_cfs=treatment_rate_cfs, + retention_rate_cfs=treatment_rate_cfs, + node_type="dry_well_facility", + ) + + return result + @staticmethod def bioinfiltration_facility_constructor( *, @@ -92,8 +112,7 @@ def bioinfiltration_facility_constructor( inf_rate_inhr: float, **kwargs: dict, ) -> Dict[str, Any]: - """This facility has incidental infiltration and a raised underdrain. - """ + """This facility has incidental infiltration and a raised underdrain.""" retention_depth_ft = safe_divide(retention_volume_cuft, area_sqft) retention_ddt_hr = safe_divide(retention_depth_ft * 12, inf_rate_inhr) @@ -192,7 +211,7 @@ def flow_and_retention_facility_constructor( return result @staticmethod - def flow_facility_constructor(**kwargs: dict,) -> Dict[str, Any]: + def flow_facility_constructor(**kwargs: dict) -> Dict[str, Any]: result = dict(node_type="flow_based_facility") @@ -206,8 +225,7 @@ def dry_weather_diversion_low_flow_facility_constructor( months_operational: str, **kwargs: dict, ) -> Dict[str, Any]: - """These are diversions, so their 'treatment' eliminates volume from the system. - """ + """These are diversions, so their 'treatment' eliminates volume from the system.""" modeled_tmnt_rate = min(treatment_rate_cfs, design_capacity_cfs) @@ -241,8 +259,7 @@ def dry_weather_treatment_low_flow_facility_constructor( months_operational: str, **kwargs: dict, ) -> Dict[str, Any]: - """These are treat and discharge facilities. - """ + """These are treat and discharge facilities.""" modeled_tmnt_rate = min(treatment_rate_cfs, design_capacity_cfs) @@ -276,8 +293,7 @@ def dw_and_low_flow_facility_constructor( months_operational: str, **kwargs: dict, ) -> Dict[str, Any]: - """These are diversions, so their 'treatment' eliminates volume from the system. - """ + """These are diversions, so their 'treatment' eliminates volume from the system.""" modeled_tmnt_rate = min(treatment_rate_cfs, design_capacity_cfs) diff --git a/nereid/nereid/src/treatment_site/tasks.py b/nereid/nereid/src/treatment_site/tasks.py index c948cd65..097c95f2 100644 --- a/nereid/nereid/src/treatment_site/tasks.py +++ b/nereid/nereid/src/treatment_site/tasks.py @@ -43,7 +43,7 @@ def initialize_treatment_sites( ) df = ( - _df.append(remainder_data) + pandas.concat([_df, pandas.DataFrame(remainder_data)]) .fillna(0) .assign( tmnt_performance_facility_type=lambda df: df["facility_type"].replace( diff --git a/nereid/nereid/src/watershed/dry_weather_loading.py b/nereid/nereid/src/watershed/dry_weather_loading.py index 9f1619d5..837ffcc2 100644 --- a/nereid/nereid/src/watershed/dry_weather_loading.py +++ b/nereid/nereid/src/watershed/dry_weather_loading.py @@ -51,7 +51,10 @@ def accumulate_dry_weather_loading( def accumulate_dry_weather_volume_by_season( - g: nx.DiGraph, data: Dict[str, Any], predecessors: List[str], season: str, + g: nx.DiGraph, + data: Dict[str, Any], + predecessors: List[str], + season: str, ) -> Dict[str, Any]: """aggregate dry weather volume for a single season diff --git a/nereid/nereid/src/watershed/solve_watershed.py b/nereid/nereid/src/watershed/solve_watershed.py index afbf285c..776d0218 100644 --- a/nereid/nereid/src/watershed/solve_watershed.py +++ b/nereid/nereid/src/watershed/solve_watershed.py @@ -29,7 +29,9 @@ def initialize_graph( - watershed: Dict[str, Any], treatment_pre_validated: bool, context: Dict[str, Any], + watershed: Dict[str, Any], + treatment_pre_validated: bool, + context: Dict[str, Any], ) -> Tuple[nx.DiGraph, List[str]]: errors: List[str] = [] @@ -52,7 +54,7 @@ def initialize_graph( ) errors.extend(treatment_facilities["errors"]) - treatment_sites = initialize_treatment_sites(watershed, context=context,) + treatment_sites = initialize_treatment_sites(watershed, context=context) errors.extend(treatment_sites["errors"]) data: Dict[str, Any] = {} @@ -73,7 +75,10 @@ def initialize_graph( return g, errors -def solve_watershed_loading(g: nx.DiGraph, context: Dict[str, Any]) -> None: +def solve_watershed_loading( + g: nx.DiGraph, + context: Dict[str, Any], +) -> None: wet_weather_parameters = init_wq_parameters( "land_surface_emc_table", context=context @@ -195,9 +200,7 @@ def solve_node( predecessors = list(g.predecessors(node)) accumulate_wet_weather_loading(g, data, predecessors, wet_weather_parameters) - accumulate_dry_weather_loading( - g, data, predecessors, dry_weather_parameters, - ) + accumulate_dry_weather_loading(g, data, predecessors, dry_weather_parameters) if "site_based" in node_type: # This sequence handles volume capture, load reductions, and also delivers @@ -219,7 +222,9 @@ def solve_node( ) elif "facility" in node_type: - if any([_type in node_type for _type in ["volume_based", "flow_based",]]): + if any( + [_type in node_type for _type in ["volume_based", "flow_based", "dry_well"]] + ): compute_volume_capture_with_nomograph(data, nomograph_map) compute_wet_weather_volume_discharge(data) compute_wet_weather_load_reduction( diff --git a/nereid/nereid/src/watershed/tasks.py b/nereid/nereid/src/watershed/tasks.py index 2e47a001..c43c3e07 100644 --- a/nereid/nereid/src/watershed/tasks.py +++ b/nereid/nereid/src/watershed/tasks.py @@ -11,7 +11,9 @@ @update_unit_registry def solve_watershed( - watershed: Dict[str, Any], treatment_pre_validated: bool, context: Dict[str, Any], + watershed: Dict[str, Any], + treatment_pre_validated: bool, + context: Dict[str, Any], ) -> Dict[str, Any]: """Main program function. This function builds the network and solves for water quality at each node in the input graph. @@ -31,7 +33,11 @@ def solve_watershed( build_nomo.cache_clear() - g, msgs = initialize_graph(watershed, treatment_pre_validated, context,) + g, msgs = initialize_graph( + watershed, + treatment_pre_validated, + context, + ) response["errors"] = [e for e in msgs if "error" in e.lower()] response["warnings"] = [w for w in msgs if "warning" in w.lower()] diff --git a/nereid/nereid/src/watershed/treatment_facility_capture.py b/nereid/nereid/src/watershed/treatment_facility_capture.py index 99eee345..dc0c519a 100644 --- a/nereid/nereid/src/watershed/treatment_facility_capture.py +++ b/nereid/nereid/src/watershed/treatment_facility_capture.py @@ -1,10 +1,12 @@ from typing import Any, Callable, Dict, List, Mapping from nereid.core.utils import safe_divide +from nereid.src.watershed.design_functions import design_intensity_inhr def compute_volume_capture_with_nomograph( - data: Dict[str, Any], nomograph_map: Mapping[str, Callable], + data: Dict[str, Any], + nomograph_map: Mapping[str, Callable], ) -> Dict[str, Any]: """Compute volume captured by a treatment facility if it treats wet weather flow via either volume or flow-based calculation strategies. @@ -17,7 +19,7 @@ def compute_volume_capture_with_nomograph( Parameters ---------- data : dict - information about the node, inclding treatment facility sizing and inflow + information about the node, including treatment facility sizing and inflow characteristics nomograph_map : mapping this mapping uses the nomograph data filepath as the key to return a 2d nomograph @@ -83,6 +85,9 @@ def compute_volume_capture_with_nomograph( elif "flow_based_facility" in node_type: data = compute_flow_based_facility(data, flow_nomo, volume_nomo) + elif "dry_well_facility" in node_type: + data = compute_dry_well_facility(data, flow_nomo, volume_nomo) + else: # pragma: no cover # this should be impossible to reach since this function is called within the # solve_watershed_loading function @@ -106,7 +111,8 @@ def compute_volume_capture_with_nomograph( def compute_volume_based_standalone_facility( - data: Dict[str, Any], volume_nomo: Callable + data: Dict[str, Any], + volume_nomo: Callable, ) -> Dict[str, Any]: """Calculate treatment and retention volume for a standalone volume-based treatment facility. Standalone means that there are not volume-based facilities @@ -178,7 +184,8 @@ def compute_volume_based_standalone_facility( def solve_volume_based_compartments( - compartments: List[Dict[str, float]], volume_nomo: Callable + compartments: List[Dict[str, float]], + volume_nomo: Callable, ) -> List[Dict[str, float]]: """Traverse a series of volume-based nomographs from the bottom compartment (retention) to the top compartment (treatment). This function accumulates the x-offset from @@ -214,7 +221,8 @@ def solve_volume_based_compartments( def compute_volume_based_nested_facility( - data: Dict[str, Any], volume_nomo: Callable + data: Dict[str, Any], + volume_nomo: Callable, ) -> Dict[str, Any]: """Process a volume based treatment facility whose performance is influenced by upstream volume based facilities. @@ -333,7 +341,10 @@ def compute_volume_based_nested_facility( def detention_vol( - tmnt_ddt: float, cumul_within_storm_vol: float, ret_vol: float, tmnt_vol: float + tmnt_ddt: float, + cumul_within_storm_vol: float, + ret_vol: float, + tmnt_vol: float, ) -> float: """This is a helper function for calculating the volume that is detained (delayed) by a treatment facility. @@ -356,7 +367,9 @@ def detention_vol( def compute_flow_based_facility( - data: Dict[str, Any], flow_nomo: Callable, volume_nomo: Callable + data: Dict[str, Any], + flow_nomo: Callable, + volume_nomo: Callable, ) -> Dict[str, Any]: """Solves volume balance for flow based treatment. these facilities *can* perform both treatment via treatment rate nomographs to reduce the effluent concentration and/or @@ -383,10 +396,11 @@ def compute_flow_based_facility( msg = f"overriding tributary_area_tc_min from '{tc}' to 5 minutes." data["node_warnings"].append(msg) - captured_fraction = float( - flow_nomo(intensity=data.get("design_intensity_inhr", 0.0), tc=tc) or 0.0 + intensity = data["design_intensity_inhr"] = design_intensity_inhr( + data.get("treatment_rate_cfs", 0.0), data["eff_area_acres_cumul"] ) + captured_fraction = float(flow_nomo(intensity=intensity, tc=tc) or 0.0) size_fraction = safe_divide( data.get("retention_volume_cuft", 0.0), data["design_volume_cuft_cumul"] ) @@ -408,8 +422,69 @@ def compute_flow_based_facility( return data +def compute_dry_well_facility( + data: Dict[str, Any], + flow_nomo: Callable, + volume_nomo: Callable, +) -> Dict[str, Any]: + """best of flow-based and volume based nomographs for bmp volume and treatment rate. + + the fate of all of the treatment for a drywell is _always_ retention. + + Parameters + ---------- + data : dict + all the current node's information. this will be treatment facilily size + information and characteristics of incoming upstream flow. + *_nomo : thinly wrapped 2D CloughTocher Interpolators + Reference: `nereid.src.nomograph.nomo` + + """ + + tc = data.get("tributary_area_tc_min") + if tc is None or tc < 5: + tc = 5 + msg = f"overriding tributary_area_tc_min from '{tc}' to 5 minutes." + data["node_warnings"].append(msg) + + # check flow nomo + intensity = data["design_intensity_inhr"] = design_intensity_inhr( + data.get("retention_rate_cfs", 0.0), data["eff_area_acres_cumul"] + ) + + flow_based_captured_fraction = float(flow_nomo(intensity=intensity, tc=tc) or 0.0) + + # check volume nomo + size_fraction = safe_divide( + data.get("retention_volume_cuft", 0.0), data["design_volume_cuft_cumul"] + ) + ret_ddt_hr = data.get("retention_ddt_hr", 0.0) + + volume_based_captured_fraction = float( + volume_nomo(size=size_fraction, ddt=ret_ddt_hr) or 0.0 + ) + + # check which is best capture + solution_type = "flow based" + if volume_based_captured_fraction > flow_based_captured_fraction: + solution_type = "volume based" + + captured_pct = ( + max(flow_based_captured_fraction, volume_based_captured_fraction) * 100 + ) + + # for dry wells, all capture is retention. + data["retained_pct"] = captured_pct + data["captured_pct"] = captured_pct + data["treated_pct"] = 0.0 + data["_nomograph_solution_status"] = f"successful; dry well {solution_type}" + + return data + + def compute_peak_flow_reduction( - data: Dict[str, Any], peak_nomo: Callable + data: Dict[str, Any], + peak_nomo: Callable, ) -> Dict[str, Any]: ret_vol_cuft = data["retention_volume_cuft"] diff --git a/nereid/nereid/src/watershed/wet_weather_loading.py b/nereid/nereid/src/watershed/wet_weather_loading.py index b701e257..16e0a6dc 100644 --- a/nereid/nereid/src/watershed/wet_weather_loading.py +++ b/nereid/nereid/src/watershed/wet_weather_loading.py @@ -121,11 +121,6 @@ def accumulate_wet_weather_loading( data.get("retention_volume_cuft", 0.0) + data["retention_volume_cuft_upstream"] ) - # calculate design intensity - data["design_intensity_inhr"] = design_intensity_inhr( - data.get("treatment_rate_cfs", 0.0), data["eff_area_acres_cumul"] - ) - # accumulate design volume data["design_volume_cuft_direct"] = design_volume_cuft( data.get("design_storm_depth_inches", 0.0), data["eff_area_acres_direct"] @@ -333,9 +328,7 @@ def compute_wet_weather_load_reduction( def check_node_results_close(data: Dict[str, Any]) -> Dict[str, Any]: - """Run a few mass balance checks on the node. - - """ + """Run a few mass balance checks on the node.""" check1 = safe_divide( ( data["runoff_volume_cuft_inflow"] diff --git a/nereid/nereid/src/wq_parameters.py b/nereid/nereid/src/wq_parameters.py index a1c8ddae..ae25a447 100644 --- a/nereid/nereid/src/wq_parameters.py +++ b/nereid/nereid/src/wq_parameters.py @@ -55,9 +55,11 @@ def init_wq_parameters(tablename: str, context: Dict[str, Any]) -> List[Dict[str """ - parameters: List[Dict[str, Any]] = context.get("project_reference_data", {}).get( - tablename, {} - ).get("parameters", []) + parameters: List[Dict[str, Any]] = ( + context.get("project_reference_data", {}) + .get(tablename, {}) + .get("parameters", []) + ) for param in parameters: diff --git a/nereid/nereid/api/templates/base.html b/nereid/nereid/static/templates/base.html similarity index 100% rename from nereid/nereid/api/templates/base.html rename to nereid/nereid/static/templates/base.html diff --git a/nereid/nereid/api/templates/display_svg.html b/nereid/nereid/static/templates/display_svg.html similarity index 100% rename from nereid/nereid/api/templates/display_svg.html rename to nereid/nereid/static/templates/display_svg.html diff --git a/nereid/nereid/tests/__init__.py b/nereid/nereid/tests/__init__.py index e69de29b..dae366cb 100644 --- a/nereid/nereid/tests/__init__.py +++ b/nereid/nereid/tests/__init__.py @@ -0,0 +1,12 @@ +from nereid.core.config import settings + + +def test(*args): # pragma: no cover + try: + import pytest + except ImportError: + raise ImportError("`pytest` is required to run the test suite") + + options = [str(settings._nereid_path)] + options.extend(list(args)) + return pytest.main(options) diff --git a/nereid/nereid/tests/conftest.py b/nereid/nereid/tests/conftest.py index 64dda4f5..9a006024 100644 --- a/nereid/nereid/tests/conftest.py +++ b/nereid/nereid/tests/conftest.py @@ -9,7 +9,7 @@ TREATMENT_FACILITY_MODELS, validate_treatment_facility_models, ) -from nereid.core.utils import get_request_context +from nereid.core.context import get_request_context from nereid.tests.utils import ( generate_random_land_surface_request_sliver, generate_random_treatment_facility_request_node, @@ -18,6 +18,11 @@ ) +@pytest.fixture(scope="session") +def async_mode(request): + return request.config.getoption("--async", False) + + @pytest.fixture def subgraph_request_dict(): graph = { @@ -400,7 +405,12 @@ def watershed_requests(contexts, subbasins, land_surface_permutations): for n_nodes, pct_tmnt in product(SIZE, PCT_TMNT): seed = numpy.random.randint(1e6) req = generate_random_watershed_solve_request( - context, subbasins, land_surface_permutations, n_nodes, pct_tmnt, seed=seed, + context, + subbasins, + land_surface_permutations, + n_nodes, + pct_tmnt, + seed=seed, ) requests[(n_nodes, pct_tmnt)] = deepcopy(req) return requests diff --git a/nereid/nereid/tests/test_api/conftest.py b/nereid/nereid/tests/test_api/conftest.py index 78c4bbd8..42fb623d 100644 --- a/nereid/nereid/tests/test_api/conftest.py +++ b/nereid/nereid/tests/test_api/conftest.py @@ -7,13 +7,17 @@ from fastapi.testclient import TestClient from nereid.core.config import settings -from nereid.main import app +from nereid.factory import create_app from nereid.src.network.utils import clean_graph_dict from nereid.tests.utils import generate_n_random_valid_watershed_graphs, get_payload @pytest.fixture(scope="module") -def client(): +def client(async_mode): + mode = "none" + if async_mode: + mode = "replace" + app = create_app(ASYNC_MODE=mode) with TestClient(app) as client: yield client @@ -54,21 +58,25 @@ def named_subgraph_responses(client): init_post_requests = [ # name, file or object, is-fast - ("subgraph_response_fast", get_payload("network_subgraph_request.json"), True), + ( + "subgraph_response_fast", + get_payload("network_subgraph_request.json"), + # True + ), ( "subgraph_response_slow", json.dumps(dict(graph=slow_graph, nodes=nodes)), - False, + # False, ), ] - for name, payload, isfast in init_post_requests: + for name, payload in init_post_requests: response = client.post(route, data=payload) responses[name] = response - result_route = response.json()["result_route"] + result_route = response.json().get("result_route") - if isfast: + if result_route: # trigger the svg render here so it's ready to get later. client.get(result_route + "/img?media_type=svg") time.sleep(0.5) @@ -95,9 +103,9 @@ def solution_sequence_response(client): response = client.post(route + f"?min_branch_size={bs}", data=payload) responses[(bs, ngraph, minmax)] = response + result_route = response.json().get("result_route") - if all([minmax == (10, 11), ngraph == 3, bs == 6]): - result_route = response.json()["result_route"] + if all([minmax == (10, 11), ngraph == 3, bs == 6, result_route]): client.get(result_route + "/img?media_type=svg") time.sleep(0.5) @@ -117,7 +125,7 @@ def land_surface_loading_responses(client, land_surface_loading_response_dicts): payload = json.dumps(ls_request) route = settings.API_LATEST + "/land_surface/loading" + f"?details={detail_tf}" response = client.post(route, data=payload) - assert response.status_code == 200, (response, detail_tf, nrows, nnodes) + assert response.status_code == 200, (response.text, detail_tf, nrows, nnodes) responses[(detail_tf, nrows, nnodes)] = response diff --git a/nereid/nereid/tests/test_api/test_land_surface_loading.py b/nereid/nereid/tests/test_api/test_land_surface_loading.py index ac8da688..2b1b1128 100644 --- a/nereid/nereid/tests/test_api/test_land_surface_loading.py +++ b/nereid/nereid/tests/test_api/test_land_surface_loading.py @@ -1,7 +1,6 @@ import pytest from nereid.api.api_v1.models import land_surface_models -from nereid.core.config import settings @pytest.mark.parametrize("details", ["true", "false"]) @@ -31,9 +30,9 @@ def test_get_land_surface_loading( post_response = land_surface_loading_responses[key] prjson = post_response.json() - if settings.FORCE_FOREGROUND: # pragma: no cover - grjson = prjson - else: + grjson = prjson + + if prjson.get("result_route"): result_route = prjson["result_route"] get_response = client.get(result_route) diff --git a/nereid/nereid/tests/test_api/test_network.py b/nereid/nereid/tests/test_api/test_network.py index 3c343668..20656291 100644 --- a/nereid/nereid/tests/test_api/test_network.py +++ b/nereid/nereid/tests/test_api/test_network.py @@ -46,10 +46,13 @@ def test_get_network_validate( post_response = named_validation_responses[post_response_name] prjson = post_response.json() - if settings.FORCE_FOREGROUND: # pragma: no cover - grjson = prjson - else: - result_route = prjson["result_route"] + grjson = prjson + result_route = prjson.get("result_route") + + if result_route: + get_route = f"{settings.API_LATEST}/task/{prjson.get('task_id', 'error$!#*&^')}" + get_response = client.get(get_route) + assert get_response.status_code == 200, (prjson, get_route) get_response = client.get(result_route) assert get_response.status_code == 200 @@ -112,20 +115,19 @@ def test_get_finished_network_subgraph( post_response = named_subgraph_responses[post_response_name] prjson = post_response.json() - if settings.FORCE_FOREGROUND: # pragma: no cover - grjson = prjson - else: - result_route = prjson["result_route"] + grjson = prjson + result_route = prjson.get("result_route") + if result_route: get_response = client.get(result_route) assert get_response.status_code == 200 grjson = get_response.json() + assert grjson["task_id"] is not None + assert grjson["result_route"] is not None assert network_models.SubgraphResponse(**prjson) assert grjson["status"].lower() == "success" - assert grjson["task_id"] is not None - assert grjson["result_route"] is not None assert grjson["data"] is not None assert len(grjson["data"]["subgraph_nodes"]) == len(exp["subgraph_nodes"]) @@ -142,7 +144,6 @@ def test_post_network_subgraph(client, named_subgraph_responses, post_response_n assert prjson["status"].lower() != "failure" -@pytest.mark.skipif(settings.FORCE_FOREGROUND, reason="tasks ran in foreground") @pytest.mark.parametrize( "post_response_name, isfast", [("subgraph_response_fast", True), ("subgraph_response_slow", False)], @@ -152,26 +153,26 @@ def test_get_render_subgraph_svg( ): post_response = named_subgraph_responses[post_response_name] + assert post_response.status_code == 200 rjson = post_response.json() - result_route = rjson["result_route"] - - svg_response = client.get(result_route + "/img") - assert svg_response.status_code == 200 - if isfast: - assert "DOCTYPE svg PUBLIC" in svg_response.content.decode() - else: - srjson = svg_response.json() - assert srjson["status"].lower() != "failure" - assert srjson["task_id"] is not None - - if isfast: - # try to cover cached retrieval by asking again + result_route = rjson.get("result_route") + if result_route: svg_response = client.get(result_route + "/img") assert svg_response.status_code == 200 + if isfast: + assert "DOCTYPE svg PUBLIC" in svg_response.content.decode() + else: + srjson = svg_response.json() + assert srjson["status"].lower() != "failure" + assert srjson["task_id"] is not None + + if isfast: + # try to cover cached retrieval by asking again + svg_response = client.get(result_route + "/img") + assert svg_response.status_code == 200 -@pytest.mark.skipif(settings.FORCE_FOREGROUND, reason="tasks ran in foreground") @pytest.mark.parametrize( "post_response_name, isfast", [("subgraph_response_fast", True), ("subgraph_response_slow", False)], @@ -181,13 +182,14 @@ def test_get_render_subgraph_svg_bad_media_type( ): post_response = named_subgraph_responses[post_response_name] + assert post_response.status_code == 200 rjson = post_response.json() - result_route = rjson["result_route"] - - svg_response = client.get(result_route + "/img?media_type=png") - assert svg_response.status_code == 400 - assert "media_type not supported" in svg_response.content.decode() + result_route = rjson.get("result_route") + if result_route: + svg_response = client.get(result_route + "/img?media_type=png") + assert svg_response.status_code == 400 + assert "media_type not supported" in svg_response.content.decode() @pytest.mark.parametrize("min_branch_size", [2, 6, 10, 50]) @@ -217,24 +219,20 @@ def test_get_solution_sequence( post_response = solution_sequence_response[key] prjson = post_response.json() - if settings.FORCE_FOREGROUND: # pragma: no cover - grjson = prjson - else: - result_route = prjson["result_route"] - + grjson = prjson + result_route = prjson.get("result_route") + if result_route: get_response = client.get(result_route) assert get_response.status_code == 200 grjson = get_response.json() + assert grjson["task_id"] == prjson["task_id"] + assert grjson["result_route"] == prjson["result_route"] assert network_models.SolutionSequenceResponse(**prjson) - assert grjson["status"].lower() == "success" - assert grjson["task_id"] == prjson["task_id"] - assert grjson["result_route"] == prjson["result_route"] assert grjson["data"] is not None -@pytest.mark.skipif(settings.FORCE_FOREGROUND, reason="tasks ran in foreground") @pytest.mark.parametrize("min_branch_size", [6]) @pytest.mark.parametrize("n_graphs", [1, 3]) @pytest.mark.parametrize("min_max", [(3, 4), (10, 11), (20, 40)]) @@ -244,33 +242,37 @@ def test_get_render_solution_sequence( key = min_branch_size, n_graphs, min_max post_response = solution_sequence_response[key] + assert post_response.status_code == 200 prjson = post_response.json() - result_route = prjson["result_route"] + result_route = prjson.get("result_route") - _ = client.get(result_route + "/img") - svg_response = client.get(result_route + "/img") + if result_route: - assert svg_response.status_code == 200 + _ = client.get(result_route + "/img") + svg_response = client.get(result_route + "/img") - if "html" in svg_response.headers["content-type"]: - assert "DOCTYPE svg PUBLIC" in svg_response.content.decode() + assert svg_response.status_code == 200 - else: - srjson = svg_response.json() - assert srjson["status"].lower() != "failure" - assert srjson["task_id"] is not None + if "html" in svg_response.headers["content-type"]: + assert "DOCTYPE svg PUBLIC" in svg_response.content.decode() + + else: + srjson = svg_response.json() + assert srjson["status"].lower() != "failure" + assert srjson["task_id"] is not None -@pytest.mark.skipif(settings.FORCE_FOREGROUND, reason="tasks ran in foreground") def test_get_render_solution_sequence_bad_media_type( client, solution_sequence_response ): key = 6, 3, (10, 11) post_response = solution_sequence_response[key] + assert post_response.status_code == 200 prjson = post_response.json() - result_route = prjson["result_route"] - svg_response = client.get(result_route + "/img?media_type=png") - assert svg_response.status_code == 400 - assert "media_type not supported" in svg_response.content.decode() + result_route = prjson.get("result_route") + if result_route: + svg_response = client.get(result_route + "/img?media_type=png") + assert svg_response.status_code == 400 + assert "media_type not supported" in svg_response.content.decode() diff --git a/nereid/nereid/tests/test_api/test_reference_data.py b/nereid/nereid/tests/test_api/test_reference_data.py index 78a1d09a..506fbd09 100644 --- a/nereid/nereid/tests/test_api/test_reference_data.py +++ b/nereid/nereid/tests/test_api/test_reference_data.py @@ -65,7 +65,12 @@ def test_ref_data_file(client, query, isvalid): @pytest.mark.parametrize( - "table, isvalid", [("", False), ("met_table", True), ("met_tables", False),], + "table, isvalid", + [ + ("", False), + ("met_table", True), + ("met_tables", False), + ], ) def test_ref_data_table(client, table, isvalid): url = settings.API_LATEST + f"/reference_data/{table}" diff --git a/nereid/nereid/tests/test_api/test_treatment_facilities.py b/nereid/nereid/tests/test_api/test_treatment_facilities.py index 1e0c7bcc..10cce829 100644 --- a/nereid/nereid/tests/test_api/test_treatment_facilities.py +++ b/nereid/nereid/tests/test_api/test_treatment_facilities.py @@ -24,15 +24,12 @@ def test_get_init_tmnt_facility_params(client, treatment_facility_responses, key post_response = treatment_facility_responses[key] prjson = post_response.json() + grjson = prjson + result_route = prjson.get("result_route") - if settings.FORCE_FOREGROUND: # pragma: no cover - grjson = prjson - else: - result_route = prjson["result_route"] - + if result_route: get_response = client.get(result_route) assert get_response.status_code == 200 - grjson = get_response.json() assert treatment_facility_models.TreatmentFacilitiesResponse(**prjson) @@ -56,12 +53,10 @@ def test_get_default_context_tmnt_facility_params( for name, post_response in default_context_treatment_facility_responses.items(): prjson = post_response.json() + grjson = prjson + result_route = prjson.get("result_route") - if settings.FORCE_FOREGROUND: # pragma: no cover - grjson = prjson - else: - result_route = prjson["result_route"] - + if result_route: get_response = client.get(result_route) assert get_response.status_code == 200 diff --git a/nereid/nereid/tests/test_api/test_utils.py b/nereid/nereid/tests/test_api/test_utils.py index 451ef99f..6e2b5b8f 100644 --- a/nereid/nereid/tests/test_api/test_utils.py +++ b/nereid/nereid/tests/test_api/test_utils.py @@ -1,7 +1,7 @@ import pytest from fastapi import HTTPException -import nereid.bg_worker as bg +# import nereid.bg_worker as bg from nereid.api.api_v1 import utils @@ -17,12 +17,12 @@ def test_get_valid_context(state, region, raises, exp): assert all([req_context[k] == v for k, v in exp.items()]) -def test_run_task_by_name(subgraph_request_dict): +# def test_run_task_by_name(subgraph_request_dict): - graph, nodes = subgraph_request_dict["graph"], subgraph_request_dict["nodes"] +# graph, nodes = subgraph_request_dict["graph"], subgraph_request_dict["nodes"] - task = bg.background_network_subgraphs.s(graph=graph, nodes=nodes) +# task = bg.background_network_subgraphs.s(graph=graph, nodes=nodes) - return utils.run_task( - task=task, router=r"¯\_(ツ)_/¯", get_route=r"¯\_(ツ)_/¯", force_foreground=True - ) +# return utils.run_task( +# task=task, router=r"¯\_(ツ)_/¯", get_route=r"¯\_(ツ)_/¯", force_foreground=True +# ) diff --git a/nereid/nereid/tests/test_api/test_watershed.py b/nereid/nereid/tests/test_api/test_watershed.py index 47e52a0b..0c81a7d6 100644 --- a/nereid/nereid/tests/test_api/test_watershed.py +++ b/nereid/nereid/tests/test_api/test_watershed.py @@ -31,10 +31,13 @@ def test_get_solve_watershed(client, watershed_responses, size, pct_tmnt): post_response = watershed_responses[key] prjson = post_response.json() - if settings.FORCE_FOREGROUND: # pragma: no cover - grjson = prjson - else: - result_route = prjson["result_route"] + grjson = prjson + result_route = prjson.get("result_route") + + if result_route: + get_route = f"{settings.API_LATEST}/task/{prjson.get('task_id', 'error$!#*&^')}" + get_response = client.get(get_route) + assert get_response.status_code == 200, (prjson, get_route) get_response = client.get(result_route) assert get_response.status_code == 200 diff --git a/nereid/nereid/tests/test_core/test_utils.py b/nereid/nereid/tests/test_core/test_utils.py index f05e674e..8cd2fa94 100644 --- a/nereid/nereid/tests/test_core/test_utils.py +++ b/nereid/nereid/tests/test_core/test_utils.py @@ -1,6 +1,7 @@ import pytest from nereid.core import utils +from nereid.core.context import get_request_context, validate_request_context @pytest.mark.parametrize( @@ -18,7 +19,7 @@ ], ) def test_get_request_context(state, region, dirname, context, exp): - req_context = utils.get_request_context(state, region, dirname, context) + req_context = get_request_context(state, region, dirname, context) assert all([k in req_context for k in exp.keys()]) @@ -44,7 +45,7 @@ def test_get_request_context(state, region, dirname, context, exp): def test_validate_request_context(contexts, key): context = contexts[key] - isvalid, msg = utils.validate_request_context(context) + isvalid, msg = validate_request_context(context) assert len(msg) > 0 diff --git a/nereid/nereid/tests/test_src/test_land_surface/test_loading.py b/nereid/nereid/tests/test_src/test_land_surface/test_loading.py index ffc88f5c..519099ea 100644 --- a/nereid/nereid/tests/test_src/test_land_surface/test_loading.py +++ b/nereid/nereid/tests/test_src/test_land_surface/test_loading.py @@ -72,7 +72,10 @@ def test_detailed_land_surface_loading_results( ) t = detailed_loading_results( - land_surfaces_df, wet_weather_parameters, dry_weather_parameters, seasons, + land_surfaces_df, + wet_weather_parameters, + dry_weather_parameters, + seasons, ) assert t["area_acres"].sum() == land_surfaces_df["area_acres"].sum() assert len(t) == len(land_surfaces_list) @@ -86,7 +89,7 @@ def test_detailed_land_surface_loading_results( assert t["runoff_volume_cuft"].sum() > 0 t = detailed_pollutant_loading_results( - land_surfaces_df, + t, wet_weather_parameters, dry_weather_parameters, seasons.keys(), diff --git a/nereid/nereid/tests/test_src/test_land_surface/test_tasks.py b/nereid/nereid/tests/test_src/test_land_surface/test_tasks.py index 625dda0b..688edad4 100644 --- a/nereid/nereid/tests/test_src/test_land_surface/test_tasks.py +++ b/nereid/nereid/tests/test_src/test_land_surface/test_tasks.py @@ -1,6 +1,6 @@ import pytest -from nereid.core.utils import get_request_context +from nereid.core.context import get_request_context from nereid.src.land_surface.tasks import land_surface_loading @@ -34,7 +34,7 @@ def test_land_surface_loading_with_err( land_surfaces["land_surfaces"][5]["surface_key"] = r"¯\_(ツ)_/¯" result = land_surface_loading(land_surfaces, details, context=get_request_context()) - assert "ERROR" in result.get("errors", [])[0] + assert "ERROR" in result.get("errors", ["nope"])[0] assert result.get("summary") is not None assert len(result.get("summary")) <= len(land_surfaces["land_surfaces"]) diff --git a/nereid/nereid/tests/test_src/test_nomograph/test_nomo.py b/nereid/nereid/tests/test_src/test_nomograph/test_nomo.py index 01e62804..925ac552 100644 --- a/nereid/nereid/tests/test_src/test_nomograph/test_nomo.py +++ b/nereid/nereid/tests/test_src/test_nomograph/test_nomo.py @@ -184,21 +184,21 @@ def test_bisection_search(fxn, seek_value, bounds, exp, converges): @pytest.mark.parametrize( "size, ddt, performance, exp", [ - (1.5, 24, None, 0.962,), - (1, 24, None, 0.899,), - (1, 24.5, None, 0.897,), - (60, 24, None, 0.99,), # size 60 is out of bounds - (2, 12, None, 0.99,), - (None, 24, 0.8, 0.668,), - (None, 500, 0.90, 3.338,), - ([1.5], [24], None, 0.967,), - (1.5, 24, None, 0.967,), - (1, 24, None, 0.899,), - (1, [24.5], None, 0.897,), - ([1, 1.5], [24.5, 24], None, [0.897, 0.967],), - ([1, 60], [24.5, 24], None, [0.897, 0.99],), - (None, [24, 24], [0.8, 0.9], [0.668, 1.01],), - ([1, 1.5], [24.5, 24], None, [0.897, 0.967],), + (1.5, 24, None, 0.962), + (1, 24, None, 0.899), + (1, 24.5, None, 0.897), + (60, 24, None, 0.99), # size 60 is out of bounds + (2, 12, None, 0.99), + (None, 24, 0.8, 0.668), + (None, 500, 0.90, 3.338), + ([1.5], [24], None, 0.967), + (1.5, 24, None, 0.967), + (1, 24, None, 0.899), + (1, [24.5], None, 0.897), + ([1, 1.5], [24.5, 24], None, [0.897, 0.967]), + ([1, 60], [24.5, 24], None, [0.897, 0.99]), + (None, [24, 24], [0.8, 0.9], [0.668, 1.01]), + ([1, 1.5], [24.5, 24], None, [0.897, 0.967]), ], ) def test_nomo_single_list_roundtrip(vol_nomo, size, ddt, performance, exp): diff --git a/nereid/nereid/tests/test_src/test_treatment_facility/test_tasks.py b/nereid/nereid/tests/test_src/test_treatment_facility/test_tasks.py index f498745c..3ae4a2e9 100644 --- a/nereid/nereid/tests/test_src/test_treatment_facility/test_tasks.py +++ b/nereid/nereid/tests/test_src/test_treatment_facility/test_tasks.py @@ -1,6 +1,6 @@ import pytest -from nereid.src.treatment_facility.tasks import initialize_treatment_facilities +from nereid.src.tasks import initialize_treatment_facilities @pytest.mark.parametrize( @@ -49,7 +49,12 @@ def test_construct_nodes_from_treatment_facility_request( ], ) def test_construct_nodes_from_treatment_facility_request_checkval( - contexts, valid_treatment_facility_dicts, ctxt_key, has_met_data, model, checkfor, + contexts, + valid_treatment_facility_dicts, + ctxt_key, + has_met_data, + model, + checkfor, ): context = contexts[ctxt_key] diff --git a/nereid/nereid/tests/test_src/test_watershed/test_solve_watershed.py b/nereid/nereid/tests/test_src/test_watershed/test_solve_watershed.py index 61ce2e26..9d641013 100644 --- a/nereid/nereid/tests/test_src/test_watershed/test_solve_watershed.py +++ b/nereid/nereid/tests/test_src/test_watershed/test_solve_watershed.py @@ -294,7 +294,28 @@ def test_solve_watershed_stable_with_subsets( "flow_nomograph": "nomographs/100_LAGUNABEACH_flow_nomo.csv", "retention_volume_cuft": 2000.0, "retention_ddt_hr": 0.22222, - "node_type": "volume_based_facility_retention_only", + "node_type": "volume_based_facility", + }, + { + "facility_type": "dry_well", + "design_storm_depth_inches": 0.85, + "is_online": True, + "tributary_area_tc_min": 0.0, # bad tc + "total_volume_cuft": 6000.0, + "treatment_rate_cfs": 0.5, + "constructor": "dry_well_facility_constructor", + "validation_fallback": "NTFacility", + "validator": "DryWellFacility", + "tmnt_performance_facility_type": "¯\\_(ツ)_/¯", + "valid_model": "DryWellFacility", + "subbasin": "10101000", + "rain_gauge": "100_LAGUNABEACH", + "et_zone": "Zone4", + "volume_nomograph": "nomographs/100_LAGUNABEACH_volume_nomo.csv", + "flow_nomograph": "nomographs/100_LAGUNABEACH_flow_nomo.csv", + "retention_volume_cuft": 4000.0, + "retention_ddt_hr": 0.22222, + "node_type": "dry_well_facility", }, { "facility_type": "dry_weather_diversion", diff --git a/nereid/nereid/tests/test_src/test_watershed/test_solve_watershed_sequence.py b/nereid/nereid/tests/test_src/test_watershed/test_solve_watershed_sequence.py index 2b51efd5..2b5260cd 100644 --- a/nereid/nereid/tests/test_src/test_watershed/test_solve_watershed_sequence.py +++ b/nereid/nereid/tests/test_src/test_watershed/test_solve_watershed_sequence.py @@ -4,10 +4,8 @@ import pandas import pytest -from nereid.src.land_surface.tasks import land_surface_loading -from nereid.src.network.tasks import solution_sequence from nereid.src.network.utils import graph_factory, nxGraph_to_dict -from nereid.src.watershed.tasks import solve_watershed +from nereid.src.tasks import land_surface_loading, solution_sequence, solve_watershed from nereid.src.watershed.utils import attrs_to_resubmit from nereid.tests.utils import check_results_dataframes @@ -50,16 +48,18 @@ def test_watershed_solve_sequence(contexts, watershed_requests, n_nodes, pct_tmn subg_request.update(subgraph) subg_request.update(previous_results) - subgraph_response_dict = solve_watershed(subg_request, False, context=context,) + subgraph_response_dict = solve_watershed(subg_request, False, context=context) subgraph_results = subgraph_response_dict["results"] presults.extend(subgraph_results) db = db.combine_first(pandas.DataFrame(subgraph_results).set_index("node_id")) response_dict = solve_watershed( - watershed=watershed_request, treatment_pre_validated=False, context=context, + watershed=watershed_request, + treatment_pre_validated=False, + context=context, ) results = response_dict["results"] + response_dict["leaf_results"] - check_db = pandas.DataFrame(results).set_index("node_id").sort_index(0) - check_results_dataframes(db.sort_index(0), check_db) + check_db = pandas.DataFrame(results).set_index("node_id").sort_index(axis=0) + check_results_dataframes(db.sort_index(axis=0), check_db) diff --git a/nereid/nereid/tests/test_src/test_watershed/test_tasks.py b/nereid/nereid/tests/test_src/test_watershed/test_tasks.py index c7a47f34..977d08d6 100644 --- a/nereid/nereid/tests/test_src/test_watershed/test_tasks.py +++ b/nereid/nereid/tests/test_src/test_watershed/test_tasks.py @@ -17,7 +17,9 @@ def test_solve_watershed_land_surface_only(contexts, watershed_requests, n_nodes watershed_request = deepcopy(watershed_requests[(n_nodes, pct_tmnt)]) context = contexts["default"] response_dict = solve_watershed( - watershed=watershed_request, treatment_pre_validated=False, context=context, + watershed=watershed_request, + treatment_pre_validated=False, + context=context, ) result = response_dict["results"] + response_dict["leaf_results"] outfall_results = [n for n in result if n["node_id"] == "0"][0] @@ -48,7 +50,7 @@ def test_solve_watershed_land_surface_only(contexts, watershed_requests, n_nodes assert abs(outfall_total - sum_individual) / outfall_total < 1e-15 -@pytest.mark.parametrize("pct_tmnt", [0.3, 0.6,]) +@pytest.mark.parametrize("pct_tmnt", [0.3, 0.6]) @pytest.mark.parametrize("n_nodes", [50, 100]) def test_solve_watershed_with_treatment( contexts, watershed_requests, n_nodes, pct_tmnt @@ -57,7 +59,9 @@ def test_solve_watershed_with_treatment( watershed_request = deepcopy(watershed_requests[(n_nodes, pct_tmnt)]) context = contexts["default"] response_dict = solve_watershed( - watershed=watershed_request, treatment_pre_validated=False, context=context, + watershed=watershed_request, + treatment_pre_validated=False, + context=context, ) result = response_dict["results"] + response_dict["leaf_results"] @@ -111,7 +115,9 @@ def test_stable_watershed_stable_subgraph_solutions( watershed_request = deepcopy(watershed_requests[(n_nodes, pct_tmnt)]) context = contexts["default"] response_dict = solve_watershed( - watershed=watershed_request, treatment_pre_validated=False, context=context, + watershed=watershed_request, + treatment_pre_validated=False, + context=context, ) results = response_dict["results"] @@ -134,7 +140,9 @@ def test_stable_watershed_stable_subgraph_solutions( new_request.update(previous_results) subgraph_response_dict = solve_watershed( - watershed=new_request, treatment_pre_validated=False, context=context, + watershed=new_request, + treatment_pre_validated=False, + context=context, ) subgraph_results = subgraph_response_dict["results"] @@ -179,7 +187,9 @@ def test_treatment_facility_waterbalance( watershed_request["treatment_facilities"] = [facility] response_dict = solve_watershed( - watershed=watershed_request, treatment_pre_validated=False, context=context, + watershed=watershed_request, + treatment_pre_validated=False, + context=context, ) results = response_dict["results"] diff --git a/nereid/nereid/tests/test_src/test_watershed/test_watershed_vs_swmm.py b/nereid/nereid/tests/test_src/test_watershed/test_watershed_vs_swmm.py index e317c224..c234cbe9 100644 --- a/nereid/nereid/tests/test_src/test_watershed/test_watershed_vs_swmm.py +++ b/nereid/nereid/tests/test_src/test_watershed/test_watershed_vs_swmm.py @@ -4,7 +4,7 @@ import pandas import pytest -from nereid.core.utils import get_request_context +from nereid.core.context import get_request_context from nereid.src.nomograph.nomo import load_nomograph_mapping from nereid.src.tmnt_performance.tasks import effluent_function_map from nereid.src.watershed.solve_watershed import solve_node diff --git a/nereid/nereid/tests/utils.py b/nereid/nereid/tests/utils.py index 06e1d3f4..3b1011d0 100644 --- a/nereid/nereid/tests/utils.py +++ b/nereid/nereid/tests/utils.py @@ -25,14 +25,15 @@ def get_payload(file): def poll_testclient_url(testclient, url, timeout=5, verbose=False): # pragma: no cover - ts = time.time() - timer = lambda: time.time() - ts + ts = time.perf_counter() + timer = lambda: time.perf_counter() - ts tries = 0 while timer() < timeout: response = testclient.get(url) - if response.json()["status"].lower() == "success": + status = response.json()["status"] + if status.lower() == "success": if verbose: print(f"\nget request polling tried: {tries} times") return response @@ -256,7 +257,7 @@ def generate_random_treatment_facility_request(node_list, context, ref_data_keys return {"treatment_facilities": nodes} -def generate_random_validated_treatment_facility_node(context,): # pragma: no cover +def generate_random_validated_treatment_facility_node(context): # pragma: no cover facility = generate_random_treatment_facility_request_node(context) validated_facility_model = validate_treatment_facility_models( @@ -320,7 +321,11 @@ def generate_random_graph_request(n_nodes, seed=0): # pragma: no cover def generate_random_watershed_solve_request_from_graph( - g, context, ref_data_keys, surface_keys, pct_tmnt=0.5, + g, + context, + ref_data_keys, + surface_keys, + pct_tmnt=0.5, ): request = {"graph": clean_graph_dict(g)} @@ -356,7 +361,12 @@ def generate_random_watershed_solve_request_from_graph( def generate_random_watershed_solve_request( - context, ref_data_keys, surface_keys, n_nodes=55, pct_tmnt=0.5, seed=42, + context, + ref_data_keys, + surface_keys, + n_nodes=55, + pct_tmnt=0.5, + seed=42, ): g = nx.relabel_nodes(nx.gnr_graph(n=n_nodes, p=0.0, seed=seed), lambda x: str(x)) diff --git a/nereid/requirements.txt b/nereid/requirements.txt index b15ad04e..158f5788 100644 --- a/nereid/requirements.txt +++ b/nereid/requirements.txt @@ -1,15 +1,15 @@ -scipy==1.7.0 -pandas==1.3.0 -networkx==2.5.1 +aiofiles==0.8.0 +celery==5.2.3 +fastapi==0.74.1 +graphviz==0.19.1 +jinja2==3.0.3 +matplotlib==3.5.1 +networkx==2.6.3 +orjson==3.6.7 +pandas==1.4.1 +pint==0.18 pydot==1.4.2 -graphviz==0.16 -matplotlib==3.4.2 -fastapi==0.65.3 -aiofiles==0.7.0 -celery==5.1.2 -jinja2==3.0.1 -redis==3.5.3 -orjson==3.5.4 -pyyaml==5.4.1 -pint==0.17 -python-dotenv==0.18.0 +python-dotenv==0.19.2 +pyyaml==6.0 +redis==4.1.4 +scipy==1.8.0 \ No newline at end of file diff --git a/nereid/requirements_server.txt b/nereid/requirements_server.txt index 8cd0fb25..c547158a 100644 --- a/nereid/requirements_server.txt +++ b/nereid/requirements_server.txt @@ -1,2 +1,2 @@ -uvicorn[standard]==0.13.4 +uvicorn[standard]>=0.12.0,<0.16.0 gunicorn==20.1.0 diff --git a/nereid/requirements_tests.txt b/nereid/requirements_tests.txt index 97156a71..9f17505f 100644 --- a/nereid/requirements_tests.txt +++ b/nereid/requirements_tests.txt @@ -1,7 +1,7 @@ -pytest==6.2.4 -coverage==5.5 -codecov==2.1.11 -requests==2.25.1 -mypy==0.910 -black==19.10b0 -isort==5.9.1 +black==22.1.0 +codecov==2.1.12 +coverage==6.3.2 +isort==5.10.1 +mypy==0.931 +pytest==7.0.1 +requests==2.27.1 \ No newline at end of file diff --git a/scripts/az-login.sh b/scripts/az-login.sh new file mode 100644 index 00000000..d9d50ac5 --- /dev/null +++ b/scripts/az-login.sh @@ -0,0 +1,5 @@ +#! /usr/bin/env sh + +set -e +source .env +az acr login --name $AZURE_CONTAINER_REGISTRY \ No newline at end of file diff --git a/scripts/build_deploy.sh b/scripts/build_deploy.sh index 293f1d86..1b1c0cf4 100644 --- a/scripts/build_deploy.sh +++ b/scripts/build_deploy.sh @@ -1,5 +1,6 @@ set -e export COMPOSE_FILE=docker-stack.yml +export COMPOSE_DOCKER_CLI_BUILD=1 docker-compose \ -f docker-compose.shared.depends.yml \ diff --git a/scripts/lint.sh b/scripts/lint.sh index 3c728973..6e9cbd42 100644 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -4,5 +4,5 @@ set -e set -x mypy nereid/nereid --install-types --non-interactive -black nereid --check +black nereid --check --diff isort nereid --check --diff diff --git a/setup.cfg b/setup.cfg index 785835c6..cc669239 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,9 +19,55 @@ classifiers = [options] packages = find: package_dir = - =nereid + = nereid include_package_data = true python_requires = >= 3.6 +install_requires = + python-dotenv>=0.14 + scipy>=1.5 + pandas>=1.1 + networkx>=2.5 + pyyaml>=5.4.1 + pint>=0.17 + matplotlib>=3.2.0 + graphviz + pydot + pydantic>=1.8.0 + orjson + pyyaml + +[options.extras_require] +app = + fastapi[all]>=0.70.0 + +async-app = + nereid[app] + celery>=5.0 + redis>=4.0.0 + +dev = + nereid[async-app] + black==22.1.0 + codecov + coverage>=6.0.0 + isort>=5.0.0 + mypy>=0.910 + pytest + requests + +[options.packages.find] +where = nereid + +[options.package_data] +nereid = + core/* + data/* + data/default_data/* + data/default_data/state/region/* + data/default_data/state/region/nomographs/* + static/* + static/logo/* + tests/test_data/* [isort] @@ -34,7 +80,7 @@ testpaths = nereid/nereid/tests [mypy] -plugins = pydantic.mypy +plugins = pydantic.mypy, numpy.typing.mypy_plugin strict_optional = True check_untyped_defs = True disallow_incomplete_defs = True @@ -46,8 +92,7 @@ warn_unreachable = True [mypy-nereid.tests.*] ignore_errors = True -[mypy-numpy.*] -ignore_missing_imports = True + [mypy-scipy.*] ignore_missing_imports = True diff --git a/setup.py b/setup.py index d7750edd..52655f12 100644 --- a/setup.py +++ b/setup.py @@ -9,25 +9,4 @@ author_email = re.search(r'__email__ = "(.*?)"', content).group(1) -setup( - version=version, - author=author, - author_email=author_email, - install_requires=[ - "python-dotenv>=0.14,<0.19", - "scipy>=1.5,<1.8", - "pandas>=1.1,<1.4", - "networkx>=2.5,<2.6", - "pydot>=1.4,<1.5", - "graphviz", - "matplotlib>=3.3,<3.5", - "fastapi>=0.65.3,<0.66", - "aiofiles>=0.6,<0.8", - "celery>=5.0,<5.2", - "jinja2>=2.11,<3.1", - "redis>=3.5,<3.6", - "orjson>=3.4,<3.6", - "pyyaml>=5.4.1,<5.5", - "pint>=0.16,<0.18", - ], -) +setup(version=version, author=author, author_email=author_email)