Ao develop

.pre-commit-config.yaml

@@ -4,7 +4,7 @@ default_language_version:
   python: python3.11
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
       - id: check-added-large-files
       - id: check-toml
@@ -14,14 +14,14 @@ repos:
       - id: end-of-file-fixer
       - id: trailing-whitespace
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.15.0
+    rev: v3.17.0
     hooks:
       - id: pyupgrade
         args:
           - --py3-plus
           - --keep-runtime-typing
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.1.11
+    rev: v0.6.7
     hooks:
       - id: ruff
         args:

Makefile

@@ -86,10 +86,10 @@ build:
 
 develop: clean stack build ## build the development environment and launch containers in background
 
-up: stack ## bring up the containers in '-d' mode
+up: stack
 	docker compose up
 
-up-d: stack
+up-d: stack ## bring up the containers in '-d' mode
 	docker compose up -d
 
 down: ## bring down the containers and detach volumes

docker-compose.build.yml

@@ -1,4 +1,3 @@
-version: "3.7"
 services:
   nereid:
     build:

docker-compose.dev.yml

@@ -1,4 +1,3 @@
-version: "3.7"
 services:
   nereid:
     command: /start-reload.sh

docker-compose.edge.build.yml

@@ -1,4 +1,3 @@
-version: "3.7"
 services:
   nereid:
     build:

docker-compose.image.yml

@@ -1,4 +1,3 @@
-version: "3.7"
 services:
   nereid:
     image: ${NEREID_ARTIFACT_REGISTRY_PATH:-nereid-}nereid:${NEREID_IMAGE_TAG:-latest}

nereid/Dockerfile.multi

@@ -4,27 +4,27 @@
 # pattern is able to reduce each image to ~250MB but takes considerable
 # time to build and is considerably more complex for scipy and pandas.
 
-FROM node:lts-bullseye as frontend
+FROM node:lts-bullseye AS frontend
 WORKDIR /app
 COPY ./nereid/static/frontend .
 RUN npm install . && npm run build
 CMD ["bash", "-c", "while true; do sleep 1; done"]
 
 
-FROM python:3.11-bullseye as nereid_install
+FROM python:3.11.10-bullseye AS nereid_install
 RUN apt-get update -y \
     && apt-get install -y --no-install-recommends graphviz \
     && rm -rf /var/lib/apt/lists/*
 WORKDIR /nereid
 CMD ["bash", "-c", "while true; do sleep 1; done"]
 
 
-FROM redis:6.2.13-alpine3.18 as redis
+FROM redis:6.2.14-alpine3.20 AS redis
 COPY redis.conf /redis.conf
 CMD ["redis-server", "/redis.conf"]
 
 
-FROM python:3.11.7-alpine3.19 as flower
+FROM python:3.11.10-alpine3.19 AS flower
 RUN apk add --no-cache ca-certificates tzdata && update-ca-certificates
 RUN pip install --no-cache-dir redis==4.6.0 flower==1.0.0 celery==5.3.4
 ENV PYTHONUNBUFFERED=1 PYTHONHASHSEED=random PYTHONDONTWRITEBYTECODE=1
@@ -45,106 +45,81 @@ EXPOSE 5555
 CMD ["celery", "flower"]
 
 
-FROM python:3.11.7-bullseye as builder
-COPY requirements/requirements_dev.txt /requirements_dev.txt
+FROM python:3.11.10-bullseye AS core-env
+RUN apt-get update && apt-get install -y build-essential curl
+ADD https://astral.sh/uv/0.4.16/install.sh /install.sh
+RUN sh /install.sh && rm /install.sh
+ENV VIRTUAL_ENV=/opt/venv \
+    PATH=/opt/venv/bin:/root/.cargo/bin/:$PATH
+COPY requirements/requirements_worker.txt /requirements_worker.txt
+RUN uv venv /opt/venv && \
+    uv pip install --no-cache -r /requirements_worker.txt
+
+
+FROM core-env AS server-env
+COPY requirements/requirements_nereid.txt /requirements_nereid.txt
 COPY requirements/requirements_server.txt /requirements_server.txt
-RUN mkdir /wheels && \
-    pip wheel \
-    --wheel-dir=/wheels \
-    -r /requirements_dev.txt \
+RUN  uv pip install --no-cache \
+    -r /requirements_nereid.txt \
     -r /requirements_server.txt
 
 
-FROM python:3.11.7-slim-bullseye as core-runtime
+FROM core-env AS test-env
+COPY requirements/requirements_dev.txt /requirements_dev.txt
+RUN  uv pip install --no-cache \
+    -r /requirements_dev.txt
+
+
+FROM python:3.11.10-slim-bullseye AS core-runtime
 RUN apt-get update -y \
     && apt-get install -y --no-install-recommends graphviz \
     && rm -rf /var/lib/apt/lists/*
 WORKDIR /nereid
-COPY ./scripts /
-RUN chmod +x /start.sh /start-reload.sh /prestart.sh /prestart-worker.sh /run-worker.sh
 ENV PYTHONPATH=/nereid
 ENV PATH=/opt/venv/bin:$PATH
-COPY ./nereid /nereid/nereid
-COPY --from=frontend /app/dist /nereid/nereid/static/frontend/dist
-
-
-# you are here, trying to get the requirements loaded, and rearranging the shared containers.
-FROM python:3.11.7-slim-bullseye as core-env
-COPY --from=builder /wheels /core
-COPY requirements/requirements_worker.txt /requirements_worker.txt
-RUN python -m venv /opt/venv
-# Make sure we use the virtualenv:
-ENV PATH=/opt/venv/bin:$PATH
-RUN pip install \
-    --no-index \
-    --no-cache-dir \
-    --find-links=/core \
-    -r /requirements_worker.txt \
-    && rm -rf /core/*
 
 
-FROM core-runtime as celeryworker
+FROM core-runtime AS celeryworker
 # Add a user with an explicit UID/GID and create necessary directories
 ENV IMG_USER=celeryworker
 RUN addgroup --gid 1000 ${IMG_USER} \
-    && adduser --system --disabled-password --uid 1000 --gid 1000 ${IMG_USER} \
-    && chown -R ${IMG_USER}:${IMG_USER} /nereid
+    && adduser --no-create-home --system --disabled-password --uid 1000 --gid 1000 ${IMG_USER}
 USER ${IMG_USER}
 COPY --from=core-env --chown=${IMG_USER} /opt/venv /opt/venv
-CMD /run-worker.sh
-
-
-FROM core-env as server-env
-COPY requirements/requirements_nereid.txt /requirements_nereid.txt
-COPY requirements/requirements_server.txt /requirements_server.txt
-COPY --from=builder /wheels /core
-RUN python -m venv /opt/venv
-# Make sure we use the virtualenv:
-ENV PATH=/opt/venv/bin:$PATH
-RUN pip install \
-    --no-index \
-    --no-cache-dir \
-    --find-links=/core \
-    -r /requirements_nereid.txt \
-    -r /requirements_server.txt \
-    && rm -rf /serve/*
+COPY --chmod=+x ./scripts /
+COPY --chown=${IMG_USER}:${IMG_USER} ./nereid /nereid/nereid
+CMD ["bash", "/run-worker.sh"]
 
 
-FROM core-runtime as nereid
+FROM core-runtime AS nereid
 COPY --from=server-env /opt/venv /opt/venv
+COPY --chmod=+x ./scripts /
+COPY ./nereid /nereid/nereid
+COPY --from=frontend /app/dist /nereid/nereid/static/frontend/dist
 COPY gunicorn_conf.py /gunicorn_conf.py
 EXPOSE 80
-CMD /start.sh
-
-
-FROM core-env as test-env
-COPY requirements/requirements_dev.txt /requirements_dev.txt
-COPY --from=builder /wheels /core
-RUN python -m venv /opt/venv
-# Make sure we use the virtualenv:
-ENV PATH=/opt/venv/bin:$PATH
-RUN pip install \
-    --no-index \
-    --no-cache-dir \
-    --find-links=/core \
-    -r /requirements_dev.txt \
-    && rm -rf /tsts/*
+CMD ["bash", "/start.sh"]
 
 
-FROM core-runtime as nereid-tests
+FROM core-runtime AS nereid-tests
 COPY --from=test-env /opt/venv /opt/venv
 COPY .coveragerc /nereid/.coveragerc
 COPY conftest.py /nereid/conftest.py
+COPY --chmod=+x ./scripts /
 ## This will make the container wait, doing nothing, but alive
 CMD ["bash", "-c", "while true; do sleep 1; done"]
 
 
-FROM python:3.12-bullseye as nereid-edge
-RUN apt-get update && apt-get install -y graphviz
-COPY requirements/requirements_dev.txt /requirements_dev.txt
-RUN awk -F"==" '{print $1}' /requirements_dev.txt > /requirements_edge.txt
-RUN cat requirements_edge.txt
-RUN pip install -r /requirements_edge.txt
+FROM python:3.12-bullseye AS nereid-edge
+RUN apt-get update -y \
+    && apt-get install -y graphviz build-essential curl \
+    && rm -rf /var/lib/apt/lists/*
+ADD https://astral.sh/uv/0.4.16/install.sh /install.sh
+RUN sh /install.sh && rm /install.sh
+ENV PATH=/opt/venv/bin:/root/.cargo/bin/:$PATH
+COPY ./requirements /
+RUN uv venv /opt/venv && \
+    uv pip install --no-cache -r /requirements_dev_unpinned.txt
 COPY ./nereid /nereid/nereid
 COPY --from=frontend /app/dist /nereid/nereid/static/frontend/dist
 COPY ./scripts /
@@ -153,10 +128,10 @@ WORKDIR /nereid
 ENV PYTHONPATH=/nereid
 
 
-FROM nereid-edge as nereid-edge-tests
+FROM nereid-edge AS nereid-edge-tests
 CMD ["bash", "-c", "while true; do sleep 1; done"]
 
 
-FROM nereid-edge as celeryworker-edge
+FROM nereid-edge AS celeryworker-edge
 ENV C_FORCE_ROOT=1
 CMD /run-worker.sh

nereid/nereid/__init__.py

@@ -1,4 +1,3 @@
-""""Water quality engine for computing graph-based land surface load aggregation and BMP effectiveness."""
-
+"""Water quality engine for computing graph-based land surface load aggregation and BMP effectiveness."""
 
 __version__ = "0.10.1"

nereid/nereid/api/docs.py

@@ -5,7 +5,10 @@
 # Swagger UI includes a supermassive curl div that is not height constrained.
 # Not sure how this hasn't been fixed, but this patches it for this project.
 def get_better_swagger_ui_html(*args, **kwargs) -> HTMLResponse:
-    body_lines = get_swagger_ui_html(**kwargs).body.decode().split("\n")
+    body = get_swagger_ui_html(**kwargs).body
+    if isinstance(body, memoryview):  # pragma: no cover
+        body = body.tobytes()
+    body_lines = body.decode().split("\n")
     ix = 4
     for i, line in enumerate(body_lines):
         if 'type="text/css" rel="stylesheet"' in line:

nereid/nereid/core/config.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 from typing import Any, Literal
 
-from pydantic import AnyHttpUrl, field_validator
+from pydantic import AnyHttpUrl, TypeAdapter, field_validator
 from pydantic_settings import BaseSettings
 
 import nereid
@@ -43,7 +43,9 @@ def create_logfile_path(cls, v: str | None) -> str | None:  # pragma: no cover
     @field_validator("ALLOW_CORS_ORIGINS", mode="after")
     @classmethod
     def stringify_cors_origins(cls, v: list[str]) -> list[str]:  # pragma: no cover
-        return [str(AnyHttpUrl(url) if url != "*" else url) for url in v]
+        anyurl = TypeAdapter(AnyHttpUrl)
+        ls = [str(anyurl.validate_python(url) if url != "*" else url) for url in v]
+        return ls
 
     model_config = {
         "env_prefix": "NEREID_",

nereid/nereid/core/log.py

@@ -1,4 +1,5 @@
 """ref: mCoding json config https://github.com/mCodingLLC/VideosSampleCode/blob/master/videos/135_modern_logging/mylogger.py"""
+
 import datetime as dt
 import json
 import logging
@@ -71,7 +72,7 @@ def _prepare_log_dict(self, record: logging.LogRecord):
 
         for key, val in record.__dict__.items():
             if key not in LOG_RECORD_BUILTIN_ATTRS:
-                message[key] = val
+                message[key] = val  # noqa [PERF403]
 
         return message
 

nereid/nereid/models/network_models.py

@@ -1,6 +1,5 @@
-from typing import Annotated
-
 from pydantic import BaseModel, Field, StrictStr, model_validator
+from typing_extensions import Annotated, Self
 
 from nereid.models.response_models import JSONAPIResponse
 from nereid.src.tasks import validate_network
@@ -125,7 +124,7 @@ class Graph(BaseModel):
 
 class ValidGraph(Graph):
     @model_validator(mode="after")
-    def validate_graph(self) -> "ValidGraph":
+    def validate_graph(self) -> Self:
         result = validate_network(self.model_dump(by_alias=True))
         assert result.get("isvalid"), result
         return self

nereid/nereid/models/treatment_site_models.py

@@ -8,7 +8,7 @@
 
 class TreatmentSite(SimpleFacilityBase):
     area_pct: float = Field(0.0, le=100.0, ge=0.0)
-    retained_pct: float = Field(0.0, le=100.0, ge=0.0)
+    retained_pct: float = Field(0.0, le=100.0, ge=0.0)  # type: ignore
     eliminate_all_dry_weather_flow_override: bool = False
 
 

nereid/nereid/models/watershed_models.py

@@ -396,9 +396,9 @@
 class Watershed(BaseModel):
     graph: Graph
     land_surfaces: list[LandSurface] | None = None
-    treatment_facilities: list[dict[str, Any]] | list[
-        STRUCTURAL_FACILITY_TYPE
-    ] | None = None
+    treatment_facilities: (
+        list[dict[str, Any]] | list[STRUCTURAL_FACILITY_TYPE] | None
+    ) = None
     treatment_sites: list[TreatmentSite] | None = None
     previous_results: list[PreviousResult] | None = None
 

nereid/nereid/src/land_surface/loading.py

@@ -1,4 +1,4 @@
-from typing import Iterable
+from typing import Iterable, Mapping
 
 import numpy
 import pandas
@@ -67,7 +67,7 @@ def detailed_volume_loading_results(df: pandas.DataFrame) -> pandas.DataFrame:
 
 
 def detailed_dry_weather_volume_loading_results(
-    df: pandas.DataFrame, seasons: dict[str, Iterable[str] | None]
+    df: pandas.DataFrame, seasons: Mapping[str, Iterable[str] | None]
 ) -> pandas.DataFrame:
     """This function aggregates the dry weather flowrate (dwf) by season according
     to the config file spec.
@@ -140,10 +140,10 @@ def detailed_loading_results(
     land_surfaces_df: pandas.DataFrame,
     wet_weather_parameters: Iterable[dict[str, str]],
     dry_weather_parameters: Iterable[dict[str, str]],
-    seasons: dict[str, Iterable[str]],
+    seasons: dict[str, Iterable[str] | None],
 ) -> pandas.DataFrame:
     results = (
-        land_surfaces_df.pipe(clean_land_surface_dataframe)
+        land_surfaces_df.pipe(clean_land_surface_dataframe)  # type: ignore
         .pipe(detailed_volume_loading_results)
         .pipe(detailed_dry_weather_volume_loading_results, seasons)
         .pipe(

nereid/nereid/src/network/render.py

@@ -7,12 +7,12 @@
 import networkx as nx
 import orjson as json
 
-Axes = Any
-Figure = Any
-
 if TYPE_CHECKING:
     from matplotlib.axes import Axes
     from matplotlib.figure import Figure
+else:
+    Axes = Any
+    Figure = Any
 
 
 def pydot_layout(*args, **kwargs) -> dict:
@@ -224,7 +224,7 @@ def fig_to_image(fig: Figure, **kwargs: Any) -> IO:
     _kwargs.update(kwargs)
 
     img = BytesIO()
-    fig.savefig(img, **_kwargs)
+    fig.savefig(img, **_kwargs)  # type: ignore[arg-type]
     img.seek(0)
 
     return img