Fix CI tests

docs/source/conf.py

@@ -75,7 +75,7 @@ def repository_root(path: PathLike = None) -> Path:
     "sphinx.ext.napoleon",
     "sphinx_autodoc_typehints",
     "sphinxext.opengraph",
-    "m2r2",
+    "sphinx_mdinclude",
 ]
 
 # sphinx_autodoc_typehints settings

examples/CERA_workflow.ipynb

@@ -40,8 +40,8 @@
    },
    "outputs": [],
    "source": [
-    "world = gpd.read_file(gpd.datasets.get_path('naturalearth_lowres'))\n",
-    "us = world[world.name.isin(['United States of America', 'Puerto Rico'])]\n",
+    "world = gpd.read_file('https://naciscdn.org/naturalearth/110m/cultural/ne_110m_admin_0_countries.zip')\n",
+    "us = world[world.NAME.isin(['United States of America', 'Puerto Rico'])]\n",
     "us_coast = us.boundary.intersection(world.unary_union.boundary)\n",
     "ax = world.plot(color='k', alpha=0.1)\n",
     "us.plot(ax=ax, color='b', alpha=0.2)\n",
@@ -206,7 +206,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "python3",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -220,7 +220,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.16"
+   "version": "3.10.14"
   }
  },
  "nbformat": 4,

examples/USGS_by_id.ipynb

@@ -82,7 +82,7 @@
     "figure, axis = plt.subplots(1, 1)\n",
     "figure.set_size_inches(12, 12 / 1.61803398875)\n",
     "\n",
-    "countries = gpd.read_file(gpd.datasets.get_path(\"naturalearth_lowres\"))\n",
+    "countries = gpd.read_file('https://naciscdn.org/naturalearth/110m/cultural/ne_110m_admin_0_countries.zip')\n",
     "_ = countries.plot(color=\"lightgrey\", ax=axis, zorder=-1)\n",
     "_ = usgs_stations.plot(ax=axis)\n",
     "_ = axis.set_xlim(usgs_stations.total_bounds[0] - 1, usgs_stations.total_bounds[2] + 1)\n",
@@ -248,9 +248,9 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "searvey",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
-   "name": "searvey"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
@@ -262,7 +262,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.9"
+   "version": "3.10.14"
   }
  },
  "nbformat": 4,

examples/USGS_data.ipynb

@@ -69,7 +69,7 @@
     "figure, axis = plt.subplots(1, 1)\n",
     "figure.set_size_inches(12, 12 / 1.61803398875)\n",
     "\n",
-    "countries = gpd.read_file(gpd.datasets.get_path(\"naturalearth_lowres\"))\n",
+    "countries = gpd.read_file('https://naciscdn.org/naturalearth/110m/cultural/ne_110m_admin_0_countries.zip')\n",
     "_ = countries.plot(color=\"lightgrey\", ax=axis, zorder=-1)\n",
     "_ = usgs_stations.plot(ax=axis)\n",
     "_ = axis.set_xlim(usgs_stations.total_bounds[0] - 1, usgs_stations.total_bounds[2] + 1)\n",
@@ -200,7 +200,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "python3",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -214,7 +214,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.9"
+   "version": "3.10.14"
   }
  },
  "nbformat": 4,

examples/coops_data.ipynb

@@ -36,7 +36,7 @@
     "import geopandas\n",
     "from matplotlib import pyplot\n",
     "\n",
-    "countries = geopandas.read_file(geopandas.datasets.get_path('naturalearth_lowres'))"
+    "countries = geopandas.read_file('https://naciscdn.org/naturalearth/110m/cultural/ne_110m_admin_0_countries.zip')"
    ]
   },
   {
@@ -279,7 +279,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.9"
+   "version": "3.10.14"
   }
  },
  "nbformat": 4,

pyproject.toml

@@ -65,7 +65,7 @@ urllib3 = "<2"   # https://github.com/kevin1024/vcrpy/issues/688
 [tool.poetry.group.docs.dependencies]
 dunamai = "*"
 furo = "*"
-m2r2 = ">=0.3.2"
+sphinx-mdinclude = "*"
 setuptools = "*"
 sphinx = "*"
 sphinxext-opengraph = "*"

requirements/requirements.txt

@@ -1,50 +1,45 @@
 annotated-types==0.7.0 ; python_version >= "3.9" and python_version < "4.0"
 anyio==4.4.0 ; python_version >= "3.9" and python_version < "4.0"
-attrs==23.2.0 ; python_version >= "3.9" and python_version < "4.0"
 beautifulsoup4==4.12.3 ; python_version >= "3.9" and python_version < "4.0"
-certifi==2024.6.2 ; python_version >= "3.9" and python_version < "4.0"
+certifi==2024.8.30 ; python_version >= "3.9" and python_version < "4.0"
 charset-normalizer==3.3.2 ; python_version >= "3.9" and python_version < "4.0"
-click-plugins==1.1.1 ; python_version >= "3.9" and python_version < "4.0"
-click==8.1.7 ; python_version >= "3.9" and python_version < "4.0"
-cligj==0.7.2 ; python_version >= "3.9" and python_version < "4"
 colorama==0.4.6 ; python_version >= "3.9" and python_version < "4.0" and platform_system == "Windows"
-dataretrieval==1.0.9 ; python_version >= "3.9" and python_version < "4.0"
+dataretrieval==1.0.10 ; python_version >= "3.9" and python_version < "4.0"
 deprecated==1.2.14 ; python_version >= "3.9" and python_version < "4.0"
 erddapy==2.2.0 ; python_version >= "3.9" and python_version < "4.0"
-exceptiongroup==1.2.1 ; python_version >= "3.9" and python_version < "4.0"
-fiona==1.9.6 ; python_version >= "3.9" and python_version < "4.0"
-geopandas==0.14.4 ; python_version >= "3.9" and python_version < "4.0"
+exceptiongroup==1.2.2 ; python_version >= "3.9" and python_version < "4.0"
+geopandas==1.0.1 ; python_version >= "3.9" and python_version < "4.0"
 h11==0.14.0 ; python_version >= "3.9" and python_version < "4.0"
 html5lib==1.1 ; python_version >= "3.9" and python_version < "4.0"
 httpcore==1.0.5 ; python_version >= "3.9" and python_version < "4.0"
-httpx==0.27.0 ; python_version >= "3.9" and python_version < "4.0"
-idna==3.7 ; python_version >= "3.9" and python_version < "4.0"
-importlib-metadata==7.1.0 ; python_version >= "3.9" and python_version < "3.10"
-importlib-resources==6.4.0 ; python_version >= "3.9" and python_version < "4.0"
-limits==3.12.0 ; python_version >= "3.9" and python_version < "4.0"
-lxml-html-clean==0.1.1 ; python_version >= "3.9" and python_version < "4.0"
-lxml==5.2.2 ; python_version >= "3.9" and python_version < "4.0"
-lxml[html-clean]==5.2.2 ; python_version >= "3.9" and python_version < "4.0"
+httpx==0.27.2 ; python_version >= "3.9" and python_version < "4.0"
+idna==3.8 ; python_version >= "3.9" and python_version < "4.0"
+importlib-resources==6.4.4 ; python_version >= "3.9" and python_version < "4.0"
+limits==3.13.0 ; python_version >= "3.9" and python_version < "4.0"
+lxml-html-clean==0.2.2 ; python_version >= "3.9" and python_version < "4.0"
+lxml==5.3.0 ; python_version >= "3.9" and python_version < "4.0"
+lxml[html-clean]==5.3.0 ; python_version >= "3.9" and python_version < "4.0"
 multifutures==0.3.2 ; python_version >= "3.9" and python_version < "4.0"
-numpy==2.0.0 ; python_version >= "3.9" and python_version < "4.0"
+numpy==2.0.2 ; python_version >= "3.9" and python_version < "4.0"
 packaging==24.1 ; python_version >= "3.9" and python_version < "4.0"
 pandas==2.2.2 ; python_version >= "3.9" and python_version < "4.0"
-pydantic-core==2.18.4 ; python_version >= "3.9" and python_version < "4.0"
-pydantic==2.7.4 ; python_version >= "3.9" and python_version < "4.0"
+pydantic-core==2.20.1 ; python_version >= "3.9" and python_version < "4.0"
+pydantic==2.8.2 ; python_version >= "3.9" and python_version < "4.0"
+pyogrio==0.9.0 ; python_version >= "3.9" and python_version < "4.0"
 pyproj==3.6.1 ; python_version >= "3.9" and python_version < "4.0"
 python-dateutil==2.9.0.post0 ; python_version >= "3.9" and python_version < "4.0"
 pytz==2024.1 ; python_version >= "3.9" and python_version < "4.0"
 requests==2.32.3 ; python_version >= "3.9" and python_version < "4.0"
-shapely==2.0.4 ; python_version >= "3.9" and python_version < "4.0"
+shapely==2.0.6 ; python_version >= "3.9" and python_version < "4.0"
 six==1.16.0 ; python_version >= "3.9" and python_version < "4.0"
 sniffio==1.3.1 ; python_version >= "3.9" and python_version < "4.0"
-soupsieve==2.5 ; python_version >= "3.9" and python_version < "4.0"
-tenacity==8.4.1 ; python_version >= "3.9" and python_version < "4.0"
-tqdm==4.66.4 ; python_version >= "3.9" and python_version < "4.0"
+soupsieve==2.6 ; python_version >= "3.9" and python_version < "4.0"
+tenacity==9.0.0 ; python_version >= "3.9" and python_version < "4.0"
+tqdm==4.66.5 ; python_version >= "3.9" and python_version < "4.0"
 typing-extensions==4.12.2 ; python_version >= "3.9" and python_version < "4.0"
 tzdata==2024.1 ; python_version >= "3.9" and python_version < "4.0"
-urllib3==1.26.18 ; python_version >= "3.9" and python_version < "4.0"
+urllib3==1.26.20 ; python_version >= "3.9" and python_version < "4.0"
 webencodings==0.5.1 ; python_version >= "3.9" and python_version < "4.0"
 wrapt==1.16.0 ; python_version >= "3.9" and python_version < "4.0"
-xarray==2024.6.0 ; python_version >= "3.9" and python_version < "4.0"
-zipp==3.19.2 ; python_version >= "3.9" and python_version < "3.10"
+xarray==2024.7.0 ; python_version >= "3.9" and python_version < "4.0"
+zipp==3.20.1 ; python_version >= "3.9" and python_version < "3.10"

searvey/_coops_api.py

@@ -32,7 +32,7 @@
 
 # constants
 COOPS_URL_TS_FORMAT = "%Y%m%d %H:%M"
-COOPS_BASE_URL = "https://tidesandcurrents.noaa.gov/api/datagetter?"
+COOPS_BASE_URL = "https://api.tidesandcurrents.noaa.gov/api/prod/datagetter?"
 
 COOPS_ProductFieldTypes = {
     "bin": int,

searvey/coops.py

@@ -5,6 +5,7 @@
 import json
 import logging
 import warnings
+from collections import defaultdict
 from datetime import datetime
 from enum import Enum
 from functools import lru_cache
@@ -80,7 +81,7 @@ class COOPS_StationMetadataSource(Enum):
     "highwater",
     "lowwater",
     "hightideflooding",
-    "ofs",
+    "ofs",  # OFS is model, not actual obs!
     "partnerstations",
 ]
 
@@ -113,6 +114,7 @@ class COOPS_Product(Enum):  # noqa: N801
         # One minute water level data for the station.
     )
     PREDICTIONS = "predictions"  # 6 minute predictions water level data for the station.*
+    OFS_WATER_LEVEL = "ofs_water_level"  # Water level model guidance at 6-minute intervals based on NOS OFS models. Data available from 2020 to present.
     DATUMS = "datums"  # datums data for the stations.
     CURRENTS = "currents"  # Currents data for currents stations.
     CURRENTS_PREDICTIONS = (
@@ -121,6 +123,17 @@ class COOPS_Product(Enum):  # noqa: N801
     )
 
 
+DATE_FMT: dict[Optional[COOPS_Product], "str"] = {COOPS_Product.PREDICTIONS: "%Y%m%d"}
+DATE_FMT = defaultdict(lambda: "%Y%m%d %H:%M", DATE_FMT)
+
+
+RSP_STN_LIST: dict["str", "str"] = {
+    "ofs": "OFSStationList",
+    "partnerstations": "PartnerStationList",
+}
+RSP_STN_LIST = defaultdict(lambda: "stations", RSP_STN_LIST)
+
+
 class COOPS_Interval(Enum):  # noqa: N801
     H = "h"  # Hourly Met data and harmonic predictions will be returned
     HILO = "hilo"  # High/Low tide predictions for all stations.
@@ -251,26 +264,20 @@ def constituents(self) -> DataFrame:
         :return: table of tidal constituents for the current station
         """
 
-        url = f"https://tidesandcurrents.noaa.gov/harcon.html?id={self.id}"
-        response = requests.get(url)
-        soup = BeautifulSoup(response.content, features="html.parser")
-        table = soup.find_all("table", {"class": "table table-striped"})
-        if len(table) > 0:
-            table = table[0]
-            columns = [field.text for field in table.find("thead").find("tr").find_all("th")]
-            constituents = []
-            for row in table.find_all("tr")[1:]:
-                constituents.append([entry.text for entry in row.find_all("td")])
-            constituents = DataFrame(constituents, columns=columns)
-            constituents.rename(columns={"Constituent #": "#"}, inplace=True)
-            constituents = constituents.astype(
-                {
-                    "#": numpy.int32,
-                    "Amplitude": numpy.float64,
-                    "Phase": numpy.float64,
-                    "Speed": numpy.float64,
+        url = f"https://api.tidesandcurrents.noaa.gov/mdapi/prod/webapi/stations/{self.id}/harcon.json"
+        df_harcon = pandas.read_json(url)
+        df_harcon = pandas.json_normalize(df_harcon["HarmonicConstituents"])
+
+        if len(df_harcon) > 0:
+            # TODO: Should local or GMT phase be used?
+            constituents = df_harcon.rename(
+                columns={
+                    "number": "#",
+                    "amplitude": "Amplitude",
+                    "phase_GMT": "Phase",
+                    "speed": "Speed",
                 }
-            )
+            )[["#", "Amplitude", "Phase", "Speed"]]
         else:
             constituents = DataFrame(columns=["#", "Amplitude", "Phase", "Speed"])
 
@@ -369,7 +376,7 @@ class COOPS_Query:
     https://api.tidesandcurrents.noaa.gov/api/prod/
     """
 
-    URL = "https://tidesandcurrents.noaa.gov/api/datagetter?"
+    URL = "https://api.tidesandcurrents.noaa.gov/api/prod/datagetter?"
 
     def __init__(
         self,
@@ -513,9 +520,10 @@ def query(self) -> Dict[str, Any]:
         product = self.product
         if isinstance(product, Enum):
             product = product.value
+        date_fmt = DATE_FMT[COOPS_Product(product)]
         start_date = self.start_date
         if start_date is not None and not isinstance(start_date, str):
-            start_date = f"{self.start_date:%Y%m%d %H:%M}"
+            start_date = f"{self.start_date:{date_fmt}}"
         datum = self.datum
         if isinstance(datum, Enum):
             datum = datum.value
@@ -533,7 +541,7 @@ def query(self) -> Dict[str, Any]:
             "station": self.station_id,
             "product": product,
             "begin_date": start_date,
-            "end_date": f"{self.end_date:%Y%m%d %H:%M}",
+            "end_date": f"{self.end_date:{date_fmt}}",
             "datum": datum,
             "units": units,
             "time_zone": time_zone,
@@ -900,6 +908,12 @@ def coops_product_within_region(
 
 
 def normalize_coops_stations(df: pandas.DataFrame) -> geopandas.GeoDataFrame:
+    # To avoid duplicate labels
+    if "lon" in df.columns:
+        df.loc[df.lon.notnull(), "lng"] = df.lon[df.lon.notnull()]
+    if "stationID" in df.columns:
+        df.loc[df.stationID.notnull(), "id"] = df.stationID[df.stationID.notnull()]
+    df = df.drop(columns=["lon", "stationID"])
     df = df.rename(
         columns={
             "id": "nos_id",
@@ -946,7 +960,9 @@ def _get_single_coops_station(station_type: str) -> pd.DataFrame:
     url = f"https://api.tidesandcurrents.noaa.gov/mdapi/prod/webapi/stations.json?expand=details&type={station_type}"
 
     df_thistype = pandas.read_json(url)
-    df_thistype = pandas.json_normalize(df_thistype["stations"])
+    data_col = RSP_STN_LIST[station_type]
+    df_thistype = pandas.json_normalize(df_thistype[data_col])
+
     df_thistype["station_type"] = station_type
 
     return df_thistype