feat(repositories): Working MetOffice Datahub Implementation

.gitignore

@@ -53,6 +53,7 @@ docs/
 # Environments
 .venv
 uv.lock
+.env
 
 # mypy
 .mypy_cache/

src/nwp_consumer/internal/entities/parameters.py

@@ -160,7 +160,7 @@ def metadata(self) -> ParameterData:
                                 "to the equilibrium vapour pressure of water",
                     units="%",
                     limits=ParameterLimits(upper=100, lower=0),
-                    alternate_shortnames=["r"],
+                    alternate_shortnames=["r", "r2"],
                 )
             case self.VISIBILITY_SL.name:
                 return ParameterData(

src/nwp_consumer/internal/entities/repometadata.py

@@ -15,6 +15,7 @@
 
 import dataclasses
 import datetime as dt
+import os
 
 import pandas as pd
 
@@ -144,6 +145,14 @@ def month_its(self, year: int, month: int) -> list[dt.datetime]:
                 its.append(dt.datetime(year, month, day, hour, tzinfo=dt.UTC))
         return its
 
+    def missing_required_envs(self) -> list[str]:
+        """Get a list of unset required environment variables.
+
+        Returns:
+            A list of missing environment variables.
+        """
+        return [var for var in self.required_env if var not in os.environ]
+
     def __str__(self) -> str:
         """Return a pretty-printed string representation of the metadata."""
         pretty: str = "".join((

src/nwp_consumer/internal/repositories/model_repositories/ceda_ftp.py

@@ -203,9 +203,11 @@ def authenticate(cls) -> ResultE["CEDAFTPModelRepository"]:
         Returns:
             A Result containing the instantiated class if successful, or an error if not.
         """
-        if all(k not in os.environ for k in cls.repository().required_env):
-            return Failure(ValueError(
-                f"Missing required environment variables: {cls.repository().required_env}",
+        missing_envs = cls.repository().missing_required_envs()
+        if len(missing_envs) > 0:
+            return Failure(OSError(
+                f"Cannot authenticate with CEDA FTP service due to "
+                f"missing required environment variables: {', '.join(missing_envs)}",
             ))
         username: str = urllib.parse.quote(os.environ["CEDA_FTP_USER"])
         password: str = urllib.parse.quote(os.environ["CEDA_FTP_PASS"])

src/nwp_consumer/internal/repositories/model_repositories/ecmwf_realtime.py

@@ -158,6 +158,12 @@ def fetch_init_data(self, it: dt.datetime) \
     @classmethod
     @override
     def authenticate(cls) -> ResultE["ECMWFRealTimeS3ModelRepository"]:
+        missing_envs = cls.repository().missing_required_envs()
+        if len(missing_envs) > 0:
+            return Failure(OSError(
+                f"Cannot authenticate with ECMWF Realtime S3 service due to "
+                f"missing required environment variables: {', '.join(missing_envs)}",
+            ))
         try:
             bucket: str = os.environ["ECMWF_REALTIME_S3_BUCKET"]
             _fs: s3fs.S3FileSystem = s3fs.S3FileSystem(

src/nwp_consumer/internal/repositories/model_repositories/mo_datahub.py

@@ -9,7 +9,24 @@
 Documented Structure
 --------------------
 
-TODO: Document filestructure
+MetOffice provide a number of models, a few of which OCF consume. Their flagship deterministic
+model us called the "Unified Model" (UM) and is run in two configurations: "Global" and "UK".
+The "Global" model has a resolution of 10km and the "UK" model has a resolution of 2km.
+
+See `https://datahub.metoffice.gov.uk/docs/f/category/atmospheric/overview`_ for more information.
+
+Data is provided on a per-order basis, so the filestructure depends on the order ID.
+For OCF's purposes, on file per parameter per step is requested.
+
+Actual Structure
+----------------
+
+The latitude and longitude increments are ascertained from the GRIB2 file's metadata:
+.. code-block:: none
+
+    iDirectionIncrementInDegrees: 0.140625
+    jDirectionIncrementInDegrees: 0.09375
+
 """
 
 import datetime as dt
@@ -53,6 +70,7 @@ def __init__(self, order_id: str, api_key: str) -> None:
         self.order_id = order_id
         self.request_url = f"{self.base_url}/{self.order_id}/latest"
 
+
     @staticmethod
     @override
     def repository() -> entities.ModelRepositoryMetadata:
@@ -73,7 +91,7 @@ def repository() -> entities.ModelRepositoryMetadata:
     def model() -> entities.ModelMetadata:
         return entities.ModelMetadata(
             name="UM-Global",
-            resolution="17km",
+            resolution="10km",
             expected_coordinates=entities.NWPDimensionCoordinateMap(
                 init_time=[],
                 step=list(range(0, 55)),
@@ -93,10 +111,12 @@ def model() -> entities.ModelMetadata:
                     ],
                 ),
                 latitude=[
-                    float(f"{lat:.4f}") for lat in np.arange(89.856, -89.856 - 0.156, -0.156)
+                    float(f"{lat:.4f}")
+                    for lat in np.arange(89.953125, -89.953125 - 0.09375, -0.09375)
                 ],
                 longitude=[
-                    float(f"{lon:.4f}") for lon in np.concatenate([np.arange(-179.87, 180, 0.234)])
+                    float(f"{lon:.4f}")
+                    for lon in np.arange(-179.929687, 179.929688 + 0.140625, 0.140625)
                 ],
             ),
         )
@@ -105,9 +125,11 @@ def model() -> entities.ModelMetadata:
     @override
     def authenticate(cls) -> ResultE["MetOfficeDatahubModelRepository"]:
         """Authenticate with the MetOffice DataHub service."""
-        if all(k not in os.environ for k in cls.repository().required_env):
-            return Failure(ValueError(
-                f"Missing required environment variables: {cls.repository().required_env}",
+        missing_envs = cls.repository().missing_required_envs()
+        if len(missing_envs) > 0:
+            return Failure(OSError(
+                f"Cannot authenticate with MetOffice DataHub service due to "
+                f"missing required environment variables: {', '.join(missing_envs)}",
             ))
         api_key: str = os.environ["METOFFICE_API_KEY"]
         order_id: str = os.environ["METOFFICE_ORDER_ID"]
@@ -250,10 +272,10 @@ def _convert(path: pathlib.Path) -> ResultE[list[xr.DataArray]]:
                 ),
             )
 
-        # Wind parameters are surfaced in the dataset as 'unknown'
+        # Some parameters are surfaced in the dataset as 'unknown'
         # and have to be differentiated via the parameterNumber attribute
         # which lines up with the last number in the GRIB2 code specified below
-        # https://datahub.metoffice.gov.uk/docs/glossary?groups=Wind&sortOrder=GRIB2_CODE
+        # https://datahub.metoffice.gov.uk/docs/glossary?sortOrder=GRIB2_CODE
         name = next(iter(ds.data_vars))
         parameter_number = ds[name].attrs["GRIB_parameterNumber"]
         match name, parameter_number:
@@ -265,10 +287,12 @@ def _convert(path: pathlib.Path) -> ResultE[list[xr.DataArray]]:
                 ds = ds.rename({name: "wdir"})
             case "unknown", 195:
                 ds = ds.rename({name: "wdir10"})
+            case "unknown", 1:
+                ds = ds.rename({name: "tcc"})
             case "unknown", _:
                 log.warning(
-                    "Encountered unknown parameter with parameterNumber %s",
-                    parameter_number,
+                    f"Encountered unknown parameter with parameterNumber {parameter_number} "
+                    f"in file '{path}'.",
                 )
 
         try:
@@ -297,8 +321,9 @@ def _convert(path: pathlib.Path) -> ResultE[list[xr.DataArray]]:
         except Exception as e:
             return Failure(
                 ValueError(
-                    f"Error processing {path} to DataArray: {e}",
+                    f"Error processing DataArray for path '{path}'. Error context: {e}",
                 ),
             )
 
+
         return Success([da])