feat(repositories): More granular grib tests

openclimatefix · Nov 27, 2024 · 0f45e00 · 0f45e00
1 parent eb0f6d4
commit 0f45e00
Show file tree

Hide file tree

Showing 14 changed files with 87 additions and 44 deletions.
diff --git a/src/nwp_consumer/internal/repositories/model_repositories/test_ceda_ftp.py b/src/nwp_consumer/internal/repositories/model_repositories/test_ceda_ftp.py
@@ -78,39 +78,49 @@ def test__convert(self) -> None:
         class TestCase:
             filename: str
             should_error: bool
+            expected_coords: entities.NWPDimensionCoordinateMap
 
         tests: list[TestCase] = [
             TestCase(
-                filename="test_UM-Global_10u-AreaC.grib",
+                filename="test_CEDAFTP_UM-Global_ssrd_20241105T00_S01-03.grib",
+                expected_coords = dataclasses.replace(
+                    CEDAFTPModelRepository.model().expected_coordinates,
+                    init_time=[dt.datetime(2024, 11, 5, 0, tzinfo=dt.UTC)],
+                    step=[1, 2, 3],
+                    variable=[entities.parameters.Parameter.DOWNWARD_SHORTWAVE_RADIATION_FLUX_GL],
+                ),
                 should_error=False,
             ),
             TestCase(
-                filename="test_UM-Global_ssrd_AreaE.grib",
+                filename="test_CEDAFTP_UM-Global_u_20241105T00_S01-03_AreaC.grib",
+                expected_coords = dataclasses.replace(
+                    CEDAFTPModelRepository.model().expected_coordinates,
+                    init_time=[dt.datetime(2024, 11, 5, 0, tzinfo=dt.UTC)],
+                    step=[1, 2, 3],
+                    variable=[entities.parameters.Parameter.WIND_U_COMPONENT_10m],
+                ),
                 should_error=False,
             ),
             TestCase(
-                filename="test_HRES-IFS_10u.grib",
+                filename="test_MODatahub_UM-Global_t2m_20241120T00_S00.grib",
+                expected_coords = CEDAFTPModelRepository.model().expected_coordinates,
                 should_error=True,
             ),
         ]
 
-        expected_coords = dataclasses.replace(
-            CEDAFTPModelRepository.model().expected_coordinates,
-            init_time=[dt.datetime(2024, 11, 5, 0, tzinfo=dt.UTC)],
-        )
 
         for t in tests:
             with self.subTest(name=t.filename):
                 # Attempt to convert the file
                 result = CEDAFTPModelRepository._convert(
-                    path=pathlib.Path(__file__).parent.absolute() / t.filename,
+                    path=pathlib.Path(__file__).parent.absolute() / "test_gribs" / t.filename,
                 )
                 region_result: ResultE[dict[str, slice]] = result.do(
                     region
                     for das in result
                     for da in das
                     for region in NWPDimensionCoordinateMap.from_xarray(da).bind(
-                        expected_coords.determine_region,
+                        t.expected_coords.determine_region,
                     )
                 )
                 if t.should_error:

diff --git a/src/nwp_consumer/internal/repositories/model_repositories/test_ecmwf_realtime.py b/src/nwp_consumer/internal/repositories/model_repositories/test_ecmwf_realtime.py
@@ -7,7 +7,7 @@
 
 from returns.result import Failure, ResultE, Success
 
-from ...entities import NWPDimensionCoordinateMap
+from ...entities import NWPDimensionCoordinateMap, Parameter
 from .ecmwf_realtime import ECMWFRealTimeS3ModelRepository
 
 if TYPE_CHECKING:
@@ -113,44 +113,52 @@ def test__convert(self) -> None:
         @dataclasses.dataclass
         class TestCase:
             filename: str
+            expected_coords: NWPDimensionCoordinateMap
             should_error: bool
 
         tests: list[TestCase] = [
             TestCase(
-                filename="test_HRES-IFS_ssrd.grib",
+                filename="test_ECMWFRealtime_HRES-IFS_ssrd_20241104T00_S60.grib",
+                expected_coords=dataclasses.replace(
+                    ECMWFRealTimeS3ModelRepository.model().expected_coordinates,
+                    init_time=[dt.datetime(2024, 11, 4, 0, tzinfo=dt.UTC)],
+                    variable=[Parameter.DOWNWARD_SHORTWAVE_RADIATION_FLUX_GL],
+                    step=[60],
+                ),
                 should_error=False,
             ),
             TestCase(
-                filename="test_HRES-IFS_10u.grib",
+                filename="test_ECMWFRealtime_HRES-IFS_10u_20241104T00_S60.grib",
+                expected_coords=dataclasses.replace(
+                    ECMWFRealTimeS3ModelRepository.model().expected_coordinates,
+                    init_time=[dt.datetime(2024, 11, 4, 0, tzinfo=dt.UTC)],
+                    variable=[Parameter.WIND_U_COMPONENT_10m],
+                    step=[60],
+                ),
                 should_error=False,
             ),
             TestCase(
-                filename="test_UM-Global_t2m.grib",
+                filename="test_NOAAS3_HRES-GFS_10u_20210509T00_S00.grib",
+                expected_coords=ECMWFRealTimeS3ModelRepository.model().expected_coordinates,
                 should_error=True,
             ),
         ]
 
-        expected_coords = dataclasses.replace(
-            ECMWFRealTimeS3ModelRepository.model().expected_coordinates,
-            init_time=[dt.datetime(2024, 11, 4, 0, tzinfo=dt.UTC)],
-        )
-
         for t in tests:
             with self.subTest(name=t.filename):
                 # Attempt to convert the file
                 result = ECMWFRealTimeS3ModelRepository._convert(
-                    path=pathlib.Path(__file__).parent.absolute() / t.filename,
+                    path=pathlib.Path(__file__).parent.absolute() / "test_gribs" / t.filename,
                 )
                 region_result: ResultE[dict[str, slice]] = result.do(
                     region
                     for das in result
                     for da in das
                     for region in NWPDimensionCoordinateMap.from_xarray(da).bind(
-                        expected_coords.determine_region,
+                        t.expected_coords.determine_region,
                     )
                 )
                 if t.should_error:
                     self.assertIsInstance(region_result, Failure, msg=f"{region_result}")
                 else:
                     self.assertIsInstance(region_result, Success, msg=f"{region_result}")
-
diff --git a/...positories/test_UM-Global_ssrd_AreaE.grib → ...TP_UM-Global_ssrd_20241105T00_S01-03.grib b/...positories/test_UM-Global_ssrd_AreaE.grib → ...TP_UM-Global_ssrd_20241105T00_S01-03.grib
diff --git a/...epositories/test_UM-Global_10u-AreaC.grib → ...UM-Global_u_20241105T00_S01-03_AreaC.grib b/...epositories/test_UM-Global_10u-AreaC.grib → ...UM-Global_u_20241105T00_S01-03_AreaC.grib
diff --git a/...model_repositories/test_HRES-IFS_10u.grib → ...ealtime_HRES-IFS_10u_20241104T00_S60.grib b/...model_repositories/test_HRES-IFS_10u.grib → ...ealtime_HRES-IFS_10u_20241104T00_S60.grib
diff --git a/...odel_repositories/test_HRES-IFS_ssrd.grib → ...altime_HRES-IFS_ssrd_20241104T00_S60.grib b/...odel_repositories/test_HRES-IFS_ssrd.grib → ...altime_HRES-IFS_ssrd_20241104T00_S60.grib
diff --git a/...odel_repositories/test_UM-Global_t2m.grib → ...atahub_UM-Global_t2m_20241120T00_S00.grib b/...odel_repositories/test_UM-Global_t2m.grib → ...atahub_UM-Global_t2m_20241120T00_S00.grib
diff --git a/...odel_repositories/test_UM-Global_10u.grib → ...atahub_UM-Global_u10_20241120T00_S17.grib b/...odel_repositories/test_UM-Global_10u.grib → ...atahub_UM-Global_u10_20241120T00_S17.grib
diff --git a/...model_repositories/test_HRES-GFS_10u.grib → ..._NOAAS3_HRES-GFS_10u_20210509T06_S00.grib b/...model_repositories/test_HRES-GFS_10u.grib → ..._NOAAS3_HRES-GFS_10u_20210509T06_S00.grib
diff --git a/...del_repositories/test_HRES-GFS_aptmp.grib → ...OAAS3_HRES-GFS_aptmp_20210509T06_S00.grib b/...del_repositories/test_HRES-GFS_aptmp.grib → ...OAAS3_HRES-GFS_aptmp_20210509T06_S00.grib
diff --git a/...model_repositories/test_HRES-GFS_lcc.grib → ..._NOAAS3_HRES-GFS_lcc_20210509T06_S00.grib b/...model_repositories/test_HRES-GFS_lcc.grib → ..._NOAAS3_HRES-GFS_lcc_20210509T06_S00.grib
diff --git a/...s/model_repositories/test_HRES-GFS_r.grib → ...st_NOAAS3_HRES-GFS_r_20210509T06_S00.grib b/...s/model_repositories/test_HRES-GFS_r.grib → ...st_NOAAS3_HRES-GFS_r_20210509T06_S00.grib
diff --git a/src/nwp_consumer/internal/repositories/model_repositories/test_mo_datahub.py b/src/nwp_consumer/internal/repositories/model_repositories/test_mo_datahub.py
@@ -6,7 +6,7 @@
 
 from returns.result import Failure, ResultE, Success
 
-from ...entities import NWPDimensionCoordinateMap
+from ...entities import NWPDimensionCoordinateMap, Parameter
 from .mo_datahub import MetOfficeDatahubModelRepository
 
 
@@ -39,40 +39,49 @@ def test__convert(self) -> None:
         @dataclasses.dataclass
         class TestCase:
             filename: str
+            expected_coords: NWPDimensionCoordinateMap
             should_error: bool
 
         tests: list[TestCase] = [
             TestCase(
-                filename="test_UM-Global_10u.grib",
+                filename="test_MODatahub_UM-Global_t2m_20241120T00_S00.grib",
+                expected_coords=dataclasses.replace(
+                    MetOfficeDatahubModelRepository.model().expected_coordinates,
+                    init_time=[dt.datetime(2024, 11, 20, 0, tzinfo=dt.UTC)],
+                    variable=[Parameter.TEMPERATURE_SL],
+                    step=[0],
+                ),
                 should_error=False,
             ),
             TestCase(
-                filename="test_UM-Global_t2m.grib",
+                filename="test_MODatahub_UM-Global_u10_20241120T00_S17.grib",
+                expected_coords=dataclasses.replace(
+                    MetOfficeDatahubModelRepository.model().expected_coordinates,
+                    init_time=[dt.datetime(2024, 11, 20, 0, tzinfo=dt.UTC)],
+                    variable=[Parameter.WIND_U_COMPONENT_10m],
+                    step=[17],
+                ),
                 should_error=False,
             ),
             TestCase(
                 filename="test_HRES-IFS_10u.grib",
+                expected_coords=MetOfficeDatahubModelRepository.model().expected_coordinates,
                 should_error=True,
             ),
         ]
 
-        expected_coords = dataclasses.replace(
-            MetOfficeDatahubModelRepository.model().expected_coordinates,
-            init_time=[dt.datetime(2024, 11, 20, 0, tzinfo=dt.UTC)],
-        )
-
         for t in tests:
             with self.subTest(name=t.filename):
                 # Attempt to convert the file
                 result = MetOfficeDatahubModelRepository._convert(
-                    path=pathlib.Path(__file__).parent.absolute() / t.filename,
+                    path=pathlib.Path(__file__).parent.absolute() / "test_gribs" / t.filename,
                 )
                 region_result: ResultE[dict[str, slice]] = result.do(
                     region
                     for das in result
                     for da in das
                     for region in NWPDimensionCoordinateMap.from_xarray(da).bind(
-                        expected_coords.determine_region,
+                        t.expected_coords.determine_region,
                     )
                 )
                 if t.should_error:

diff --git a/src/nwp_consumer/internal/repositories/model_repositories/test_noaa_s3.py b/src/nwp_consumer/internal/repositories/model_repositories/test_noaa_s3.py
@@ -8,7 +8,7 @@
 import s3fs
 from returns.result import Failure, ResultE, Success
 
-from ...entities import NWPDimensionCoordinateMap
+from ...entities import NWPDimensionCoordinateMap, Parameter
 from .noaa_s3 import NOAAS3ModelRepository
 
 if TYPE_CHECKING:
@@ -117,48 +117,64 @@ def test__convert(self) -> None:
         @dataclasses.dataclass
         class TestCase:
             filename: str
+            expected_coords: NWPDimensionCoordinateMap
             should_error: bool
 
         tests: list[TestCase] = [
             TestCase(
-                filename="test_HRES-GFS_10u.grib",
+                filename="test_NOAAS3_HRES-GFS_10u_20210509T06_S00.grib",
+                expected_coords=dataclasses.replace(
+                    NOAAS3ModelRepository.model().expected_coordinates,
+                    init_time=[dt.datetime(2021, 5, 9, 6, tzinfo=dt.UTC)],
+                    variable=[Parameter.WIND_U_COMPONENT_10m],
+                    step=[0],
+                ),
                 should_error=False,
             ),
             TestCase(
-                filename="test_HRES-GFS_lcc.grib",
+                filename="test_NOAAS3_HRES-GFS_lcc_20210509T06_S00.grib",
+                expected_coords=dataclasses.replace(
+                    NOAAS3ModelRepository.model().expected_coordinates,
+                    init_time=[dt.datetime(2021, 5, 9, 6, tzinfo=dt.UTC)],
+                    variable=[Parameter.CLOUD_COVER_LOW],
+                    step=[0],
+                ),
                 should_error=False,
             ),
             TestCase(
-                filename="test_HRES-GFS_r.grib",
+                filename="test_NOAAS3_HRES-GFS_r_20210509T06_S00.grib",
+                expected_coords=dataclasses.replace(
+                    NOAAS3ModelRepository.model().expected_coordinates,
+                    init_time=[dt.datetime(2021, 5, 9, 6, tzinfo=dt.UTC)],
+                    variable=[Parameter.RELATIVE_HUMIDITY_SL],
+                    step=[0],
+                ),
                 should_error=False,
             ),
             TestCase(
-                filename="test_HRES-GFS_aptmp.grib",
+                filename="test_NOAAS3_HRES-GFS_aptmp_20210509T06_S00.grib",
+                expected_coords=NOAAS3ModelRepository.model().expected_coordinates,
                 should_error=True,
             ),
             TestCase(
-                filename="test_UM-Global_t2m.grib",
+                filename="test_MODatahub_UM-Global_t2m_20241120T00_S00.grib",
+                expected_coords=NOAAS3ModelRepository.model().expected_coordinates,
                 should_error=True,
             ),
         ]
 
-        expected_coords = dataclasses.replace(
-            NOAAS3ModelRepository.model().expected_coordinates,
-            init_time=[dt.datetime(2021, 5, 9, 6, tzinfo=dt.UTC)],
-        )
-
         for t in tests:
             with self.subTest(name=t.filename):
                 # Attempt to convert the file
                 result = NOAAS3ModelRepository._convert(
-                    path=pathlib.Path(__file__).parent.absolute() / t.filename,
+                    path=pathlib.Path(__file__).parent.absolute() / "test_gribs" / t.filename,
                 )
                 region_result: ResultE[dict[str, slice]] = result.do(
                     region
                     for das in result
                     for da in das
                     for region in NWPDimensionCoordinateMap.from_xarray(da).bind(
-                        expected_coords.determine_region,
+                        t.expected_coords.determine_region,
                     )
                 )
                 if t.should_error: