📊 climate: monthly era5 surface temperature update (#3946)

dag/climate.yml

@@ -32,7 +32,7 @@ steps:
   # Copernicus Climate Change Service - Surface temperature.
   #
   data://meadow/climate/2025-01-07/surface_temperature:
-  - snapshot://climate/2025-01-07/surface_temperature.zip
+  - snapshot://climate/2025-02-07/surface_temperature.zip
   - snapshot://countries/2023-12-27/world_bank.zip
   data://garden/climate/2025-01-07/surface_temperature:
     - data://meadow/climate/2025-01-07/surface_temperature

etl/steps/data/grapher/climate/2025-01-07/surface_global_monthly_anomaly.meta.override.yml → etl/steps/data/grapher/climate/2025-01-07/surface_global_monthly_anomaly.meta.yml

@@ -1,3 +1,6 @@
+dataset:
+  title: Global annual temperature anomalies
+
 tables:
   surface_global_monthly_anomaly:
     variables:

etl/steps/data/grapher/climate/2025-01-07/surface_temperature_annual_average.py

@@ -58,7 +58,12 @@ def run(dest_dir: str) -> None:
     # Replace the decadal values with NaN for all years except the start of each decade
     combined.loc[combined["year"] % 10 != 0, ["temperature_anomaly_decadal", "temperature_2m_decadal"]] = np.nan
     combined = combined.drop(columns=["decade"])
-    # Filter rows where the year is less than or equal to 2024
+    # Remove the latest as it's often not representation of the full year
+    latest_year = combined["year"].max()
+    combined = combined[combined["year"] != latest_year]
+
+    # Set decadal values to NaN for non-decadal years
+    combined.loc[combined["year"] % 10 != 0, ["temperature_anomaly_decadal", "temperature_2m_decadal"]] = np.nan
     combined = combined.set_index(["year", "country"], verify_integrity=True)
 
     # Save outputs.

snapshots/climate/2025-02-07/surface_temperature.py

@@ -0,0 +1,55 @@
+"""Script to create a snapshot of the monthly averaged surface temperature data from 1950 to present from the Copernicus Climate Change Service.
+
+The script assumes that the data is available on the CDS API.
+Instructions on how to access the API on a Mac are here: https://confluence.ecmwf.int/display/CKB/How+to+install+and+use+CDS+API+on+macOS
+
+More information on how to access the data is here: hhttps://cds.climate.copernicus.eu/datasets/reanalysis-era5-single-levels-monthly-means?tab=overview
+
+The data is downloaded as a NetCDF file. Tutorials for using the Copernicus API are here and work with the NETCDF format are here: https://ecmwf-projects.github.io/copernicus-training-c3s/cds-tutorial.html
+"""
+
+import tempfile
+from pathlib import Path
+
+# CDS API
+import cdsapi
+import click
+
+from etl.snapshot import Snapshot
+
+# Version for current snapshot dataset.
+SNAPSHOT_VERSION = Path(__file__).parent.name
+
+
+@click.command()
+@click.option("--upload/--skip-upload", default=True, type=bool, help="Upload dataset to Snapshot")
+def main(upload: bool) -> None:
+    # Create a new snapshot.
+    snap = Snapshot(f"climate/{SNAPSHOT_VERSION}/surface_temperature.zip")
+
+    # Save data as a compressed temporary file.
+    with tempfile.TemporaryDirectory() as temp_dir:
+        output_file = Path(temp_dir) / "era5_monthly_t2m_eur.nc"
+
+        client = cdsapi.Client()
+
+        dataset = "reanalysis-era5-single-levels-monthly-means"
+        request = {
+            "product_type": ["monthly_averaged_reanalysis"],
+            "variable": ["2m_temperature"],
+            "year": [str(year) for year in range(1940, 2026)],
+            "month": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"],
+            "time": "00:00",
+            "area": [90, -180, -90, 180],
+            "data_format": "grib",
+            "download_format": "zip",
+        }
+
+        client.retrieve(dataset, request, output_file)
+
+        # Upload snapshot.
+        snap.create_snapshot(filename=output_file, upload=upload)
+
+
+if __name__ == "__main__":
+    main()

snapshots/climate/2025-02-07/surface_temperature.zip.dvc

@@ -0,0 +1,26 @@
+meta:
+  origin:
+    title_snapshot: ERA5 Monthly Averaged Data on Single Levels from 1940 to Present - Monthly Averages of 2m Surface Temperature
+    title: ERA5 monthly averaged data on single levels from 1940 to present
+    description: |-
+      ERA5 is the latest climate reanalysis produced by ECMWF, providing hourly data on many atmospheric, land-surface and sea-state parameters together with estimates of uncertainty.
+
+      ERA5 data are available in the Climate Data Store on regular latitude-longitude grids at 0.25° x 0.25° resolution, with atmospheric parameters on 37 pressure levels.
+
+      ERA5 is available from 1940 and continues to be extended forward in time, with daily updates being made available 5 days behind real time
+
+      Initial release data, i.e., data no more than three months behind real time, are called ERA5T.
+    producer: Contains modified Copernicus Climate Change Service information
+    version_producer: 2
+    citation_full: |-
+      Hersbach, H., Bell, B., Berrisford, P., Biavati, G., Horányi, A., Muñoz Sabater, J., Nicolas, J., Peubey, C., Radu, R., Rozum, I., Schepers, D., Simmons, A., Soci, C., Dee, D., Thépaut, J-N. (2023): ERA5 monthly averaged data on single levels from 1940 to present. Copernicus Climate Change Service (C3S) Climate Data Store (CDS), DOI: 10.24381/cds.f17050d7 (Accessed on 07-Feb-2025)
+    url_main: https://cds.climate.copernicus.eu/datasets/reanalysis-era5-single-levels-monthly-means?tab=overview
+    date_accessed: 2025-02-07
+    date_published: 2025-02-06
+    license:
+      name: Copernicus License
+      url: https://cds.climate.copernicus.eu/datasets/reanalysis-era5-single-levels-monthly-means?tab=overview
+outs:
+  - md5: 77e63577901ad6c2a762f77eb139936b
+    size: 1712706017
+    path: surface_temperature.zip