From c638b10320d1d74d68bc680066f2c22e4e42bd5f Mon Sep 17 00:00:00 2001 From: devsjc <47188100+devsjc@users.noreply.github.com> Date: Thu, 12 Sep 2024 16:58:52 +0100 Subject: [PATCH] chore(ceda-global): Update asset specification --- local_archives/nwp/ceda/ceda_global.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/local_archives/nwp/ceda/ceda_global.py b/local_archives/nwp/ceda/ceda_global.py index 5ab1039..a67a4cd 100644 --- a/local_archives/nwp/ceda/ceda_global.py +++ b/local_archives/nwp/ceda/ceda_global.py @@ -11,14 +11,27 @@ @dg.asset( name="zarr_archive", + description="".join(( + "Zarr archive of NWP data from the Met Office's Global model. ", + "Sourced via FTP from CEDA: ", + "https://catalogue.ceda.ac.uk/uuid/86df725b793b4b4cb0ca0646686bd783/\n", + "This asset is updated monthly, and surfaced as a Zarr Directory Store ", + "for each month. It is downloaded using the nwp-consumer: ", + "https://github.com/openclimatefix/nwp-consumer", + )), key_prefix=["nwp", "ceda", "global"], metadata={ "archive_folder": dg.MetadataValue.text(f"{ZARR_FOLDER}/nwp/ceda/global"), "area": dg.MetadataValue.text("global"), "source": dg.MetadataValue.text("ceda"), + "expected_runtime": dg.MetadataValue.text("6 hours"), + }, + compute_kind="docker", + automation_condition=dg.AutomationCondition.eager(), + tags={ + "dagster/max_runtime": str(60 * 60 * 10), # Should take 6 ish hours + "dagster/priority": "1", }, - compute_kind="download", - op_tags={"dagster/max_runtime": int(60 * 100)}, partitions_def=dg.MonthlyPartitionsDefinition( start_date="2019-01-01", end_offset=-3,