From c86b08b20cf18a275f192d8b202508fa35783676 Mon Sep 17 00:00:00 2001 From: devsjc <47188100+devsjc@users.noreply.github.com> Date: Thu, 10 Oct 2024 10:43:03 +0100 Subject: [PATCH 1/3] fix(datatailor): Better logging for customisation creation --- satip/eumetsat.py | 74 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 54 insertions(+), 20 deletions(-) diff --git a/satip/eumetsat.py b/satip/eumetsat.py index afc4656d..7e8d2868 100644 --- a/satip/eumetsat.py +++ b/satip/eumetsat.py @@ -636,29 +636,63 @@ def create_and_download_datatailor_data( datatailor = eumdac.DataTailor(eumdac.AccessToken((self.user_key, self.user_secret))) - # sometimes the customisation fails first time, so we try twice - # This is from Data Tailor only allowing 3 customizations at once - # So this should then continue until it is created successfully - created_customization = False - # 5 minute timeout here - start = datetime.datetime.now() - while not created_customization and (datetime.datetime.now() - start).seconds < 600: - try: - num_running_customizations = 0 - for customisation in datatailor.customisations: - if customisation.status in ['INACTIVE']: # Clear stuck ones + # Attempt to create customisation. This is attempted for 5 minutes, + # as other running customisations can block the creation, hence we + # allow them time to finish. + customisation: eumdac.Customization | None = None + log.debug("Attempting to create DataTailor customisation") + start: datatime.datetime = datetime.datetime.now() + attempt: int = 1 + # 5 minute timeout + while (datetime.datetime.now() - start).seconds < 300: + running_customisations: list[eumdac.Customisation] = [ + c for c in datatailor.customisations if c.status in ['RUNNING', 'QUEUED', 'INACTIVE'] + ] + inactive_customisations: list[eumdac.Customisation] = [ + c for c in running_customisations if c.status == 'INACTIVE' + ] + log.debug( + f"Attempt {attempt}: Found {len(running_customisations)} running customisations, " + f"of which {len(inactive_customisations)} are inactive.", + ) + + # Try to kill any inactive customisations + killed: int = 0 + if len(inactive_customisations) > 0: + log.debug( + f"Attempt {attempt}: Clearing {len(inactive_customisations)} inactive customisations...", + parent="DownloadManager", + ) + try: + for i, customisation in enumerate(inactive_customisations): customisation.kill() customisation.delete() - if customisation.status in ['RUNNING','QUEUED', 'INACTIVE']: - num_running_customizations += 1 - if num_running_customizations < 3: + killed += 1 + except Exception as e: + log.debug( + f"Attempt {attempt}: Error clearing inactive customisation " + f"{i} of {len(inactive_customisations)}: {e}", + parent="DownloadManager", + ) + + # If that left enough space for a new customisation, try to make it + if len(running_customisations) - killed < 3: + try: customisation = datatailor.new_customisation(dataset_id, chain=chain) - created_customization = True - except Exception: - log.debug("Customization not made successfully, so " - "trying again after less than 3 customizations") - time.sleep(3) - continue + log.debug(f"Attempt {attempt}: Created customisation {customisation}") + break + except Exception as e: + log.debug(f"Attempt {attempt}: Error creating customisation: {e}") + else: + log.debug( + f"Attempt {attempt}: Too many running customisations, trying again in 10 seconds", + parent="DownloadManager", + ) + time.sleep(10) + + if customisation is None: + log.error("Failed to create customisation, exiting", parent="DownloadManager") + return sleep_time = 5 # seconds log.debug(f"Customisation: {customisation}", parent="DownloadManager") From a4fb072912124beafd0a7b948a1300b8c32e2c06 Mon Sep 17 00:00:00 2001 From: devsjc <47188100+devsjc@users.noreply.github.com> Date: Thu, 10 Oct 2024 10:49:35 +0100 Subject: [PATCH 2/3] fix(datatailor): Linting --- satip/eumetsat.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/satip/eumetsat.py b/satip/eumetsat.py index 7e8d2868..6d6232a6 100644 --- a/satip/eumetsat.py +++ b/satip/eumetsat.py @@ -641,26 +641,30 @@ def create_and_download_datatailor_data( # allow them time to finish. customisation: eumdac.Customization | None = None log.debug("Attempting to create DataTailor customisation") - start: datatime.datetime = datetime.datetime.now() + start: datetime.datetime = datetime.datetime.now() attempt: int = 1 # 5 minute timeout while (datetime.datetime.now() - start).seconds < 300: running_customisations: list[eumdac.Customisation] = [ - c for c in datatailor.customisations if c.status in ['RUNNING', 'QUEUED', 'INACTIVE'] + c for c in datatailor.customisations + if c.status in ['RUNNING', 'QUEUED', 'INACTIVE'] ] inactive_customisations: list[eumdac.Customisation] = [ - c for c in running_customisations if c.status == 'INACTIVE' + c for c in running_customisations + if c.status == 'INACTIVE' ] log.debug( - f"Attempt {attempt}: Found {len(running_customisations)} running customisations, " - f"of which {len(inactive_customisations)} are inactive.", + f"Attempt {attempt}: Found {len(running_customisations)} " + f"running customisations, of which " + f"{len(inactive_customisations)} are inactive.", ) # Try to kill any inactive customisations killed: int = 0 if len(inactive_customisations) > 0: log.debug( - f"Attempt {attempt}: Clearing {len(inactive_customisations)} inactive customisations...", + f"Attempt {attempt}: Clearing {len(inactive_customisations)} " + f"inactive customisations...", parent="DownloadManager", ) try: @@ -685,7 +689,8 @@ def create_and_download_datatailor_data( log.debug(f"Attempt {attempt}: Error creating customisation: {e}") else: log.debug( - f"Attempt {attempt}: Too many running customisations, trying again in 10 seconds", + f"Attempt {attempt}: Too many running customisations. " + f"Trying again in 10 seconds", parent="DownloadManager", ) time.sleep(10) From 6d33881ab5e422ae1dfe1438da507ac9804c0855 Mon Sep 17 00:00:00 2001 From: devsjc <47188100+devsjc@users.noreply.github.com> Date: Thu, 10 Oct 2024 10:52:15 +0100 Subject: [PATCH 3/3] fix(workflow): Update marketplace actions --- .github/workflows/plots.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/plots.yml b/.github/workflows/plots.yml index e248bffa..bca43fe8 100644 --- a/.github/workflows/plots.yml +++ b/.github/workflows/plots.yml @@ -11,10 +11,10 @@ jobs: runs-on: ${{ matrix.os }} steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -52,7 +52,7 @@ jobs: cml-send-comment report.md - name: Archive plots - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: plots path: |