fix(provider): wekeo_ecmwf default dates (#1288)

docs/notebooks/api_user_guide/2_providers_products_available.ipynb

@@ -412,6 +412,9 @@
        " 'S3_ERR',\n",
        " 'S3_ERR_BC002',\n",
        " 'S3_LAN',\n",
+       " 'S3_LAN_HY',\n",
+       " 'S3_LAN_LI',\n",
+       " 'S3_LAN_SI',\n",
        " 'S3_OLCI_L2LFR',\n",
        " 'S3_OLCI_L2LRR',\n",
        " 'S3_OLCI_L2WFR',\n",
@@ -460,6 +463,7 @@
        " 'S5P_L2_CLOUD',\n",
        " 'S5P_L2_CO',\n",
        " 'S5P_L2_HCHO',\n",
+       " 'S5P_L2_IR_ALL',\n",
        " 'S5P_L2_NO2',\n",
        " 'S5P_L2_NP_BD3',\n",
        " 'S5P_L2_NP_BD6',\n",

eodag/plugins/search/base.py

@@ -179,6 +179,27 @@ def get_product_type_def_params(
         else:
             return {}
 
+    def get_product_type_cfg_value(self, key: str, default: Any = None) -> Any:
+        """
+        Get the value of a configuration option specific to the current product type.
+
+        This method retrieves the value of a configuration option from the
+        `product_type_config` attribute. If the option is not found, the provided
+        default value is returned.
+
+        :param key: The configuration option key.
+        :type key: str
+        :param default: The default value to be returned if the option is not found (default is None).
+        :type default: Any
+
+        :return: The value of the specified configuration option or the default value.
+        :rtype: Any
+        """
+        product_type_cfg = getattr(self.config, "product_type_config", {})
+        non_none_cfg = {k: v for k, v in product_type_cfg.items() if v}
+
+        return non_none_cfg.get(key, default)
+
     def get_metadata_mapping(
         self, product_type: Optional[str] = None
     ) -> Dict[str, Union[str, List[str]]]:

eodag/plugins/search/build_search_result.py

@@ -339,24 +339,6 @@ def build_query_string(
             self, product_type=product_type, **available_properties
         )
 
-    def get_product_type_cfg(self, key: str, default: Any = None) -> Any:
-        """
-        Get the value of a configuration option specific to the current product type.
-
-        This method retrieves the value of a configuration option from the
-        `_product_type_config` attribute. If the option is not found, the provided
-        default value is returned.
-
-        :param key: The configuration option key.
-        :param default: The default value to be returned if the option is not found (default is None).
-
-        :return: The value of the specified configuration option or the default value.
-        """
-        product_type_cfg = getattr(self.config, "product_type_config", {})
-        non_none_cfg = {k: v for k, v in product_type_cfg.items() if v}
-
-        return non_none_cfg.get(key, default)
-
     def _preprocess_search_params(self, params: Dict[str, Any]) -> None:
         """Preprocess search parameters before making a request to the CDS API.
 
@@ -397,7 +379,7 @@ def _preprocess_search_params(self, params: Dict[str, Any]) -> None:
 
         # dates
         mission_start_dt = datetime.fromisoformat(
-            self.get_product_type_cfg(
+            self.get_product_type_cfg_value(
                 "missionStartDate", DEFAULT_MISSION_START_DATE
             ).replace(
                 "Z", "+00:00"

eodag/plugins/search/qssearch.py

@@ -20,6 +20,7 @@
 import logging
 import re
 from copy import copy as copy_copy
+from datetime import datetime
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -48,6 +49,7 @@
 import orjson
 import requests
 import yaml
+from dateutil.utils import today
 from jsonpath_ng import JSONPath
 from lxml import etree
 from pydantic import create_model
@@ -1274,6 +1276,82 @@ def normalize_results(
 class PostJsonSearch(QueryStringSearch):
     """A specialisation of a QueryStringSearch that uses POST method"""
 
+    def _get_default_end_date_from_start_date(
+        self, start_datetime: str, product_type: str
+    ) -> str:
+        default_end_date = self.config.products.get(product_type, {}).get(
+            "_default_end_date", None
+        )
+        if default_end_date:
+            return default_end_date
+        try:
+            start_date = datetime.fromisoformat(start_datetime)
+        except ValueError:
+            start_date = datetime.strptime(start_datetime, "%Y-%m-%dT%H:%M:%SZ")
+        product_type_conf = self.config.products[product_type]
+        if (
+            "metadata_mapping" in product_type_conf
+            and "startTimeFromAscendingNode" in product_type_conf["metadata_mapping"]
+        ):
+            mapping = product_type_conf["metadata_mapping"][
+                "startTimeFromAscendingNode"
+            ]
+            if isinstance(mapping, list) and "year" in mapping[0]:
+                # if date is mapped to year/month/(day), use end_date = start_date to avoid large requests
+                end_date = start_date
+                return end_date.isoformat()
+        return self.get_product_type_cfg_value("missionEndDate", today().isoformat())
+
+    def _check_date_params(self, keywords: Dict[str, Any], product_type: str) -> None:
+        """checks if start and end date are present in the keywords and adds them if not"""
+        if (
+            "startTimeFromAscendingNode"
+            and "completionTimeFromAscendingNode" in keywords
+        ):
+            return
+        # start time given, end time missing
+        if "startTimeFromAscendingNode" in keywords:
+            keywords[
+                "completionTimeFromAscendingNode"
+            ] = self._get_default_end_date_from_start_date(
+                keywords["startTimeFromAscendingNode"], product_type
+            )
+            return
+        product_type_conf = self.config.products[product_type]
+        if (
+            "metadata_mapping" in product_type_conf
+            and "startTimeFromAscendingNode" in product_type_conf["metadata_mapping"]
+        ):
+            mapping = product_type_conf["metadata_mapping"][
+                "startTimeFromAscendingNode"
+            ]
+            if isinstance(mapping, list):
+                # get time parameters (date, year, month, ...) from metadata mapping
+                input_mapping = mapping[0].replace("{{", "").replace("}}", "")
+                time_params = [
+                    values.split(":")[0].strip() for values in input_mapping.split(",")
+                ]
+                time_params = [
+                    tp.replace('"', "").replace("'", "") for tp in time_params
+                ]
+                # if startTime is not given but other time params (e.g. year/month/(day)) are given,
+                # no default date is required
+                in_keywords = True
+                for tp in time_params:
+                    if tp not in keywords:
+                        in_keywords = False
+                if not in_keywords:
+                    keywords[
+                        "startTimeFromAscendingNode"
+                    ] = self.get_product_type_cfg_value(
+                        "missionStartDate", today().isoformat()
+                    )
+                    keywords[
+                        "completionTimeFromAscendingNode"
+                    ] = self._get_default_end_date_from_start_date(
+                        keywords["startTimeFromAscendingNode"], product_type
+                    )
+
     def query(
         self,
         prep: PreparedSearch = PreparedSearch(),
@@ -1323,6 +1401,8 @@ def query(
                     and isinstance(self.config.metadata_mapping[k], list)
                 }
             )
+            if getattr(self.config, "dates_required", False):
+                self._check_date_params(keywords, product_type)
 
             qp, _ = self.build_query_string(product_type, **keywords)
 

eodag/resources/providers.yml

@@ -4725,7 +4725,7 @@
     need_auth: true
     auth_error_code: 401
     results_entry: 'features'
-    two_passes_id_search: true
+    dates_required: true
     pagination:
       total_items_nb_key_path: '$.properties.totalResults'
       next_page_query_obj: '{{"itemsPerPage":{items_per_page},"startIndex":{skip}}}'
@@ -4736,6 +4736,7 @@
     constraints_file_url: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/queryable/{dataset}'
     constraints_file_dataset_key: productType
     constraints_entry: constraints
+    stop_without_constraints_entry_key: true
     metadata_mapping:
       productType:
         - '{{"dataset_id": "{productType}"}}'
@@ -4748,10 +4749,10 @@
         - '{{"productIdentifier": "{id}"}}'
         - '{$.id#remove_extension}'
       startTimeFromAscendingNode:
-        - '{{"startDate": "{startTimeFromAscendingNode}"}}'
+        - '{{"dtstart": "{startTimeFromAscendingNode#to_iso_utc_datetime}"}}'
         - '$.properties.startdate'
       completionTimeFromAscendingNode:
-        - '{{"completionDate": "{completionTimeFromAscendingNode}"}}'
+        - '{{"dtend": "{completionTimeFromAscendingNode#to_iso_utc_datetime}"}}'
         - '$.properties.enddate'
       downloadLink: '$.properties.location'
       title: '$.id'
@@ -4867,6 +4868,15 @@
       day:
         - '{{"day": {day}}}'
         - '$.null'
+      hyear:
+        - '{{"year": {hyear}}}'
+        - '$.null'
+      hmonth:
+        - '{{"month": {hmonth}}}'
+        - '$.null'
+      hday:
+        - '{{"day": {hday}}}'
+        - '$.null'
       satellite:
         - '{{"satellite": {satellite}}}'
         - '$.null'
@@ -4967,13 +4977,18 @@
       productType: EO:ECMWF:DAT:SATELLITE_SEA_ICE_THICKNESS
       satellite:
         - envisat
-      cdr_type:
-        - cdr
+      cdr_type: cdr
       variable: all
       version: '3_0'
       format: zip
       metadata_mapping:
         id: '$.id'
+        variable:
+          - '{{"variable": "{variable}"}}'
+          - '$.null'
+        version:
+          - '{{"version": "{version}"}}'
+          - '$.null'
         <<: *month_year
         defaultGeometry: 'POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))'
         orderLink: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{downloadLink}","product_id":"{id}", "dataset_id": "EO:ECMWF:DAT:SATELLITE_SEA_ICE_THICKNESS"}}'
@@ -5195,6 +5210,9 @@
       version: "wgms_fog_2022_09"
       metadata_mapping:
         id: '$.id'
+        hydrological_year:
+          - '{{"hydrological_year": {hydrological_year}}}'
+          - '$.null'
         startTimeFromAscendingNode:
           - |
             {{
@@ -5401,12 +5419,6 @@
         - '00:00'
       metadata_mapping:
         id: '$.id'
-        startTimeFromAscendingNode:
-          - '{{"dtstart": "{startTimeFromAscendingNode#to_iso_utc_datetime}"}}'
-          - '$.properties.startdate'
-        completionTimeFromAscendingNode:
-          - '{{"dtend": "{completionTimeFromAscendingNode#to_iso_utc_datetime}"}}'
-          - '$.properties.enddate'
         <<: *variable_list
         defaultGeometry: 'POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))'
         orderLink: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{downloadLink}","product_id":"{id}", "dataset_id": "EO:ECMWF:DAT:CAMS_GLOBAL_REANALYSIS_EAC4"}}'
@@ -5538,12 +5550,6 @@
         - '0'
       metadata_mapping:
         id: '$.id'
-        startTimeFromAscendingNode:
-          - '{{"dtstart": "{startTimeFromAscendingNode#to_iso_utc_datetime}"}}'
-          - '$.properties.startdate'
-        completionTimeFromAscendingNode:
-          - '{{"dtend": "{completionTimeFromAscendingNode#to_iso_utc_datetime}"}}'
-          - '$.properties.enddate'
         <<: *variable_list
         defaultGeometry: 'POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))'
         orderLink: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{downloadLink}","product_id":"{id}", "dataset_id": "EO:ECMWF:DAT:CAMS_GLOBAL_GHG_REANALYSIS_EGG4"}}'
@@ -5584,12 +5590,6 @@
         - '0'
       metadata_mapping:
         id: '$.id'
-        startTimeFromAscendingNode:
-          - '{{"dtstart": "{startTimeFromAscendingNode#to_iso_utc_datetime}"}}'
-          - '$.properties.startdate'
-        completionTimeFromAscendingNode:
-          - '{{"dtend": "{completionTimeFromAscendingNode#to_iso_utc_datetime}"}}'
-          - '$.properties.enddate'
         type:
           - '{{"type": {type}}}'
           - '$.null'
@@ -5609,12 +5609,6 @@
         - '0'
       metadata_mapping:
         id: '$.id'
-        startTimeFromAscendingNode:
-          - '{{"dtstart": "{startTimeFromAscendingNode#to_iso_utc_datetime}"}}'
-          - '$.properties.startdate'
-        completionTimeFromAscendingNode:
-          - '{{"dtend": "{completionTimeFromAscendingNode#to_iso_utc_datetime}"}}'
-          - '$.properties.enddate'
         type:
           - '{{"type": {type}}}'
           - '$.null'
@@ -5630,12 +5624,6 @@
       format: csv
       metadata_mapping:
         id: '$.id'
-        startTimeFromAscendingNode:
-          - '{{"dtstart": "{startTimeFromAscendingNode#to_iso_utc_datetime}"}}'
-          - '$.properties.startdate'
-        completionTimeFromAscendingNode:
-          - '{{"dtend": "{completionTimeFromAscendingNode#to_iso_utc_datetime}"}}'
-          - '$.properties.enddate'
         step:
           - '{{"time_step": "{step}"}}'
           - '$.null'