Speed up ESGF search (#1512)

* Speed up ESGF search

* Update test

* fix indent and add feature branch for GA test

* remove current feature branch for GA test

Co-authored-by: Valeriu Predoi <valeriu.predoi@gmail.com>
Co-authored-by: Manuel Schlund <manuel.schlund@dlr.de>

esmvalcore/esgf/_search.py

@@ -86,9 +86,7 @@ def esgf_search_files(facets):
 
     results = context.search(
         batch_size=500,
-        # enable ignore_facet_check once the following issue has been fixed:
-        # https://github.com/ESGF/esgf-pyclient/issues/75
-        # ignore_facet_check=True,
+        ignore_facet_check=True,
     )
 
     files = ESGFFile._from_results(results, facets)

package/meta.yaml

@@ -43,7 +43,6 @@ requirements:
     - cftime
     - cf-units>=3.0.0
     - cython  # required by cf-units but not automatically installed
-    - defusedxml  # required by esgf-pyclient
     - esgf-pyclient
     - esmpy!=8.1.0  # see github.com/ESMValGroup/ESMValCore/issues/1208
     - fiona
@@ -53,7 +52,6 @@ requirements:
     - isodate
     - jinja2
     - mpich<3.4  # prevent external build
-    - myproxyclient  # required by esgf-pyclient
     - nc-time-axis
     - nested-lookup
     - netCDF4
@@ -64,7 +62,6 @@ requirements:
     - pydot
     - pyyaml
     - requests
-    - requests_cache<0.6  # https://github.com/ESGF/esgf-pyclient/issues/71
     - shapely
     - yamale
 

setup.py

@@ -32,9 +32,7 @@
     'install': [
         'cf-units>=3.0.0',
         'dask[array]',
-        'esgf-pyclient',
-        # pin requests_cache: https://github.com/ESGF/esgf-pyclient/issues/71
-        'requests_cache<0.6',
+        'esgf-pyclient>=0.3.1',
         'esmpy!=8.1.0',  # see github.com/ESMValGroup/ESMValCore/issues/1208
         'fiona',
         'fire',

tests/unit/esgf/test_search.py

@@ -117,10 +117,8 @@ def get_mock_connection(facets, results):
     class MockFileSearchContext:
         def search(self, **kwargs):
             assert kwargs['batch_size'] == 500
-            # enable ignore_facet_check once the following issue has been
-            # fixed: https://github.com/ESGF/esgf-pyclient/issues/75
-            # assert kwargs['ignore_facet_check']
-            assert len(kwargs) == 1
+            assert kwargs['ignore_facet_check']
+            assert len(kwargs) == 2
             return results
 
     class MockConnection: