From 805ced271e80bd2d186f1e80bc8b4ba4b2387da8 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Mon, 28 Feb 2022 16:26:05 +0100 Subject: [PATCH 1/4] Speed up ESGF search --- esmvalcore/esgf/_search.py | 4 +--- package/meta.yaml | 3 --- setup.py | 4 +--- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/esmvalcore/esgf/_search.py b/esmvalcore/esgf/_search.py index 0eabe8fb4d..ae2f5590c0 100644 --- a/esmvalcore/esgf/_search.py +++ b/esmvalcore/esgf/_search.py @@ -86,9 +86,7 @@ def esgf_search_files(facets): results = context.search( batch_size=500, - # enable ignore_facet_check once the following issue has been fixed: - # https://github.com/ESGF/esgf-pyclient/issues/75 - # ignore_facet_check=True, + ignore_facet_check=True, ) files = ESGFFile._from_results(results, facets) diff --git a/package/meta.yaml b/package/meta.yaml index c997adfb11..88635d1669 100644 --- a/package/meta.yaml +++ b/package/meta.yaml @@ -43,7 +43,6 @@ requirements: - cftime - cf-units>=3.0.0 - cython # required by cf-units but not automatically installed - - defusedxml # required by esgf-pyclient - esgf-pyclient - esmpy!=8.1.0 # see github.com/ESMValGroup/ESMValCore/issues/1208 - fiona @@ -53,7 +52,6 @@ requirements: - isodate - jinja2 - mpich<3.4 # prevent external build - - myproxyclient # required by esgf-pyclient - nc-time-axis - nested-lookup - netCDF4 @@ -64,7 +62,6 @@ requirements: - pydot - pyyaml - requests - - requests_cache<0.6 # https://github.com/ESGF/esgf-pyclient/issues/71 - shapely - yamale diff --git a/setup.py b/setup.py index cd05d726cd..dcf21e941e 100755 --- a/setup.py +++ b/setup.py @@ -32,9 +32,7 @@ 'install': [ 'cf-units>=3.0.0', 'dask[array]', - 'esgf-pyclient', - # pin requests_cache: https://github.com/ESGF/esgf-pyclient/issues/71 - 'requests_cache<0.6', + 'esgf-pyclient>=0.3.1', 'esmpy!=8.1.0', # see github.com/ESMValGroup/ESMValCore/issues/1208 'fiona', 'fire', From 1741236a27af626161ea988cc1bdca1715452cb2 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Mon, 28 Feb 2022 16:58:07 +0100 Subject: [PATCH 2/4] Update test --- tests/unit/esgf/test_search.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/unit/esgf/test_search.py b/tests/unit/esgf/test_search.py index 525aa88ad0..5f87f241a9 100644 --- a/tests/unit/esgf/test_search.py +++ b/tests/unit/esgf/test_search.py @@ -117,10 +117,8 @@ def get_mock_connection(facets, results): class MockFileSearchContext: def search(self, **kwargs): assert kwargs['batch_size'] == 500 - # enable ignore_facet_check once the following issue has been - # fixed: https://github.com/ESGF/esgf-pyclient/issues/75 - # assert kwargs['ignore_facet_check'] - assert len(kwargs) == 1 + assert kwargs['ignore_facet_check'] + assert len(kwargs) == 2 return results class MockConnection: From cd230d2e3fd2a1d1c35febb962aa8dae7aa99ea9 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 1 Mar 2022 15:46:05 +0000 Subject: [PATCH 3/4] fix indent and add feature branch for GA test --- .github/workflows/run-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 3d7ef7254c..f98faf3a5f 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -20,7 +20,8 @@ on: # turn it on only if needed push: branches: - - main + - main + - speedup-esgf-search # run the test only if the PR is to main # turn it on if required #pull_request: From 190a3e228107369eb65d3c7b80c4abc8cd8db35f Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 1 Mar 2022 16:30:36 +0000 Subject: [PATCH 4/4] remove current feature branch for GA test --- .github/workflows/run-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index f98faf3a5f..8c6bdd5b2d 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -21,7 +21,6 @@ on: push: branches: - main - - speedup-esgf-search # run the test only if the PR is to main # turn it on if required #pull_request: