Merge pull request #2681 from szampier/refactor_eso_download

Refactor ESO authentication and download

CHANGES.rst

@@ -4,6 +4,16 @@
 New Tools and Services
 ----------------------
 
+eso
+^^^
+
+- Authenticate with ESO using APIs and tokens instead of HTML forms. [#2681]
+- Discontinue usage of old Request Handler for dataset retrieval in favor of new dataportal API. [#2681]
+- Local reimplementation of astroquery's ``_download_file`` to fix some issues and avoid sending a HEAD request
+  just to get the original filename. [#1580]
+- Restore support for .Z files. [#1818]
+- Update tests and documentation.
+
 ipac.irsa
 ^^^^^^^^^
 

astroquery/eso/tests/data/oidc_token.json

@@ -0,0 +1,7 @@
+{
+  "access_token": "some-access-token",
+  "id_token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2Nzg2Mjg5NTl9.qqKrC1MesQQmLtqsFOm2kxe4f_Nqo4EPqgpup30c6Mg",
+  "token_type": "bearer",
+  "expires_in": 28800,
+  "scope": ""
+}

astroquery/eso/tests/setup_package.py

@@ -8,5 +8,8 @@ def get_package_data():
     paths = [os.path.join('data', '*.pickle'),
              os.path.join('data', '*.html'),
              os.path.join('data', '*.tbl'),
+             os.path.join('data', '*.xml'),
+             os.path.join('data', '*.json'),
+             os.path.join('data', '*.fits*')
              ]
     return {'astroquery.eso.tests': paths}

astroquery/eso/tests/test_eso.py

@@ -1,7 +1,10 @@
 # Licensed under a 3-clause BSD style license - see LICENSE.rst
 import os
-from astroquery.utils.mocks import MockResponse
+import sys
+
+import pytest
 
+from astroquery.utils.mocks import MockResponse
 from ...eso import Eso
 
 DATA_DIR = os.path.join(os.path.dirname(__file__), 'data')
@@ -11,21 +14,21 @@ def data_path(filename):
     return os.path.join(DATA_DIR, filename)
 
 
-DATA_FILES = {'GET': {'http://archive.eso.org/wdb/wdb/eso/eso_archive_main/form':
-                      'main_query_form.html',
-                      'http://archive.eso.org/wdb/wdb/eso/amber/form':
-                      'amber_query_form.html',
-                      'http://archive.eso.org/wdb/wdb/adp/phase3_main/form':
-                      'vvv_sgra_form.html',
-                      },
-              'POST': {'http://archive.eso.org/wdb/wdb/eso/eso_archive_main/query':
-                       'main_sgra_query.tbl',
-                       'http://archive.eso.org/wdb/wdb/eso/amber/query':
-                       'amber_sgra_query.tbl',
-                       'http://archive.eso.org/wdb/wdb/adp/phase3_main/query':
-                       'vvv_sgra_survey_response.tbl',
-                       }
-              }
+DATA_FILES = {
+    'GET':
+        {
+            'http://archive.eso.org/wdb/wdb/eso/eso_archive_main/form': 'main_query_form.html',
+            'http://archive.eso.org/wdb/wdb/eso/amber/form': 'amber_query_form.html',
+            'http://archive.eso.org/wdb/wdb/adp/phase3_main/form': 'vvv_sgra_form.html',
+            Eso.AUTH_URL: 'oidc_token.json',
+        },
+    'POST':
+        {
+            'http://archive.eso.org/wdb/wdb/eso/eso_archive_main/query': 'main_sgra_query.tbl',
+            'http://archive.eso.org/wdb/wdb/eso/amber/query': 'amber_sgra_query.tbl',
+            'http://archive.eso.org/wdb/wdb/adp/phase3_main/query': 'vvv_sgra_survey_response.tbl',
+        }
+}
 
 
 def eso_request(request_type, url, **kwargs):
@@ -34,6 +37,32 @@ def eso_request(request_type, url, **kwargs):
     return response
 
 
+def download_request(url, **kwargs):
+    filename = 'testfile.fits.Z'
+    with open(data_path(filename), 'rb') as f:
+        header = {'Content-Disposition': f'filename={filename}'}
+        response = MockResponse(content=f.read(), url=url, headers=header)
+    return response
+
+
+def calselector_request(url, **kwargs):
+    is_multipart = len(kwargs['data']['dp_id']) > 1
+    if is_multipart:
+        filename = 'FORS2.2021-01-02T00_59_12.533_raw2raw_multipart.xml'
+        header = {
+            'Content-Type': 'multipart/form-data; boundary=uFQlfs9nBIDEAIoz0_ZM-O2SXKsZ2iSd4h7H;charset=UTF-8'
+        }
+    else:
+        filename = 'FORS2.2021-01-02T00_59_12.533_raw2raw.xml'
+        header = {
+            'Content-Disposition': f'filename="{filename}"',
+            'Content-Type': 'application/xml; content=calselector'
+        }
+    with open(data_path(filename), 'rb') as f:
+        response = MockResponse(content=f.read(), url=url, headers=header)
+    return response
+
+
 # @pytest.fixture
 # def patch_get(request):
 #    mp = request.getfixturevalue("monkeypatch")
@@ -92,3 +121,60 @@ def test_vvv(monkeypatch):
     assert result_s is not None
     assert 'Object' in result_s.colnames
     assert 'b333' in result_s['Object']
+
+
+def test_authenticate(monkeypatch):
+    eso = Eso()
+    monkeypatch.setattr(eso, '_request', eso_request)
+    eso.cache_location = DATA_DIR
+    authenticated = eso._authenticate(username="someuser", password="somepassword")
+    assert authenticated is True
+
+
+def test_download(monkeypatch):
+    eso = Eso()
+    fileid = 'testfile'
+    destination = os.path.join(DATA_DIR, 'downloads')
+    filename = os.path.join(destination, f"{fileid}.fits.Z")
+    os.makedirs(destination, exist_ok=True)
+    monkeypatch.setattr(eso._session, 'get', download_request)
+    downloaded_files = eso.retrieve_data([fileid], destination=destination, unzip=False)
+    assert len(downloaded_files) == 1
+    assert downloaded_files[0] == filename
+
+
+@pytest.mark.skipif(sys.platform == "win32", reason="gunzip not available on Windows")
+def test_unzip():
+    eso = Eso()
+    filename = os.path.join(DATA_DIR, 'testfile.fits.Z')
+    uncompressed_filename = os.path.join(DATA_DIR, 'testfile.fits')
+    uncompressed_files = eso._unzip_files([filename])
+    assert len(uncompressed_files) == 1
+    assert uncompressed_files[0] == uncompressed_filename
+
+
+def test_cached_file():
+    eso = Eso()
+    filename = os.path.join(DATA_DIR, 'testfile.fits.Z')
+    assert eso._find_cached_file(filename) is True
+    assert eso._find_cached_file("non_existent_filename") is False
+
+
+def test_calselector(monkeypatch):
+    eso = Eso()
+    dataset = 'FORS2.2021-01-02T00:59:12.533'
+    monkeypatch.setattr(eso._session, 'post', calselector_request)
+    result = eso.get_associated_files([dataset], savexml=True, destination=data_path('downloads'))
+    assert isinstance(result, list)
+    assert len(result) == 50
+    assert dataset not in result
+
+
+def test_calselector_multipart(monkeypatch):
+    eso = Eso()
+    datasets = ['FORS2.2021-01-02T00:59:12.533', 'FORS2.2021-01-02T00:59:12.534']
+    monkeypatch.setattr(eso._session, 'post', calselector_request)
+    result = eso.get_associated_files(datasets, savexml=False, destination=data_path('downloads'))
+    assert isinstance(result, list)
+    assert len(result) == 99
+    assert datasets[0] not in result and datasets[1] not in result

astroquery/eso/tests/test_eso_remote.py

@@ -1,10 +1,10 @@
 # Licensed under a 3-clause BSD style license - see LICENSE.rst
+
 import numpy as np
 import pytest
-import warnings
 
-from astroquery.exceptions import LoginError, NoResultsWarning
 from astroquery.eso import Eso
+from astroquery.exceptions import NoResultsWarning
 
 instrument_list = [u'fors1', u'fors2', u'sphere', u'vimos', u'omegacam',
                    u'hawki', u'isaac', u'naco', u'visir', u'vircam', u'apex',
@@ -68,18 +68,6 @@ def test_multisurvey(self, tmp_path):
         assert 'b333_414_58214' in result_s['Object']
         assert 'Pistol-Star' in result_s['Object']
 
-    def test_nologin(self):
-        # WARNING: this test will fail if you haven't cleared your cache and
-        # you have downloaded this file!
-        eso = Eso()
-
-        with pytest.raises(LoginError) as exc:
-            eso.retrieve_data('AMBER.2006-03-14T07:40:19.830')
-
-        assert (exc.value.args[0]
-                == ("If you do not pass a username to login(), you should "
-                    "configure a default one!"))
-
     def test_empty_return(self):
         # test for empty return with an object from the North
         eso = Eso()
@@ -112,40 +100,28 @@ def test_list_instruments(self):
         # we only care about the sets matching
         assert set(inst) == set(instrument_list)
 
-    @pytest.mark.skipif('not Eso.USERNAME')
     def test_retrieve_data(self):
         eso = Eso()
-        eso.login()
-        result = eso.retrieve_data(["MIDI.2014-07-25T02:03:11.561"])
-        assert len(result) > 0
-        assert "MIDI.2014-07-25T02:03:11.561" in result[0]
-        result = eso.retrieve_data("MIDI.2014-07-25T02:03:11.561")
-        assert isinstance(result, str)
-        result = eso.retrieve_data("MIDI.2014-07-25T02:03:11.561",
-                                   request_all_objects=True)
+        file_id = 'AMBER.2006-03-14T07:40:19.830'
+        result = eso.retrieve_data(file_id)
         assert isinstance(result, str)
+        assert file_id in result
 
     @pytest.mark.skipif('not Eso.USERNAME')
-    def test_retrieve_data_twice(self):
+    def test_retrieve_data_authenticated(self):
         eso = Eso()
         eso.login()
-        eso.retrieve_data("MIDI.2014-07-25T02:03:11.561")
-        eso.retrieve_data("AMBER.2006-03-14T07:40:19.830")
+        file_id = 'AMBER.2006-03-14T07:40:19.830'
+        result = eso.retrieve_data(file_id)
+        assert isinstance(result, str)
+        assert file_id in result
 
-    @pytest.mark.skipif('not Eso.USERNAME')
-    def test_retrieve_data_and_calib(self):
+    def test_retrieve_data_list(self):
         eso = Eso()
-        eso.login()
-        result = eso.retrieve_data(["FORS2.2016-06-22T01:44:01.585"],
-                                   with_calib='raw')
-        assert len(result) == 59
-        # Try again, from cache this time
-        result = eso.retrieve_data(["FORS2.2016-06-22T01:44:01.585"],
-                                   with_calib='raw')
-        # Here we get only 1 file path for the science file: as this file
-        # exists, no request is made to get the associated calibrations file
-        # list.
-        assert len(result) == 1
+        datasets = ['MIDI.2014-07-25T02:03:11.561', 'AMBER.2006-03-14T07:40:19.830']
+        result = eso.retrieve_data(datasets)
+        assert isinstance(result, list)
+        assert len(result) == 2
 
     # TODO: remove filter when https://github.com/astropy/astroquery/issues/2539 is fixed
     @pytest.mark.filterwarnings("ignore::pytest.PytestUnraisableExceptionWarning")
@@ -173,14 +149,13 @@ def test_each_instrument_SgrAstar(self, tmp_path):
         instruments = eso.list_instruments(cache=False)
 
         for instrument in instruments:
-            with warnings.catch_warnings(record=True) as record:
-                result_i = eso.query_instrument(instrument, coord1=266.41681662,
-                                                coord2=-29.00782497, cache=False)
+            try:
+                result = eso.query_instrument(instrument, coord1=266.41681662, coord2=-29.00782497, cache=False)
+            except NoResultsWarning:
                 # Sometimes there are ResourceWarnings, we ignore those for this test
-                if len(record) > 0 and NoResultsWarning in {record[i].category for i in range(len(record))}:
-                    assert result_i is None
-                else:
-                    assert len(result_i) > 0
+                pass
+            else:
+                assert len(result) > 0
 
     def test_each_survey_and_SgrAstar(self, tmp_path):
         eso = Eso()

astroquery/utils/mocks.py

@@ -1,6 +1,9 @@
 # Licensed under a 3-clause BSD style license - see LICENSE.rst
 
 import json
+from functools import partial
+from io import BytesIO
+
 
 # The MockResponse class is currently relied upon in code and thus
 # temporarily got moved here to avoid adding pytest as a
@@ -12,24 +15,34 @@ class MockResponse:
     A mocked/non-remote version of `astroquery.query.AstroResponse`
     """
 
-    def __init__(self, content=None, *, url=None, headers={}, content_type=None,
+    def __init__(self, content=None, *, url=None, headers=None, content_type=None,
                  stream=False, auth=None, status_code=200, verify=True,
                  allow_redirects=True, json=None):
         assert content is None or hasattr(content, 'decode')
         self.content = content
         self.raw = content
-        self.headers = headers
+        self.headers = headers or {}
         if content_type is not None:
             self.headers.update({'Content-Type': content_type})
         self.url = url
         self.auth = auth
         self.status_code = status_code
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
+
     def iter_lines(self):
         content = self.content.split(b"\n")
         for line in content:
             yield line
 
+    def iter_content(self, chunk_size):
+        stream = BytesIO(self.content)
+        return iter(partial(stream.read, chunk_size), b'')
+
     def raise_for_status(self):
         pass