From baf95edd23ee04c585acf7082fffc15ed9e16ff8 Mon Sep 17 00:00:00 2001 From: Abdu Zoghbi Date: Mon, 13 May 2024 11:31:58 -0400 Subject: [PATCH] deprecate HeasarcBrowse --- astroquery/heasarc/__init__.py | 3 +- astroquery/heasarc/core.py | 11 +- astroquery/heasarc/heasarc_browse.py | 429 ------------------ astroquery/heasarc/tests/conftest.py | 136 ------ astroquery/heasarc/tests/setup_package.py | 11 - .../heasarc/tests/test_heasarcBrowse.py | 1 - .../tests/test_heasarcBrowse_remote.py | 127 ------ .../tests/test_heasarcBrowse_remote_isdc.py | 211 --------- .../heasarc/tests/test_heasarc_remote.py | 4 +- docs/heasarc/heasarc.rst | 296 ------------ 10 files changed, 7 insertions(+), 1222 deletions(-) delete mode 100644 astroquery/heasarc/heasarc_browse.py delete mode 100644 astroquery/heasarc/tests/conftest.py delete mode 100644 astroquery/heasarc/tests/setup_package.py delete mode 100644 astroquery/heasarc/tests/test_heasarcBrowse.py delete mode 100644 astroquery/heasarc/tests/test_heasarcBrowse_remote.py delete mode 100644 astroquery/heasarc/tests/test_heasarcBrowse_remote_isdc.py diff --git a/astroquery/heasarc/__init__.py b/astroquery/heasarc/__init__.py index 7db561b455..251a681bae 100644 --- a/astroquery/heasarc/__init__.py +++ b/astroquery/heasarc/__init__.py @@ -47,9 +47,8 @@ class Conf(_config.ConfigNamespace): conf = Conf() -from .core import Heasarc, HeasarcClass, HeasarcBrowseClass, HeasarcBrowse +from .core import Heasarc, HeasarcClass __all__ = ['Heasarc', 'HeasarcClass', - 'HeasarcBrowseClass', 'HeasarcBrowse', 'Conf', 'conf', ] diff --git a/astroquery/heasarc/core.py b/astroquery/heasarc/core.py index bf68da48e1..0d095e2ad0 100644 --- a/astroquery/heasarc/core.py +++ b/astroquery/heasarc/core.py @@ -19,8 +19,6 @@ from ..exceptions import InvalidQueryError, NoResultsWarning from . import conf -from .heasarc_browse import HeasarcBrowseClass - @async_to_sync class HeasarcClass(BaseQuery): @@ -98,7 +96,7 @@ def _get_default_cols(self, table_name): """ meta = self._meta[ (self._meta['table'] == table_name) - and (self._meta['par'] != '') + & (self._meta['par'] != '') ] meta.sort('value') defaults = meta['par'] @@ -119,7 +117,7 @@ def get_default_radius(self, table_name): """ meta = self._meta[ (self._meta['table'] == table_name) - and (self._meta['par'] == '') + & (self._meta['par'] == '') ] radius = np.double(meta['value'][0]) * u.arcmin return radius @@ -503,8 +501,8 @@ def get_links(self, query_result=None, tablename=None): if tablename is None: tablename = self._tablename - if ( - not isinstance(tablename, str) + if not ( + isinstance(tablename, str) and tablename in self.tap.tables.keys() ): raise ValueError(f'Unknown table name: {tablename}') @@ -747,4 +745,3 @@ def _s3_tree_download(client, bucket_name, path, local): Heasarc = HeasarcClass() -HeasarcBrowse = HeasarcBrowseClass() diff --git a/astroquery/heasarc/heasarc_browse.py b/astroquery/heasarc/heasarc_browse.py deleted file mode 100644 index c6aa597032..0000000000 --- a/astroquery/heasarc/heasarc_browse.py +++ /dev/null @@ -1,429 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst - -""" -HEASARC -==== - - -Module to query the NASA High Energy Archive HEASARC. -""" - -from typing import Union -import warnings -from io import StringIO, BytesIO -from astropy.table import Table -from astropy.io import fits -from astropy import coordinates -from astropy import units as u -from ..query import BaseQuery -from ..utils import commons, async_to_sync -from ..exceptions import InvalidQueryError, NoResultsWarning -from . import conf - - -__all__ = ['HeasarcBrowseClass'] - - -def Table_read(*args, **kwargs): - if commons.ASTROPY_LT_5_1: - return Table.read(*args, **kwargs) - else: - return Table.read(*args, **kwargs, unit_parse_strict='silent') - - -@async_to_sync -class HeasarcBrowseClass(BaseQuery): - - """ - HEASARC query class. - - For a list of available HEASARC mission tables, visit: - https://heasarc.gsfc.nasa.gov/cgi-bin/W3Browse/w3catindex.pl - """ - - URL = conf.server - TIMEOUT = conf.timeout - coord_systems = ['fk5', 'fk4', 'equatorial', 'galactic'] - - def query_async(self, request_payload, *, cache=True, url=None): - """ - Submit a query based on a given request_payload. This allows detailed - control of the query to be submitted. - - cache (bool) defaults to True. If set overrides global caching behavior. - See :ref:`caching documentation `. - """ - - if url is None: - url = conf.server - - response = self._request('GET', url, params=request_payload, - timeout=self.TIMEOUT, cache=cache) - return response - - def query_mission_list(self, *, cache=True, get_query_payload=False): - """ - Returns a list of all available mission tables with descriptions - - cache (bool) defaults to True. If set overrides global caching behavior. - See :ref:`caching documentation `. - """ - request_payload = self._args_to_payload( - entry='none', - mission='xxx', - displaymode='BatchDisplay' - ) - - if get_query_payload: - return request_payload - - # Parse the results specially (it's ascii format, not fits) - response = self.query_async( - request_payload, - url=conf.server, - cache=cache - ) - data_str = response.text - data_str = data_str.replace('Table xxx does not seem to exist!\n\n\n\nAvailable tables:\n', '') - table = Table.read(data_str, format='ascii.fixed_width_two_line', - delimiter='+', header_start=1, position_line=2, - data_start=3, data_end=-1) - return table - - def query_mission_cols(self, mission, *, cache=True, get_query_payload=False, - **kwargs): - """ - Returns a list containing the names of columns that can be returned for - a given mission table. By default all column names are returned. - - Parameters - ---------- - mission : str - Mission table (short name) to search from - fields : str, optional - Return format for columns from the server available options: - * Standard : Return default table columns - * All (default) : Return all table columns - * : User defined csv list of columns to be returned - cache : bool, optional - Defaults to True. If set overrides global caching behavior. - See :ref:`caching documentation `. - All other parameters have no effect - """ - - response = self.query_region_async(position=coordinates.SkyCoord(10, 10, unit='deg', frame='fk5'), - mission=mission, - radius='361 degree', - cache=cache, - get_query_payload=get_query_payload, - resultsmax=1, - fields='All') - - # Return payload if requested - if get_query_payload: - return response - - return self._parse_result(response).colnames - - def query_object_async(self, object_name, mission, *, - cache=True, get_query_payload=False, - **kwargs): - """ - Query around a specific object within a given mission catalog - - Parameters - ---------- - object_name : str - Object to query around. To set search radius use the 'radius' - parameter. - mission : str - Mission table to search from - cache : bool - Defaults to True. If set overrides global caching behavior. - See :ref:`caching documentation `. - **kwargs : - see `~astroquery.heasarc.HeasarcClass._args_to_payload` for list - of additional parameters that can be used to refine search query - """ - request_payload = self._args_to_payload( - mission=mission, - entry=object_name, - **kwargs - ) - - # Return payload if requested - if get_query_payload: - return request_payload - - return self.query_async(request_payload, cache=cache) - - def query_region_async(self, position: Union[coordinates.SkyCoord, str], - mission, radius, *, cache=True, get_query_payload=False, - **kwargs): - """ - Query around specific set of coordinates within a given mission - catalog. Method first converts the supplied coordinates into the FK5 - reference frame and searches for sources from there. Because of this, - returned offset coordinates may look different than the ones supplied. - - Parameters - ---------- - position : `astropy.coordinates.SkyCoord` or str - The position around which to search. It may be specified as a - string in which case it is resolved using online services or as - the appropriate `astropy.coordinates` object. ICRS coordinates - may also be entered as a string. - (adapted from nrao module) - mission : str - Mission table to search from - radius : - Astropy Quantity object, or a string that can be parsed into one. - e.g., '1 degree' or 1*u.degree. - cache : bool - Defaults to True. If set overrides global caching behavior. - See :ref:`caching documentation `. - **kwargs : - see `~astroquery.heasarc.HeasarcClass._args_to_payload` for list - of additional parameters that can be used to refine search query - """ - # Convert the coordinates to FK5 - c = commons.parse_coordinates(position).transform_to(coordinates.FK5) - kwargs['coordsys'] = 'fk5' - kwargs['equinox'] = 2000 - - # Generate the request - # Fixed string representation of coordinates ensures that request payload - # does not depend on python/astropy version for the same input coordinates - request_payload = self._args_to_payload( - mission=mission, - entry=f"{c.ra.degree:.10f},{c.dec.degree:.10f}", - radius=u.Quantity(radius), - **kwargs - ) - - # Return payload if requested - if get_query_payload: - return request_payload - - # Submit the request - return self.query_async(request_payload, cache=cache) - - def _old_w3query_fallback(self, content): - # old w3query (such as that used in ISDC) return very strange fits, with all ints - - fits_content = fits.open(BytesIO(content)) - - for col in fits_content[1].columns: - if col.disp is not None: - col.format = col.disp - else: - col.format = str(col.format).replace("I", "A") - - tmp_out = BytesIO() - fits_content.writeto(tmp_out) - tmp_out.seek(0) - - return Table_read(tmp_out) - - def _fallback(self, text): - """ - Blank columns which have to be converted to float or in fail so - lets fix that by replacing with -1's - """ - - data = StringIO(text) - header = fits.getheader(data, 1) # Get header for column info - colstart = [y for x, y in header.items() if "TBCOL" in x] - collens = [int(float(y[1:])) - for x, y in header.items() if "TFORM" in x] - - new_table = [] - - old_table = text.split("END")[-1].strip() - for line in old_table.split("\n"): - newline = [] - for n, tup in enumerate(zip(colstart, collens), start=1): - cstart, clen = tup - part = line[cstart - 1:cstart + clen] - newline.append(part) - if len(part.strip()) == 0: - if header["TFORM%i" % n][0] in ["F", "I"]: - # extra space is required to separate column - newline[-1] = "-1".rjust(clen) + " " - new_table.append("".join(newline)) - - data = StringIO(text.replace(old_table, "\n".join(new_table))) - - return Table_read(data, hdu=1) - - def _blank_table_fallback(self, data): - """ - In late 2022, we started seeing examples where the null result came - back as a FITS file with an ImageHDU and no BinTableHDU. - """ - with fits.open(data) as fh: - comments = fh[1].header['COMMENT'] - emptytable = Table() - emptytable.meta['COMMENT'] = comments - warnings.warn(NoResultsWarning("No matching rows were found in the query.")) - return emptytable - - def _parse_result(self, response, *, verbose=False): - # if verbose is False then suppress any VOTable related warnings - if not verbose: - commons.suppress_vo_warnings() - - if "BATCH_RETRIEVAL_MSG ERROR:" in response.text: - raise InvalidQueryError("One or more inputs is not recognized by HEASARC. " - "Check that the object name is in GRB, SIMBAD+Sesame, or " - "NED format and that the mission name is as listed in " - "query_mission_list().") - elif "Software error:" in response.text: - raise InvalidQueryError("Unspecified error from HEASARC database. " - "\nCheck error message: \n{!s}".format(response.text)) - elif "NO MATCHING ROWS" in response.text: - warnings.warn(NoResultsWarning("No matching rows were found in the query.")) - return Table() - - if "XTENSION= 'IMAGE '" in response.text: - data = BytesIO(response.content) - return self._blank_table_fallback(data) - - try: - data = BytesIO(response.content) - return Table_read(data, hdu=1) - except ValueError: - try: - return self._fallback(response.text) - except Exception: - return self._old_w3query_fallback(response.content) - - def _args_to_payload(self, **kwargs): - """ - Generates the payload based on user supplied arguments - - Parameters - ---------- - mission : str - Mission table to query - entry : str, optional - Object or position for center of query. A blank value will return - all entries in the mission table. Acceptable formats: - * Object name : Name of object, e.g. 'Crab' - * Coordinates : X,Y coordinates, either as 'degrees,degrees' or - 'hh mm ss,dd mm ss' - fields : str, optional - Return format for columns from the server available options: - * Standard (default) : Return default table columns - * All : Return all table columns - * : User defined csv list of columns to be - returned - radius : float (arcmin), optional - Astropy Quantity object, or a string that can be parsed into one. - e.g., '1 degree' or 1*u.degree. - coordsys: str, optional - If 'entry' is a set of coordinates, this specifies the coordinate - system used to interpret them. By default, equatorial coordinates - are assumed. Possible values: - * 'fk5' (FK5 J2000 equatorial coordinates) - * 'fk4' (FK4 B1950 equatorial coordinates) - * 'equatorial' (equatorial coordinates, `equinox` param - determines epoch) - * 'galactic' (Galactic coordinates) - equinox : int, optional - Epoch by which to interpret supplied equatorial coordinates - (defaults to 2000, ignored if `coordsys` is not 'equatorial') - resultmax : int, optional - Set maximum query results to be returned - sortvar : str, optional - Set the name of the column by which to sort the results. By default - the results are sorted by distance from queried object/position - - displaymode : str, optional - Return format from server. Since the user does not interact with - this directly, it's best to leave this alone - action : str, optional - Type of action to be taken (defaults to 'Query') - """ - # User-facing parameters are lower case, while parameters as passed to the - # HEASARC service are capitalized according to the HEASARC requirements. - # The necessary transformations are done in this function. - - # Define the basic query for this object - mission = kwargs.pop('mission') - - request_payload = dict( - tablehead=('name=BATCHRETRIEVALCATALOG_2.0 {}' - .format(mission)), - Entry=kwargs.pop('entry', 'none'), - Action=kwargs.pop('action', 'Query'), - displaymode=kwargs.pop('displaymode', 'FitsDisplay'), - resultsmax=kwargs.pop('resultsmax', '10') - ) - - # Fill in optional information for refined queries - - # Handle queries involving coordinates - coordsys = kwargs.pop('coordsys', 'fk5') - equinox = kwargs.pop('equinox', None) - - if coordsys.lower() == 'fk5': - request_payload['Coordinates'] = 'Equatorial: R.A. Dec' - - elif coordsys.lower() == 'fk4': - request_payload['Coordinates'] = 'Equatorial: R.A. Dec' - request_payload['equinox'] = 1950 - - elif coordsys.lower() == 'equatorial': - request_payload['Coordinates'] = 'Equatorial: R.A. Dec' - - if equinox is not None: - request_payload['Equinox'] = str(equinox) - - elif coordsys.lower() == 'galactic': - request_payload['Coordinates'] = 'Galactic: LII BII' - - else: - raise ValueError("'coordsys' parameter must be one of {!s}" - .format(self.coord_systems)) - - # Specify which table columns are to be returned - fields = kwargs.pop('fields', None) - if fields is not None: - if fields.lower() == 'standard': - request_payload['Fields'] = 'Standard' - elif fields.lower() == 'all': - request_payload['Fields'] = 'All' - else: - request_payload['varon'] = fields.lower().split(',') - - # Set search radius (arcmin) - radius = kwargs.pop('radius', None) - if radius is not None: - request_payload['Radius'] = "{}".format(u.Quantity(radius).to(u.arcmin)) - - # Maximum number of results to be returned - resultmax = kwargs.pop('resultmax', None) - if resultmax is not None: - request_payload['ResultMax'] = int(resultmax) - - # Set variable for sorting results - sortvar = kwargs.pop('sortvar', None) - if sortvar is not None: - request_payload['sortvar'] = sortvar.lower() - - # Time range variable - _time = kwargs.pop('time', None) - if _time is not None: - request_payload['Time'] = _time - - if len(kwargs) > 0: - mission_fields = [k.lower() for k in self.query_mission_cols(mission=mission)] - - for k, v in kwargs.items(): - if k.lower() in mission_fields: - request_payload['bparam_' + k.lower()] = v - else: - raise ValueError(f"unknown parameter '{k}' provided, must be one of {mission_fields}") - - return request_payload diff --git a/astroquery/heasarc/tests/conftest.py b/astroquery/heasarc/tests/conftest.py deleted file mode 100644 index e459ec8c2d..0000000000 --- a/astroquery/heasarc/tests/conftest.py +++ /dev/null @@ -1,136 +0,0 @@ -import json -import os -import glob -import hashlib -import requests -import pytest -from astropy.coordinates import SkyCoord -from ... import log - -""" -This is an attempt to reduce code duplication between remote and local tests. - -if there is test data: - runs as usual, except all tests have two versions with the same code: remote and local -else - runs remote test patched so that the test data is stored in a temporary directory. - advice is given to copy the newly generated test data into the repository -""" - - -# The quasar 3C 273 -skycoord_3C_273 = SkyCoord("12h29m06.70s +02d03m08.7s", frame="icrs") - - -class MockResponse: - def __init__(self, text): - self.text = text - self.content = text.encode() - - -def data_path(filename, output=False): - if output: - data_dir = os.path.join( - os.getenv("TMPDIR", "/tmp"), "astroquery-heasarc-saved-data" - ) - os.makedirs(data_dir, exist_ok=True) - else: - data_dir = os.path.join(os.path.dirname(__file__), "data") - - return os.path.join(data_dir, filename + ".dat") - - -def fileid_for_request(url, params): - return hashlib.md5(str((url, sorted(params.items()))).encode()).hexdigest()[:8] - - -def filename_for_request(url, params, output=False): - fileid = fileid_for_request(url, params) - - filename = data_path(fileid, output=output) - log.debug(f'constructed filename {filename} for request: ' - + json.dumps(dict(url=url, params=params), sort_keys=True, indent=4)) - - return filename - - -def get_mockreturn(session, method, url, params=None, timeout=10, **kwargs): - - filename = filename_for_request(url, params) - try: - with open(filename, 'rt') as infile: - content = infile.read() - except FileNotFoundError: - log.error( - f'no stored mock data in {filename} for url="{url}" and params="{params}", ' - f'perhaps you need to clean test data and regenerate it? ' - f'It will be regenerated automatically if cleaned, try `rm -fv astroquery/heasarc/tests/data/* ./build`' - ) - raise - - return MockResponse(content) - - -def save_response_of_get(session, method, url, params=None, timeout=10, **kwargs): - # _original_request is a monkeypatch-added attribute in patch_get - text = requests.Session._original_request( - session, method, url, params=params, timeout=timeout - ).text - - filename = filename_for_request(url, params, output=True) - - with open(filename, "wt") as f: - log.info(f'saving output to {filename} for url="{url}" and params="{params}"') - log.warning( - f"you may want to run `cp -fv {os.path.dirname(filename)}/* astroquery/heasarc/tests/data/; rm -rfv build`" - "`git add astroquery/heasarc/tests/data/*`." - ) - f.write(text) - - return MockResponse(text) - - -@pytest.fixture() -def patch_get(request): - """ - If the mode is not remote, patch `requests.Session` to either return saved local data - or run save data new local data - """ - mode = request.param - mp = request.getfixturevalue("monkeypatch") - - if mode != "remote": - requests.Session._original_request = requests.Session.request - mp.setattr( - requests.Session, - "request", - {"save": save_response_of_get, "local": get_mockreturn}[mode], - ) - - mp.assume_fileid_for_request = lambda patched_fileid_for_request: \ - mp.setattr('astroquery.heasarc.tests.conftest.fileid_for_request', patched_fileid_for_request) - - mp.reset_default_fileid_for_request = lambda: \ - mp.delattr('astroquery.heasarc.tests.conftest.fileid_for_request') - - return mp - - -def have_mock_data(): - return len(glob.glob(data_path("*"))) > 0 - - -parametrization_local_save_remote = pytest.mark.parametrize( - "patch_get", [ - pytest.param("local", marks=[ - pytest.mark.skipif(not have_mock_data(), - reason="No test data found. If remote_data is allowed, we'll generate some.")]), - pytest.param("save", marks=[ - pytest.mark.remote_data, - pytest.mark.skipif(have_mock_data(), - reason="found some test data: please delete them to save again.")]), - pytest.param("remote", marks=[ - pytest.mark.remote_data, - pytest.mark.skipif(not have_mock_data(), - reason="No test data found, [save] will run remote tests and save data.")])], - indirect=True) diff --git a/astroquery/heasarc/tests/setup_package.py b/astroquery/heasarc/tests/setup_package.py deleted file mode 100644 index 7ff81553fa..0000000000 --- a/astroquery/heasarc/tests/setup_package.py +++ /dev/null @@ -1,11 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst - - -import os - - -def get_package_data(): - paths = [ - os.path.join("data", "*.dat"), - ] - return {"astroquery.heasarc.tests": paths} diff --git a/astroquery/heasarc/tests/test_heasarcBrowse.py b/astroquery/heasarc/tests/test_heasarcBrowse.py deleted file mode 100644 index 9dce85d06f..0000000000 --- a/astroquery/heasarc/tests/test_heasarcBrowse.py +++ /dev/null @@ -1 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst diff --git a/astroquery/heasarc/tests/test_heasarcBrowse_remote.py b/astroquery/heasarc/tests/test_heasarcBrowse_remote.py deleted file mode 100644 index 0089aaa140..0000000000 --- a/astroquery/heasarc/tests/test_heasarcBrowse_remote.py +++ /dev/null @@ -1,127 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst - -import pytest -import requests - -from ...heasarc import HeasarcBrowse as Heasarc - -from .conftest import MockResponse, parametrization_local_save_remote, skycoord_3C_273 - -from astroquery.exceptions import NoResultsWarning - -from astropy.coordinates import SkyCoord -from astropy import units as u - - -@parametrization_local_save_remote -class TestHeasarcBrowse: - - @pytest.fixture(autouse=True) - def _patch_get(self, patch_get): - return patch_get - - def test_custom_args(self): - object_name = 'Crab' - mission = 'intscw' - - heasarc = Heasarc() - - table = heasarc.query_object(object_name, - mission=mission, - radius='1 degree', - time="2020-09-01 .. 2020-12-01", - resultmax=10, - good_isgri=">1000", - cache=False - ) - assert len(table) > 0 - - def test_filter_custom_args(self): - object_name = 'Crab' - mission = 'intscw' - - heasarc = Heasarc() - - with pytest.raises(ValueError): - heasarc.query_object(object_name, - mission=mission, - radius='1 degree', - time="2020-09-01 .. 2020-12-01", - resultmax=10, - very_good_isgri=">1000", - ) - - def test_basic_example(self): - mission = 'rosmaster' - object_name = '3c273' - - heasarc = Heasarc() - table = heasarc.query_object(object_name, mission=mission) - - assert len(table) == 63 - - def test_mission_list(self): - heasarc = Heasarc() - missions = heasarc.query_mission_list() - - # Assert that there are indeed a large number of tables - # Number of tables could change, but should be > 900 (currently 956) - assert len(missions) > 900 - - def test_mission_cols(self): - heasarc = Heasarc() - mission = 'rosmaster' - cols = heasarc.query_mission_cols(mission=mission) - - assert len(cols) == 29 - - # Test that the cols list contains known names - assert 'EXPOSURE' in cols - assert 'RA' in cols - assert 'DEC' in cols - assert 'SEARCH_OFFSET_' in cols - - def test_query_object_async(self): - mission = 'rosmaster' - object_name = '3c273' - - heasarc = Heasarc() - response = heasarc.query_object_async(object_name, mission=mission) - assert response is not None - assert isinstance(response, (requests.models.Response, MockResponse)) - - def test_query_region_async(self): - heasarc = Heasarc() - mission = 'rosmaster' - response = heasarc.query_region_async( - skycoord_3C_273, mission=mission, radius="1 degree") - assert response is not None - assert isinstance(response, (requests.models.Response, MockResponse)) - - def test_query_region(self): - heasarc = Heasarc() - mission = 'rosmaster' - - table = heasarc.query_region( - skycoord_3C_273, mission=mission, radius="1 degree") - - assert len(table) == 63 - - def test_query_region_nohits(self): - """ - Regression test for #2560: HEASARC returns a FITS file as a null result - """ - heasarc = Heasarc() - - with pytest.warns(NoResultsWarning, match='No matching rows were found in the query.'): - # This was an example coordinate that returned nothing - # Since Fermi is still active, it is possible that sometime in the - # future an event will occur here. - table = heasarc.query_region(SkyCoord(0.28136*u.deg, -0.09789*u.deg, frame='fk5'), - mission='fermilpsc', radius=0.1*u.deg) - - assert len(table) == 0 - # this is to check that the header comments got parsed correctly - # I'm not certain that they will always be returned in the same order, - # so it may be necessary in the future to change this part of the test - assert 'heasarc_fermilpsc' in table.meta['COMMENT'][0] diff --git a/astroquery/heasarc/tests/test_heasarcBrowse_remote_isdc.py b/astroquery/heasarc/tests/test_heasarcBrowse_remote_isdc.py deleted file mode 100644 index 254c0e1289..0000000000 --- a/astroquery/heasarc/tests/test_heasarcBrowse_remote_isdc.py +++ /dev/null @@ -1,211 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst - -import pytest -import requests -from astropy.time import Time, TimeDelta -import astropy.units as u - -from ...heasarc import HeasarcBrowse as Heasarc, Conf -from .conftest import MockResponse, parametrization_local_save_remote, skycoord_3C_273 - - -@parametrization_local_save_remote -class TestHeasarcBrowseISDC: - - @pytest.fixture(autouse=True) - def _patch_get(self, patch_get): - return patch_get - - @property - def isdc_context(self): - return Conf.server.set_temp( - 'https://www.isdc.unige.ch/browse/w3query.pl' - ) - - def test_custom_args(self): - object_name = 'Crab' - mission = 'integral_rev3_scw' - - heasarc = Heasarc() - - with self.isdc_context: - table = heasarc.query_object( - object_name, - mission=mission, - radius='1 degree', - time="2020-09-01 .. 2020-12-01", - resultmax=10, - good_isgri=">1000", - cache=False - ) - assert len(table) > 0 - - def test_filter_custom_args(self): - object_name = 'Crab' - mission = 'integral_rev3_scw' - - heasarc = Heasarc() - - with self.isdc_context: - with pytest.raises(ValueError): - heasarc.query_object( - object_name, - mission=mission, - radius='1 degree', - time="2020-09-01 .. 2020-12-01", - resultmax=10, - very_good_isgri=">1000", - ) - - def test_basic_time(self): - object_name = 'Crab' - - heasarc = Heasarc() - - def Q(mission): - return heasarc.query_object( - object_name, - mission=mission, - radius='1 degree', - time="2020-09-01 .. 2020-12-01", - resultmax=10000 - ) - - with self.isdc_context: - table_isdc = Q('integral_rev3_scw') - - table_heasarc = Q('intscw') - - assert len(table_isdc) == 11 - assert len(table_isdc) == len(table_heasarc) - - def test_compare_time(self, patch_get): - patch_get.assume_fileid_for_request( - lambda url, params: f"last-month-{params['tablehead'].split()[-1]}") - - object_name = 'Crab' - - heasarc = Heasarc() - - month_ago = (Time.now() - TimeDelta(30 * u.day)).isot[:10] - today = Time.now().isot[:10] - T = month_ago + " .. " + today - - def Q(mission): - return heasarc.query_object( - object_name, - mission=mission, - time=T, - resultmax=10000, - radius='1000 deg' - ) - - with self.isdc_context: - table_isdc = Q('integral_rev3_scw') - - table_heasarc = Q('intscw') - - # heasarc synchronizes twice a month, and it might or might not be the same at the request time - assert len(table_isdc) >= len(table_heasarc) - - def test_ra_validity(self): - object_name = 'Crab' - - heasarc = Heasarc() - - T = "2020-01-01 03:56:30 .. 2020-01-01 04:55:10" - - with self.isdc_context: - table_isdc = heasarc.query_object( - object_name, - mission='integral_rev3_scw', - time=T, - resultmax=10000, - radius='1000 deg' - ) - - table_heasarc = heasarc.query_object( - object_name, - mission='intscw', - time=T, - resultmax=10000, - radius='1000 deg' - ) - - assert len(table_isdc) == len(table_heasarc) == 1 - - assert table_isdc['SCW_ID'] == table_heasarc['SCW_ID'] - - assert 'RA' in table_heasarc.columns - assert 'RA_X' in table_isdc.columns - - assert abs(float(table_isdc['RA_X'][0]) - float(table_heasarc['RA'][0])) < 0.0001 - - def test_basic_example(self): - mission = 'integral_rev3_scw' - object_name = '3c273' - - heasarc = Heasarc() - - with self.isdc_context: - table = heasarc.query_object( - object_name, - mission=mission, - radius='1 degree' - ) - - assert len(table) >= 274 - - def test_mission_list(self): - heasarc = Heasarc() - - with self.isdc_context: - missions = heasarc.query_mission_list() - - # The number changes - assert len(missions) >= 5 - - def test_mission_cols(self): - heasarc = Heasarc() - mission = 'integral_rev3_scw' - - with self.isdc_context: - cols = heasarc.query_mission_cols(mission=mission) - - assert len(cols) == 35 - - # Test that the cols list contains known names - assert 'SCW_ID' in cols - assert 'GOOD_ISGRI' in cols - assert 'RA_X' in cols - assert 'DEC_X' in cols - assert 'SEARCH_OFFSET_' in cols - - def test_query_object_async(self): - mission = 'integral_rev3_scw' - object_name = '3c273' - - heasarc = Heasarc() - response = heasarc.query_object_async(object_name, mission=mission) - assert response is not None - assert isinstance(response, (requests.models.Response, MockResponse)) - - def test_query_region_async(self): - heasarc = Heasarc() - mission = 'integral_rev3_scw' - - with self.isdc_context: - response = heasarc.query_region_async( - skycoord_3C_273, mission=mission, radius="1 degree") - assert response is not None - assert isinstance(response, (requests.models.Response, MockResponse)) - - def test_query_region(self): - heasarc = Heasarc() - mission = 'integral_rev3_scw' - - with self.isdc_context: - table = heasarc.query_region( - skycoord_3C_273, mission=mission, radius="1 degree") - - assert len(table) >= 274 diff --git a/astroquery/heasarc/tests/test_heasarc_remote.py b/astroquery/heasarc/tests/test_heasarc_remote.py index ed00e90774..4b74735ef5 100644 --- a/astroquery/heasarc/tests/test_heasarc_remote.py +++ b/astroquery/heasarc/tests/test_heasarc_remote.py @@ -213,8 +213,8 @@ def test_download_data__s3_folder(self, slash): tab = Table( { "aws": [ - ("s3://nasa-heasarc/rxte/data/archive/AO10/" - "P91129/91129-01-68-00A/stdprod{slash}") + (f"s3://nasa-heasarc/rxte/data/archive/AO10/" + f"P91129/91129-01-68-00A/stdprod{slash}") ] } ) diff --git a/docs/heasarc/heasarc.rst b/docs/heasarc/heasarc.rst index a261dfc71a..8e2d81a0d3 100644 --- a/docs/heasarc/heasarc.rst +++ b/docs/heasarc/heasarc.rst @@ -16,7 +16,6 @@ Virtual Observatory protocols with the Xamin interface, which offers more powerful search options than the old Browse interface. - :ref:`Heasarc Main Interface`. -- :ref:`Old Browse Interface`. .. _Heasarc Main Interface: @@ -279,301 +278,6 @@ in the XMM master table ``xmmmaster``: xmm_revolution XMM-Newton Revolution (Orbit) Number status Status of Observation: scheduled, observed, processed, archived - -.. _Old Browse Interface: - -Old Browse Interface -==================== - -.. warning:: - The old Browse interface has limited support from the Heasarc, - please consider using the main `~astroquery.heasarc.HeasarcClass` interface. - -Getting lists of available datasets ------------------------------------ - -There are two ways to obtain a list of objects. The first is by querying around -an object by name: - -.. doctest-remote-data:: - - >>> from astroquery.heasarc import Heasarc - >>> heasarc = Heasarc() - >>> mission = 'rosmaster' - >>> object_name = '3c273' - >>> table = heasarc.query_object(object_name, mission=mission) - >>> table[:3].pprint(max_width=120) - SEQ_ID INSTRUMENT EXPOSURE NAME ... DEC START_TIME END_TIME SEARCH_OFFSET_ - s ... deg mjd mjd - ---------------- ---------- -------- -------------------- ... ------ ---------------- ---------------- ---------------- - RH701576N00 HRI 68154 3C 273 ... 2.0500 49704.3090856482 49724.6236226852 0.190 (3c273)\n - RP600242A01 PSPCB 24822 GIOVANELLI-HAYNES CL ... 1.6000 48980.6468865741 48982.9284837963 34.236 (3c273)\n - RH700234N00 HRI 17230 3C 273 ... 2.0500 48629.2693055556 48632.4716782407 0.190 (3c273)\n - -Alternatively, a query can also be conducted around a specific set of sky -coordinates: - -.. doctest-remote-data:: - - >>> from astroquery.heasarc import Heasarc - >>> from astropy.coordinates import SkyCoord - >>> heasarc = Heasarc() - >>> mission = 'rosmaster' - >>> coords = SkyCoord('12h29m06.70s +02d03m08.7s', frame='icrs') - >>> table = heasarc.query_region(coords, mission=mission, radius='1 degree') - >>> table[:3].pprint(max_width=120) - SEQ_ID INSTRUMENT EXPOSURE ... START_TIME END_TIME SEARCH_OFFSET_ - s ... mjd mjd - ---------------- ---------- -------- ... ---------------- ---------------- -------------------------------------- - RH701576N00 HRI 68154 ... 49704.3090856482 49724.6236226852 0.191 (187.2779228198,2.0524148595)\n - RP600242A01 PSPCB 24822 ... 48980.6468865741 48982.9284837963 34.237 (187.2779228198,2.0524148595)\n - RH700234N00 HRI 17230 ... 48629.2693055556 48632.4716782407 0.191 (187.2779228198,2.0524148595)\n - -Note that the :meth:`~astroquery.heasarc.HeasarcClass.query_region` converts -the passed coordinates to the FK5 reference frame before submitting the query. - - -Modifying returned table columns --------------------------------- - -Each table has a set of default columns that are returned when querying the -database. You can return all available columns for a given mission by specifying -the ``fields`` parameter in either of the above queries. For exampe: - -.. doctest-remote-data:: - - >>> table = heasarc.query_object(object_name='3c273', mission='rosmaster', fields='All') - -will return all available columns from the ``rosmaster`` mission table. -Alternatively, a comma-separated list of column names can also be provided to -specify which columns will be returned: - -.. doctest-remote-data:: - - >>> table = heasarc.query_object(object_name='3c273', mission='rosmaster', fields='EXPOSURE,RA,DEC') - >>> table[:3].pprint() - EXPOSURE RA DEC SEARCH_OFFSET_ - s deg deg - -------- -------- ------ ---------------- - 68154 187.2800 2.0500 0.190 (3c273)\n - 24822 186.9300 1.6000 34.236 (3c273)\n - 17230 187.2800 2.0500 0.190 (3c273)\n - -Note that the ``SEARCH_OFFSET_`` column will always be included in the results. -If a column name is passed to the ``fields`` parameter which does not exist in -the requested mission table, the query will fail. To obtain a list of available -columns for a given mission table, do the following: - -.. doctest-remote-data:: - - >>> cols = heasarc.query_mission_cols(mission='rosmaster') - >>> print(cols) - ['SEQ_ID', 'INSTRUMENT', 'EXPOSURE', 'NAME', 'RA', 'DEC', 'START_TIME', 'END_TIME', 'AO', 'BII', 'CLASS', 'FILTER', 'FITS_TYPE', 'INDEX_ID', 'LII', 'PI_FNAME', 'PI_LNAME', 'PROC_REV', 'PROPOSAL_NUMBER', 'QA_NUMBER', 'RDAY_BEGIN', 'RDAY_END', 'REQUESTED_EXPOSURE', 'ROLL', 'ROR', 'SITE', 'SUBJ_CAT', 'TITLE', 'SEARCH_OFFSET_'] - - -Additional query parameters ---------------------------- - -By default, the :meth:`~astroquery.heasarc.HeasarcClass.query_object` method -returns all entries within approximately one degree of the specified object. -This can be modified by supplying the ``radius`` parameter. This parameter -takes a distance to look for objects. The following modifies the search radius -to 120 arcmin: - -.. doctest-remote-data:: - - >>> from astroquery.heasarc import Heasarc - >>> heasarc = Heasarc() - >>> table = heasarc.query_object(object_name, mission='rosmaster', radius='120 arcmin') - -``radius`` takes an angular distance specified as an astropy Quantity object, -or a string that can be parsed into one (e.g., '1 degree' or 1*u.degree). The -following are equivalent: - -.. doctest-remote-data:: - - >>> table = heasarc.query_object(object_name, mission='rosmaster', radius='120 arcmin') - >>> table = heasarc.query_object(object_name, mission='rosmaster', radius='2 degree') - ... - >>> from astropy import units as u - >>> table = heasarc.query_object(object_name, mission='rosmaster', radius=120*u.arcmin) - >>> table = heasarc.query_object(object_name, mission='rosmaster', radius=2*u.degree) - -As per the astroquery specifications, the :meth:`~astroquery.heasarc.HeasarcClass.query_region` -method requires the user to supply the radius parameter. - -The results can also be sorted by the value in a given column using the ``sortvar`` -parameter. The following sorts the results by the value in the 'EXPOSURE' column. - -.. doctest-remote-data:: - - >>> table = heasarc.query_object(object_name, mission='rosmaster', sortvar='EXPOSURE') - >>> table[:3].pprint() - SEQ_ID INSTRUMENT EXPOSURE ... END_TIME SEARCH_OFFSET_ - s ... mjd - ---------------- ---------- -------- ... ---------------- ---------------- - RH120001N00 HRI 0 ... 48079.8913773148 0.496 (3c273)\n - RH701979N00 HRI 354 ... 49726.0977083333 0.190 (3c273)\n - RP141520N00 PSPCB 485 ... 49540.0447569444 0.496 (3c273)\n - -Setting the ``resultmax`` parameter controls the maximum number of results to be -returned. The following will store only the first 10 results: - -.. doctest-remote-data:: - - >>> table = heasarc.query_object(object_name, mission='rosmaster', resultmax=10) - -All of the above parameters can be mixed and matched to refine the query results. - -It is also possible to select time range: - -.. doctest-remote-data:: - - >>> from astroquery.heasarc import Heasarc - >>> heasarc = Heasarc() - >>> table = heasarc.query_region('3C273', mission="numaster", radius='1 degree', time='2019-01-01 .. 2020-01-01') - >>> table.pprint(max_width=120) - NAME RA DEC ... ISSUE_FLAG SEARCH_OFFSET_ - deg deg ... - -------------------------------------------------- -------- ------ ... ---------- ------------------------------------- - 3C273 187.2473 2.0362 ... 0 2.077 (187.2779220936,2.0523864234)\n - - -Getting list of available missions ----------------------------------- - -The `~astroquery.heasarc.HeasarcClass.query_mission_list` method will return a list of available missions -that can be queried. - -.. doctest-remote-data:: - - >>> from astroquery.heasarc import Heasarc - >>> heasarc = Heasarc() - >>> table = heasarc.query_mission_list() - >>> table.pprint() #doctest: +IGNORE_OUTPUT - Mission Table Table Description - -------------- ---------- -------------------------------------------------------------------------------- - GALAXY CATALOG a2pic HEAO 1 A2 Piccinotti Catalog - GALAXY CATALOG abell Abell Clusters - GALAXY CATALOG abellzcat Abell Clusters Measured Redshifts Catalog - GALAXY CATALOG acceptcat Archive of Chandra Cluster Entropy Profile Tables (ACCEPT) Catalog - GALAXY CATALOG agnsdssxm2 Sloan Digital Sky Survey/XMM-Newton Type1 AGN X-Ray and Radio Properties Catalog - GALAXY CATALOG agnsdssxmm Sloan Digital Sky Survey/XMM-Newton AGN Spectral Properties Catalog - GALAXY CATALOG allwiseagn AllWISE Catalog of Mid-IR AGNs - GALAXY CATALOG arxa Atlas of Radio/X-Ray Associations (ARXA) - GALAXY CATALOG ascaegclus ASCA Elliptical Galaxies and Galaxy Clusters Catalog - GALAXY CATALOG asiagosn Asiago Supernova Catalog (Dynamic Version) - GALAXY CATALOG baxgalclus BAX X-Ray Galaxy Clusters and Groups Catalog - GALAXY CATALOG cbatpicagn CGRO BATSE-Observed Piccinotti Sample of Active Galactic Nuclei - GALAXY CATALOG ccosrssfag Chandra COSMOS Radio-Selected Star-Forming Galaxies and AGN Catalog - GALAXY CATALOG cfa2s CfA Redshift Survey: South Galactic Cap Data - GALAXY CATALOG cgmw Candidate Galaxies Behind the Milky Way - GALAXY CATALOG cosmosvlba COSMOS Field VLBA Observations 1.4-GHz Source Catalog - GALAXY CATALOG cosxfirmwc COSMOS Field X-Ray & FIR Detected AGN Multiwavelength Properties Catalog - GALAXY CATALOG denisigal First DENIS I-band Extragalactic Catalog - GALAXY CATALOG eingalcat Catalog of Galaxies Observed by the Einstein Observatory IPC & HRI - GALAXY CATALOG eingalclus Einstein Observatory Clusters of Galaxies Catalog - GALAXY CATALOG esouppsala ESO-Uppsala ESO(B) Survey - GALAXY CATALOG etgalxray Early-Type Galaxies X-Ray Luminosities Catalog - GALAXY CATALOG exgalemobj Hewitt & Burbidge (1991) Catalog of Extragalactic Emission-Line Objects - GALAXY CATALOG fricat FIRST Catalog of FR I Radio Galaxies - GALAXY CATALOG friicat FIRST Catalog of FR II Radio Galaxies - GALAXY CATALOG fsvsclustr Faint Sky Variability Survey Catalog of Galaxy Clusters and Rich Groups - ... ... ... - xmm-newton xmmlss10ks XMM-Newton Large-Scale Structure Uniform 10-ksec Exposure X-Ray Source Catalog - xmm-newton xmmlssclas XMM-Newton Large-Scale Structure Optical Counterparts and Redshifts - xmm-newton xmmlssdeep XMM-Newton Large-Scale Structure Deep Full-Exposure X-Ray Source Catalog - xmm-newton xmmlssoid XMM-Newton Large-Scale Structure Optical Identifications Catalog - xmm-newton xmmmaster XMM-Newton Master Log & Public Archive - xmm-newton xmmobstars XMM-Newton OB Stars Catalog - xmm-newton xmmomcat XMM-Newton OM Object Catalog - xmm-newton xmmomcdfs XMM-Newton Optical Monitor Chandra Deep Field-South UV Catalog - xmm-newton xmmomobj XMM-Newton OM Objects (2008 Version) - xmm-newton xmmomsuob XMM-Newton Optical Monitor SUSS Catalog, v4.1: Observation IDs - xmm-newton xmmomsuss XMM-Newton Optical Monitor Serendipitous UV Source Survey Catalog, v4.1 - xmm-newton xmmsdssgce 2XMMi/SDSS Galaxy Cluster Survey Extension - xmm-newton xmmsdssgcs 2XMMi/SDSS Galaxy Cluster Survey - xmm-newton xmmslewcln XMM-Newton Slew Survey Clean Source Catalog, v2.0 - xmm-newton xmmslewegs XMM-Newton Slew Survey Extragalactic Sample - xmm-newton xmmslewful XMM-Newton Slew Survey Full Source Catalog, v2.0 - xmm-newton xmmssc XMM-Newton Serendipitous Source Catalog (4XMM-DR10 Version) - xmm-newton xmmsscgps XMM-Newton Survey Science Center Survey of the Galactic Plane - xmm-newton xmmssclwbd XMM-Newton 2XMMi-DR3 Selected Source Detections Catalog - xmm-newton xmmssclwbs XMM-Newton 2XMMi-DR3 Selected Source Classifications Catalog - xmm-newton xmmstack XMM-Newton Serendipitous Source Catalog from Stacked Observations (4XMM-DR10s) - xmm-newton xmmstackob XMM-Newton Serendipitous Source Catalog from Stacked Observations: Obs. Data - xmm-newton xmmt2flare 2XMM Flares Detected from Tycho-2 Stars - xmm-newton xmmvaragn Ensemble X-Ray Variability of AGN in 2XMMi-DR3 - xmm-newton xmmxassist XMM-Newton XAssist Source List - xmm-newton xms XMM-Newton Medium Sensitivity Survey (XMS) Source Catalog - xmm-newton xwas XMM-Newton Wide Angle Survey - Length = 1160 rows - - -The returned table includes both the names and a short description of each -mission table. - - -Using alternative HEASARC servers ---------------------------------- - -It is possible to set alternative locations for HEASARC servers. One such location -is hosted by `INTEGRAL Science Data Center `_, and has further -tables listing the most recent INTEGRAL data. - -.. doctest-remote-data:: - - >>> from astroquery.heasarc import Heasarc, Conf - >>> heasarc = Heasarc() - >>> Conf.server.set('https://www.isdc.unige.ch/browse/w3query.pl') - >>> table = heasarc.query_mission_list() - >>> table.pprint(max_width=120) - Mission Table Table Description - ------------- ---------------------- ----------------------------------------------- - CTASST1M-REV1 cta_sst1m_rev1_run Run - FACT-REV1 fact_rev1_run Run - INTEGRAL-REV3 integral_rev3_prop Proposals - INTEGRAL-REV3 integral_rev3_prop_obs Proposal Information and Observation Parameters - INTEGRAL-REV3 integral_rev3_scw SCW - Science Window Data - >>> - >>> table = heasarc.query_object('Crab', mission='integral_rev3_scw', - ... radius='361 degree', time="2022-12-01 .. 2022-12-31", - ... sortvar='START_DATE', resultmax=100000) - >>> table.pprint() # doctest: +IGNORE_OUTPUT - SCW_ID SCW_VER SCW_TYPE ... GOOD_OMC DSIZE SEARCH_OFFSET_ - ------------ ------- -------- ... -------- --------- ----------------- - 258300400010 001 POINTING ... 0 123494400 5199.027 (CRAB)\n - 258400320021 001 SLEW ... 0 5799936 5082.095 (CRAB)\n - 258400260021 001 SLEW ... 0 5791744 5104.388 (CRAB)\n - 258400350010 001 POINTING ... 0 123146240 5167.027 (CRAB)\n - 258700350021 001 SLEW ... 0 5750784 5120.836 (CRAB)\n - 258400330010 001 POINTING ... 0 123179008 5067.991 (CRAB)\n - 258400260010 001 POINTING ... 0 123371520 5093.007 (CRAB)\n - ... ... ... ... ... ... ... - 258400270021 001 SLEW ... 0 126386176 5114.308 (CRAB)\n - 258400270010 001 POINTING ... 0 1200128 5113.839 (CRAB)\n - 258700360010 001 POINTING ... 0 122130432 5136.165 (CRAB)\n - 258200770010 001 POINTING ... 0 1490944 4184.684 (CRAB)\n - 258200770021 001 SLEW ... 0 962560 4184.587 (CRAB)\n - 258200780010 001 POINTING ... 0 1585152 4184.378 (CRAB)\n - 258700340021 001 SLEW ... 0 5779456 5181.635 (CRAB)\n - Length = 1601 rows - - -Troubleshooting -=============== - -If you are repeatedly getting failed queries, or bad/out-of-date results, try clearing your cache: - -.. code-block:: python - - >>> from astroquery.heasarc import Heasarc - >>> Heasarc.clear_cache() - -If this function is unavailable, upgrade your version of astroquery. -The ``clear_cache`` function was introduced in version 0.4.7.dev8479. - - Reference/API =============