From 0ad006dfa67b150627dfaf59386d19a41e6a6a1e Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Wed, 27 Nov 2024 13:28:41 +0100 Subject: [PATCH 01/75] Switch to TAP list_surveys and list_instruments --- astroquery/eso/core.py | 48 +++++++++++-------------- astroquery/eso/tests/test_eso_remote.py | 7 ++-- 2 files changed, 26 insertions(+), 29 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index 421e6fd81a..cb06f9b1f2 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -26,6 +26,7 @@ from ..exceptions import RemoteServiceError, NoResultsWarning, LoginError from ..query import QueryWithLogin from ..utils import schema +import pyvo __doctest_skip__ = ['EsoClass.*'] @@ -305,6 +306,7 @@ def _get_auth_header(self) -> Dict[str, str]: else: return {} + # TODO remove hardcoded values def list_instruments(self, *, cache=True): """ List all the available instrument-specific queries offered by the ESO archive. @@ -317,18 +319,20 @@ def list_instruments(self, *, cache=True): """ if self._instrument_list is None: - url = "http://archive.eso.org/cms/eso-data/instrument-specific-query-forms.html" - instrument_list_response = self._request("GET", url, cache=cache) - root = BeautifulSoup(instrument_list_response.content, 'html5lib') + url = "http://archive.eso.org/tap_obs" self._instrument_list = [] - for element in root.select("div[id=col3] a[href]"): - href = element.attrs["href"] - if u"http://archive.eso.org/wdb/wdb/eso" in href: - instrument = href.split("/")[-2] - if instrument not in self._instrument_list: - self._instrument_list.append(instrument) + tap = pyvo.dal.TAPService(url) + query = """ + select table_name + from TAP_SCHEMA.tables + where schema_name='ist' + order by table_name + """ + res = tap.search(query)["table_name"].data + self._instrument_list = list(map(lambda x: x.split(".")[1], res)) return self._instrument_list + # TODO remove hardcoded values def list_surveys(self, *, cache=True): """ List all the available surveys (phase 3) in the ESO archive. @@ -340,25 +344,15 @@ def list_surveys(self, *, cache=True): See :ref:`caching documentation `. """ if self._survey_list is None: - survey_list_response = self._request( - "GET", "http://archive.eso.org/wdb/wdb/adp/phase3_main/form", - cache=cache) - root = BeautifulSoup(survey_list_response.content, 'html5lib') + url = "http://archive.eso.org/tap_obs" self._survey_list = [] - collections_table = root.find('table', id='collections_table') - other_collections = root.find('select', id='collection_name_option') - # it is possible to have empty collections or other collections... - collection_elts = (collections_table.find_all('input', type='checkbox') - if collections_table is not None - else []) - other_elts = (other_collections.find_all('option') - if other_collections is not None - else []) - for element in (collection_elts + other_elts): - if 'value' in element.attrs: - survey = element.attrs['value'] - if survey and survey not in self._survey_list and 'Any' not in survey: - self._survey_list.append(survey) + tap = pyvo.dal.TAPService(url) + query = """ + SELECT distinct obs_collection from ivoa.ObsCore + """ + + res = tap.search(query) + self._survey_list = list(res["obs_collection"].data) return self._survey_list def query_surveys(self, *, surveys='', cache=True, diff --git a/astroquery/eso/tests/test_eso_remote.py b/astroquery/eso/tests/test_eso_remote.py index 5e5d362985..b356215efc 100644 --- a/astroquery/eso/tests/test_eso_remote.py +++ b/astroquery/eso/tests/test_eso_remote.py @@ -6,11 +6,14 @@ from astroquery.eso import Eso from astroquery.exceptions import NoResultsWarning -instrument_list = [u'fors1', u'fors2', u'sphere', u'vimos', u'omegacam', +instrument_list = [ + u'fors1', u'fors2', u'sphere', u'vimos', u'omegacam', u'hawki', u'isaac', u'naco', u'visir', u'vircam', u'apex', u'giraffe', u'uves', u'xshooter', u'muse', u'crires', u'kmos', u'sinfoni', u'amber', u'midi', u'pionier', - u'gravity', u'espresso', u'wlgsu', u'matisse', u'eris'] + u'gravity', u'espresso', u'wlgsu', u'matisse', u'eris', + u'fiat', + ] # Some tests take too long, leading to travis timeouts # TODO: make this a configuration item From ea9162bebec399c5af5fbb9a023f5122535b4b02 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Wed, 27 Nov 2024 13:58:18 +0100 Subject: [PATCH 02/75] Define tap_url as static method --- astroquery/eso/core.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index cb06f9b1f2..bbdb316868 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -76,6 +76,14 @@ class EsoClass(QueryWithLogin): DOWNLOAD_URL = "https://dataportal.eso.org/dataPortal/file/" AUTH_URL = "https://www.eso.org/sso/oidc/token" GUNZIP = "gunzip" + USE_DEV_TAP = False + + @staticmethod + def tap_url(): + url = "http://archive.eso.org/tap_obs" + if EsoClass.USE_DEV_TAP: + url = "http://dfidev5.hq.eso.org:8123/tap_obs" + return url def __init__(self): super().__init__() @@ -319,9 +327,8 @@ def list_instruments(self, *, cache=True): """ if self._instrument_list is None: - url = "http://archive.eso.org/tap_obs" self._instrument_list = [] - tap = pyvo.dal.TAPService(url) + tap = pyvo.dal.TAPService(EsoClass.tap_url()) query = """ select table_name from TAP_SCHEMA.tables @@ -344,9 +351,8 @@ def list_surveys(self, *, cache=True): See :ref:`caching documentation `. """ if self._survey_list is None: - url = "http://archive.eso.org/tap_obs" self._survey_list = [] - tap = pyvo.dal.TAPService(url) + tap = pyvo.dal.TAPService(EsoClass.tap_url()) query = """ SELECT distinct obs_collection from ivoa.ObsCore """ From cda3ad919fae1a0ec95335f0c48b14548562e77e Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Wed, 27 Nov 2024 14:31:59 +0100 Subject: [PATCH 03/75] Deselect ALMA at query level - TEMPORAL SOLUTION --- astroquery/eso/core.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index bbdb316868..32e1bfdf9e 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -76,7 +76,7 @@ class EsoClass(QueryWithLogin): DOWNLOAD_URL = "https://dataportal.eso.org/dataPortal/file/" AUTH_URL = "https://www.eso.org/sso/oidc/token" GUNZIP = "gunzip" - USE_DEV_TAP = False + USE_DEV_TAP = True @staticmethod def tap_url(): @@ -340,6 +340,7 @@ def list_instruments(self, *, cache=True): return self._instrument_list # TODO remove hardcoded values + #TODO call it list_collections def list_surveys(self, *, cache=True): """ List all the available surveys (phase 3) in the ESO archive. @@ -354,7 +355,7 @@ def list_surveys(self, *, cache=True): self._survey_list = [] tap = pyvo.dal.TAPService(EsoClass.tap_url()) query = """ - SELECT distinct obs_collection from ivoa.ObsCore + SELECT distinct obs_collection from ivoa.ObsCore where obs_collection != 'ALMA' """ res = tap.search(query) From ba6ba41feed345c43c4160bd0305c95a9c41348a Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Wed, 27 Nov 2024 15:06:40 +0100 Subject: [PATCH 04/75] Mark list_surveys as deprecated; define list_collections --- astroquery/eso/core.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index 32e1bfdf9e..b63107e8ff 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -2,6 +2,7 @@ import base64 import email +import functools import json import os.path import re @@ -31,6 +32,18 @@ __doctest_skip__ = ['EsoClass.*'] +def eso_deprecated(func): + @functools.wraps(func) + def func_with_warnings(*args, **kwargs): + warnings.simplefilter('always', DeprecationWarning) # turn off filter + warnings.warn("Call to deprecated function {}.".format(func.__name__), + category=DeprecationWarning, + stacklevel=2) + warnings.simplefilter('default', DeprecationWarning) # reset filter + return func(*args, **kwargs) + return func_with_warnings + + def _check_response(content): """ Check the response from an ESO service query for various types of error @@ -340,8 +353,7 @@ def list_instruments(self, *, cache=True): return self._instrument_list # TODO remove hardcoded values - #TODO call it list_collections - def list_surveys(self, *, cache=True): + def list_collections(self, *, cache=True): """ List all the available surveys (phase 3) in the ESO archive. Returns @@ -362,6 +374,11 @@ def list_surveys(self, *, cache=True): self._survey_list = list(res["obs_collection"].data) return self._survey_list + @eso_deprecated + def list_surveys(self, *args, **kwargs): + return self.list_collections(*args, **kwargs) + + def query_surveys(self, *, surveys='', cache=True, help=False, open_form=False, **kwargs): """ From e882d8c63aa61637111e3e11b6c4ecd4354f1e8a Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Wed, 27 Nov 2024 15:20:23 +0100 Subject: [PATCH 05/75] Add name of the new function to deprecation warning --- astroquery/eso/core.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index b63107e8ff..d259246496 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -31,12 +31,20 @@ __doctest_skip__ = ['EsoClass.*'] +def decorator_with_params(dec): + def layer(*args, **kwargs): + def repl(f): + return dec(f, *args, **kwargs) + return repl + return layer -def eso_deprecated(func): + +@decorator_with_params +def eso_deprecated(func, new_function): @functools.wraps(func) def func_with_warnings(*args, **kwargs): warnings.simplefilter('always', DeprecationWarning) # turn off filter - warnings.warn("Call to deprecated function {}.".format(func.__name__), + warnings.warn(f"\n\nCall to deprecated function *{func.__name__}*. Please use *{new_function}* instead.", category=DeprecationWarning, stacklevel=2) warnings.simplefilter('default', DeprecationWarning) # reset filter @@ -374,7 +382,7 @@ def list_collections(self, *, cache=True): self._survey_list = list(res["obs_collection"].data) return self._survey_list - @eso_deprecated + @eso_deprecated(new_function="list_collections") def list_surveys(self, *args, **kwargs): return self.list_collections(*args, **kwargs) From 93e253fb9d6d78f75fcc60a5741dfc0c3570da10 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Wed, 27 Nov 2024 15:30:26 +0100 Subject: [PATCH 06/75] Deprecate list_ and query_ _surveys in favor of _collections --- astroquery/eso/core.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index d259246496..06d01634a7 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -382,12 +382,8 @@ def list_collections(self, *, cache=True): self._survey_list = list(res["obs_collection"].data) return self._survey_list - @eso_deprecated(new_function="list_collections") - def list_surveys(self, *args, **kwargs): - return self.list_collections(*args, **kwargs) - - def query_surveys(self, *, surveys='', cache=True, + def query_collections(self, *, surveys='', cache=True, help=False, open_form=False, **kwargs): """ Query survey Phase 3 data contained in the ESO archive. @@ -1050,4 +1046,14 @@ def _print_surveys_help(self, url, *, cache=True): return result_string + @eso_deprecated(new_function="list_collections") + def list_surveys(self, *args, **kwargs): + return self.list_collections(*args, **kwargs) + + + @eso_deprecated(new_function="query_collections") + def query_surveys(self, *args, **kwargs): + return self.query_collections(*args, **kwargs) + + Eso = EsoClass() From b36255230a1d4a95fb767852cdf118ad32716826 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Thu, 28 Nov 2024 16:29:20 +0100 Subject: [PATCH 07/75] Replace 'survey' by 'collection' accross eso module --- astroquery/eso/core.py | 48 ++++++++++++------------- astroquery/eso/tests/test_eso_remote.py | 36 +++++++++---------- docs/eso/eso.rst | 18 +++++----- 3 files changed, 51 insertions(+), 51 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index 06d01634a7..aa98a11bf6 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -109,7 +109,7 @@ def tap_url(): def __init__(self): super().__init__() self._instrument_list = None - self._survey_list = None + self._collection_list = None self._auth_info: Optional[AuthInfo] = None def _activate_form(self, response, *, form_index=0, form_id=None, inputs={}, @@ -189,7 +189,7 @@ def _activate_form(self, response, *, form_index=0, form_id=None, inputs={}, value = form_elem.select( 'option[value]')[0].get('value') else: - # survey form just uses text, not value + # form just uses text, not value for option in form_elem.select('option'): if option.get('selected') is not None: value = str(option.string) @@ -362,39 +362,39 @@ def list_instruments(self, *, cache=True): # TODO remove hardcoded values def list_collections(self, *, cache=True): - """ List all the available surveys (phase 3) in the ESO archive. + """ List all the available collections (phase 3) in the ESO archive. Returns ------- - survey_list : list of strings + collection_list : list of strings cache : bool Defaults to True. If set overrides global caching behavior. See :ref:`caching documentation `. """ - if self._survey_list is None: - self._survey_list = [] + if self._collection_list is None: + self._collection_list = [] tap = pyvo.dal.TAPService(EsoClass.tap_url()) query = """ SELECT distinct obs_collection from ivoa.ObsCore where obs_collection != 'ALMA' """ res = tap.search(query) - self._survey_list = list(res["obs_collection"].data) - return self._survey_list + self._collection_list = list(res["obs_collection"].data) + return self._collection_list - def query_collections(self, *, surveys='', cache=True, + def query_collections(self, *, collections='', cache=True, help=False, open_form=False, **kwargs): """ - Query survey Phase 3 data contained in the ESO archive. + Query collection Phase 3 data contained in the ESO archive. Parameters ---------- - survey : string or list - Name of the survey(s) to query. Should beone or more of the - names returned by `~astroquery.eso.EsoClass.list_surveys`. If + collection : string or list + Name of the collection(s) to query. Should beone or more of the + names returned by `~astroquery.eso.EsoClass.list_collections`. If specified as a string, should be a comma-separated list of - survey names. + collection names. cache : bool Defaults to True. If set overrides global caching behavior. See :ref:`caching documentation `. @@ -403,7 +403,7 @@ def query_collections(self, *, surveys='', cache=True, ------- table : `~astropy.table.Table` or `None` A table representing the data available in the archive for the - specified survey, matching the constraints specified in ``kwargs``. + specified collection, matching the constraints specified in ``kwargs``. The number of rows returned is capped by the ROW_LIMIT configuration item. `None` is returned when the query has no results. @@ -414,24 +414,24 @@ def query_collections(self, *, surveys='', cache=True, if open_form: webbrowser.open(url) elif help: - self._print_surveys_help(url, cache=cache) + self._print_collections_help(url, cache=cache) else: - survey_form = self._request("GET", url, cache=cache) + collection_form = self._request("GET", url, cache=cache) query_dict = kwargs query_dict["wdbo"] = "csv/download" - if isinstance(surveys, str): - surveys = surveys.split(",") - query_dict['collection_name'] = surveys + if isinstance(collections, str): + collections = collections.split(",") + query_dict['collection_name'] = collections if self.ROW_LIMIT >= 0: query_dict["max_rows_returned"] = int(self.ROW_LIMIT) else: query_dict["max_rows_returned"] = 10000 - survey_response = self._activate_form(survey_form, form_index=0, + collection_response = self._activate_form(collection_form, form_index=0, form_id='queryform', inputs=query_dict, cache=cache) - content = survey_response.content + content = collection_response.content # First line is always garbage content = content.split(b'\n', 1)[1] log.debug("Response content:\n{0}".format(content)) @@ -972,12 +972,12 @@ def _print_query_help(self, url, *, cache=True): log.info("\n".join(result_string)) return result_string - def _print_surveys_help(self, url, *, cache=True): + def _print_collections_help(self, url, *, cache=True): """ Download a form and print it in a quasi-human-readable way """ log.info("List of the parameters accepted by the " - "surveys query.") + "collections query.") log.info("The presence of a column in the result table can be " "controlled if prefixed with a [ ] checkbox.") log.info("The default columns in the result table are shown as " diff --git a/astroquery/eso/tests/test_eso_remote.py b/astroquery/eso/tests/test_eso_remote.py index b356215efc..8e3a2266df 100644 --- a/astroquery/eso/tests/test_eso_remote.py +++ b/astroquery/eso/tests/test_eso_remote.py @@ -19,7 +19,7 @@ # TODO: make this a configuration item SKIP_SLOW = True -SGRA_SURVEYS = ['195.B-0283', 'GIRAFFE', 'HARPS', 'HAWKI', 'KMOS', +SGRA_COLLECTIONS = ['195.B-0283', 'GIRAFFE', 'HARPS', 'HAWKI', 'KMOS', 'ERIS-SPIFFIER', 'MW-BULGE-PSFPHOT', 'VPHASplus', 'VVV', 'VVVX', 'XSHOOTER'] @@ -37,23 +37,23 @@ def test_SgrAstar(self, tmp_path): result_i = eso.query_instrument('midi', coord1=266.41681662, coord2=-29.00782497, cache=False) - surveys = eso.list_surveys(cache=False) - assert len(surveys) > 0 - # result_s = eso.query_surveys('VVV', target='Sgr A*') + collections = eso.list_collections(cache=False) + assert len(collections) > 0 + # result_s = eso.query_collections('VVV', target='Sgr A*') # Equivalent, does not depend on SESAME: - result_s = eso.query_surveys(surveys='VVV', coord1=266.41681662, + result_s = eso.query_collections(collections='VVV', coord1=266.41681662, coord2=-29.00782497, box='01 00 00', cache=False) assert 'midi' in instruments assert result_i is not None - assert 'VVV' in surveys + assert 'VVV' in collections assert result_s is not None assert 'Object' in result_s.colnames assert 'b333' in result_s['Object'] - def test_multisurvey(self, tmp_path): + def test_multicollection(self, tmp_path): eso = Eso() eso.cache_location = tmp_path @@ -61,7 +61,7 @@ def test_multisurvey(self, tmp_path): # first b333 was at 157 # first pistol....? - result_s = eso.query_surveys(surveys=['VVV', 'XSHOOTER'], + result_s = eso.query_collections(collections=['VVV', 'XSHOOTER'], coord1=266.41681662, coord2=-29.00782497, box='01 00 00', @@ -75,12 +75,12 @@ def test_multisurvey(self, tmp_path): def test_empty_return(self): # test for empty return with an object from the North eso = Eso() - surveys = eso.list_surveys(cache=False) - assert len(surveys) > 0 + collections = eso.list_collections(cache=False) + assert len(collections) > 0 # Avoid SESAME with pytest.warns(NoResultsWarning): - result_s = eso.query_surveys(surveys=surveys[0], coord1=202.469575, + result_s = eso.query_collections(collections=collections[0], coord1=202.469575, coord2=47.195258, cache=False) assert result_s is None @@ -161,28 +161,28 @@ def test_each_instrument_SgrAstar(self, tmp_path): else: assert len(result) > 0 - def test_each_survey_and_SgrAstar(self, tmp_path): + def test_each_collection_and_SgrAstar(self, tmp_path): eso = Eso() eso.cache_location = tmp_path eso.ROW_LIMIT = 5 - surveys = eso.list_surveys(cache=False) - for survey in surveys: - if survey in SGRA_SURVEYS: - result_s = eso.query_surveys(surveys=survey, coord1=266.41681662, + collections = eso.list_collections(cache=False) + for collection in collections: + if collection in SGRA_COLLECTIONS: + result_s = eso.query_collections(collections=collection, coord1=266.41681662, coord2=-29.00782497, box='01 00 00', cache=False) assert len(result_s) > 0 else: with pytest.warns(NoResultsWarning): - result_s = eso.query_surveys(surveys=survey, coord1=266.41681662, + result_s = eso.query_collections(collections=collection, coord1=266.41681662, coord2=-29.00782497, box='01 00 00', cache=False) assert result_s is None - generic_result = eso.query_surveys(surveys=survey) + generic_result = eso.query_collections(collections=collection) assert len(generic_result) > 0 def test_mixed_case_instrument(self, tmp_path): diff --git a/docs/eso/eso.rst b/docs/eso/eso.rst index 80f25c31ec..089a3f8bd4 100644 --- a/docs/eso/eso.rst +++ b/docs/eso/eso.rst @@ -11,7 +11,7 @@ This is a python interface for querying the ESO archive web service. For now, it supports the following: - listing available instruments -- listing available surveys (phase 3) +- listing available collections (phase 3) - searching all instrument specific raw data: http://archive.eso.org/cms/eso-data/instrument-specific-query-forms.html - searching data products (phase 3): http://archive.eso.org/wdb/wdb/adp/phase3_main/form - downloading data by dataset identifiers: http://archive.eso.org/cms/eso-data/eso-data-direct-retrieval.html @@ -276,28 +276,28 @@ Query the ESO archive for reduced data ====================================== In addition to raw data, ESO makes available processed data. -In this section, we show how to obtain these processed survey data from the archive. +In this section, we show how to obtain these processed collection data from the archive. -Identify available surveys +Identify available collections -------------------------- -The list of available surveys can be obtained with :meth:`astroquery.eso.EsoClass.list_surveys` as follows: +The list of available collections can be obtained with :meth:`astroquery.eso.EsoClass.list_collections` as follows: .. doctest-remote-data:: - >>> surveys = eso.list_surveys() + >>> collections = eso.list_collections() -Query a specific survey with constraints +Query a specific collection with constraints ---------------------------------------- -Let's assume that we work with the ``HARPS`` survey, and that we are interested in +Let's assume that we work with the ``HARPS`` collection, and that we are interested in target ``HD203608``. The archive can be queried as follows: .. doctest-remote-data:: - >>> table = eso.query_surveys(surveys='HARPS', cache=False, target="HD203608") + >>> table = eso.query_collections(collections='HARPS', cache=False, target="HD203608") The returned table has an ``ARCFILE`` column. It can be used to retrieve the datasets with :meth:`astroquery.eso.EsoClass.retrieve_data` (see next section). @@ -356,7 +356,7 @@ Downloading datasets from the archive ===================================== Continuing from the query with constraints example, the first two datasets are selected, -using their data product IDs ``DP.ID`` (or ``ARCFILE`` for surveys), and retrieved from the ESO archive. +using their data product IDs ``DP.ID`` (or ``ARCFILE`` for collections), and retrieved from the ESO archive. .. doctest-skip:: From 105c8cfa686e9715066f7c949c850451fa9b4e58 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Fri, 29 Nov 2024 11:51:22 +0100 Subject: [PATCH 08/75] Make Pep8-compliant --- astroquery/eso/core.py | 12 ++++----- astroquery/eso/tests/test_eso_remote.py | 36 ++++++++++++------------- 2 files changed, 23 insertions(+), 25 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index aa98a11bf6..f3b465c839 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -31,6 +31,7 @@ __doctest_skip__ = ['EsoClass.*'] + def decorator_with_params(dec): def layer(*args, **kwargs): def repl(f): @@ -101,7 +102,7 @@ class EsoClass(QueryWithLogin): @staticmethod def tap_url(): - url = "http://archive.eso.org/tap_obs" + url = "http://archive.eso.org/tap_obs" if EsoClass.USE_DEV_TAP: url = "http://dfidev5.hq.eso.org:8123/tap_obs" return url @@ -382,9 +383,8 @@ def list_collections(self, *, cache=True): self._collection_list = list(res["obs_collection"].data) return self._collection_list - def query_collections(self, *, collections='', cache=True, - help=False, open_form=False, **kwargs): + help=False, open_form=False, **kwargs): """ Query collection Phase 3 data contained in the ESO archive. @@ -428,8 +428,8 @@ def query_collections(self, *, collections='', cache=True, query_dict["max_rows_returned"] = 10000 collection_response = self._activate_form(collection_form, form_index=0, - form_id='queryform', - inputs=query_dict, cache=cache) + form_id='queryform', + inputs=query_dict, cache=cache) content = collection_response.content # First line is always garbage @@ -1045,12 +1045,10 @@ def _print_collections_help(self, url, *, cache=True): log.info("\n".join(result_string)) return result_string - @eso_deprecated(new_function="list_collections") def list_surveys(self, *args, **kwargs): return self.list_collections(*args, **kwargs) - @eso_deprecated(new_function="query_collections") def query_surveys(self, *args, **kwargs): return self.query_collections(*args, **kwargs) diff --git a/astroquery/eso/tests/test_eso_remote.py b/astroquery/eso/tests/test_eso_remote.py index 8e3a2266df..b304b970bd 100644 --- a/astroquery/eso/tests/test_eso_remote.py +++ b/astroquery/eso/tests/test_eso_remote.py @@ -6,8 +6,7 @@ from astroquery.eso import Eso from astroquery.exceptions import NoResultsWarning -instrument_list = [ - u'fors1', u'fors2', u'sphere', u'vimos', u'omegacam', +instrument_list = [u'fors1', u'fors2', u'sphere', u'vimos', u'omegacam', u'hawki', u'isaac', u'naco', u'visir', u'vircam', u'apex', u'giraffe', u'uves', u'xshooter', u'muse', u'crires', u'kmos', u'sinfoni', u'amber', u'midi', u'pionier', @@ -20,8 +19,9 @@ SKIP_SLOW = True SGRA_COLLECTIONS = ['195.B-0283', 'GIRAFFE', 'HARPS', 'HAWKI', 'KMOS', - 'ERIS-SPIFFIER', - 'MW-BULGE-PSFPHOT', 'VPHASplus', 'VVV', 'VVVX', 'XSHOOTER'] + 'ERIS-SPIFFIER', + 'MW-BULGE-PSFPHOT', 'VPHASplus', 'VVV', 'VVVX', 'XSHOOTER' + ] @pytest.mark.remote_data @@ -42,9 +42,9 @@ def test_SgrAstar(self, tmp_path): # result_s = eso.query_collections('VVV', target='Sgr A*') # Equivalent, does not depend on SESAME: result_s = eso.query_collections(collections='VVV', coord1=266.41681662, - coord2=-29.00782497, - box='01 00 00', - cache=False) + coord2=-29.00782497, + box='01 00 00', + cache=False) assert 'midi' in instruments assert result_i is not None @@ -62,10 +62,10 @@ def test_multicollection(self, tmp_path): # first pistol....? result_s = eso.query_collections(collections=['VVV', 'XSHOOTER'], - coord1=266.41681662, - coord2=-29.00782497, - box='01 00 00', - cache=False) + coord1=266.41681662, + coord2=-29.00782497, + box='01 00 00', + cache=False) assert result_s is not None assert 'Object' in result_s.colnames @@ -81,7 +81,7 @@ def test_empty_return(self): # Avoid SESAME with pytest.warns(NoResultsWarning): result_s = eso.query_collections(collections=collections[0], coord1=202.469575, - coord2=47.195258, cache=False) + coord2=47.195258, cache=False) assert result_s is None @@ -170,16 +170,16 @@ def test_each_collection_and_SgrAstar(self, tmp_path): for collection in collections: if collection in SGRA_COLLECTIONS: result_s = eso.query_collections(collections=collection, coord1=266.41681662, - coord2=-29.00782497, - box='01 00 00', - cache=False) + coord2=-29.00782497, + box='01 00 00', + cache=False) assert len(result_s) > 0 else: with pytest.warns(NoResultsWarning): result_s = eso.query_collections(collections=collection, coord1=266.41681662, - coord2=-29.00782497, - box='01 00 00', - cache=False) + coord2=-29.00782497, + box='01 00 00', + cache=False) assert result_s is None generic_result = eso.query_collections(collections=collection) From 91c98e198ca035fc08406c472be771d5c6c4b6bf Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Fri, 29 Nov 2024 12:05:13 +0100 Subject: [PATCH 09/75] Use astroquery deprecated decorator --- astroquery/eso/core.py | 29 +++++------------------------ 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index f3b465c839..a25cb35129 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -19,7 +19,7 @@ import keyring import requests.exceptions from astropy.table import Table, Column -from astropy.utils.decorators import deprecated_renamed_argument +from astropy.utils.decorators import deprecated, deprecated_renamed_argument from bs4 import BeautifulSoup from astroquery import log @@ -32,27 +32,6 @@ __doctest_skip__ = ['EsoClass.*'] -def decorator_with_params(dec): - def layer(*args, **kwargs): - def repl(f): - return dec(f, *args, **kwargs) - return repl - return layer - - -@decorator_with_params -def eso_deprecated(func, new_function): - @functools.wraps(func) - def func_with_warnings(*args, **kwargs): - warnings.simplefilter('always', DeprecationWarning) # turn off filter - warnings.warn(f"\n\nCall to deprecated function *{func.__name__}*. Please use *{new_function}* instead.", - category=DeprecationWarning, - stacklevel=2) - warnings.simplefilter('default', DeprecationWarning) # reset filter - return func(*args, **kwargs) - return func_with_warnings - - def _check_response(content): """ Check the response from an ESO service query for various types of error @@ -1045,11 +1024,13 @@ def _print_collections_help(self, url, *, cache=True): log.info("\n".join(result_string)) return result_string - @eso_deprecated(new_function="list_collections") + @deprecated(since="v0.4.7", message=("The ESO list_surveys function is deprecated," + "Use the list_collections function instead.")) def list_surveys(self, *args, **kwargs): return self.list_collections(*args, **kwargs) - @eso_deprecated(new_function="query_collections") + @deprecated(since="v0.4.7", message=("The ESO query_surveys function is deprecated," + "Use the query_collections function instead.")) def query_surveys(self, *args, **kwargs): return self.query_collections(*args, **kwargs) From d0e9310cf01c641ba432e374b7b4efc668a58bfb Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Fri, 29 Nov 2024 13:27:23 +0100 Subject: [PATCH 10/75] Accept argument 'surveys=' when calling *_surveys() deprecated functions --- astroquery/eso/core.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index a25cb35129..03cef6c502 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -1024,14 +1024,18 @@ def _print_collections_help(self, url, *, cache=True): log.info("\n".join(result_string)) return result_string - @deprecated(since="v0.4.7", message=("The ESO list_surveys function is deprecated," + @deprecated(since="v0.4.8", message=("The ESO list_surveys function is deprecated," "Use the list_collections function instead.")) def list_surveys(self, *args, **kwargs): + if "surveys" in kwargs.keys(): + kwargs["collections"] = kwargs["surveys"] return self.list_collections(*args, **kwargs) - @deprecated(since="v0.4.7", message=("The ESO query_surveys function is deprecated," + @deprecated(since="v0.4.8", message=("The ESO query_surveys function is deprecated," "Use the query_collections function instead.")) def query_surveys(self, *args, **kwargs): + if "surveys" in kwargs.keys(): + kwargs["collections"] = kwargs["surveys"] return self.query_collections(*args, **kwargs) From 55657b0a4f6c6dabb22c7cbee25a6e4e0c8bf1d6 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Fri, 29 Nov 2024 13:31:34 +0100 Subject: [PATCH 11/75] Remove unused imports --- astroquery/eso/core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index 03cef6c502..6a367f96b4 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -2,7 +2,6 @@ import base64 import email -import functools import json import os.path import re From 90646e1c65ddb3bec5cd81cf2f8a249ffd5637ca Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Fri, 29 Nov 2024 13:43:07 +0100 Subject: [PATCH 12/75] query_surveys --> query_collections in test_eso.py --- astroquery/eso/tests/test_eso.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/astroquery/eso/tests/test_eso.py b/astroquery/eso/tests/test_eso.py index ef8b9efafd..30e082d442 100644 --- a/astroquery/eso/tests/test_eso.py +++ b/astroquery/eso/tests/test_eso.py @@ -115,10 +115,10 @@ def test_vvv(monkeypatch): monkeypatch.setattr(eso, '_request', eso_request) eso.cache_location = DATA_DIR - result_s = eso.query_surveys(surveys='VVV', - coord1=266.41681662, coord2=-29.00782497, - box='01 00 00', - ) + result_s = eso.query_collections(collections='VVV', + coord1=266.41681662, coord2=-29.00782497, + box='01 00 00', + ) assert result_s is not None assert 'Object' in result_s.colnames assert 'b333' in result_s['Object'] From 0f099cef8cddcaab5f53444f529a5f3c43a942da Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Fri, 29 Nov 2024 14:16:30 +0100 Subject: [PATCH 13/75] Update CHANGES.rst --- CHANGES.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 16ebe881f4..37d6b58002 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -78,6 +78,11 @@ esasky - Added support for eROSITA downloads. [#3111] +eso +^^^ + +- Deprecate functions ``list_surveys`` and ``query_surveys`` in favor of ``list_collections`` and ``query_collections``. [#3138] + gaia ^^^^ From 7098338fee4538e369a6f1f9d242f11c9516905b Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Fri, 29 Nov 2024 14:33:04 +0100 Subject: [PATCH 14/75] Add test_tap_url() --- astroquery/eso/tests/test_eso.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/astroquery/eso/tests/test_eso.py b/astroquery/eso/tests/test_eso.py index 30e082d442..c5578ada53 100644 --- a/astroquery/eso/tests/test_eso.py +++ b/astroquery/eso/tests/test_eso.py @@ -6,7 +6,7 @@ import pytest from astroquery.utils.mocks import MockResponse -from ...eso import Eso +from ...eso import Eso, EsoClass DATA_DIR = os.path.join(os.path.dirname(__file__), 'data') @@ -182,3 +182,11 @@ def test_calselector_multipart(monkeypatch, tmp_path): assert isinstance(result, list) assert len(result) == 99 assert datasets[0] not in result and datasets[1] not in result + + +def test_tap_url(): + url = EsoClass.tap_url() + if EsoClass.USE_DEV_TAP: + assert url == "http://dfidev5.hq.eso.org:8123/tap_obs" + else: + assert url == "http://archive.eso.org/tap_obs" From 4a09d75cf54917b72f49528f4b3ee8eae62f76d9 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Mon, 2 Dec 2024 10:56:32 +0100 Subject: [PATCH 15/75] Remove hard-coded queries iside user-exposed functions --- astroquery/eso/core.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index 6a367f96b4..64cd5a098d 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -314,7 +314,12 @@ def _get_auth_header(self) -> Dict[str, str]: else: return {} - # TODO remove hardcoded values + def _query_tap_service(self, query_str, col_name): + tap = pyvo.dal.TAPService(EsoClass.tap_url()) + query = query_str + res = tap.search(query)[col_name].data + return res + def list_instruments(self, *, cache=True): """ List all the available instrument-specific queries offered by the ESO archive. @@ -328,18 +333,17 @@ def list_instruments(self, *, cache=True): """ if self._instrument_list is None: self._instrument_list = [] - tap = pyvo.dal.TAPService(EsoClass.tap_url()) - query = """ + query_str = """ select table_name from TAP_SCHEMA.tables where schema_name='ist' order by table_name """ - res = tap.search(query)["table_name"].data + col_name = "table_name" + res = self._query_tap_service(query_str, col_name) self._instrument_list = list(map(lambda x: x.split(".")[1], res)) return self._instrument_list - # TODO remove hardcoded values def list_collections(self, *, cache=True): """ List all the available collections (phase 3) in the ESO archive. @@ -352,15 +356,20 @@ def list_collections(self, *, cache=True): """ if self._collection_list is None: self._collection_list = [] - tap = pyvo.dal.TAPService(EsoClass.tap_url()) - query = """ + query_str = """ SELECT distinct obs_collection from ivoa.ObsCore where obs_collection != 'ALMA' """ + col_name = "obs_collection" + res = self._query_tap_service(query_str, col_name) - res = tap.search(query) - self._collection_list = list(res["obs_collection"].data) + self._collection_list = list(res) return self._collection_list + # JM - Example queries: + # select * from ivoa.ObsCore where data_collection = 'AMBRE' and facility_name = 'something' + # select * from ivoa.ObsCore where data_collection = some_collection and key1 = val1 and key2=val2 and ... + # Extra stuff she mentioned: + # select * from TAP_SCHEMA.columns where table_name = 'ivoa.ObsCore' def query_collections(self, *, collections='', cache=True, help=False, open_form=False, **kwargs): """ From 97a4ad5e6d805d7aec3e48b35a7e0426b585174f Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Mon, 2 Dec 2024 12:41:12 +0100 Subject: [PATCH 16/75] Remove function _activate_form --- astroquery/eso/core.py | 143 ----------------------------------------- 1 file changed, 143 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index 64cd5a098d..7d121c423e 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -91,149 +91,6 @@ def __init__(self): self._collection_list = None self._auth_info: Optional[AuthInfo] = None - def _activate_form(self, response, *, form_index=0, form_id=None, inputs={}, - cache=True, method=None): - """ - Parameters - ---------- - method: None or str - Can be used to override the form-specified method - """ - # Extract form from response - root = BeautifulSoup(response.content, 'html5lib') - if form_id is None: - form = root.find_all('form')[form_index] - else: - form = root.find_all('form', id=form_id)[form_index] - # Construct base url - form_action = form.get('action') - if "://" in form_action: - url = form_action - elif form_action.startswith('/'): - url = '/'.join(response.url.split('/', 3)[:3]) + form_action - else: - url = response.url.rsplit('/', 1)[0] + '/' + form_action - # Identify payload format - fmt = None - form_method = form.get('method').lower() - if form_method == 'get': - fmt = 'get' # get(url, params=payload) - elif form_method == 'post': - if 'enctype' in form.attrs: - if form.attrs['enctype'] == 'multipart/form-data': - fmt = 'multipart/form-data' # post(url, files=payload) - elif form.attrs['enctype'] == 'application/x-www-form-urlencoded': - fmt = 'application/x-www-form-urlencoded' # post(url, data=payload) - else: - raise Exception("enctype={0} is not supported!".format(form.attrs['enctype'])) - else: - fmt = 'application/x-www-form-urlencoded' # post(url, data=payload) - # Extract payload from form - payload = [] - for form_elem in form.find_all(['input', 'select', 'textarea']): - value = None - is_file = False - tag_name = form_elem.name - key = form_elem.get('name') - if tag_name == 'input': - is_file = (form_elem.get('type') == 'file') - value = form_elem.get('value') - if form_elem.get('type') in ['checkbox', 'radio']: - if form_elem.has_attr('checked'): - if not value: - value = 'on' - else: - value = None - elif tag_name == 'select': - if form_elem.get('multiple') is not None: - value = [] - if form_elem.select('option[value]'): - for option in form_elem.select('option[value]'): - if option.get('selected') is not None: - value.append(option.get('value')) - else: - for option in form_elem.select('option'): - if option.get('selected') is not None: - # bs4 NavigableString types have bad, - # undesirable properties that result - # in recursion errors when caching - value.append(str(option.string)) - else: - if form_elem.select('option[value]'): - for option in form_elem.select('option[value]'): - if option.get('selected') is not None: - value = option.get('value') - # select the first option field if none is selected - if value is None: - value = form_elem.select( - 'option[value]')[0].get('value') - else: - # form just uses text, not value - for option in form_elem.select('option'): - if option.get('selected') is not None: - value = str(option.string) - # select the first option field if none is selected - if value is None: - value = str(form_elem.select('option')[0].string) - - if key in inputs: - if isinstance(inputs[key], list): - # list input is accepted (for array uploads) - value = inputs[key] - else: - value = str(inputs[key]) - - if (key is not None): # and (value is not None): - if fmt == 'multipart/form-data': - if is_file: - payload.append( - (key, ('', '', 'application/octet-stream'))) - else: - if type(value) is list: - for v in value: - entry = (key, ('', v)) - # Prevent redundant key, value pairs - # (can happen if the form repeats them) - if entry not in payload: - payload.append(entry) - elif value is None: - entry = (key, ('', '')) - if entry not in payload: - payload.append(entry) - else: - entry = (key, ('', value)) - if entry not in payload: - payload.append(entry) - else: - if type(value) is list: - for v in value: - entry = (key, v) - if entry not in payload: - payload.append(entry) - else: - entry = (key, value) - if entry not in payload: - payload.append(entry) - - # for future debugging - self._payload = payload - log.debug("Form: payload={0}".format(payload)) - - if method is not None: - fmt = method - - log.debug("Method/format = {0}".format(fmt)) - - # Send payload - if fmt == 'get': - response = self._request("GET", url, params=payload, cache=cache) - elif fmt == 'multipart/form-data': - response = self._request("POST", url, files=payload, cache=cache) - elif fmt == 'application/x-www-form-urlencoded': - response = self._request("POST", url, data=payload, cache=cache) - - return response - def _authenticate(self, *, username: str, password: str) -> bool: """ Get the access token from ESO SSO provider From 851ee40b398810f8f7a6a909b5f82e06fbcf3710 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Tue, 3 Dec 2024 17:49:55 +0100 Subject: [PATCH 17/75] Develop py2adql function with tests --- astroquery/eso/core.py | 157 +++++++++++-------------------- astroquery/eso/tests/test_eso.py | 80 ++++++++++++++++ astroquery/eso/utils.py | 41 ++++++++ 3 files changed, 175 insertions(+), 103 deletions(-) create mode 100644 astroquery/eso/utils.py diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index 7d121c423e..363fc94b7f 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -14,6 +14,7 @@ from io import BytesIO from typing import List, Optional, Tuple, Dict, Set +import astropy.table import astropy.utils.data import keyring import requests.exceptions @@ -26,6 +27,7 @@ from ..exceptions import RemoteServiceError, NoResultsWarning, LoginError from ..query import QueryWithLogin from ..utils import schema +from .utils import py2adql, _split_str_as_list_of_str import pyvo __doctest_skip__ = ['EsoClass.*'] @@ -85,6 +87,13 @@ def tap_url(): url = "http://dfidev5.hq.eso.org:8123/tap_obs" return url + @staticmethod + def tap_columns(table_name="ivoa.ObsCore"): + # TODO + # return a list with the available columns + query = f"select * from TAP_SCHEMA.columns where table_name = '{table_name}'" + return query + def __init__(self): super().__init__() self._instrument_list = None @@ -171,7 +180,27 @@ def _get_auth_header(self) -> Dict[str, str]: else: return {} + # TODO - delete or rewrite + def _print_collections_help(self): + """ + Prints help + """ + log.info("List of the parameters accepted by the " + "collections query.") + log.info("The presence of a column in the result table can be " + "controlled if prefixed with a [ ] checkbox.") + log.info("The default columns in the result table are shown as " + "already ticked: [x].") + + result_string = [] + + log.info("\n".join(result_string)) + return result_string + def _query_tap_service(self, query_str, col_name): + """ + returns an astropy.table.Table # JM TODO + """ tap = pyvo.dal.TAPService(EsoClass.tap_url()) query = query_str res = tap.search(query)[col_name].data @@ -227,8 +256,9 @@ def list_collections(self, *, cache=True): # select * from ivoa.ObsCore where data_collection = some_collection and key1 = val1 and key2=val2 and ... # Extra stuff she mentioned: # select * from TAP_SCHEMA.columns where table_name = 'ivoa.ObsCore' + @deprecated_renamed_argument(old_name='open_form', new_name=None, since='0.4.8') def query_collections(self, *, collections='', cache=True, - help=False, open_form=False, **kwargs): + help=False, open_form=None, **kwargs): """ Query collection Phase 3 data contained in the ESO archive. @@ -253,38 +283,30 @@ def query_collections(self, *, collections='', cache=True, results. """ + if isinstance(collections, str): + collections = _split_str_as_list_of_str(collections) + table_to_return = None # Return an astropy.table.Table or None + if help: + self._print_collections_help() - url = "http://archive.eso.org/wdb/wdb/adp/phase3_main/form" - if open_form: - webbrowser.open(url) - elif help: - self._print_collections_help(url, cache=cache) - else: - collection_form = self._request("GET", url, cache=cache) - query_dict = kwargs - query_dict["wdbo"] = "csv/download" - if isinstance(collections, str): - collections = collections.split(",") - query_dict['collection_name'] = collections - if self.ROW_LIMIT >= 0: - query_dict["max_rows_returned"] = int(self.ROW_LIMIT) - else: - query_dict["max_rows_returned"] = 10000 + tap = pyvo.dal.TAPService(EsoClass.tap_url()) + collections = list(map(lambda x: f"'{x.strip()}'", collections)) + where_collections_str = "obs_collection in (" + ", ".join(collections) + ")" + where_constraints_strlist = [f"{k} = {v}" for k, v in kwargs.items()] + where_constraints = [where_collections_str] + where_constraints_strlist + query = py2adql(table="ivoa.ObsCore", columns='*', where_constraints=where_constraints) - collection_response = self._activate_form(collection_form, form_index=0, - form_id='queryform', - inputs=query_dict, cache=cache) + # select * from ivoa.ObsCore where obs_collection = 'AMBRE' OR obs_collection = 'ALCOHOLS' + try: + table_to_return = tap.search(query, maxrec=self.ROW_LIMIT).to_table() + except Exception as e: + raise e # TODO catch properly the relevant exception - content = collection_response.content - # First line is always garbage - content = content.split(b'\n', 1)[1] - log.debug("Response content:\n{0}".format(content)) - if _check_response(content): - table = Table.read(BytesIO(content), format="ascii.csv", - comment="^#") - return table - else: - warnings.warn("Query returned no results", NoResultsWarning) + if len(table_to_return) < 1: + warnings.warn("Query returned no results", NoResultsWarning) + table_to_return = None + + return table_to_return def query_main(self, *, column_filters={}, columns=[], open_form=False, help=False, cache=True, **kwargs): @@ -816,84 +838,12 @@ def _print_query_help(self, url, *, cache=True): log.info("\n".join(result_string)) return result_string - def _print_collections_help(self, url, *, cache=True): - """ - Download a form and print it in a quasi-human-readable way - """ - log.info("List of the parameters accepted by the " - "collections query.") - log.info("The presence of a column in the result table can be " - "controlled if prefixed with a [ ] checkbox.") - log.info("The default columns in the result table are shown as " - "already ticked: [x].") - - result_string = [] - - resp = self._request("GET", url, cache=cache) - doc = BeautifulSoup(resp.content, 'html5lib') - form = doc.select("html body form")[0] - - # hovertext from different labels are used to give more info on forms - helptext_dict = {abbr['title'].split(":")[0].strip(): ":".join(abbr['title'].split(":")[1:]) - for abbr in form.find_all('abbr') - if 'title' in abbr.attrs and ":" in abbr['title']} - - for fieldset in form.select('fieldset'): - legend = fieldset.select('legend') - if len(legend) > 1: - raise ValueError("Form parsing error: too many legends.") - elif len(legend) == 0: - continue - section_title = "\n\n" + "".join(legend[0].stripped_strings) + "\n" - - result_string.append(section_title) - - for section in fieldset.select('table'): - - checkbox_name = "" - checkbox_value = "" - for tag in section.next_elements: - if tag.name == u"table": - break - elif tag.name == u"input": - if tag.get(u'type') == u"checkbox": - checkbox_name = tag['name'] - checkbox_value = (u"[x]" - if ('checked' in tag.attrs) - else u"[ ]") - name = "" - value = "" - else: - name = tag['name'] - value = "" - elif tag.name == u"select": - options = [] - for option in tag.select("option"): - options += ["{0} ({1})".format(option['value'] if 'value' in option else "", - "".join(option.stripped_strings))] - name = tag[u"name"] - value = ", ".join(options) - else: - name = "" - value = "" - if u"tab_" + name == checkbox_name: - checkbox = checkbox_value - else: - checkbox = " " - if name != u"": - result_string.append("{0} {1}: {2}" - .format(checkbox, name, value)) - if name.strip() in helptext_dict: - result_string.append(helptext_dict[name.strip()]) - - log.info("\n".join(result_string)) - return result_string - @deprecated(since="v0.4.8", message=("The ESO list_surveys function is deprecated," "Use the list_collections function instead.")) def list_surveys(self, *args, **kwargs): if "surveys" in kwargs.keys(): kwargs["collections"] = kwargs["surveys"] + del(kwargs["surveys"]) return self.list_collections(*args, **kwargs) @deprecated(since="v0.4.8", message=("The ESO query_surveys function is deprecated," @@ -901,6 +851,7 @@ def list_surveys(self, *args, **kwargs): def query_surveys(self, *args, **kwargs): if "surveys" in kwargs.keys(): kwargs["collections"] = kwargs["surveys"] + del(kwargs["surveys"]) return self.query_collections(*args, **kwargs) diff --git a/astroquery/eso/tests/test_eso.py b/astroquery/eso/tests/test_eso.py index c5578ada53..7f25b65d8f 100644 --- a/astroquery/eso/tests/test_eso.py +++ b/astroquery/eso/tests/test_eso.py @@ -7,6 +7,7 @@ from astroquery.utils.mocks import MockResponse from ...eso import Eso, EsoClass +from ...eso.utils import py2adql DATA_DIR = os.path.join(os.path.dirname(__file__), 'data') @@ -190,3 +191,82 @@ def test_tap_url(): assert url == "http://dfidev5.hq.eso.org:8123/tap_obs" else: assert url == "http://archive.eso.org/tap_obs" + + +def test_py2adql(): + # Example query: + # + # SELECT + # target_name, dp_id, s_ra, s_dec, t_exptime, em_min, em_max, + # dataproduct_type, instrument_name, obstech, abmaglim, + # proposal_id, obs_collection + # FROM + # ivoa.ObsCore + # WHERE + # intersects(s_region, circle('ICRS', 109.668246, -24.558700, 0.001389))=1 + # AND + # dataproduct_type in ('spectrum') + # AND + # em_min>4.0e-7 + # AND + # em_max<1.2e-6 + # ORDER BY SNR DESC + # + + # Simple tests + q = py2adql('ivoa.ObsCore') + eq = "select * from ivoa.ObsCore" + assert eq == q, f"Expected:\n{eq}\n\nObtained:\n{q}\n\n" + + q = py2adql('ivoa.ObsCore', columns='*') + eq = "select * from ivoa.ObsCore" + assert eq == q, f"Expected:\n{eq}\n\nObtained:\n{q}\n\n" + + q = py2adql('pinko.Pallino', ['pippo', 'tizio', 'caio']) + eq = "select pippo, tizio, caio from pinko.Pallino" + assert eq == q, f"Expected:\n{eq}\n\nObtained:\n{q}\n\n" + + q = py2adql('pinko.Pallino', ['pippo', 'tizio', 'caio']) + eq = "select pippo, tizio, caio from pinko.Pallino" + assert eq == q, f"Expected:\n{eq}\n\nObtained:\n{q}\n\n" + + q = py2adql('pinko.Pallino', ['pippo', 'tizio', 'caio'], + where_constraints=['asdf > 1', 'asdf < 2', 'asdf = 3', 'asdf != 4'], + intersects=(1, 2, 3, 4), order_by='order_col') + eq = "select pippo, tizio, caio from pinko.Pallino " + \ + "where asdf > 1 and asdf < 2 and asdf = 3 and asdf != 4 and " + \ + "intersects(1, circle('ICRS', 2, 3, 4))=1 order by order_col desc" # JM check here!! + assert eq == q, f"Expected:\n{eq}\n\nObtained:\n{q}\n\n" + + q = py2adql('pinko.Pallino', ['pippo', 'tizio', 'caio'], + where_constraints=["asdf = 'ASDF'", "bcd = 'BCD'"], + intersects=(1, 2, 3, 4), order_by='order_col') + eq = "select pippo, tizio, caio from pinko.Pallino " + \ + "where asdf = 'ASDF' and bcd = 'BCD' and " + \ + "intersects(1, circle('ICRS', 2, 3, 4))=1 order by order_col desc" # JM check here!! + assert eq == q, f"Expected:\n{eq}\n\nObtained:\n{q}\n\n" + + # All arguments + columns = 'target_name, dp_id, s_ra, s_dec, t_exptime, em_min, em_max, ' + \ + 'dataproduct_type, instrument_name, obstech, abmaglim, ' + \ + 'proposal_id, obs_collection' + table = 'ivoa.ObsCore' + and_c_list = ['em_min>4.0e-7', 'em_max<1.2e-6', 'asdasdads'] + intersects_tuple = ("s_region", 109.668246, -24.5587, 0.001389) + + q = py2adql(columns=columns, table=table, + where_constraints=and_c_list, intersects=intersects_tuple, + order_by='snr', order_by_desc=True) + expected_query = 'select ' + columns + ' from ' + table + \ + ' where ' + and_c_list[0] + ' and ' + and_c_list[1] + ' and ' + and_c_list[2] + \ + ' and ' + "intersects(s_region, circle('ICRS', 109.668246, -24.5587, 0.001389))=1 order by snr desc" + assert expected_query == q, f"Expected:\n{expected_query}\n\nObtained:\n{q}\n\n" + + # All arguments + q = py2adql(columns=columns, table=table, + where_constraints=and_c_list, intersects=intersects_tuple, + order_by='snr', order_by_desc=False) + expected_query = 'select ' + columns + ' from ' + table + \ + ' where ' + and_c_list[0] + ' and ' + and_c_list[1] + ' and ' + and_c_list[2] + \ + ' and ' + "intersects(s_region, circle('ICRS', 109.668246, -24.5587, 0.001389))=1 order by snr asc" + assert expected_query == q, f"Expected:\n{expected_query}\n\nObtained:\n{q}\n\n" diff --git a/astroquery/eso/utils.py b/astroquery/eso/utils.py new file mode 100644 index 0000000000..6629210382 --- /dev/null +++ b/astroquery/eso/utils.py @@ -0,0 +1,41 @@ +#!/usr/bin/env/python +from typing import Union + + +def _split_str_as_list_of_str(column_str: str): + if column_str == '': + column_list = [] + else: + column_list = list(map(lambda x: x.strip(), column_str.split(','))) + return column_list + + +def py2adql(table: str, columns: Union[list, str] = [], where_constraints=[], + intersects: tuple[str, float, float, float] = (), + order_by: str = '', order_by_desc=True): + # Initialize / validate + query_string = None + where_constraints = where_constraints[:] # do not modify the original list + if isinstance(columns, str): + columns = _split_str_as_list_of_str(columns) + if columns == []: + columns = ['*'] + + if len(intersects) > 0: + where_constraints += [ + f"intersects({intersects[0]}, circle('ICRS', {intersects[1]}, {intersects[2]}, {intersects[3]}))=1"] + # INTERSECTS( arg1, arg2 ) is a function that return 1 if + # the two geographies arg1 and arg2 intersect in at least one point. + # http://archive.eso.org/tap_obs/examples + + # Build the query + query_string = 'select ' + ', '.join(columns) + ' from ' + table + if len(where_constraints) > 0: + where_string = ' where ' + ' and '.join(where_constraints) + query_string += where_string + + if len(order_by) > 0: + order_string = ' order by ' + order_by + (' desc ' if order_by_desc else ' asc ') + query_string += order_string + + return query_string.strip() From 474c3bd9d69b53652954163c18056abafb7665c0 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Wed, 4 Dec 2024 15:44:39 +0100 Subject: [PATCH 18/75] Handle coord1 and coord2 arguments in query_collections --- astroquery/eso/core.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index 363fc94b7f..a214510a46 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -283,6 +283,7 @@ def query_collections(self, *, collections='', cache=True, results. """ + kwd = dict(kwargs) # not to modify the original kwargs if isinstance(collections, str): collections = _split_str_as_list_of_str(collections) table_to_return = None # Return an astropy.table.Table or None @@ -292,11 +293,21 @@ def query_collections(self, *, collections='', cache=True, tap = pyvo.dal.TAPService(EsoClass.tap_url()) collections = list(map(lambda x: f"'{x.strip()}'", collections)) where_collections_str = "obs_collection in (" + ", ".join(collections) + ")" - where_constraints_strlist = [f"{k} = {v}" for k, v in kwargs.items()] + + coord_constraint = [] + if ('coord1' in kwd.keys()) and ('coord2' in kwd.keys()): + c1, c2 = kwd['coord1'], kwd["coord2"] + del kwd['coord1'], kwd['coord2'] + # TODO make size a parameter + c_size = 0.01 + coord_constraint = \ + [f"intersects(circle('ICRS', {c1}, {c2}, {c_size}), s_region)=1"] + # http://archive.eso.org/tap_obs/examples + + where_constraints_strlist = [f"{k} = {v}" for k, v in kwd.items()] + coord_constraint where_constraints = [where_collections_str] + where_constraints_strlist query = py2adql(table="ivoa.ObsCore", columns='*', where_constraints=where_constraints) - # select * from ivoa.ObsCore where obs_collection = 'AMBRE' OR obs_collection = 'ALCOHOLS' try: table_to_return = tap.search(query, maxrec=self.ROW_LIMIT).to_table() except Exception as e: @@ -843,7 +854,7 @@ def _print_query_help(self, url, *, cache=True): def list_surveys(self, *args, **kwargs): if "surveys" in kwargs.keys(): kwargs["collections"] = kwargs["surveys"] - del(kwargs["surveys"]) + del (kwargs["surveys"]) return self.list_collections(*args, **kwargs) @deprecated(since="v0.4.8", message=("The ESO query_surveys function is deprecated," @@ -851,7 +862,7 @@ def list_surveys(self, *args, **kwargs): def query_surveys(self, *args, **kwargs): if "surveys" in kwargs.keys(): kwargs["collections"] = kwargs["surveys"] - del(kwargs["surveys"]) + del (kwargs["surveys"]) return self.query_collections(*args, **kwargs) From e57f0570c8310157854d8a80bd7ef60c404cce2c Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Wed, 4 Dec 2024 16:46:43 +0100 Subject: [PATCH 19/75] Remove intersects arg from py2adql --- astroquery/eso/utils.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/astroquery/eso/utils.py b/astroquery/eso/utils.py index 6629210382..f07cc609a1 100644 --- a/astroquery/eso/utils.py +++ b/astroquery/eso/utils.py @@ -11,7 +11,6 @@ def _split_str_as_list_of_str(column_str: str): def py2adql(table: str, columns: Union[list, str] = [], where_constraints=[], - intersects: tuple[str, float, float, float] = (), order_by: str = '', order_by_desc=True): # Initialize / validate query_string = None @@ -21,13 +20,6 @@ def py2adql(table: str, columns: Union[list, str] = [], where_constraints=[], if columns == []: columns = ['*'] - if len(intersects) > 0: - where_constraints += [ - f"intersects({intersects[0]}, circle('ICRS', {intersects[1]}, {intersects[2]}, {intersects[3]}))=1"] - # INTERSECTS( arg1, arg2 ) is a function that return 1 if - # the two geographies arg1 and arg2 intersect in at least one point. - # http://archive.eso.org/tap_obs/examples - # Build the query query_string = 'select ' + ', '.join(columns) + ' from ' + table if len(where_constraints) > 0: From 40603692afe469fca7a0730057b2cebbf752df58 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Thu, 5 Dec 2024 11:16:08 +0100 Subject: [PATCH 20/75] Write TAP enabled query_collections; pass existing tests --- astroquery/eso/core.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index a214510a46..1d0e346490 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -284,6 +284,11 @@ def query_collections(self, *, collections='', cache=True, """ kwd = dict(kwargs) # not to modify the original kwargs + c_size = 0.1775 # so that even HARPS fits to pass the tests + if 'box' in kwd.keys(): + # TODO make c_size a parameter + c_size = 0.1775 # so that even HARPS fits to pass the tests + del kwd['box'] if isinstance(collections, str): collections = _split_str_as_list_of_str(collections) table_to_return = None # Return an astropy.table.Table or None @@ -298,8 +303,6 @@ def query_collections(self, *, collections='', cache=True, if ('coord1' in kwd.keys()) and ('coord2' in kwd.keys()): c1, c2 = kwd['coord1'], kwd["coord2"] del kwd['coord1'], kwd['coord2'] - # TODO make size a parameter - c_size = 0.01 coord_constraint = \ [f"intersects(circle('ICRS', {c1}, {c2}, {c_size}), s_region)=1"] # http://archive.eso.org/tap_obs/examples @@ -310,8 +313,10 @@ def query_collections(self, *, collections='', cache=True, try: table_to_return = tap.search(query, maxrec=self.ROW_LIMIT).to_table() + except pyvo.dal.exceptions.DALQueryError: + raise pyvo.dal.exceptions.DALQueryError(f"\n\nError executing the following query:\n\n{query}\n\n") except Exception as e: - raise e # TODO catch properly the relevant exception + raise Exception(f"\n\nUnknown exception {e} while executing the following query: \n\n{query}\n\n") if len(table_to_return) < 1: warnings.warn("Query returned no results", NoResultsWarning) From ecbd1f216be8a42b6b3f0db1f70d5e8a855cfe4a Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Thu, 5 Dec 2024 11:19:46 +0100 Subject: [PATCH 21/75] Remove intersects arg from py2adql test - not its responsibility --- astroquery/eso/tests/test_eso.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/astroquery/eso/tests/test_eso.py b/astroquery/eso/tests/test_eso.py index 7f25b65d8f..a6323bb751 100644 --- a/astroquery/eso/tests/test_eso.py +++ b/astroquery/eso/tests/test_eso.py @@ -232,18 +232,18 @@ def test_py2adql(): q = py2adql('pinko.Pallino', ['pippo', 'tizio', 'caio'], where_constraints=['asdf > 1', 'asdf < 2', 'asdf = 3', 'asdf != 4'], - intersects=(1, 2, 3, 4), order_by='order_col') + order_by='order_col') eq = "select pippo, tizio, caio from pinko.Pallino " + \ - "where asdf > 1 and asdf < 2 and asdf = 3 and asdf != 4 and " + \ - "intersects(1, circle('ICRS', 2, 3, 4))=1 order by order_col desc" # JM check here!! + "where asdf > 1 and asdf < 2 and asdf = 3 and asdf != 4 " + \ + "order by order_col desc" # JM check here!! assert eq == q, f"Expected:\n{eq}\n\nObtained:\n{q}\n\n" q = py2adql('pinko.Pallino', ['pippo', 'tizio', 'caio'], where_constraints=["asdf = 'ASDF'", "bcd = 'BCD'"], - intersects=(1, 2, 3, 4), order_by='order_col') + order_by='order_col') eq = "select pippo, tizio, caio from pinko.Pallino " + \ - "where asdf = 'ASDF' and bcd = 'BCD' and " + \ - "intersects(1, circle('ICRS', 2, 3, 4))=1 order by order_col desc" # JM check here!! + "where asdf = 'ASDF' and bcd = 'BCD' " + \ + "order by order_col desc" # JM check here!! assert eq == q, f"Expected:\n{eq}\n\nObtained:\n{q}\n\n" # All arguments @@ -252,21 +252,20 @@ def test_py2adql(): 'proposal_id, obs_collection' table = 'ivoa.ObsCore' and_c_list = ['em_min>4.0e-7', 'em_max<1.2e-6', 'asdasdads'] - intersects_tuple = ("s_region", 109.668246, -24.5587, 0.001389) q = py2adql(columns=columns, table=table, - where_constraints=and_c_list, intersects=intersects_tuple, + where_constraints=and_c_list, order_by='snr', order_by_desc=True) expected_query = 'select ' + columns + ' from ' + table + \ ' where ' + and_c_list[0] + ' and ' + and_c_list[1] + ' and ' + and_c_list[2] + \ - ' and ' + "intersects(s_region, circle('ICRS', 109.668246, -24.5587, 0.001389))=1 order by snr desc" + " order by snr desc" assert expected_query == q, f"Expected:\n{expected_query}\n\nObtained:\n{q}\n\n" # All arguments q = py2adql(columns=columns, table=table, - where_constraints=and_c_list, intersects=intersects_tuple, + where_constraints=and_c_list, order_by='snr', order_by_desc=False) expected_query = 'select ' + columns + ' from ' + table + \ ' where ' + and_c_list[0] + ' and ' + and_c_list[1] + ' and ' + and_c_list[2] + \ - ' and ' + "intersects(s_region, circle('ICRS', 109.668246, -24.5587, 0.001389))=1 order by snr asc" + " order by snr asc" assert expected_query == q, f"Expected:\n{expected_query}\n\nObtained:\n{q}\n\n" From 9abf461ba3b712c5c79dd938d730974057a40e1c Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Thu, 5 Dec 2024 11:21:14 +0100 Subject: [PATCH 22/75] Add ATLASGAL to SGRA_COLLECTIONS --- astroquery/eso/tests/test_eso_remote.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/astroquery/eso/tests/test_eso_remote.py b/astroquery/eso/tests/test_eso_remote.py index b304b970bd..bd25060e94 100644 --- a/astroquery/eso/tests/test_eso_remote.py +++ b/astroquery/eso/tests/test_eso_remote.py @@ -18,9 +18,18 @@ # TODO: make this a configuration item SKIP_SLOW = True -SGRA_COLLECTIONS = ['195.B-0283', 'GIRAFFE', 'HARPS', 'HAWKI', 'KMOS', +SGRA_COLLECTIONS = ['195.B-0283', + 'ATLASGAL', 'ERIS-SPIFFIER', - 'MW-BULGE-PSFPHOT', 'VPHASplus', 'VVV', 'VVVX', 'XSHOOTER' + 'GIRAFFE', + 'HARPS', + 'HAWKI', + 'KMOS', + 'MW-BULGE-PSFPHOT', + 'VPHASplus', + 'VVV', + 'VVVX', + 'XSHOOTER' ] From cc51bc66eec8d76a5bfde93111a35f153b8206a4 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Thu, 5 Dec 2024 12:26:31 +0100 Subject: [PATCH 23/75] Ignore DALOverflowWarning thrown on truncated long query results --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.cfg b/setup.cfg index 7f008e445f..d7b3c8ea6a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -85,6 +85,8 @@ filterwarnings = ignore:numpy.core:DeprecationWarning # SHA module deprecation/defunct ignore:The upstream SHA API has been changed:UserWarning +# Enable TAP-querying a small number of records without raising a pyvo.dal.exceptions.DALOverflowWarning + ignore:Partial result set. Potential causes MAXREC, async storage space, etc.:pyvo.dal.exceptions.DALOverflowWarning markers = bigdata: marks tests that are expected to trigger a large download (deselect with '-m "not bigdata"') From 883f349c1bb236751664dc710d5473c7787bebfe Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Thu, 5 Dec 2024 13:53:25 +0100 Subject: [PATCH 24/75] Write test for func query_tap_service --- astroquery/eso/tests/test_eso_remote.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/astroquery/eso/tests/test_eso_remote.py b/astroquery/eso/tests/test_eso_remote.py index bd25060e94..519da54d4a 100644 --- a/astroquery/eso/tests/test_eso_remote.py +++ b/astroquery/eso/tests/test_eso_remote.py @@ -5,6 +5,7 @@ from astroquery.eso import Eso from astroquery.exceptions import NoResultsWarning +from astropy.table import Table instrument_list = [u'fors1', u'fors2', u'sphere', u'vimos', u'omegacam', u'hawki', u'isaac', u'naco', u'visir', u'vircam', u'apex', @@ -35,6 +36,12 @@ @pytest.mark.remote_data class TestEso: + def test_query_tap_service(self): + eso = Eso() + t = eso._query_tap_service("select * from ivoa.ObsCore") + assert type(t) is Table, f"Expected type {type(Table)}; Obtained {type(t)}" + assert len(t) > 0, "Table length is zero" + def test_SgrAstar(self, tmp_path): eso = Eso() eso.cache_location = tmp_path From 2ccb253e97162301b4819c983ec21c8a83afba7e Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Thu, 5 Dec 2024 13:56:42 +0100 Subject: [PATCH 25/75] Define and use _query_tap_service --- astroquery/eso/core.py | 49 ++++++++++++++++-------------------------- 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index 1d0e346490..e639a6d799 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -33,19 +33,6 @@ __doctest_skip__ = ['EsoClass.*'] -def _check_response(content): - """ - Check the response from an ESO service query for various types of error - - If all is OK, return True - """ - if b"NETWORKPROBLEM" in content: - raise RemoteServiceError("The query resulted in a network " - "problem; the service may be offline.") - elif b"# No data returned !" not in content: - return True - - class CalSelectorError(Exception): """ Raised on failure to parse CalSelector's response. @@ -197,14 +184,23 @@ def _print_collections_help(self): log.info("\n".join(result_string)) return result_string - def _query_tap_service(self, query_str, col_name): + def _query_tap_service(self, query_str: str): """ - returns an astropy.table.Table # JM TODO + returns an astropy.table.Table from an adql query string + Example use: + eso._query_tap_service("Select * from ivoa.ObsCore") """ tap = pyvo.dal.TAPService(EsoClass.tap_url()) - query = query_str - res = tap.search(query)[col_name].data - return res + table_to_return = None + + try: + table_to_return = tap.search(query_str).to_table() + except pyvo.dal.exceptions.DALQueryError: + raise pyvo.dal.exceptions.DALQueryError(f"\n\nError executing the following query:\n\n{query_str}\n\n") + except Exception as e: + raise Exception(f"\n\nUnknown exception {e} while executing the following query: \n\n{query_str}\n\n") + + return table_to_return def list_instruments(self, *, cache=True): """ List all the available instrument-specific queries offered by the ESO archive. @@ -225,8 +221,7 @@ def list_instruments(self, *, cache=True): where schema_name='ist' order by table_name """ - col_name = "table_name" - res = self._query_tap_service(query_str, col_name) + res = self._query_tap_service(query_str)["table_name"].data self._instrument_list = list(map(lambda x: x.split(".")[1], res)) return self._instrument_list @@ -240,18 +235,18 @@ def list_collections(self, *, cache=True): Defaults to True. If set overrides global caching behavior. See :ref:`caching documentation `. """ + # TODO include ALMA if self._collection_list is None: self._collection_list = [] query_str = """ SELECT distinct obs_collection from ivoa.ObsCore where obs_collection != 'ALMA' """ - col_name = "obs_collection" - res = self._query_tap_service(query_str, col_name) + res = self._query_tap_service(query_str)["obs_collection"].data self._collection_list = list(res) return self._collection_list - # JM - Example queries: + # JM - Example queries # select * from ivoa.ObsCore where data_collection = 'AMBRE' and facility_name = 'something' # select * from ivoa.ObsCore where data_collection = some_collection and key1 = val1 and key2=val2 and ... # Extra stuff she mentioned: @@ -295,7 +290,6 @@ def query_collections(self, *, collections='', cache=True, if help: self._print_collections_help() - tap = pyvo.dal.TAPService(EsoClass.tap_url()) collections = list(map(lambda x: f"'{x.strip()}'", collections)) where_collections_str = "obs_collection in (" + ", ".join(collections) + ")" @@ -311,12 +305,7 @@ def query_collections(self, *, collections='', cache=True, where_constraints = [where_collections_str] + where_constraints_strlist query = py2adql(table="ivoa.ObsCore", columns='*', where_constraints=where_constraints) - try: - table_to_return = tap.search(query, maxrec=self.ROW_LIMIT).to_table() - except pyvo.dal.exceptions.DALQueryError: - raise pyvo.dal.exceptions.DALQueryError(f"\n\nError executing the following query:\n\n{query}\n\n") - except Exception as e: - raise Exception(f"\n\nUnknown exception {e} while executing the following query: \n\n{query}\n\n") + table_to_return = self._query_tap_service(query_str=query) if len(table_to_return) < 1: warnings.warn("Query returned no results", NoResultsWarning) From 68ed74ec01f58fe0010a8e4095397bbd0a07a82a Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Thu, 5 Dec 2024 15:24:42 +0100 Subject: [PATCH 26/75] fwd query_instrument and _collections to query_instrument_or_collection --- astroquery/eso/core.py | 174 ++++++++++++++++------------------------- 1 file changed, 66 insertions(+), 108 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index e639a6d799..11e3dbad76 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -246,64 +246,76 @@ def list_collections(self, *, cache=True): self._collection_list = list(res) return self._collection_list - # JM - Example queries - # select * from ivoa.ObsCore where data_collection = 'AMBRE' and facility_name = 'something' - # select * from ivoa.ObsCore where data_collection = some_collection and key1 = val1 and key2=val2 and ... - # Extra stuff she mentioned: - # select * from TAP_SCHEMA.columns where table_name = 'ivoa.ObsCore' @deprecated_renamed_argument(old_name='open_form', new_name=None, since='0.4.8') - def query_collections(self, *, collections='', cache=True, - help=False, open_form=None, **kwargs): + def _query_instrument_or_collection(self, i_true_c_false: bool, instmnt_or_clctn_name, *, column_filters={}, + columns=[], open_form=None, help=False, cache=True, **kwargs): + # TODO + # Refactor. This function has the same logic as query_collections """ - Query collection Phase 3 data contained in the ESO archive. + Query instrument-specific raw data contained in the ESO archive. Parameters ---------- - collection : string or list - Name of the collection(s) to query. Should beone or more of the - names returned by `~astroquery.eso.EsoClass.list_collections`. If - specified as a string, should be a comma-separated list of - collection names. + instrument : string + Name of the instrument to query, one of the names returned by + `~astroquery.eso.EsoClass.list_instruments`. + column_filters : dict + Constraints applied to the query. + columns : list of strings + Columns returned by the query. + open_form : bool + If `True`, opens in your default browser the query form + for the requested instrument. + help : bool + If `True`, prints all the parameters accepted in + ``column_filters`` and ``columns`` for the requested + ``instrument``. cache : bool Defaults to True. If set overrides global caching behavior. See :ref:`caching documentation `. Returns ------- - table : `~astropy.table.Table` or `None` + table : `~astropy.table.Table` A table representing the data available in the archive for the - specified collection, matching the constraints specified in ``kwargs``. - The number of rows returned is capped by the ROW_LIMIT - configuration item. `None` is returned when the query has no - results. + specified instrument, matching the constraints specified in + ``kwargs``. The number of rows returned is capped by the + ROW_LIMIT configuration item. """ - kwd = dict(kwargs) # not to modify the original kwargs + if help: + h = self._query_tap_service( + "select column_name, datatype from TAP_SCHEMA.columns where table_name = 'ivoa.ObsCore'") + log.info(f"Columns present in the table: {h}") + return + + filters = {**dict(kwargs), **column_filters} c_size = 0.1775 # so that even HARPS fits to pass the tests - if 'box' in kwd.keys(): + if 'box' in filters.keys(): # TODO make c_size a parameter c_size = 0.1775 # so that even HARPS fits to pass the tests - del kwd['box'] - if isinstance(collections, str): - collections = _split_str_as_list_of_str(collections) + del filters['box'] + if isinstance(instmnt_or_clctn_name, str): + instmnt_or_clctn_name = _split_str_as_list_of_str(instmnt_or_clctn_name) table_to_return = None # Return an astropy.table.Table or None if help: self._print_collections_help() - collections = list(map(lambda x: f"'{x.strip()}'", collections)) - where_collections_str = "obs_collection in (" + ", ".join(collections) + ")" + instmnt_or_clctn_name = list(map(lambda x: f"'{x.strip()}'", instmnt_or_clctn_name)) + column_name = "instrument_name" if i_true_c_false else "obs_collection" + where_collections_str = f"{column_name} in (" + ", ".join(instmnt_or_clctn_name) + ")" coord_constraint = [] - if ('coord1' in kwd.keys()) and ('coord2' in kwd.keys()): - c1, c2 = kwd['coord1'], kwd["coord2"] - del kwd['coord1'], kwd['coord2'] + if ('coord1' in filters.keys()) and ('coord2' in filters.keys()): + c1, c2 = filters['coord1'], filters["coord2"] + del filters['coord1'], filters['coord2'] coord_constraint = \ [f"intersects(circle('ICRS', {c1}, {c2}, {c_size}), s_region)=1"] # http://archive.eso.org/tap_obs/examples - where_constraints_strlist = [f"{k} = {v}" for k, v in kwd.items()] + coord_constraint + where_constraints_strlist = [f"{k} = {v}" for k, v in filters.items()] + coord_constraint where_constraints = [where_collections_str] + where_constraints_strlist - query = py2adql(table="ivoa.ObsCore", columns='*', where_constraints=where_constraints) + query = py2adql(table="ivoa.ObsCore", columns=columns, where_constraints=where_constraints) table_to_return = self._query_tap_service(query_str=query) @@ -313,79 +325,29 @@ def query_collections(self, *, collections='', cache=True, return table_to_return - def query_main(self, *, column_filters={}, columns=[], - open_form=False, help=False, cache=True, **kwargs): - """ - Query raw data contained in the ESO archive. - - Parameters - ---------- - column_filters : dict - Constraints applied to the query. - columns : list of strings - Columns returned by the query. - open_form : bool - If `True`, opens in your default browser the query form - for the requested instrument. - help : bool - If `True`, prints all the parameters accepted in - ``column_filters`` and ``columns`` for the requested - ``instrument``. - cache : bool - Defaults to True. If set overrides global caching behavior. - See :ref:`caching documentation `. - - Returns - ------- - table : `~astropy.table.Table` - A table representing the data available in the archive for the - specified instrument, matching the constraints specified in - ``kwargs``. The number of rows returned is capped by the - ROW_LIMIT configuration item. - - """ - url = self.QUERY_INSTRUMENT_URL + "/eso_archive_main/form" - return self._query(url, column_filters=column_filters, columns=columns, - open_form=open_form, help=help, cache=cache, **kwargs) - + @deprecated_renamed_argument(old_name='open_form', new_name=None, since='0.4.8') def query_instrument(self, instrument, *, column_filters={}, columns=[], open_form=False, help=False, cache=True, **kwargs): - """ - Query instrument-specific raw data contained in the ESO archive. - - Parameters - ---------- - instrument : string - Name of the instrument to query, one of the names returned by - `~astroquery.eso.EsoClass.list_instruments`. - column_filters : dict - Constraints applied to the query. - columns : list of strings - Columns returned by the query. - open_form : bool - If `True`, opens in your default browser the query form - for the requested instrument. - help : bool - If `True`, prints all the parameters accepted in - ``column_filters`` and ``columns`` for the requested - ``instrument``. - cache : bool - Defaults to True. If set overrides global caching behavior. - See :ref:`caching documentation `. - - Returns - ------- - table : `~astropy.table.Table` - A table representing the data available in the archive for the - specified instrument, matching the constraints specified in - ``kwargs``. The number of rows returned is capped by the - ROW_LIMIT configuration item. + _ = self._query_instrument_or_collection(i_true_c_false=True, + instmnt_or_clctn_name=instrument, + column_filters=column_filters, + columns=columns, + help=help, + cache=cache, + **kwargs) + return _ - """ - - url = self.QUERY_INSTRUMENT_URL + '/{0}/form'.format(instrument.lower()) - return self._query(url, column_filters=column_filters, columns=columns, - open_form=open_form, help=help, cache=cache, **kwargs) + @deprecated_renamed_argument(old_name='open_form', new_name=None, since='0.4.8') + def query_collections(self, collections, *, column_filters={}, columns=[], + open_form=None, help=False, cache=True, **kwargs): + _ = self._query_instrument_or_collection(i_true_c_false=False, + instmnt_or_clctn_name=collections, + column_filters=column_filters, + columns=columns, + help=help, + cache=cache, + **kwargs) + return _ def _query(self, url, *, column_filters={}, columns=[], open_form=False, help=False, cache=True, **kwargs): @@ -425,12 +387,8 @@ def _query(self, url, *, column_filters={}, columns=[], # First line is always garbage content = content.split(b'\n', 1)[1] log.debug("Response content:\n{0}".format(content)) - if _check_response(content): - table = Table.read(BytesIO(content), format="ascii.csv", - comment='^#') - return table - else: - warnings.warn("Query returned no results", NoResultsWarning) + table = Table.read(BytesIO(content), format="ascii.csv", comment='^#') + return table def get_headers(self, product_ids, *, cache=True): """ @@ -763,7 +721,7 @@ def query_apex_quicklooks(self, *, project_id=None, help=False, cache=cache, method='application/x-www-form-urlencoded') content = apex_response.content - if _check_response(content): + if True: # First line is always garbage content = content.split(b'\n', 1)[1] try: @@ -778,7 +736,7 @@ def query_apex_quicklooks(self, *, project_id=None, help=False, comment="#") else: raise ex - else: + else: # this function is going to be replaced soon raise RemoteServiceError("Query returned no results") return table From b66d987504f514c248bf1b2f1aff4be9f421b7e4 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Thu, 5 Dec 2024 15:37:47 +0100 Subject: [PATCH 27/75] Ignore DALOverflowWarning with more granularity --- astroquery/eso/tests/test_eso_remote.py | 2 ++ setup.cfg | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/astroquery/eso/tests/test_eso_remote.py b/astroquery/eso/tests/test_eso_remote.py index 519da54d4a..8652f9a5c9 100644 --- a/astroquery/eso/tests/test_eso_remote.py +++ b/astroquery/eso/tests/test_eso_remote.py @@ -177,6 +177,8 @@ def test_each_instrument_SgrAstar(self, tmp_path): else: assert len(result) > 0 + # TODO Ignore OverflowWarning -- how? + @pytest.mark.filterwarnings("ignore::pyvo.dal.exceptions.DALOverflowWarning") def test_each_collection_and_SgrAstar(self, tmp_path): eso = Eso() eso.cache_location = tmp_path diff --git a/setup.cfg b/setup.cfg index d7b3c8ea6a..7f008e445f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -85,8 +85,6 @@ filterwarnings = ignore:numpy.core:DeprecationWarning # SHA module deprecation/defunct ignore:The upstream SHA API has been changed:UserWarning -# Enable TAP-querying a small number of records without raising a pyvo.dal.exceptions.DALOverflowWarning - ignore:Partial result set. Potential causes MAXREC, async storage space, etc.:pyvo.dal.exceptions.DALOverflowWarning markers = bigdata: marks tests that are expected to trigger a large download (deselect with '-m "not bigdata"') From c790eee5862c3f407c28e6c920c28129d0450382 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Thu, 5 Dec 2024 15:44:18 +0100 Subject: [PATCH 28/75] Remove unused code --- astroquery/eso/core.py | 27 +++------------------------ 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index 11e3dbad76..0f88d2372f 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -167,22 +167,6 @@ def _get_auth_header(self) -> Dict[str, str]: else: return {} - # TODO - delete or rewrite - def _print_collections_help(self): - """ - Prints help - """ - log.info("List of the parameters accepted by the " - "collections query.") - log.info("The presence of a column in the result table can be " - "controlled if prefixed with a [ ] checkbox.") - log.info("The default columns in the result table are shown as " - "already ticked: [x].") - - result_string = [] - - log.info("\n".join(result_string)) - return result_string def _query_tap_service(self, query_str: str): """ @@ -246,13 +230,10 @@ def list_collections(self, *, cache=True): self._collection_list = list(res) return self._collection_list - @deprecated_renamed_argument(old_name='open_form', new_name=None, since='0.4.8') def _query_instrument_or_collection(self, i_true_c_false: bool, instmnt_or_clctn_name, *, column_filters={}, - columns=[], open_form=None, help=False, cache=True, **kwargs): - # TODO - # Refactor. This function has the same logic as query_collections + columns=[], help=False, cache=True, **kwargs): """ - Query instrument-specific raw data contained in the ESO archive. + Query instrument- or collection-specific raw data contained in the ESO archive. Parameters ---------- @@ -281,8 +262,8 @@ def _query_instrument_or_collection(self, i_true_c_false: bool, instmnt_or_clctn specified instrument, matching the constraints specified in ``kwargs``. The number of rows returned is capped by the ROW_LIMIT configuration item. - """ + if help: h = self._query_tap_service( "select column_name, datatype from TAP_SCHEMA.columns where table_name = 'ivoa.ObsCore'") @@ -298,8 +279,6 @@ def _query_instrument_or_collection(self, i_true_c_false: bool, instmnt_or_clctn if isinstance(instmnt_or_clctn_name, str): instmnt_or_clctn_name = _split_str_as_list_of_str(instmnt_or_clctn_name) table_to_return = None # Return an astropy.table.Table or None - if help: - self._print_collections_help() instmnt_or_clctn_name = list(map(lambda x: f"'{x.strip()}'", instmnt_or_clctn_name)) column_name = "instrument_name" if i_true_c_false else "obs_collection" From 52cbe5063ab6ac035b44eb37ccce563df8908ba7 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Fri, 6 Dec 2024 09:25:29 +0100 Subject: [PATCH 29/75] Take instrument or collection data from the corresponding table --- astroquery/eso/core.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index 0f88d2372f..81de9ffd4b 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -66,6 +66,9 @@ class EsoClass(QueryWithLogin): AUTH_URL = "https://www.eso.org/sso/oidc/token" GUNZIP = "gunzip" USE_DEV_TAP = True + # TODO + # INSTRUMENTS_COLUMN_NAME = + # COLLECTIONS_COLUMN_NAME = @staticmethod def tap_url(): @@ -167,7 +170,6 @@ def _get_auth_header(self) -> Dict[str, str]: else: return {} - def _query_tap_service(self, query_str: str): """ returns an astropy.table.Table from an adql query string @@ -233,7 +235,9 @@ def list_collections(self, *, cache=True): def _query_instrument_or_collection(self, i_true_c_false: bool, instmnt_or_clctn_name, *, column_filters={}, columns=[], help=False, cache=True, **kwargs): """ - Query instrument- or collection-specific raw data contained in the ESO archive. + Query instrument- or collection-specific data contained in the ESO archive. + - instrument-specific data is raw + - collection-specific data is processed Parameters ---------- @@ -263,10 +267,18 @@ def _query_instrument_or_collection(self, i_true_c_false: bool, instmnt_or_clctn ``kwargs``. The number of rows returned is capped by the ROW_LIMIT configuration item. """ + # False for collections, True for Instruments + source_table_dict = {True: "dbo.raw", + False: "ivoa.ObsCore"} + column_name_dict = {True: "instrument", + False: "obs_collection"} + # TODO - these queries are the same, parameterize only what changes, use the py2adql func + help_query_dict = {True: "select column_name, datatype from TAP_SCHEMA.columns where table_name = 'dbo.raw'", + False: "select column_name, datatype from TAP_SCHEMA.columns" + + "where table_name = 'ivoa.ObsCore'"} if help: - h = self._query_tap_service( - "select column_name, datatype from TAP_SCHEMA.columns where table_name = 'ivoa.ObsCore'") + h = self._query_tap_service(help_query_dict[i_true_c_false]) log.info(f"Columns present in the table: {h}") return @@ -281,8 +293,7 @@ def _query_instrument_or_collection(self, i_true_c_false: bool, instmnt_or_clctn table_to_return = None # Return an astropy.table.Table or None instmnt_or_clctn_name = list(map(lambda x: f"'{x.strip()}'", instmnt_or_clctn_name)) - column_name = "instrument_name" if i_true_c_false else "obs_collection" - where_collections_str = f"{column_name} in (" + ", ".join(instmnt_or_clctn_name) + ")" + where_collections_str = f"{column_name_dict[i_true_c_false]} in (" + ", ".join(instmnt_or_clctn_name) + ")" coord_constraint = [] if ('coord1' in filters.keys()) and ('coord2' in filters.keys()): @@ -294,7 +305,7 @@ def _query_instrument_or_collection(self, i_true_c_false: bool, instmnt_or_clctn where_constraints_strlist = [f"{k} = {v}" for k, v in filters.items()] + coord_constraint where_constraints = [where_collections_str] + where_constraints_strlist - query = py2adql(table="ivoa.ObsCore", columns=columns, where_constraints=where_constraints) + query = py2adql(table=source_table_dict[i_true_c_false], columns=columns, where_constraints=where_constraints) table_to_return = self._query_tap_service(query_str=query) From 280927af71b3ea7d8e2cbd278c0095a991a349b2 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Fri, 6 Dec 2024 09:26:43 +0100 Subject: [PATCH 30/75] Ignore DALOverflowWarning on remote tests --- astroquery/eso/tests/test_eso_remote.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/astroquery/eso/tests/test_eso_remote.py b/astroquery/eso/tests/test_eso_remote.py index 8652f9a5c9..5cc3f9cd23 100644 --- a/astroquery/eso/tests/test_eso_remote.py +++ b/astroquery/eso/tests/test_eso_remote.py @@ -36,12 +36,14 @@ @pytest.mark.remote_data class TestEso: + @pytest.mark.filterwarnings("ignore::pyvo.dal.exceptions.DALOverflowWarning") def test_query_tap_service(self): eso = Eso() t = eso._query_tap_service("select * from ivoa.ObsCore") assert type(t) is Table, f"Expected type {type(Table)}; Obtained {type(t)}" assert len(t) > 0, "Table length is zero" + @pytest.mark.filterwarnings("ignore::pyvo.dal.exceptions.DALOverflowWarning") def test_SgrAstar(self, tmp_path): eso = Eso() eso.cache_location = tmp_path @@ -66,9 +68,16 @@ def test_SgrAstar(self, tmp_path): assert result_i is not None assert 'VVV' in collections assert result_s is not None - assert 'Object' in result_s.colnames - assert 'b333' in result_s['Object'] + # From obs.raw, we have "object" (when query_instruments) + # object: Target designation as given by the astronomer, though at times overwritten by the obeservatory, especially for CALIB observations. Compare with the similar field called "target".) + + # From ivoa.ObsCore, we have "target_name" (when query_collections) + # target_name: The target name as assigned by the Principal Investigator; ref. Ref. OBJECT keyword in ESO SDP standard. For spectroscopic public surveys, the value shall be set to the survey source identifier... + assert 'target_name' in result_s.colnames + assert 'b333' in result_s['target_name'] + + @pytest.mark.filterwarnings("ignore::pyvo.dal.exceptions.DALOverflowWarning") def test_multicollection(self, tmp_path): eso = Eso() @@ -88,6 +97,7 @@ def test_multicollection(self, tmp_path): assert 'b333_414_58214' in result_s['Object'] assert 'Pistol-Star' in result_s['Object'] + @pytest.mark.filterwarnings("ignore::pyvo.dal.exceptions.DALOverflowWarning") def test_empty_return(self): # test for empty return with an object from the North eso = Eso() @@ -101,6 +111,7 @@ def test_empty_return(self): assert result_s is None + @pytest.mark.filterwarnings("ignore::pyvo.dal.exceptions.DALOverflowWarning") def test_SgrAstar_remotevslocal(self, tmp_path): eso = Eso() # Remote version @@ -112,6 +123,7 @@ def test_SgrAstar_remotevslocal(self, tmp_path): coord2=-29.00782497, cache=True) assert all(result1 == result2) + @pytest.mark.filterwarnings("ignore::pyvo.dal.exceptions.DALOverflowWarning") def test_list_instruments(self): # If this test fails, we may simply need to update it @@ -120,6 +132,7 @@ def test_list_instruments(self): # we only care about the sets matching assert set(inst) == set(instrument_list) + @pytest.mark.filterwarnings("ignore::pyvo.dal.exceptions.DALOverflowWarning") def test_retrieve_data(self): eso = Eso() file_id = 'AMBER.2006-03-14T07:40:19.830' @@ -136,6 +149,7 @@ def test_retrieve_data_authenticated(self): assert isinstance(result, str) assert file_id in result + @pytest.mark.filterwarnings("ignore::pyvo.dal.exceptions.DALOverflowWarning") def test_retrieve_data_list(self): eso = Eso() datasets = ['MIDI.2014-07-25T02:03:11.561', 'AMBER.2006-03-14T07:40:19.830'] @@ -146,10 +160,12 @@ def test_retrieve_data_list(self): # TODO: remove filter when https://github.com/astropy/astroquery/issues/2539 is fixed @pytest.mark.filterwarnings("ignore::pytest.PytestUnraisableExceptionWarning") @pytest.mark.parametrize('instrument', instrument_list) + @pytest.mark.filterwarnings("ignore::pyvo.dal.exceptions.DALOverflowWarning") def test_help(self, instrument): eso = Eso() eso.query_instrument(instrument, help=True) + @pytest.mark.filterwarnings("ignore::pyvo.dal.exceptions.DALOverflowWarning") def test_apex_retrieval(self): eso = Eso() @@ -162,6 +178,7 @@ def test_apex_retrieval(self): assert np.all(tbl == tblb) + @pytest.mark.filterwarnings("ignore::pyvo.dal.exceptions.DALOverflowWarning") def test_each_instrument_SgrAstar(self, tmp_path): eso = Eso() eso.cache_location = tmp_path @@ -203,6 +220,7 @@ def test_each_collection_and_SgrAstar(self, tmp_path): generic_result = eso.query_collections(collections=collection) assert len(generic_result) > 0 + @pytest.mark.filterwarnings("ignore::pyvo.dal.exceptions.DALOverflowWarning") def test_mixed_case_instrument(self, tmp_path): eso = Eso() eso.cache_location = tmp_path From 0a3e9f188c8f61361dbe5561fe710d66c43bd708 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Fri, 6 Dec 2024 17:11:00 +0100 Subject: [PATCH 31/75] Issue queries based on new QueryOnField class --- astroquery/eso/core.py | 58 ++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index 81de9ffd4b..1f43f18bc0 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -57,6 +57,21 @@ def expired(self) -> bool: return time.time() > self.expiration_time - 600 +class QueryOnField(): + table_name = "" + column_name = "" + + +class QueryOnInstrument(): + table_name = "dbo.raw" + column_name = "instrument" + + +class QueryOnCollection(): + table_name = "ivoa.ObsCore" + column_name = "obs_collection" + + class EsoClass(QueryWithLogin): ROW_LIMIT = conf.row_limit USERNAME = conf.username @@ -66,9 +81,6 @@ class EsoClass(QueryWithLogin): AUTH_URL = "https://www.eso.org/sso/oidc/token" GUNZIP = "gunzip" USE_DEV_TAP = True - # TODO - # INSTRUMENTS_COLUMN_NAME = - # COLLECTIONS_COLUMN_NAME = @staticmethod def tap_url(): @@ -77,13 +89,6 @@ def tap_url(): url = "http://dfidev5.hq.eso.org:8123/tap_obs" return url - @staticmethod - def tap_columns(table_name="ivoa.ObsCore"): - # TODO - # return a list with the available columns - query = f"select * from TAP_SCHEMA.columns where table_name = '{table_name}'" - return query - def __init__(self): super().__init__() self._instrument_list = None @@ -224,15 +229,17 @@ def list_collections(self, *, cache=True): # TODO include ALMA if self._collection_list is None: self._collection_list = [] - query_str = """ - SELECT distinct obs_collection from ivoa.ObsCore where obs_collection != 'ALMA' + c = QueryOnCollection.column_name + t = QueryOnCollection.table_name + query_str = f""" + SELECT distinct {c} from {t} where {c} != 'ALMA' """ - res = self._query_tap_service(query_str)["obs_collection"].data + res = self._query_tap_service(query_str)[c].data self._collection_list = list(res) return self._collection_list - def _query_instrument_or_collection(self, i_true_c_false: bool, instmnt_or_clctn_name, *, column_filters={}, + def _query_instrument_or_collection(self, query_on: QueryOnField, instmnt_or_clctn_name, *, column_filters={}, columns=[], help=False, cache=True, **kwargs): """ Query instrument- or collection-specific data contained in the ESO archive. @@ -267,18 +274,10 @@ def _query_instrument_or_collection(self, i_true_c_false: bool, instmnt_or_clctn ``kwargs``. The number of rows returned is capped by the ROW_LIMIT configuration item. """ - # False for collections, True for Instruments - source_table_dict = {True: "dbo.raw", - False: "ivoa.ObsCore"} - column_name_dict = {True: "instrument", - False: "obs_collection"} - # TODO - these queries are the same, parameterize only what changes, use the py2adql func - help_query_dict = {True: "select column_name, datatype from TAP_SCHEMA.columns where table_name = 'dbo.raw'", - False: "select column_name, datatype from TAP_SCHEMA.columns" - + "where table_name = 'ivoa.ObsCore'"} - + help_query = f"select column_name, datatype from TAP_SCHEMA.columns where table_name = '{query_on.table_name}'" + # TODO - move help printing to its own function if help: - h = self._query_tap_service(help_query_dict[i_true_c_false]) + h = self._query_tap_service(help_query) log.info(f"Columns present in the table: {h}") return @@ -293,7 +292,7 @@ def _query_instrument_or_collection(self, i_true_c_false: bool, instmnt_or_clctn table_to_return = None # Return an astropy.table.Table or None instmnt_or_clctn_name = list(map(lambda x: f"'{x.strip()}'", instmnt_or_clctn_name)) - where_collections_str = f"{column_name_dict[i_true_c_false]} in (" + ", ".join(instmnt_or_clctn_name) + ")" + where_collections_str = f"{query_on.column_name} in (" + ", ".join(instmnt_or_clctn_name) + ")" coord_constraint = [] if ('coord1' in filters.keys()) and ('coord2' in filters.keys()): @@ -305,7 +304,7 @@ def _query_instrument_or_collection(self, i_true_c_false: bool, instmnt_or_clctn where_constraints_strlist = [f"{k} = {v}" for k, v in filters.items()] + coord_constraint where_constraints = [where_collections_str] + where_constraints_strlist - query = py2adql(table=source_table_dict[i_true_c_false], columns=columns, where_constraints=where_constraints) + query = py2adql(table=query_on.table_name, columns=columns, where_constraints=where_constraints) table_to_return = self._query_tap_service(query_str=query) @@ -318,7 +317,7 @@ def _query_instrument_or_collection(self, i_true_c_false: bool, instmnt_or_clctn @deprecated_renamed_argument(old_name='open_form', new_name=None, since='0.4.8') def query_instrument(self, instrument, *, column_filters={}, columns=[], open_form=False, help=False, cache=True, **kwargs): - _ = self._query_instrument_or_collection(i_true_c_false=True, + _ = self._query_instrument_or_collection(query_on=QueryOnInstrument(), instmnt_or_clctn_name=instrument, column_filters=column_filters, columns=columns, @@ -330,7 +329,7 @@ def query_instrument(self, instrument, *, column_filters={}, columns=[], @deprecated_renamed_argument(old_name='open_form', new_name=None, since='0.4.8') def query_collections(self, collections, *, column_filters={}, columns=[], open_form=None, help=False, cache=True, **kwargs): - _ = self._query_instrument_or_collection(i_true_c_false=False, + _ = self._query_instrument_or_collection(query_on=QueryOnCollection(), instmnt_or_clctn_name=collections, column_filters=column_filters, columns=columns, @@ -351,7 +350,6 @@ def _query(self, url, *, column_filters={}, columns=[], instrument_form = self._request("GET", url, cache=cache) query_dict = {} query_dict.update(column_filters) - # TODO: replace this with individually parsed kwargs query_dict.update(kwargs) query_dict["wdbo"] = "csv/download" From 1e5c18804b0d00034d789beacb9d4c4d4ec79f6d Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Mon, 9 Dec 2024 15:25:28 +0100 Subject: [PATCH 32/75] Remove unused function _query --- astroquery/eso/core.py | 53 +++--------------------------------------- 1 file changed, 3 insertions(+), 50 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index 1f43f18bc0..569d2c2803 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -206,12 +206,7 @@ def list_instruments(self, *, cache=True): """ if self._instrument_list is None: self._instrument_list = [] - query_str = """ - select table_name - from TAP_SCHEMA.tables - where schema_name='ist' - order by table_name - """ + query_str = "select table_name from TAP_SCHEMA.tables where schema_name='ist' order by table_name" res = self._query_tap_service(query_str)["table_name"].data self._instrument_list = list(map(lambda x: x.split(".")[1], res)) return self._instrument_list @@ -231,9 +226,7 @@ def list_collections(self, *, cache=True): self._collection_list = [] c = QueryOnCollection.column_name t = QueryOnCollection.table_name - query_str = f""" - SELECT distinct {c} from {t} where {c} != 'ALMA' - """ + query_str = f"select distinct {c} from {t} where {c} != 'ALMA'" res = self._query_tap_service(query_str)[c].data self._collection_list = list(res) @@ -274,9 +267,9 @@ def _query_instrument_or_collection(self, query_on: QueryOnField, instmnt_or_clc ``kwargs``. The number of rows returned is capped by the ROW_LIMIT configuration item. """ - help_query = f"select column_name, datatype from TAP_SCHEMA.columns where table_name = '{query_on.table_name}'" # TODO - move help printing to its own function if help: + help_query = f"select column_name, datatype from TAP_SCHEMA.columns where table_name = '{query_on.table_name}'" h = self._query_tap_service(help_query) log.info(f"Columns present in the table: {h}") return @@ -338,46 +331,6 @@ def query_collections(self, collections, *, column_filters={}, columns=[], **kwargs) return _ - def _query(self, url, *, column_filters={}, columns=[], - open_form=False, help=False, cache=True, **kwargs): - - table = None - if open_form: - webbrowser.open(url) - elif help: - self._print_query_help(url) - else: - instrument_form = self._request("GET", url, cache=cache) - query_dict = {} - query_dict.update(column_filters) - query_dict.update(kwargs) - query_dict["wdbo"] = "csv/download" - - # Default to returning the DP.ID since it is needed for header - # acquisition - query_dict['tab_dp_id'] = kwargs.pop('tab_dp_id', 'on') - - for k in columns: - query_dict["tab_" + k] = True - if self.ROW_LIMIT >= 0: - query_dict["max_rows_returned"] = int(self.ROW_LIMIT) - else: - query_dict["max_rows_returned"] = 10000 - # used to be form 0, but now there's a new 'logout' form at the top - # (form_index = -1 and 0 both work now that form_id is included) - instrument_response = self._activate_form(instrument_form, - form_index=-1, - form_id='queryform', - inputs=query_dict, - cache=cache) - - content = instrument_response.content - # First line is always garbage - content = content.split(b'\n', 1)[1] - log.debug("Response content:\n{0}".format(content)) - table = Table.read(BytesIO(content), format="ascii.csv", comment='^#') - return table - def get_headers(self, product_ids, *, cache=True): """ Get the headers associated to a list of data product IDs From 596b11cc92b5b455bd6b2cd948ac82e666b9d199 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Mon, 9 Dec 2024 18:01:28 +0100 Subject: [PATCH 33/75] Modify test_multicollection to something less arbitrary --- astroquery/eso/tests/test_eso_remote.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/astroquery/eso/tests/test_eso_remote.py b/astroquery/eso/tests/test_eso_remote.py index 5cc3f9cd23..e8c213dfa5 100644 --- a/astroquery/eso/tests/test_eso_remote.py +++ b/astroquery/eso/tests/test_eso_remote.py @@ -86,16 +86,24 @@ def test_multicollection(self, tmp_path): # first b333 was at 157 # first pistol....? - result_s = eso.query_collections(collections=['VVV', 'XSHOOTER'], + test_collections = ['VVV', 'XSHOOTER'] + result_s = eso.query_collections(collections=test_collections, coord1=266.41681662, coord2=-29.00782497, box='01 00 00', cache=False) assert result_s is not None - assert 'Object' in result_s.colnames - assert 'b333_414_58214' in result_s['Object'] - assert 'Pistol-Star' in result_s['Object'] + assert 'target_name' in result_s.colnames + + from collections import Counter + counts = Counter(result_s["obs_collection"].data) + for tc in test_collections: + assert counts[tc] > 0, f"{tc} : collection not present in results" + + # TODO - Confirm that these tests are really necessary. + # assert 'b333' in result_s['target_name'] + # assert 'Pistol-Star' in result_s['target_name'] @pytest.mark.filterwarnings("ignore::pyvo.dal.exceptions.DALOverflowWarning") def test_empty_return(self): From fa89ba2026343390d12d0d3dd4e709fd2376d652 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Mon, 9 Dec 2024 18:06:29 +0100 Subject: [PATCH 34/75] Make PEP8 linter happy --- astroquery/eso/core.py | 3 ++- astroquery/eso/tests/test_eso_remote.py | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index 569d2c2803..e731b49055 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -269,7 +269,8 @@ def _query_instrument_or_collection(self, query_on: QueryOnField, instmnt_or_clc """ # TODO - move help printing to its own function if help: - help_query = f"select column_name, datatype from TAP_SCHEMA.columns where table_name = '{query_on.table_name}'" + help_query = \ + f"select column_name, datatype from TAP_SCHEMA.columns where table_name = '{query_on.table_name}'" h = self._query_tap_service(help_query) log.info(f"Columns present in the table: {h}") return diff --git a/astroquery/eso/tests/test_eso_remote.py b/astroquery/eso/tests/test_eso_remote.py index e8c213dfa5..a20d0ac3f4 100644 --- a/astroquery/eso/tests/test_eso_remote.py +++ b/astroquery/eso/tests/test_eso_remote.py @@ -70,10 +70,14 @@ def test_SgrAstar(self, tmp_path): assert result_s is not None # From obs.raw, we have "object" (when query_instruments) - # object: Target designation as given by the astronomer, though at times overwritten by the obeservatory, especially for CALIB observations. Compare with the similar field called "target".) + # object: Target designation as given by the astronomer, + # though at times overwritten by the obeservatory, + # especially for CALIB observations. Compare with the similar field called "target".) # From ivoa.ObsCore, we have "target_name" (when query_collections) - # target_name: The target name as assigned by the Principal Investigator; ref. Ref. OBJECT keyword in ESO SDP standard. For spectroscopic public surveys, the value shall be set to the survey source identifier... + # target_name: The target name as assigned by the Principal Investigator; + # ref. Ref. OBJECT keyword in ESO SDP standard. + # For spectroscopic public surveys, the value shall be set to the survey source identifier... assert 'target_name' in result_s.colnames assert 'b333' in result_s['target_name'] From 9b629e94425b5e4b3810d6fce305f16c91228726 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Wed, 11 Dec 2024 09:36:04 +0100 Subject: [PATCH 35/75] Replace mutable default arguments by None --- astroquery/eso/core.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index e731b49055..6875c18321 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -232,8 +232,8 @@ def list_collections(self, *, cache=True): self._collection_list = list(res) return self._collection_list - def _query_instrument_or_collection(self, query_on: QueryOnField, instmnt_or_clctn_name, *, column_filters={}, - columns=[], help=False, cache=True, **kwargs): + def _query_instrument_or_collection(self, query_on: QueryOnField, instmnt_or_clctn_name, *, column_filters=None, + columns=None, help=False, cache=True, **kwargs): """ Query instrument- or collection-specific data contained in the ESO archive. - instrument-specific data is raw @@ -267,6 +267,9 @@ def _query_instrument_or_collection(self, query_on: QueryOnField, instmnt_or_clc ``kwargs``. The number of rows returned is capped by the ROW_LIMIT configuration item. """ + column_filters = column_filters or {} + columns = columns or [] + # TODO - move help printing to its own function if help: help_query = \ @@ -296,6 +299,8 @@ def _query_instrument_or_collection(self, query_on: QueryOnField, instmnt_or_clc [f"intersects(circle('ICRS', {c1}, {c2}, {c_size}), s_region)=1"] # http://archive.eso.org/tap_obs/examples + # TODO + # Check whether v is string or number, put in single quotes if string where_constraints_strlist = [f"{k} = {v}" for k, v in filters.items()] + coord_constraint where_constraints = [where_collections_str] + where_constraints_strlist query = py2adql(table=query_on.table_name, columns=columns, where_constraints=where_constraints) @@ -309,8 +314,10 @@ def _query_instrument_or_collection(self, query_on: QueryOnField, instmnt_or_clc return table_to_return @deprecated_renamed_argument(old_name='open_form', new_name=None, since='0.4.8') - def query_instrument(self, instrument, *, column_filters={}, columns=[], + def query_instrument(self, instrument, *, column_filters=None, columns=None, open_form=False, help=False, cache=True, **kwargs): + column_filters = column_filters or {} + columns = columns or [] _ = self._query_instrument_or_collection(query_on=QueryOnInstrument(), instmnt_or_clctn_name=instrument, column_filters=column_filters, @@ -321,8 +328,10 @@ def query_instrument(self, instrument, *, column_filters={}, columns=[], return _ @deprecated_renamed_argument(old_name='open_form', new_name=None, since='0.4.8') - def query_collections(self, collections, *, column_filters={}, columns=[], + def query_collections(self, collections, *, column_filters=None, columns=None, open_form=None, help=False, cache=True, **kwargs): + column_filters = column_filters or {} + columns = columns or [] _ = self._query_instrument_or_collection(query_on=QueryOnCollection(), instmnt_or_clctn_name=collections, column_filters=column_filters, From 95bdd20c4ccb4262d8ee5347f83e24b0ba7cf50c Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Wed, 11 Dec 2024 09:56:53 +0100 Subject: [PATCH 36/75] Add type hints to query_instruments and query_collections --- astroquery/eso/core.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index 6875c18321..f385f91756 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -12,7 +12,7 @@ import webbrowser import xml.etree.ElementTree as ET from io import BytesIO -from typing import List, Optional, Tuple, Dict, Set +from typing import List, Optional, Tuple, Dict, Set, Union import astropy.table import astropy.utils.data @@ -232,8 +232,10 @@ def list_collections(self, *, cache=True): self._collection_list = list(res) return self._collection_list - def _query_instrument_or_collection(self, query_on: QueryOnField, instmnt_or_clctn_name, *, column_filters=None, - columns=None, help=False, cache=True, **kwargs): + def _query_instrument_or_collection(self, + query_on: QueryOnField, primary_filter: Union[List[str], str], *, + column_filters: Dict = None, + columns: Union[List, str] = None, help=False, cache=True, **kwargs): """ Query instrument- or collection-specific data contained in the ESO archive. - instrument-specific data is raw @@ -284,12 +286,12 @@ def _query_instrument_or_collection(self, query_on: QueryOnField, instmnt_or_clc # TODO make c_size a parameter c_size = 0.1775 # so that even HARPS fits to pass the tests del filters['box'] - if isinstance(instmnt_or_clctn_name, str): - instmnt_or_clctn_name = _split_str_as_list_of_str(instmnt_or_clctn_name) + if isinstance(primary_filter, str): + primary_filter = _split_str_as_list_of_str(primary_filter) table_to_return = None # Return an astropy.table.Table or None - instmnt_or_clctn_name = list(map(lambda x: f"'{x.strip()}'", instmnt_or_clctn_name)) - where_collections_str = f"{query_on.column_name} in (" + ", ".join(instmnt_or_clctn_name) + ")" + primary_filter = list(map(lambda x: f"'{x.strip()}'", primary_filter)) + where_collections_str = f"{query_on.column_name} in (" + ", ".join(primary_filter) + ")" coord_constraint = [] if ('coord1' in filters.keys()) and ('coord2' in filters.keys()): @@ -314,12 +316,11 @@ def _query_instrument_or_collection(self, query_on: QueryOnField, instmnt_or_clc return table_to_return @deprecated_renamed_argument(old_name='open_form', new_name=None, since='0.4.8') - def query_instrument(self, instrument, *, column_filters=None, columns=None, + def query_instrument(self, instrument: Union[List, str] = None, *, + column_filters: Dict = None, columns: Union[List, str] = None, open_form=False, help=False, cache=True, **kwargs): - column_filters = column_filters or {} - columns = columns or [] _ = self._query_instrument_or_collection(query_on=QueryOnInstrument(), - instmnt_or_clctn_name=instrument, + primary_filter=instrument, column_filters=column_filters, columns=columns, help=help, @@ -328,12 +329,13 @@ def query_instrument(self, instrument, *, column_filters=None, columns=None, return _ @deprecated_renamed_argument(old_name='open_form', new_name=None, since='0.4.8') - def query_collections(self, collections, *, column_filters=None, columns=None, - open_form=None, help=False, cache=True, **kwargs): + def query_collections(self, collections: Union[List, str] = None, *, + column_filters: Dict = None, columns: Union[List, str] = None, + open_form=False, help=False, cache=True, **kwargs): column_filters = column_filters or {} columns = columns or [] _ = self._query_instrument_or_collection(query_on=QueryOnCollection(), - instmnt_or_clctn_name=collections, + primary_filter=collections, column_filters=column_filters, columns=columns, help=help, From 6a7de6c8b1c73359bfcb6bca28347edda8d824c7 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Wed, 11 Dec 2024 10:22:35 +0100 Subject: [PATCH 37/75] Rename *_list attributes; add return types --- astroquery/eso/core.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index f385f91756..833b2c8ecf 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -83,7 +83,7 @@ class EsoClass(QueryWithLogin): USE_DEV_TAP = True @staticmethod - def tap_url(): + def tap_url() -> str: url = "http://archive.eso.org/tap_obs" if EsoClass.USE_DEV_TAP: url = "http://dfidev5.hq.eso.org:8123/tap_obs" @@ -91,8 +91,8 @@ def tap_url(): def __init__(self): super().__init__() - self._instrument_list = None - self._collection_list = None + self._instruments = None + self._collections = None self._auth_info: Optional[AuthInfo] = None def _authenticate(self, *, username: str, password: str) -> bool: @@ -175,7 +175,7 @@ def _get_auth_header(self) -> Dict[str, str]: else: return {} - def _query_tap_service(self, query_str: str): + def _query_tap_service(self, query_str: str) -> Optional[astropy.table.Table]: """ returns an astropy.table.Table from an adql query string Example use: @@ -204,12 +204,12 @@ def list_instruments(self, *, cache=True): See :ref:`caching documentation `. """ - if self._instrument_list is None: - self._instrument_list = [] + if self._instruments is None: + self._instruments = [] query_str = "select table_name from TAP_SCHEMA.tables where schema_name='ist' order by table_name" res = self._query_tap_service(query_str)["table_name"].data - self._instrument_list = list(map(lambda x: x.split(".")[1], res)) - return self._instrument_list + self._instruments = list(map(lambda x: x.split(".")[1], res)) + return self._instruments def list_collections(self, *, cache=True): """ List all the available collections (phase 3) in the ESO archive. @@ -222,15 +222,15 @@ def list_collections(self, *, cache=True): See :ref:`caching documentation `. """ # TODO include ALMA - if self._collection_list is None: - self._collection_list = [] + if self._collections is None: + self._collections = [] c = QueryOnCollection.column_name t = QueryOnCollection.table_name query_str = f"select distinct {c} from {t} where {c} != 'ALMA'" res = self._query_tap_service(query_str)[c].data - self._collection_list = list(res) - return self._collection_list + self._collections = list(res) + return self._collections def _query_instrument_or_collection(self, query_on: QueryOnField, primary_filter: Union[List[str], str], *, From daf45fbf67c80bc13da62f3c3e7dfb005a924ece Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Wed, 11 Dec 2024 10:37:50 +0100 Subject: [PATCH 38/75] Add return types --- astroquery/eso/core.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index 833b2c8ecf..72c795cd92 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -193,7 +193,7 @@ def _query_tap_service(self, query_str: str) -> Optional[astropy.table.Table]: return table_to_return - def list_instruments(self, *, cache=True): + def list_instruments(self, *, cache=True) -> List[str]: """ List all the available instrument-specific queries offered by the ESO archive. Returns @@ -211,7 +211,7 @@ def list_instruments(self, *, cache=True): self._instruments = list(map(lambda x: x.split(".")[1], res)) return self._instruments - def list_collections(self, *, cache=True): + def list_collections(self, *, cache=True) -> List[str]: """ List all the available collections (phase 3) in the ESO archive. Returns @@ -235,7 +235,8 @@ def list_collections(self, *, cache=True): def _query_instrument_or_collection(self, query_on: QueryOnField, primary_filter: Union[List[str], str], *, column_filters: Dict = None, - columns: Union[List, str] = None, help=False, cache=True, **kwargs): + columns: Union[List, str] = None, help=False, cache=True, + **kwargs) -> astropy.table.Table: """ Query instrument- or collection-specific data contained in the ESO archive. - instrument-specific data is raw @@ -318,7 +319,8 @@ def _query_instrument_or_collection(self, @deprecated_renamed_argument(old_name='open_form', new_name=None, since='0.4.8') def query_instrument(self, instrument: Union[List, str] = None, *, column_filters: Dict = None, columns: Union[List, str] = None, - open_form=False, help=False, cache=True, **kwargs): + open_form=False, help=False, cache=True, + **kwargs) -> astropy.table.Table: _ = self._query_instrument_or_collection(query_on=QueryOnInstrument(), primary_filter=instrument, column_filters=column_filters, @@ -331,7 +333,8 @@ def query_instrument(self, instrument: Union[List, str] = None, *, @deprecated_renamed_argument(old_name='open_form', new_name=None, since='0.4.8') def query_collections(self, collections: Union[List, str] = None, *, column_filters: Dict = None, columns: Union[List, str] = None, - open_form=False, help=False, cache=True, **kwargs): + open_form=False, help=False, cache=True, + **kwargs) -> astropy.table.Table: column_filters = column_filters or {} columns = columns or [] _ = self._query_instrument_or_collection(query_on=QueryOnCollection(), From 93c9e64c6bf42a34bbedaad0b30b93baa97f8641 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Wed, 11 Dec 2024 10:56:56 +0100 Subject: [PATCH 39/75] Refactor QueryOnField classes as dataclasses --- astroquery/eso/core.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index 72c795cd92..00a97742a1 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -13,6 +13,7 @@ import xml.etree.ElementTree as ET from io import BytesIO from typing import List, Optional, Tuple, Dict, Set, Union +from dataclasses import dataclass import astropy.table import astropy.utils.data @@ -57,19 +58,20 @@ def expired(self) -> bool: return time.time() > self.expiration_time - 600 -class QueryOnField(): - table_name = "" - column_name = "" +@dataclass +class QueryOnField: + table_name: str + column_name: str -class QueryOnInstrument(): - table_name = "dbo.raw" - column_name = "instrument" +QueryOnInstrument = QueryOnField( + table_name="dbo.raw", + column_name="instrument") -class QueryOnCollection(): - table_name = "ivoa.ObsCore" - column_name = "obs_collection" +QueryOnCollection = QueryOnField( + table_name="ivoa.ObsCore", + column_name="obs_collection") class EsoClass(QueryWithLogin): From 823a18a58bb7366ed2431593486ac1d46ec82adf Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Wed, 11 Dec 2024 14:20:20 +0100 Subject: [PATCH 40/75] correctly pass the dataclass to query_* function --- astroquery/eso/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index 00a97742a1..d2064814cd 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -323,7 +323,7 @@ def query_instrument(self, instrument: Union[List, str] = None, *, column_filters: Dict = None, columns: Union[List, str] = None, open_form=False, help=False, cache=True, **kwargs) -> astropy.table.Table: - _ = self._query_instrument_or_collection(query_on=QueryOnInstrument(), + _ = self._query_instrument_or_collection(query_on=QueryOnInstrument, primary_filter=instrument, column_filters=column_filters, columns=columns, @@ -339,7 +339,7 @@ def query_collections(self, collections: Union[List, str] = None, *, **kwargs) -> astropy.table.Table: column_filters = column_filters or {} columns = columns or [] - _ = self._query_instrument_or_collection(query_on=QueryOnCollection(), + _ = self._query_instrument_or_collection(query_on=QueryOnCollection, primary_filter=collections, column_filters=column_filters, columns=columns, From 602743db9676dd4727ddb642af48022c1af4cfbb Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Wed, 11 Dec 2024 14:23:04 +0100 Subject: [PATCH 41/75] Annotate types of EsoClass.collections and instruments --- astroquery/eso/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index d2064814cd..f2f905acfd 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -93,8 +93,8 @@ def tap_url() -> str: def __init__(self): super().__init__() - self._instruments = None - self._collections = None + self._instruments: Optional[List[str]] = None + self._collections: Optional[List[str]] = None self._auth_info: Optional[AuthInfo] = None def _authenticate(self, *, username: str, password: str) -> bool: From a06e6bbacb43fa3c437737ac4db023c7f3b70abd Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Wed, 11 Dec 2024 15:02:31 +0100 Subject: [PATCH 42/75] Add type hints to py2adql --- astroquery/eso/utils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/astroquery/eso/utils.py b/astroquery/eso/utils.py index f07cc609a1..86c6701057 100644 --- a/astroquery/eso/utils.py +++ b/astroquery/eso/utils.py @@ -1,5 +1,5 @@ #!/usr/bin/env/python -from typing import Union +from typing import Union, List def _split_str_as_list_of_str(column_str: str): @@ -10,20 +10,20 @@ def _split_str_as_list_of_str(column_str: str): return column_list -def py2adql(table: str, columns: Union[list, str] = [], where_constraints=[], +def py2adql(table: str, columns: Union[List, str] = None, where_constraints: List = None, order_by: str = '', order_by_desc=True): # Initialize / validate query_string = None - where_constraints = where_constraints[:] # do not modify the original list + wc = [] if where_constraints is None else where_constraints[:] # do not modify the original list if isinstance(columns, str): columns = _split_str_as_list_of_str(columns) - if columns == []: + if columns is None or len(columns) < 1: columns = ['*'] # Build the query query_string = 'select ' + ', '.join(columns) + ' from ' + table - if len(where_constraints) > 0: - where_string = ' where ' + ' and '.join(where_constraints) + if len(wc) > 0: + where_string = ' where ' + ' and '.join(wc) query_string += where_string if len(order_by) > 0: From 6f4ad2fb06eb472b927b246976c3dda0e26183ae Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Wed, 11 Dec 2024 16:26:14 +0100 Subject: [PATCH 43/75] Write utils function sanitize_val; Add top arg in py2adql --- astroquery/eso/utils.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/astroquery/eso/utils.py b/astroquery/eso/utils.py index 86c6701057..9b0b8d8bc1 100644 --- a/astroquery/eso/utils.py +++ b/astroquery/eso/utils.py @@ -10,8 +10,15 @@ def _split_str_as_list_of_str(column_str: str): return column_list +def sanitize_val(x): + if isinstance(x, str): + return f"'{x}'" + else: + return f"{x}" + + def py2adql(table: str, columns: Union[List, str] = None, where_constraints: List = None, - order_by: str = '', order_by_desc=True): + order_by: str = '', order_by_desc=True, top: int = None): # Initialize / validate query_string = None wc = [] if where_constraints is None else where_constraints[:] # do not modify the original list @@ -21,7 +28,7 @@ def py2adql(table: str, columns: Union[List, str] = None, where_constraints: Lis columns = ['*'] # Build the query - query_string = 'select ' + ', '.join(columns) + ' from ' + table + query_string = ', '.join(columns) + ' from ' + table if len(wc) > 0: where_string = ' where ' + ' and '.join(wc) query_string += where_string @@ -30,4 +37,9 @@ def py2adql(table: str, columns: Union[List, str] = None, where_constraints: Lis order_string = ' order by ' + order_by + (' desc ' if order_by_desc else ' asc ') query_string += order_string + if top is not None: + query_string = "select top " + str(top) + query_string + else: + query_string = "select " + query_string + return query_string.strip() From 455f38ae84fc1083fd5d8f41c5e9a6c8cbe29057 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Wed, 11 Dec 2024 16:28:15 +0100 Subject: [PATCH 44/75] Write TAP version of query_main; sanitize constraints vals --- astroquery/eso/core.py | 58 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index f2f905acfd..623a12d070 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -28,7 +28,7 @@ from ..exceptions import RemoteServiceError, NoResultsWarning, LoginError from ..query import QueryWithLogin from ..utils import schema -from .utils import py2adql, _split_str_as_list_of_str +from .utils import py2adql, _split_str_as_list_of_str, sanitize_val import pyvo __doctest_skip__ = ['EsoClass.*'] @@ -306,9 +306,10 @@ def _query_instrument_or_collection(self, # TODO # Check whether v is string or number, put in single quotes if string - where_constraints_strlist = [f"{k} = {v}" for k, v in filters.items()] + coord_constraint + where_constraints_strlist = [f"{k} = {sanitize_val(v)}" for k, v in filters.items()] + coord_constraint where_constraints = [where_collections_str] + where_constraints_strlist - query = py2adql(table=query_on.table_name, columns=columns, where_constraints=where_constraints) + query = py2adql(table=query_on.table_name, columns=columns, where_constraints=where_constraints, + top=self.ROW_LIMIT) table_to_return = self._query_tap_service(query_str=query) @@ -348,6 +349,57 @@ def query_collections(self, collections: Union[List, str] = None, *, **kwargs) return _ + def query_main(self, *, column_filters={}, columns=[], + open_form=False, help=False, cache=True, **kwargs): + """ + Query raw data contained in the ESO archive. + + Parameters + ---------- + column_filters : dict + Constraints applied to the query. + columns : list of strings + Columns returned by the query. + open_form : bool + If `True`, opens in your default browser the query form + for the requested instrument. + help : bool + If `True`, prints all the parameters accepted in + ``column_filters`` and ``columns`` for the requested + ``instrument``. + cache : bool + Defaults to True. If set overrides global caching behavior. + See :ref:`caching documentation `. + + Returns + ------- + table : `~astropy.table.Table` + A table representing the data available in the archive for the + specified instrument, matching the constraints specified in + ``kwargs``. The number of rows returned is capped by the + ROW_LIMIT configuration item. + + """ + column_filters = column_filters or {} + columns = columns or [] + filters = {**dict(kwargs), **column_filters} + + where_constraints_strlist = [f"{k} = {sanitize_val(v)}" for k, v in filters.items()] + where_constraints = where_constraints_strlist + + query = py2adql(table=QueryOnInstrument.table_name, + columns=columns, + where_constraints=where_constraints, + top=self.ROW_LIMIT) + + table_to_return = self._query_tap_service(query_str=query) + + if len(table_to_return) < 1: + warnings.warn("Query returned no results", NoResultsWarning) + table_to_return = None + + return table_to_return + def get_headers(self, product_ids, *, cache=True): """ Get the headers associated to a list of data product IDs From 919b41109300a9d77b2a6a11fb5c5c12330e8335 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Thu, 12 Dec 2024 10:29:26 +0100 Subject: [PATCH 45/75] correct typo in query string --- astroquery/eso/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astroquery/eso/utils.py b/astroquery/eso/utils.py index 9b0b8d8bc1..8ce492e582 100644 --- a/astroquery/eso/utils.py +++ b/astroquery/eso/utils.py @@ -38,7 +38,7 @@ def py2adql(table: str, columns: Union[List, str] = None, where_constraints: Lis query_string += order_string if top is not None: - query_string = "select top " + str(top) + query_string + query_string = f"select top {top} " + query_string else: query_string = "select " + query_string From 0f1768e3690de1ba4d12c59c8d269dbc65d4cd65 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Thu, 12 Dec 2024 12:03:52 +0100 Subject: [PATCH 46/75] Add test_main_SgrAstar --- astroquery/eso/tests/test_eso_remote.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/astroquery/eso/tests/test_eso_remote.py b/astroquery/eso/tests/test_eso_remote.py index a20d0ac3f4..a38f0f0062 100644 --- a/astroquery/eso/tests/test_eso_remote.py +++ b/astroquery/eso/tests/test_eso_remote.py @@ -244,3 +244,16 @@ def test_mixed_case_instrument(self, tmp_path): coord2=-29.00782497, cache=False) assert np.all(result1 == result2) + + @pytest.mark.filterwarnings("ignore::pyvo.dal.exceptions.DALOverflowWarning") + def test_main_SgrAstar(self): + eso = Eso() + eso.ROW_LIMIT = 5 + + # the failure should occur here + result = eso.query_main(target='SGR A', object='SGR A') + + # test that max_results = 5 + assert len(result) == 5 + assert 'SGR A' in result['object'] + assert 'SGR A' in result['target'] From b3bdfbc97fc2c98a041f6b404b3c3c63e7f6ffa5 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Thu, 12 Dec 2024 12:15:37 +0100 Subject: [PATCH 47/75] Row limit handled at query level. No filterwarnings needed --- astroquery/eso/tests/test_eso_remote.py | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/astroquery/eso/tests/test_eso_remote.py b/astroquery/eso/tests/test_eso_remote.py index a38f0f0062..5d2e44d4e6 100644 --- a/astroquery/eso/tests/test_eso_remote.py +++ b/astroquery/eso/tests/test_eso_remote.py @@ -36,14 +36,15 @@ @pytest.mark.remote_data class TestEso: - @pytest.mark.filterwarnings("ignore::pyvo.dal.exceptions.DALOverflowWarning") def test_query_tap_service(self): eso = Eso() - t = eso._query_tap_service("select * from ivoa.ObsCore") + eso.ROW_LIMIT = 7 + t = eso._query_tap_service(f"select top {eso.ROW_LIMIT} * from ivoa.ObsCore") + lt = len(t) assert type(t) is Table, f"Expected type {type(Table)}; Obtained {type(t)}" assert len(t) > 0, "Table length is zero" + assert len(t) == eso.ROW_LIMIT, f"Table length is {lt}, expected {eso.ROW_LIMIT}" - @pytest.mark.filterwarnings("ignore::pyvo.dal.exceptions.DALOverflowWarning") def test_SgrAstar(self, tmp_path): eso = Eso() eso.cache_location = tmp_path @@ -81,7 +82,6 @@ def test_SgrAstar(self, tmp_path): assert 'target_name' in result_s.colnames assert 'b333' in result_s['target_name'] - @pytest.mark.filterwarnings("ignore::pyvo.dal.exceptions.DALOverflowWarning") def test_multicollection(self, tmp_path): eso = Eso() @@ -109,7 +109,6 @@ def test_multicollection(self, tmp_path): # assert 'b333' in result_s['target_name'] # assert 'Pistol-Star' in result_s['target_name'] - @pytest.mark.filterwarnings("ignore::pyvo.dal.exceptions.DALOverflowWarning") def test_empty_return(self): # test for empty return with an object from the North eso = Eso() @@ -123,7 +122,6 @@ def test_empty_return(self): assert result_s is None - @pytest.mark.filterwarnings("ignore::pyvo.dal.exceptions.DALOverflowWarning") def test_SgrAstar_remotevslocal(self, tmp_path): eso = Eso() # Remote version @@ -135,7 +133,6 @@ def test_SgrAstar_remotevslocal(self, tmp_path): coord2=-29.00782497, cache=True) assert all(result1 == result2) - @pytest.mark.filterwarnings("ignore::pyvo.dal.exceptions.DALOverflowWarning") def test_list_instruments(self): # If this test fails, we may simply need to update it @@ -144,7 +141,6 @@ def test_list_instruments(self): # we only care about the sets matching assert set(inst) == set(instrument_list) - @pytest.mark.filterwarnings("ignore::pyvo.dal.exceptions.DALOverflowWarning") def test_retrieve_data(self): eso = Eso() file_id = 'AMBER.2006-03-14T07:40:19.830' @@ -161,7 +157,6 @@ def test_retrieve_data_authenticated(self): assert isinstance(result, str) assert file_id in result - @pytest.mark.filterwarnings("ignore::pyvo.dal.exceptions.DALOverflowWarning") def test_retrieve_data_list(self): eso = Eso() datasets = ['MIDI.2014-07-25T02:03:11.561', 'AMBER.2006-03-14T07:40:19.830'] @@ -172,12 +167,10 @@ def test_retrieve_data_list(self): # TODO: remove filter when https://github.com/astropy/astroquery/issues/2539 is fixed @pytest.mark.filterwarnings("ignore::pytest.PytestUnraisableExceptionWarning") @pytest.mark.parametrize('instrument', instrument_list) - @pytest.mark.filterwarnings("ignore::pyvo.dal.exceptions.DALOverflowWarning") def test_help(self, instrument): eso = Eso() eso.query_instrument(instrument, help=True) - @pytest.mark.filterwarnings("ignore::pyvo.dal.exceptions.DALOverflowWarning") def test_apex_retrieval(self): eso = Eso() @@ -190,7 +183,6 @@ def test_apex_retrieval(self): assert np.all(tbl == tblb) - @pytest.mark.filterwarnings("ignore::pyvo.dal.exceptions.DALOverflowWarning") def test_each_instrument_SgrAstar(self, tmp_path): eso = Eso() eso.cache_location = tmp_path @@ -206,8 +198,6 @@ def test_each_instrument_SgrAstar(self, tmp_path): else: assert len(result) > 0 - # TODO Ignore OverflowWarning -- how? - @pytest.mark.filterwarnings("ignore::pyvo.dal.exceptions.DALOverflowWarning") def test_each_collection_and_SgrAstar(self, tmp_path): eso = Eso() eso.cache_location = tmp_path @@ -232,7 +222,6 @@ def test_each_collection_and_SgrAstar(self, tmp_path): generic_result = eso.query_collections(collections=collection) assert len(generic_result) > 0 - @pytest.mark.filterwarnings("ignore::pyvo.dal.exceptions.DALOverflowWarning") def test_mixed_case_instrument(self, tmp_path): eso = Eso() eso.cache_location = tmp_path @@ -245,7 +234,6 @@ def test_mixed_case_instrument(self, tmp_path): assert np.all(result1 == result2) - @pytest.mark.filterwarnings("ignore::pyvo.dal.exceptions.DALOverflowWarning") def test_main_SgrAstar(self): eso = Eso() eso.ROW_LIMIT = 5 From 6033a3542df708bada344a5f84ab56d94321922b Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Thu, 12 Dec 2024 17:33:41 +0100 Subject: [PATCH 48/75] query_main is always against dbo.raw --- astroquery/eso/core.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index 623a12d070..a136a199b6 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -65,7 +65,7 @@ class QueryOnField: QueryOnInstrument = QueryOnField( - table_name="dbo.raw", + table_name="dbo.raw", # This should be ist. column_name="instrument") @@ -185,7 +185,7 @@ def _query_tap_service(self, query_str: str) -> Optional[astropy.table.Table]: """ tap = pyvo.dal.TAPService(EsoClass.tap_url()) table_to_return = None - + # TODO add url to documentation in the exception try: table_to_return = tap.search(query_str).to_table() except pyvo.dal.exceptions.DALQueryError: @@ -287,7 +287,7 @@ def _query_instrument_or_collection(self, c_size = 0.1775 # so that even HARPS fits to pass the tests if 'box' in filters.keys(): # TODO make c_size a parameter - c_size = 0.1775 # so that even HARPS fits to pass the tests + c_size = c_size del filters['box'] if isinstance(primary_filter, str): primary_filter = _split_str_as_list_of_str(primary_filter) @@ -387,7 +387,7 @@ def query_main(self, *, column_filters={}, columns=[], where_constraints_strlist = [f"{k} = {sanitize_val(v)}" for k, v in filters.items()] where_constraints = where_constraints_strlist - query = py2adql(table=QueryOnInstrument.table_name, + query = py2adql(table="dbo.raw", columns=columns, where_constraints=where_constraints, top=self.ROW_LIMIT) From a75deb053d191147bdd532343762f33b9c624cc1 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Thu, 12 Dec 2024 17:38:52 +0100 Subject: [PATCH 49/75] Add assert messages --- astroquery/eso/tests/test_eso_remote.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/astroquery/eso/tests/test_eso_remote.py b/astroquery/eso/tests/test_eso_remote.py index 5d2e44d4e6..8f4e742bcb 100644 --- a/astroquery/eso/tests/test_eso_remote.py +++ b/astroquery/eso/tests/test_eso_remote.py @@ -196,7 +196,8 @@ def test_each_instrument_SgrAstar(self, tmp_path): # Sometimes there are ResourceWarnings, we ignore those for this test pass else: - assert len(result) > 0 + assert result is not None, f"query_instrument({instrument}) returned None" + assert len(result) > 0, f"query_instrument({instrument}) returned no records" def test_each_collection_and_SgrAstar(self, tmp_path): eso = Eso() @@ -220,7 +221,8 @@ def test_each_collection_and_SgrAstar(self, tmp_path): assert result_s is None generic_result = eso.query_collections(collections=collection) - assert len(generic_result) > 0 + assert generic_result is not None, f"query_collection({collection}) returned None" + assert len(generic_result) > 0, f"query_collection({collection}) returned no records" def test_mixed_case_instrument(self, tmp_path): eso = Eso() From a7b2b8fe8723d93fa6329b222820324c06457216 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Thu, 12 Dec 2024 17:40:45 +0100 Subject: [PATCH 50/75] Make linter happy --- astroquery/eso/tests/test_eso.py | 1 - 1 file changed, 1 deletion(-) diff --git a/astroquery/eso/tests/test_eso.py b/astroquery/eso/tests/test_eso.py index a6323bb751..87e8ed50b2 100644 --- a/astroquery/eso/tests/test_eso.py +++ b/astroquery/eso/tests/test_eso.py @@ -95,7 +95,6 @@ def test_amber_SgrAstar(monkeypatch): def test_main_SgrAstar(monkeypatch): # Local caching prevents a remote query here - eso = Eso() # monkeypatch instructions from https://pytest.org/latest/monkeypatch.html From e91483e0dcb0489209d0dfc228e5e30ae6e2a1fa Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Fri, 13 Dec 2024 10:51:35 +0100 Subject: [PATCH 51/75] Add cache functionality to tap queries --- astroquery/eso/core.py | 81 +++++++++++++++++++++++++++++++++++------ astroquery/eso/utils.py | 19 ++++++++++ 2 files changed, 89 insertions(+), 11 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index a136a199b6..4fcb655dc8 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -4,6 +4,7 @@ import email import json import os.path +import pickle import re import shutil import subprocess @@ -14,21 +15,23 @@ from io import BytesIO from typing import List, Optional, Tuple, Dict, Set, Union from dataclasses import dataclass +from datetime import datetime, timezone, timedelta import astropy.table import astropy.utils.data +import astropy.units as u import keyring import requests.exceptions from astropy.table import Table, Column from astropy.utils.decorators import deprecated, deprecated_renamed_argument from bs4 import BeautifulSoup -from astroquery import log +from astroquery import log, cache_conf from . import conf from ..exceptions import RemoteServiceError, NoResultsWarning, LoginError from ..query import QueryWithLogin from ..utils import schema -from .utils import py2adql, _split_str_as_list_of_str, sanitize_val +from .utils import py2adql, _split_str_as_list_of_str, sanitize_val, to_cache, hash import pyvo __doctest_skip__ = ['EsoClass.*'] @@ -84,6 +87,25 @@ class EsoClass(QueryWithLogin): GUNZIP = "gunzip" USE_DEV_TAP = True + def __init__(self, timeout=None): + super().__init__() + self._instruments: Optional[List[str]] = None + self._collections: Optional[List[str]] = None + self._auth_info: Optional[AuthInfo] = None + self.timeout = timeout + self._hash = None + + @property + def timeout(self): + return self._timeout + + @timeout.setter + def timeout(self, value): + if hasattr(value, 'to'): + self._timeout = value.to(u.s).value + else: + self._timeout = value + @staticmethod def tap_url() -> str: url = "http://archive.eso.org/tap_obs" @@ -91,11 +113,35 @@ def tap_url() -> str: url = "http://dfidev5.hq.eso.org:8123/tap_obs" return url - def __init__(self): - super().__init__() - self._instruments: Optional[List[str]] = None - self._collections: Optional[List[str]] = None - self._auth_info: Optional[AuthInfo] = None + def request_file(self, query_str: str): + h = hash(query_str=query_str, url=self.tap_url()) + fn = self.cache_location.joinpath(h + ".pickle") + return fn + + def from_cache(self, query_str, cache_timeout): + request_file = self.request_file(query_str) + print(f"from cache: {request_file}") + try: + if cache_timeout is None: + expired = False + else: + current_time = datetime.now(timezone.utc) + cache_time = datetime.fromtimestamp(request_file.stat().st_mtime, timezone.utc) + expired = current_time-cache_time > timedelta(seconds=cache_timeout) + if not expired: + with open(request_file, "rb") as f: + # TODO rename respones to table + response = pickle.load(f) + if not isinstance(response, Table): + response = None + else: + log.debug(f"Cache expired for {request_file}...") + response = None + except FileNotFoundError: + response = None + if response: + log.debug("Retrieved data from {0}".format(request_file)) + return response def _authenticate(self, *, username: str, password: str) -> bool: """ @@ -177,17 +223,30 @@ def _get_auth_header(self) -> Dict[str, str]: else: return {} - def _query_tap_service(self, query_str: str) -> Optional[astropy.table.Table]: + def _query_tap_service(self, query_str: str, cache: Optional[bool] = None) -> Optional[astropy.table.Table]: """ returns an astropy.table.Table from an adql query string Example use: eso._query_tap_service("Select * from ivoa.ObsCore") """ + if cache is None: # Global caching not overridden + cache = cache_conf.cache_active + tap = pyvo.dal.TAPService(EsoClass.tap_url()) table_to_return = None + # TODO add url to documentation in the exception try: - table_to_return = tap.search(query_str).to_table() + + if not cache: + with cache_conf.set_temp("cache_active", False): + table_to_return = tap.search(query_str).to_table() + else: + table_to_return = self.from_cache(query_str, cache_conf.cache_timeout) + if not table_to_return: + table_to_return = tap.search(query_str).to_table() + to_cache(table_to_return, self.request_file(query_str=query_str)) + except pyvo.dal.exceptions.DALQueryError: raise pyvo.dal.exceptions.DALQueryError(f"\n\nError executing the following query:\n\n{query_str}\n\n") except Exception as e: @@ -311,9 +370,9 @@ def _query_instrument_or_collection(self, query = py2adql(table=query_on.table_name, columns=columns, where_constraints=where_constraints, top=self.ROW_LIMIT) - table_to_return = self._query_tap_service(query_str=query) + table_to_return = self._query_tap_service(query_str=query, cache=cache) - if len(table_to_return) < 1: + if table_to_return is None or len(table_to_return) < 1: warnings.warn("Query returned no results", NoResultsWarning) table_to_return = None diff --git a/astroquery/eso/utils.py b/astroquery/eso/utils.py index 8ce492e582..0dc6c6c02e 100644 --- a/astroquery/eso/utils.py +++ b/astroquery/eso/utils.py @@ -1,5 +1,9 @@ #!/usr/bin/env/python from typing import Union, List +import pickle +import copy +from astroquery import log +import hashlib def _split_str_as_list_of_str(column_str: str): @@ -43,3 +47,18 @@ def py2adql(table: str, columns: Union[List, str] = None, where_constraints: Lis query_string = "select " + query_string return query_string.strip() + + +def hash(query_str: str, url: str): + request_key = (query_str, url) + h = hashlib.sha224(pickle.dumps(request_key)).hexdigest() + return h + + +def to_cache(table, cache_file): + print(f"to_cache cache: {cache_file}") + log.debug("Caching data to {0}".format(cache_file)) + + table = copy.deepcopy(table) + with open(cache_file, "wb") as f: + pickle.dump(table, f, protocol=4) From 538f3c3e22b5fb1892fc665d10f1f59137085170 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Fri, 13 Dec 2024 12:49:05 +0100 Subject: [PATCH 52/75] Try mimicking the old warning behaviour --- astroquery/eso/core.py | 39 ++++++++++++++------------------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index 4fcb655dc8..df29e2742f 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -252,7 +252,10 @@ def _query_tap_service(self, query_str: str, cache: Optional[bool] = None) -> Op except Exception as e: raise Exception(f"\n\nUnknown exception {e} while executing the following query: \n\n{query_str}\n\n") - return table_to_return + if len(table_to_return) > 0: + return table_to_return + else: + warnings.warn("Query returned no results", NoResultsWarning) def list_instruments(self, *, cache=True) -> List[str]: """ List all the available instrument-specific queries offered by the ESO archive. @@ -350,7 +353,6 @@ def _query_instrument_or_collection(self, del filters['box'] if isinstance(primary_filter, str): primary_filter = _split_str_as_list_of_str(primary_filter) - table_to_return = None # Return an astropy.table.Table or None primary_filter = list(map(lambda x: f"'{x.strip()}'", primary_filter)) where_collections_str = f"{query_on.column_name} in (" + ", ".join(primary_filter) + ")" @@ -370,27 +372,20 @@ def _query_instrument_or_collection(self, query = py2adql(table=query_on.table_name, columns=columns, where_constraints=where_constraints, top=self.ROW_LIMIT) - table_to_return = self._query_tap_service(query_str=query, cache=cache) - - if table_to_return is None or len(table_to_return) < 1: - warnings.warn("Query returned no results", NoResultsWarning) - table_to_return = None - - return table_to_return + return self._query_tap_service(query_str=query, cache=cache) @deprecated_renamed_argument(old_name='open_form', new_name=None, since='0.4.8') def query_instrument(self, instrument: Union[List, str] = None, *, column_filters: Dict = None, columns: Union[List, str] = None, open_form=False, help=False, cache=True, **kwargs) -> astropy.table.Table: - _ = self._query_instrument_or_collection(query_on=QueryOnInstrument, - primary_filter=instrument, - column_filters=column_filters, - columns=columns, - help=help, - cache=cache, - **kwargs) - return _ + return self._query_instrument_or_collection(query_on=QueryOnInstrument, + primary_filter=instrument, + column_filters=column_filters, + columns=columns, + help=help, + cache=cache, + **kwargs) @deprecated_renamed_argument(old_name='open_form', new_name=None, since='0.4.8') def query_collections(self, collections: Union[List, str] = None, *, @@ -399,14 +394,13 @@ def query_collections(self, collections: Union[List, str] = None, *, **kwargs) -> astropy.table.Table: column_filters = column_filters or {} columns = columns or [] - _ = self._query_instrument_or_collection(query_on=QueryOnCollection, + return self._query_instrument_or_collection(query_on=QueryOnCollection, primary_filter=collections, column_filters=column_filters, columns=columns, help=help, cache=cache, **kwargs) - return _ def query_main(self, *, column_filters={}, columns=[], open_form=False, help=False, cache=True, **kwargs): @@ -451,13 +445,8 @@ def query_main(self, *, column_filters={}, columns=[], where_constraints=where_constraints, top=self.ROW_LIMIT) - table_to_return = self._query_tap_service(query_str=query) - - if len(table_to_return) < 1: - warnings.warn("Query returned no results", NoResultsWarning) - table_to_return = None + return self._query_tap_service(query_str=query) - return table_to_return def get_headers(self, product_ids, *, cache=True): """ From a74edfbd316592baf274a7115337e054a14b07c7 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Fri, 13 Dec 2024 13:20:55 +0100 Subject: [PATCH 53/75] Provisional fix to apex query --- astroquery/eso/core.py | 162 +++++++++++++++++++++++++++++++++++++--- astroquery/eso/utils.py | 14 ++++ 2 files changed, 166 insertions(+), 10 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index df29e2742f..ccd5672a58 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -31,7 +31,7 @@ from ..exceptions import RemoteServiceError, NoResultsWarning, LoginError from ..query import QueryWithLogin from ..utils import schema -from .utils import py2adql, _split_str_as_list_of_str, sanitize_val, to_cache, hash +from .utils import py2adql, _split_str_as_list_of_str, sanitize_val, to_cache, hash, _check_response import pyvo __doctest_skip__ = ['EsoClass.*'] @@ -395,12 +395,12 @@ def query_collections(self, collections: Union[List, str] = None, *, column_filters = column_filters or {} columns = columns or [] return self._query_instrument_or_collection(query_on=QueryOnCollection, - primary_filter=collections, - column_filters=column_filters, - columns=columns, - help=help, - cache=cache, - **kwargs) + primary_filter=collections, + column_filters=column_filters, + columns=columns, + help=help, + cache=cache, + **kwargs) def query_main(self, *, column_filters={}, columns=[], open_form=False, help=False, cache=True, **kwargs): @@ -447,7 +447,6 @@ def query_main(self, *, column_filters={}, columns=[], return self._query_tap_service(query_str=query) - def get_headers(self, product_ids, *, cache=True): """ Get the headers associated to a list of data product IDs @@ -746,6 +745,149 @@ def retrieve_data(self, datasets, *, continuation=False, destination=None, with_ log.info("Done!") return files[0] if files and len(files) == 1 and return_string else files + def _activate_form(self, response, *, form_index=0, form_id=None, inputs={}, + cache=True, method=None): + """ + Parameters + ---------- + method: None or str + Can be used to override the form-specified method + """ + # Extract form from response + root = BeautifulSoup(response.content, 'html5lib') + if form_id is None: + form = root.find_all('form')[form_index] + else: + form = root.find_all('form', id=form_id)[form_index] + # Construct base url + form_action = form.get('action') + if "://" in form_action: + url = form_action + elif form_action.startswith('/'): + url = '/'.join(response.url.split('/', 3)[:3]) + form_action + else: + url = response.url.rsplit('/', 1)[0] + '/' + form_action + # Identify payload format + fmt = None + form_method = form.get('method').lower() + if form_method == 'get': + fmt = 'get' # get(url, params=payload) + elif form_method == 'post': + if 'enctype' in form.attrs: + if form.attrs['enctype'] == 'multipart/form-data': + fmt = 'multipart/form-data' # post(url, files=payload) + elif form.attrs['enctype'] == 'application/x-www-form-urlencoded': + fmt = 'application/x-www-form-urlencoded' # post(url, data=payload) + else: + raise Exception("enctype={0} is not supported!".format(form.attrs['enctype'])) + else: + fmt = 'application/x-www-form-urlencoded' # post(url, data=payload) + # Extract payload from form + payload = [] + for form_elem in form.find_all(['input', 'select', 'textarea']): + value = None + is_file = False + tag_name = form_elem.name + key = form_elem.get('name') + if tag_name == 'input': + is_file = (form_elem.get('type') == 'file') + value = form_elem.get('value') + if form_elem.get('type') in ['checkbox', 'radio']: + if form_elem.has_attr('checked'): + if not value: + value = 'on' + else: + value = None + elif tag_name == 'select': + if form_elem.get('multiple') is not None: + value = [] + if form_elem.select('option[value]'): + for option in form_elem.select('option[value]'): + if option.get('selected') is not None: + value.append(option.get('value')) + else: + for option in form_elem.select('option'): + if option.get('selected') is not None: + # bs4 NavigableString types have bad, + # undesirable properties that result + # in recursion errors when caching + value.append(str(option.string)) + else: + if form_elem.select('option[value]'): + for option in form_elem.select('option[value]'): + if option.get('selected') is not None: + value = option.get('value') + # select the first option field if none is selected + if value is None: + value = form_elem.select( + 'option[value]')[0].get('value') + else: + # survey form just uses text, not value + for option in form_elem.select('option'): + if option.get('selected') is not None: + value = str(option.string) + # select the first option field if none is selected + if value is None: + value = str(form_elem.select('option')[0].string) + + if key in inputs: + if isinstance(inputs[key], list): + # list input is accepted (for array uploads) + value = inputs[key] + else: + value = str(inputs[key]) + + if (key is not None): # and (value is not None): + if fmt == 'multipart/form-data': + if is_file: + payload.append( + (key, ('', '', 'application/octet-stream'))) + else: + if type(value) is list: + for v in value: + entry = (key, ('', v)) + # Prevent redundant key, value pairs + # (can happen if the form repeats them) + if entry not in payload: + payload.append(entry) + elif value is None: + entry = (key, ('', '')) + if entry not in payload: + payload.append(entry) + else: + entry = (key, ('', value)) + if entry not in payload: + payload.append(entry) + else: + if type(value) is list: + for v in value: + entry = (key, v) + if entry not in payload: + payload.append(entry) + else: + entry = (key, value) + if entry not in payload: + payload.append(entry) + + # for future debugging + self._payload = payload + log.debug("Form: payload={0}".format(payload)) + + if method is not None: + fmt = method + + log.debug("Method/format = {0}".format(fmt)) + + # Send payload + if fmt == 'get': + response = self._request("GET", url, params=payload, cache=cache) + elif fmt == 'multipart/form-data': + response = self._request("POST", url, files=payload, cache=cache) + elif fmt == 'application/x-www-form-urlencoded': + response = self._request("POST", url, data=payload, cache=cache) + + return response + def query_apex_quicklooks(self, *, project_id=None, help=False, open_form=False, cache=True, **kwargs): """ @@ -779,7 +921,7 @@ def query_apex_quicklooks(self, *, project_id=None, help=False, cache=cache, method='application/x-www-form-urlencoded') content = apex_response.content - if True: + if _check_response(content): # First line is always garbage content = content.split(b'\n', 1)[1] try: @@ -794,7 +936,7 @@ def query_apex_quicklooks(self, *, project_id=None, help=False, comment="#") else: raise ex - else: # this function is going to be replaced soon + else: raise RemoteServiceError("Query returned no results") return table diff --git a/astroquery/eso/utils.py b/astroquery/eso/utils.py index 0dc6c6c02e..b9d018e298 100644 --- a/astroquery/eso/utils.py +++ b/astroquery/eso/utils.py @@ -4,6 +4,20 @@ import copy from astroquery import log import hashlib +from ..exceptions import RemoteServiceError + + +def _check_response(content): # required only by apex_quicklooks + """ + Check the response from an ESO service query for various types of error + + If all is OK, return True + """ + if b"NETWORKPROBLEM" in content: + raise RemoteServiceError("The query resulted in a network " + "problem; the service may be offline.") + elif b"# No data returned !" not in content: + return True def _split_str_as_list_of_str(column_str: str): From 28e397b51bfef8ef4759e0e6a78d3a2bea2341be Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Fri, 13 Dec 2024 16:16:21 +0100 Subject: [PATCH 54/75] Use t1.values_equal(t2) for table equality --- astroquery/eso/core.py | 1 - astroquery/eso/tests/test_eso_remote.py | 2 +- astroquery/eso/utils.py | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index ccd5672a58..cf007e25ec 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -120,7 +120,6 @@ def request_file(self, query_str: str): def from_cache(self, query_str, cache_timeout): request_file = self.request_file(query_str) - print(f"from cache: {request_file}") try: if cache_timeout is None: expired = False diff --git a/astroquery/eso/tests/test_eso_remote.py b/astroquery/eso/tests/test_eso_remote.py index 8f4e742bcb..539ed014cf 100644 --- a/astroquery/eso/tests/test_eso_remote.py +++ b/astroquery/eso/tests/test_eso_remote.py @@ -131,7 +131,7 @@ def test_SgrAstar_remotevslocal(self, tmp_path): eso.cache_location = tmp_path result2 = eso.query_instrument('gravity', coord1=266.41681662, coord2=-29.00782497, cache=True) - assert all(result1 == result2) + assert all(result1.values_equal(result2)) def test_list_instruments(self): # If this test fails, we may simply need to update it diff --git a/astroquery/eso/utils.py b/astroquery/eso/utils.py index b9d018e298..942cd037b5 100644 --- a/astroquery/eso/utils.py +++ b/astroquery/eso/utils.py @@ -70,7 +70,6 @@ def hash(query_str: str, url: str): def to_cache(table, cache_file): - print(f"to_cache cache: {cache_file}") log.debug("Caching data to {0}".format(cache_file)) table = copy.deepcopy(table) From 0a90e349ebf72e7df547048d16e9ed548c673bcd Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Mon, 16 Dec 2024 09:57:36 +0100 Subject: [PATCH 55/75] Test table equality via t1.values_equal(t2) --- astroquery/eso/tests/test_eso_remote.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astroquery/eso/tests/test_eso_remote.py b/astroquery/eso/tests/test_eso_remote.py index 539ed014cf..a12eeef54b 100644 --- a/astroquery/eso/tests/test_eso_remote.py +++ b/astroquery/eso/tests/test_eso_remote.py @@ -234,7 +234,7 @@ def test_mixed_case_instrument(self, tmp_path): result2 = eso.query_instrument('MiDi', coord1=266.41681662, coord2=-29.00782497, cache=False) - assert np.all(result1 == result2) + assert all(result1.values_equal(result2)) def test_main_SgrAstar(self): eso = Eso() From ea39d2cce6b7efaa83b13a20a1cd47784f593470 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Mon, 16 Dec 2024 17:12:18 +0100 Subject: [PATCH 56/75] Do not exclude ALMA from collection queries --- astroquery/eso/core.py | 3 +-- astroquery/eso/tests/test_eso_remote.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index cf007e25ec..88d8024798 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -284,12 +284,11 @@ def list_collections(self, *, cache=True) -> List[str]: Defaults to True. If set overrides global caching behavior. See :ref:`caching documentation `. """ - # TODO include ALMA if self._collections is None: self._collections = [] c = QueryOnCollection.column_name t = QueryOnCollection.table_name - query_str = f"select distinct {c} from {t} where {c} != 'ALMA'" + query_str = f"select distinct {c} from {t}" res = self._query_tap_service(query_str)[c].data self._collections = list(res) diff --git a/astroquery/eso/tests/test_eso_remote.py b/astroquery/eso/tests/test_eso_remote.py index a12eeef54b..e66735d62e 100644 --- a/astroquery/eso/tests/test_eso_remote.py +++ b/astroquery/eso/tests/test_eso_remote.py @@ -20,6 +20,7 @@ SKIP_SLOW = True SGRA_COLLECTIONS = ['195.B-0283', + 'ALMA', 'ATLASGAL', 'ERIS-SPIFFIER', 'GIRAFFE', From 50d73ab55a7ab22f4005aa824575932a777498a8 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Thu, 16 Jan 2025 16:59:17 +0100 Subject: [PATCH 57/75] Rename request --> table in function from_cache(...) --- astroquery/eso/core.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index 88d8024798..7cacc1d8b8 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -119,28 +119,27 @@ def request_file(self, query_str: str): return fn def from_cache(self, query_str, cache_timeout): - request_file = self.request_file(query_str) + table_file = self.request_file(query_str) try: if cache_timeout is None: expired = False else: current_time = datetime.now(timezone.utc) - cache_time = datetime.fromtimestamp(request_file.stat().st_mtime, timezone.utc) + cache_time = datetime.fromtimestamp(table_file.stat().st_mtime, timezone.utc) expired = current_time-cache_time > timedelta(seconds=cache_timeout) if not expired: - with open(request_file, "rb") as f: - # TODO rename respones to table - response = pickle.load(f) - if not isinstance(response, Table): - response = None + with open(table_file, "rb") as f: + cached_table = pickle.load(f) + if not isinstance(cached_table, Table): + cached_table = None else: - log.debug(f"Cache expired for {request_file}...") - response = None + log.debug(f"Cache expired for {table_file}...") + cached_table = None except FileNotFoundError: - response = None - if response: - log.debug("Retrieved data from {0}".format(request_file)) - return response + cached_table = None + if cached_table: + log.debug("Retrieved data from {0}".format(table_file)) + return cached_table def _authenticate(self, *, username: str, password: str) -> bool: """ From db0b2a3ecfd8b617498593a7e2032c7edf4cce41 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Fri, 17 Jan 2025 09:47:08 +0100 Subject: [PATCH 58/75] Remove unused data files --- .../eso/tests/data/amber_query_form.html | 546 ------------ .../eso/tests/data/amber_sgra_query.tbl | 60 -- .../eso/tests/data/main_query_form.html | 341 -------- astroquery/eso/tests/data/main_sgra_query.tbl | 60 -- astroquery/eso/tests/data/vvv_sgra_form.html | 822 ------------------ .../tests/data/vvv_sgra_survey_response.tbl | 87 -- 6 files changed, 1916 deletions(-) delete mode 100644 astroquery/eso/tests/data/amber_query_form.html delete mode 100644 astroquery/eso/tests/data/amber_sgra_query.tbl delete mode 100644 astroquery/eso/tests/data/main_query_form.html delete mode 100644 astroquery/eso/tests/data/main_sgra_query.tbl delete mode 100644 astroquery/eso/tests/data/vvv_sgra_form.html delete mode 100644 astroquery/eso/tests/data/vvv_sgra_survey_response.tbl diff --git a/astroquery/eso/tests/data/amber_query_form.html b/astroquery/eso/tests/data/amber_query_form.html deleted file mode 100644 index 1be90fe1d6..0000000000 --- a/astroquery/eso/tests/data/amber_query_form.html +++ /dev/null @@ -1,546 +0,0 @@ - - - - -ESO Science Archive - AMBER Data Products - - - - - - - -
- - - - - - - - - - -
ESO ESOSPECIAL ACCESS
SCIENCE ARCHIVE FACILITY		AMBER Raw Data
Query Form
- - - - - - - - -
How to use? -Other Instruments -Other Instruments -Archive FAQArchive Facility HOMEESO HOME
-
-
- - -
-Description -
- This form provides access to observations (raw data) performed with -AMBER -since 2004 that are stored on-line in the -Science Archive Facility in Garching. -Find more information about AMBER data on the -Quality Control web pages. -Paranal ambient conditions can be queried separately from here.
- -
- -
- - - -
-
Special Access Info [open]
- -		 -
- -
-
-
-
- -
-
-
- - -
-
Output preferences: -
-
Return max rows.

-

Target Information
    Target name.......................:   
-    Coordinate System.................:      RA:      DEC:   RA: sexagesimal=hours,decimal=degrees
-    Search Box........................:      Equatorial Output Format:  
- Input Target List.................:  
-
Observation and proposal parameters
 DATE OBS..........................:   (YYYY MM(M) DD of night begin [12:00 UT], DD MM(M) YYYY also acceptable)
-  OR give a query range using the following two fields (start/end dates)
    Start.............................:       End:   
- ProgId............................:   PPP.C-NNNN   (e.g. 080.A-0123*)
- Program Type......................:       SV Mode:  
- PI/CoI............................:   
- Proposal Title....................:  
-
Generic File Information
 DP.ID.............................:   archive filename of FITS file (e.g. AMBER.2010-09-04T09:40:45.174)
- OB.ID.............................:   identification number of OB within ESO system
- OBS.TARG.NAME.....................:   Observation block target name
-
 DPR.CATG..........................:   Data category (e.g. SCIENCE)
- DPR.TYPE..........................:   Observation type    User defined input:   (e.g. 2P2V or 3P2V)
- DPR.TECH..........................:   Observation technique    Request all interferometry files 
-
 TPL.NAME..........................:   AMBER template name
- TPL.NEXP..........................:   total number of exposures within the template
- TPL.START.........................:   starting time of template (UT)
-
Instrument Specific Information
 DEL.FT.SENSOR.....................:    Fringe Tracker Sensor Name
- DEL.FT.STATUS.....................:    Fringe Tracker Status (could be ON or OFF)
- DET.NTEL..........................:   Number of telescopes
- ISS.CONF.STATION1.................:   Station of telescope 1
- ISS.CONF.STATION2.................:   Station of telescope 2
- ISS.CONF.STATION3.................:   Station of telescope 3
- OCS.OBS.MODE......................:   Observation mode
- OCS.OBS.SPECCONF..................:   Spectral Configuration
- OCS.OBS.TYPE......................:   Observation type
- INS.GRAT1.WLEN....................:   Grating central wavelength [nm]
-
Ambient Parameters
 DIMM S-avg........................:   [arcsec] DIMM Seeing average over the exposure (FWHM at 0.5mue)
- Airmass...........................:   +/- 0.1
- Night?............................:   Night Exposure ?
- Moon Illumination.................:   Moon illumination during the exposure (percentage, negative when moon below the horizon)
-
Result set
    Sort by...........................: 
Extra Columns on Tabular Output
-
-
- - Use tabular output even if only one row is returned.
- - Use full-screen output even if more than one row is returned.
-
- -
-
- - - - - - - - - - -
- - ESO HOME - - ARCHIVE HOME - - ARCHIVE FAQ
- wdb 3.0h - 21-JUL-2017 ...... - - Send comments to archive@eso.org -
-
-
- - diff --git a/astroquery/eso/tests/data/amber_sgra_query.tbl b/astroquery/eso/tests/data/amber_sgra_query.tbl deleted file mode 100644 index e7efc35cfd..0000000000 --- a/astroquery/eso/tests/data/amber_sgra_query.tbl +++ /dev/null @@ -1,60 +0,0 @@ - -# SIMBAD coordinates for Sgr A* : 17 45 40.0, -29 00 28.1. -# -Object,RA,DEC,Target Ra Dec,Target l b,ProgId,DP.ID,OB.ID,DPR.CATG,DPR.TYPE,DPR.TECH,ISS.CONF.STATION1,ISS.CONF.STATION2,ISS.CONF.STATION3,INS.GRAT1.WLEN,DIMM S-avg -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:40:03.741,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.64 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:40:19.830,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.64 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:40:35.374,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.64 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:40:50.932,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.68 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:41:07.444,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.68 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:41:24.179,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.68 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:41:39.523,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.68 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:41:55.312,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.69 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:42:12.060,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.69 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:42:29.119,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.69 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:42:44.370,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.69 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:42:59.649,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.69 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:43:16.399,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.69 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:43:32.910,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.69 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:43:48.941,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.69 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:44:04.843,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.77 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:44:21.541,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.77 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:44:38.309,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.77 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:44:53.798,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.81 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:45:10.049,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.80 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:45:26.987,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.80 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:45:43.433,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.80 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:45:58.674,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.78 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:46:14.474,200156177,SCIENCE,"CPTPIST,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.79 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:46:37.269,200156177,SCIENCE,"CPTPIST,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.79 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:47:01.474,200156177,SCIENCE,"FRNSRC,BASE31",INTERFEROMETRY,U1,U3,U4,-1.000,0.80 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:50:20.362,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.74 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:50:33.223,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.74 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:50:46.632,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.74 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:50:59.556,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.74 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:51:14.547,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.78 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:51:27.935,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.77 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:51:41.670,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.77 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:51:54.480,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.77 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:52:07.271,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.77 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:52:21.824,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.76 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:52:35.051,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.76 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:52:48.928,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.76 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:53:02.492,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.76 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:53:24.007,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.76 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:53:37.858,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.76 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:53:51.137,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.76 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:54:05.106,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.76 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:54:26.021,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.77 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:54:39.202,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.77 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:54:52.656,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.77 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:55:06.685,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.77 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:55:26.262,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.78 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:55:39.382,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.80 [0.01] -GC_IRS7,266.417056,-29.006140,17:45:40.09 -29:00:22.1,359.945774 -0.045458,076.B-0863(A),AMBER.2006-03-14T07:55:52.259,200156177,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,U1,U3,U4,-1.000,0.80 [0.01] - -# A maximum of 50 records were found matching the provided criteria - any remaining rows were ignored. -# -# -# -# \ No newline at end of file diff --git a/astroquery/eso/tests/data/main_query_form.html b/astroquery/eso/tests/data/main_query_form.html deleted file mode 100644 index 766c641da5..0000000000 --- a/astroquery/eso/tests/data/main_query_form.html +++ /dev/null @@ -1,341 +0,0 @@ - - - - -ESO Archive Raw Data - - - - - - - -
- - - - - - - - - -
ESOSPECIAL ACCESS
SCIENCE ARCHIVE FACILITY		Observational Raw Data
Query Form
- - - - - - - - - -
How to use? -Instrument-specific Interfaces -Instruments-specific Interfaces -ESO Archive OverviewArchive FAQArchive Facility HOMEESO HOME
-
-
-
-
-Description -

-To request data you have to register as an -ESO/ST-ECF Archive user. A request can be submitted -following the instructions in the results table.
-

-
- - -
-
Output preferences: -
-
Return max rows.

-

 Input File..........:  
- Export: 
- HDR: 
-    Note................:  
-
Target Information
    Target..............:   
-    Search Box..........:   If Simbad/Ned name or coordinates given
-    RA..................:      Format:   J2000 (RA 05 34;Dec +22)
-    DEC.................:   (J2000)
- Target Ra, Dec: 
-    Output Format.......:  
- Stat PLOT: 
-    Export File Format..:  
- Night...............:   (YYYY MM(M) DD of night begin [12:00 UT], DD MM(M) YYYY also acceptable)
-    nightlog............:  
-   OR give a query range using the following start/end dates:
    Start...............:       End:   
-
Program Information
 Program_ID..........:  
-    survey..............:  
- PI/CoI..............:  
- s/v.................:  
- Title...............:  
- Prog_Type...........:  
-
Observing Information
 Instrument..........:  
- Stat Ins: 
- Category............:  
- Type................:  
- Mode................:  
- Dataset ID..........:  
- Orig Name...........:  
- Release_Date........:  
- OB_Name.............:  
- OB_ID...............:  
- TPL ID..............:  
- TPL START...........:  
-
Instrumental Setup
 Exptime.............:   (seconds)
- Stat Exp: 
- Filter..............:   (e.g. R*)
- Grism...............:   (e.g. GRIS_600*)
- Grating.............:   (e.g. CD*)
- Slit................:   (e.g. ~lslit1* [see also the help button])
-
Extra Columns
 MJD-OBS: 
- Airmass: 
- Ambient: 
- INFO................:  
-
Result Set
    Sort by.............:  
-    aladin_colour.......: 
Extra Columns on Tabular Output
-
-
- - Use tabular output even if only one row is returned.
- - Use full-screen output even if more than one row is returned.
-
- -
-
- - - - - - - - - - -
- - ESO HOME - - ARCHIVE HOME - - ARCHIVE FAQ
- wdb 3.0h - 21-JUL-2017 ...... - - Send comments to archive@eso.org -
-
-
- - diff --git a/astroquery/eso/tests/data/main_sgra_query.tbl b/astroquery/eso/tests/data/main_sgra_query.tbl deleted file mode 100644 index 41b0428a50..0000000000 --- a/astroquery/eso/tests/data/main_sgra_query.tbl +++ /dev/null @@ -1,60 +0,0 @@ - -# SIMBAD coordinates for Sgr A* : 17 45 40.0, -29 00 28.1. -# -OBJECT,RA,DEC,Program_ID,Instrument,Category,Type,Mode,Dataset ID,Release_Date,TPL ID,TPL START,Exptime,Filter,MJD-OBS,Airmass -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:40:03.741,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:21:16, 0.200,,53808.319488, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:40:19.830,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:21:16, 0.200,,53808.319674, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:40:35.374,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:21:16, 0.200,,53808.319854, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:40:50.932,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:21:16, 0.200,,53808.320034, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:41:07.444,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:21:16, 0.200,,53808.320225, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:41:24.179,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:21:16, 0.200,,53808.320419, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:41:39.523,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:21:16, 0.200,,53808.320596, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:41:55.312,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:21:16, 0.200,,53808.320779, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:42:12.060,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:21:16, 0.200,,53808.320973, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:42:29.119,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:21:16, 0.200,,53808.321170, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:42:44.370,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:21:16, 0.200,,53808.321347, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:42:59.649,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:21:16, 0.200,,53808.321524, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:43:16.399,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:21:16, 0.200,,53808.321718, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:43:32.910,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:21:16, 0.200,,53808.321909, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:43:48.941,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:21:16, 0.200,,53808.322094, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:44:04.843,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:21:16, 0.200,,53808.322278, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:44:21.541,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:21:16, 0.200,,53808.322472, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:44:38.309,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:21:16, 0.200,,53808.322666, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:44:53.798,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:21:16, 0.200,,53808.322845, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:45:10.049,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:21:16, 0.200,,53808.323033, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:45:26.987,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:21:16, 0.200,,53808.323229, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:45:43.433,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:21:16, 0.200,,53808.323419, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:45:58.674,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:21:16, 0.200,,53808.323596, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"CPTPIST,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:46:14.474,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:21:16, 0.200,,53808.323779, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"CPTPIST,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:46:37.269,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:21:16, 0.200,,53808.324042, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE31",INTERFEROMETRY,AMBER.2006-03-14T07:47:01.474,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:21:16, 0.200,,53808.324323, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:50:20.362,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:48:12, 0.100,,53808.326625, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:50:33.223,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:48:12, 0.100,,53808.326773, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:50:46.632,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:48:12, 0.100,,53808.326929, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:50:59.556,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:48:12, 0.100,,53808.327078, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:51:14.547,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:48:12, 0.100,,53808.327252, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:51:27.935,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:48:12, 0.100,,53808.327407, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:51:41.670,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:48:12, 0.100,,53808.327566, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:51:54.480,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:48:12, 0.100,,53808.327714, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:52:07.271,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:48:12, 0.100,,53808.327862, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:52:21.824,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:48:12, 0.100,,53808.328030, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:52:35.051,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:48:12, 0.100,,53808.328183, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:52:48.928,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:48:12, 0.100,,53808.328344, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:53:02.492,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:48:12, 0.100,,53808.328501, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:53:24.007,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:48:12, 0.100,,53808.328750, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:53:37.858,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:48:12, 0.100,,53808.328910, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:53:51.137,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:48:12, 0.100,,53808.329064, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:54:05.106,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:48:12, 0.100,,53808.329226, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:54:26.021,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:48:12, 0.100,,53808.329468, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:54:39.202,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:48:12, 0.100,,53808.329620, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:54:52.656,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:48:12, 0.100,,53808.329776, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:55:06.685,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:48:12, 0.100,,53808.329938, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:55:26.262,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:48:12, 0.100,,53808.330165, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:55:39.382,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:48:12, 0.100,,53808.330317, -GC_IRS7,17:45:40.09,-29:00:22.1,076.B-0863(A),AMBER,SCIENCE,"FRNSRC,BASE12",INTERFEROMETRY,AMBER.2006-03-14T07:55:52.259,Mar 14 2007,AMBER_3Tstd_acq,2006-03-14T07:48:12, 0.100,,53808.330466, - -# A maximum of 50 records were found matching the provided criteria - any remaining rows were ignored. -# -# -# -# \ No newline at end of file diff --git a/astroquery/eso/tests/data/vvv_sgra_form.html b/astroquery/eso/tests/data/vvv_sgra_form.html deleted file mode 100644 index d469a145b6..0000000000 --- a/astroquery/eso/tests/data/vvv_sgra_form.html +++ /dev/null @@ -1,822 +0,0 @@ - - - - -ESO Science Archive - Data Products - - - - - - - - - - - - - - - - - - - - - - - - -
ESO - - - - - - - - - -
- GENERIC  - - SPECTRAL  - - IMAGING  - - VISTA  -
PHASE3 ARCHIVE INTERFACES
- - - - - - - -
-HELP -REDUCED DATA TYPES DESCRIPTION -FAQ -DATA RELEASES
-
-		Generic Data Products
Query Form
-
-Please be aware of an important announcement about the flux calibration of XSHOOTER UVB data products.
-This form provides access to reduced or fully calibrated data sets, and derived catalogs, -that were contributed by PIs of ESO programmes or produced by ESO (using ESO calibration pipelines with the best available calibration data), and then -integrated into the ESO Science Archive Facility starting April 2011, through the -Phase 3 process. Included are optical, infrared, and APEX (millimetre, submillimetre) data products. -Each available data set is fully described; please see the list of contributed data releases and pipeline-processed data streams including their corresponding descriptions. -This form allows generic query constraints for all types of data products. -More specific forms, customised for each particular data type, are available for optical and infrared imaging, -for spectra, -and for VISTA data products. -Other data not yet migrated to the Phase 3 infrastructure are available via different user interfaces; please check the archive home page. -

Note: The FITS format of the spectra retrievable via this query form complies to the ESO Science Data Product standard [PDF]. The 1d spectra help page provides a list of tools that support this format and a quick guide to read and display these spectra in IDL and IRAF.

-
-