in case local resolver fails, fallback to the external resolver

cdci_data_analysis/analysis/drupal_helper.py

@@ -21,6 +21,8 @@
 from enum import Enum, auto
 from astropy.coordinates import SkyCoord, Angle
 from astropy import units as u
+from astroquery.simbad import Simbad
+import xml.etree.ElementTree as ET
 
 from cdci_data_analysis.analysis import tokenHelper
 from ..analysis.exceptions import RequestNotUnderstood, InternalError, RequestNotAuthorized
@@ -551,11 +553,14 @@
         if update_astro_entity:
             auto_update = kwargs.pop('auto_update', 'False') == 'True'
             if auto_update is True:
-                name_resolver_url = disp_conf.name_resolver_url
+                local_name_resolver_url = disp_conf.local_name_resolver_url
+                external_name_resolver_url = disp_conf.external_name_resolver_url
                 entities_portal_url = disp_conf.entities_portal_url
-                resolved_obj = resolve_name(name_resolver_url=name_resolver_url,
+                resolved_obj = resolve_name(local_name_resolver_url=local_name_resolver_url,
+                                            external_name_resolver_url=external_name_resolver_url,
                                             entities_portal_url=entities_portal_url,
-                                            name=src_name)
+                                            name=src_name,
+                                            sentry_dsn=sentry_dsn)
                 if resolved_obj is not None:
                     msg = ''
                     if 'message' in resolved_obj:
@@ -1488,39 +1493,127 @@
     return False
 
 
-def resolve_name(name_resolver_url: str, entities_portal_url: str = None, name: str = None):
+def resolve_name(local_name_resolver_url: str, external_name_resolver_url: str, entities_portal_url: str = None, name: str = None, sentry_dsn=None):
     resolved_obj = {}
     if name is not None:
         quoted_name = urllib.parse.quote(name.strip())
-        res = requests.get(name_resolver_url.format(quoted_name))
-        if res.status_code == 200:
-            returned_resolved_obj = res.json()
-            if 'success' in returned_resolved_obj:
+        local_name_resolver_url_formatted = local_name_resolver_url.format(quoted_name)
+        try:
+            res = requests.get(local_name_resolver_url_formatted)
+            if res.status_code == 200:
+                returned_resolved_obj = res.json()
+                if 'success' in returned_resolved_obj:
+                    resolved_obj['name'] = name.replace('_', ' ')
+                    if returned_resolved_obj['success']:
+                        logger.info(f"object {name} successfully resolved")
+                        if 'ra' in returned_resolved_obj:
+                            resolved_obj['RA'] = float(returned_resolved_obj['ra'])
+                        if 'dec' in returned_resolved_obj:
+                            resolved_obj['DEC'] = float(returned_resolved_obj['dec'])
+                        if 'object_ids' in returned_resolved_obj:
+                            resolved_obj['object_ids'] = returned_resolved_obj['object_ids']
+                        if 'object_type' in returned_resolved_obj:
+                            resolved_obj['object_type'] = returned_resolved_obj['object_type']
+                        resolved_obj['entity_portal_link'] = entities_portal_url.format(quoted_name)
+                        resolved_obj['message'] = f'{name} successfully resolved'
+                    elif not returned_resolved_obj['success']:
+                        logger.info(f"resolution of the object {name} unsuccessful")
+                        resolved_obj['message'] = f'{name} could not be resolved'
+            else:
+                logger.warning("There seems to be some problem in completing the request for the resolution of the object"
+                               f" \"{name}\" using the local resolver.\n"
+                               f"The request lead to the error {res.text}, "
+                               "this might be due to an error in the url or the service "
+                               "requested is currently not available. The external resolver will be used.")
+                if sentry_dsn is not None:
+                    sentry.capture_message(f'Failed to resolve object "{name}" using the local resolver. '
+                                           f'URL: {local_name_resolver_url_formatted} '
+                                           f'Status Code: {res.status_code} '
+                                           f'Response: {res.text}')
+        except (ConnectionError,
+                requests.exceptions.ConnectionError,
+                requests.exceptions.Timeout) as e:
+            logger.warning(f'An exception occurred while trying to resolve the object "{name}" using the local resolver. '
+                           f'using the url: {local_name_resolver_url_formatted}. Exception details: {str(e)}')
+            if sentry_dsn is not None:
+                sentry.capture_message(f'An exception occurred while trying to resolve the object "{name}" using the local resolver. '
+                                       f'URL: {local_name_resolver_url_formatted} '
+                                       f"Exception details: {str(e)}")
+        external_name_resolver_url_formatted = external_name_resolver_url.format(quoted_name)
+        try:
+            res = requests.get(external_name_resolver_url_formatted)
+            if res.status_code == 200:
+                root = ET.fromstring(res.text)
                 resolved_obj['name'] = name.replace('_', ' ')
-                if returned_resolved_obj['success']:
-                    logger.info(f"object {name} successfully resolved")
-                    if 'ra' in returned_resolved_obj:
-                        resolved_obj['RA'] = float(returned_resolved_obj['ra'])
-                    if 'dec' in returned_resolved_obj:
-                        resolved_obj['DEC'] = float(returned_resolved_obj['dec'])
-                    if 'object_ids' in returned_resolved_obj:
-                        resolved_obj['object_ids'] = returned_resolved_obj['object_ids']
-                    if 'object_type' in returned_resolved_obj:
-                        resolved_obj['object_type'] = returned_resolved_obj['object_type']
-                    resolved_obj['entity_portal_link'] = entities_portal_url.format(quoted_name)
-                    resolved_obj['message'] = f'{name} successfully resolved'
-                elif not returned_resolved_obj['success']:
-                    logger.info(f"resolution of the object {name} unsuccessful")
+                resolver_tag = root.find('.//Resolver')
+                if resolver_tag is not None:
+                    ra_tag = resolver_tag.find('.//jradeg')
+                    dec_tag = resolver_tag.find('.//jdedeg')
+                    if ra_tag is None or dec_tag is None:
+                        info_tag = root.find('.//INFO')
+                        resolved_obj['message'] = f'{name} could not be resolved'
+                        if info_tag is not None:
+                            message_info = info_tag.text
+                            resolved_obj['message'] += f': {message_info}'
+                    else:
+                        resolved_obj['RA'] = float(ra_tag.text)
+                        resolved_obj['DEC'] = float(dec_tag.text)
+                        resolved_obj['entity_portal_link'] = entities_portal_url.format(quoted_name)
+
+                        try:
+                            Simbad.add_votable_fields("otype")
+                            result_table = Simbad.query_object(quoted_name)
+                            object_type = str(result_table[0]['OTYPE']).strip()
+                            resolved_obj['object_type'] = object_type
+                        except Exception as e:
+                            logger.warning(f"An exception occurred while using Simbad to query the object \"{name}\" "
+                                           f"while using the external resolver:\n{str(e)}")
+                            resolved_obj['object_type'] = None
+                        try:
+                            object_ids_table = Simbad.query_objectids(name)
+                            source_ids_list = object_ids_table['ID'].tolist()
+                            resolved_obj['object_ids'] = source_ids_list
+                        except Exception as e:
+                            logger.warning(f"An exception occurred while using Simbad to query the object ids for the object \"{name}\" "
+                                           f"while using the external resolver:\n{str(e)}")
+                            resolved_obj['object_ids'] = None
+                else:
+                    warning_msg = ("There seems to be some problem in completing the request for the resolution of the object"
+                                   f" \"{name}\" using the external resolver.")
                     resolved_obj['message'] = f'{name} could not be resolved'
-        else:
-            logger.warning(f"there seems to be some problem in completing the request for the resolution of the object: {name}\n"
-                           f"the request lead to the error {res.text}, "
-                           "this might be due to an error in the url or the service "
-                           "requested is currently not available, "
-                           "please check your request and try to issue it again")
-            raise InternalError('issue when performing a request to the local resolver',
-                                status_code=500,
-                                payload={'drupal_helper_error_message': res.text})
+                    info_tag = root.find('.//INFO')
+                    if info_tag is not None:
+                        warning_msg += (f"The request lead to the error {info_tag.text}, "
+                                       "this might be due to an error in the name of the object that ha been provided.")
+                        resolved_obj['message'] += f': {info_tag.text}'
+                    logger.warning(warning_msg)
+                    if sentry_dsn is not None:
+                        sentry.capture_message(f'Failed to resolve object "{name}" using the external resolver. '
+                                               f'URL: {external_name_resolver_url_formatted} '
+                                               f'Status Code: {res.status_code} '
+                                               f'Response: {res.text}'
+                                               f"Info returned from the resolver: {resolved_obj['message']}")
+            else:
+                logger.warning("There seems to be some problem in completing the request for the resolution of the object"
+                               f" \"{name}\" using the external resolver.\n"
+                               f"The request lead to the error {res.text}, "
+                               "this might be due to an error in the url or the service "
+                               "requested is currently not available. The object could not be resolved.")
+                if sentry_dsn is not None:
+                    sentry.capture_message(f'Failed to resolve object "{name}" using the external resolver. '
+                                           f'URL: {external_name_resolver_url_formatted} '
+                                           f'Status Code: {res.status_code} '
+                                           f'Response: {res.text}')
+                resolved_obj['message'] = f'{name} could not be resolved: {res.text}'
+        except (ConnectionError,
+                requests.exceptions.ConnectionError,
+                requests.exceptions.Timeout) as e:
+            logger.warning(f'An exception occurred while trying to resolve the object "{name}" using the local resolver. '
+                           f'using the url: {external_name_resolver_url_formatted}. Exception details: {str(e)}')
+            if sentry_dsn is not None:
+                sentry.capture_message(f'An exception occurred while trying to resolve the object "{name}" using the external resolver. '
+                                       f'URL: {external_name_resolver_url_formatted} '
+                                       f"Exception details: {str(e)}")
     return resolved_obj
 
 

cdci_data_analysis/config_dir/conf_env.yml.example

@@ -115,8 +115,10 @@ dispatcher:
         product_gallery_secret_key: PRODUCT_GALLERY_SECRET_KEY
         # timezone used within the drupal configuration, these two values have to be always aligned
         product_gallery_timezone: PRODUCT_GALLERY_SECRET_KEY
-        # url of the name resolver
-        name_resolver_url: NAME_RESOLVER_URL
+        # url of the local name resolver
+        local_name_resolver_url: NAME_RESOLVER_URL
+        # url of the external name resolver
+        external_name_resolver_url: NAME_RESOLVER_URL
         # url of the online catalog for astrophysical entities
         entities_portal_url: ENTITIES_PORTAL_URL
         # url for the conversion of a given time, in UTC format, to the correspondent REVNUM

cdci_data_analysis/configurer.py

@@ -260,7 +260,10 @@ def __init__(self, cfg_dict, origin=None):
                                      disp_dict.get('product_gallery_options', {}).get('product_gallery_secret_key', None),
                                      disp_dict.get('product_gallery_options', {}).get('product_gallery_timezone',
                                                                                       "Europe/Zurich"),
-                                     disp_dict.get('product_gallery_options', {}).get('name_resolver_url', 'https://resolver-prod.obsuks1.unige.ch/api/v1.1/byname/{}'),
+                                     disp_dict.get('product_gallery_options', {}).get('local_name_resolver_url',
+                                                                                      'https://resolver-prod.obsuks1.unige.ch/api/v1.1/byname/{}'),
+                                     disp_dict.get('product_gallery_options', {}).get('external_name_resolver_url',
+                                                                                      'http://cdsweb.u-strasbg.fr/cgi-bin/nph-sesame/-oxp/NSV?{}'),
                                      disp_dict.get('product_gallery_options', {}).get('entities_portal_url', 'http://cdsportal.u-strasbg.fr/?target={}'),
                                      disp_dict.get('product_gallery_options', {}).get('converttime_revnum_service_url', 'https://www.astro.unige.ch/mmoda/dispatch-data/gw/timesystem/api/v1.0/converttime/UTC/{}/REVNUM'),
                                      disp_dict.get('renku_options', {}).get('renku_gitlab_repository_url', None),
@@ -338,7 +341,8 @@ def set_conf_dispatcher(self,
                             product_gallery_url,
                             product_gallery_secret_key,
                             product_gallery_timezone,
-                            name_resolver_url,
+                            local_name_resolver_url,
+                            external_name_resolver_url,
                             entities_portal_url,
                             converttime_revnum_service_url,
                             renku_gitlab_repository_url,
@@ -389,7 +393,8 @@ def set_conf_dispatcher(self,
         self.product_gallery_url = product_gallery_url
         self.product_gallery_secret_key = product_gallery_secret_key
         self.product_gallery_timezone = product_gallery_timezone
-        self.name_resolver_url = name_resolver_url
+        self.local_name_resolver_url = local_name_resolver_url
+        self.external_name_resolver_url = external_name_resolver_url
         self.entities_portal_url = entities_portal_url
         self.converttime_revnum_service_url = converttime_revnum_service_url
         self.renku_gitlab_repository_url = renku_gitlab_repository_url

cdci_data_analysis/flask_app/app.py

@@ -650,12 +650,17 @@
 
     name = par_dic.get('name', None)
 
-    name_resolver_url = app_config.name_resolver_url
+    local_name_resolver_url = app_config.local_name_resolver_url
+    external_name_resolver_url = app_config.external_name_resolver_url
     entities_portal_url = app_config.entities_portal_url
 
-    resolve_object = drupal_helper.resolve_name(name_resolver_url=name_resolver_url,
+    sentry_dsn = sentry.sentry_url
+
+    resolve_object = drupal_helper.resolve_name(local_name_resolver_url=local_name_resolver_url,
+                                                external_name_resolver_url=external_name_resolver_url,
                                                 entities_portal_url=entities_portal_url,
-                                                name=name)
+                                                name=name,
+                                                sentry_dsn=sentry_dsn)
 
     return resolve_object