in case local resolver fails, fallback to the external resolver

cdci_data_analysis/analysis/drupal_helper.py

@@ -21,6 +21,7 @@
 from enum import Enum, auto
 from astropy.coordinates import SkyCoord, Angle
 from astropy import units as u
+import xml.etree.ElementTree as ET
 
 from cdci_data_analysis.analysis import tokenHelper
 from ..analysis.exceptions import RequestNotUnderstood, InternalError, RequestNotAuthorized
@@ -551,11 +552,14 @@ def post_content_to_gallery(decoded_token,
         if update_astro_entity:
             auto_update = kwargs.pop('auto_update', 'False') == 'True'
             if auto_update is True:
-                name_resolver_url = disp_conf.name_resolver_url
+                local_name_resolver_url = disp_conf.local_name_resolver_url
+                external_name_resolver_url = disp_conf.external_name_resolver_url
                 entities_portal_url = disp_conf.entities_portal_url
-                resolved_obj = resolve_name(name_resolver_url=name_resolver_url,
+                resolved_obj = resolve_name(local_name_resolver_url=local_name_resolver_url,
+                                            external_name_resolver_url=external_name_resolver_url,
                                             entities_portal_url=entities_portal_url,
-                                            name=src_name)
+                                            name=src_name,
+                                            sentry_dsn=sentry_dsn)
                 if resolved_obj is not None:
                     msg = ''
                     if 'message' in resolved_obj:
@@ -1488,39 +1492,98 @@ def check_matching_coords(source_1_name, source_1_coord_ra, source_1_coord_dec,
     return False
 
 
-def resolve_name(name_resolver_url: str, entities_portal_url: str = None, name: str = None):
+def resolve_name(local_name_resolver_url: str, external_name_resolver_url: str, entities_portal_url: str = None, name: str = None, sentry_dsn=None):
     resolved_obj = {}
     if name is not None:
         quoted_name = urllib.parse.quote(name.strip())
-        res = requests.get(name_resolver_url.format(quoted_name))
+        local_name_resolver_url_formatted = local_name_resolver_url.format(quoted_name)
+        try:
+            res = requests.get(local_name_resolver_url_formatted)
+            if res.status_code == 200:
+                returned_resolved_obj = res.json()
+                if 'success' in returned_resolved_obj:
+                    resolved_obj['name'] = name.replace('_', ' ')
+                    if returned_resolved_obj['success']:
+                        logger.info(f"object {name} successfully resolved")
+                        if 'ra' in returned_resolved_obj:
+                            resolved_obj['RA'] = float(returned_resolved_obj['ra'])
+                        if 'dec' in returned_resolved_obj:
+                            resolved_obj['DEC'] = float(returned_resolved_obj['dec'])
+                        if 'object_ids' in returned_resolved_obj:
+                            resolved_obj['object_ids'] = returned_resolved_obj['object_ids']
+                        if 'object_type' in returned_resolved_obj:
+                            resolved_obj['object_type'] = returned_resolved_obj['object_type']
+                        resolved_obj['entity_portal_link'] = entities_portal_url.format(quoted_name)
+                        resolved_obj['message'] = f'{name} successfully resolved'
+                    elif not returned_resolved_obj['success']:
+                        logger.info(f"resolution of the object {name} unsuccessful")
+                        resolved_obj['message'] = f'{name} could not be resolved'
+            else:
+                logger.warning("There seems to be some problem in completing the request for the resolution of the object"
+                               f" \"{name}\" using the local resolver.\n"
+                               f"The request lead to the error {res.text}, "
+                               "this might be due to an error in the url or the service "
+                               "requested is currently not available. The external resolver will be used.")
+                if sentry_dsn is not None:
+                    sentry.capture_message(f'Failed to resolve object "{name}" using the local resolver. '
+                                           f'URL: {local_name_resolver_url_formatted} '
+                                           f'Status Code: {res.status_code} '
+                                           f'Response: {res.text}')
+        except (ConnectionError,
+                requests.exceptions.ConnectionError,
+                requests.exceptions.Timeout) as e:
+            logger.warning(f'An exception occurred while trying to resolve the object "{name}" using the local resolver. '
+                           f'using the url: {local_name_resolver_url_formatted}. Exception details: {str(e)}')
+            if sentry_dsn is not None:
+                sentry.capture_message(f'An exception occurred while trying to resolve the object "{name}" using the local resolver. '
+                                       f'URL: {local_name_resolver_url_formatted} '
+                                       f"Exception details: {str(e)}")
+        res = requests.get(external_name_resolver_url.format(quoted_name))
         if res.status_code == 200:
-            returned_resolved_obj = res.json()
-            if 'success' in returned_resolved_obj:
-                resolved_obj['name'] = name.replace('_', ' ')
-                if returned_resolved_obj['success']:
-                    logger.info(f"object {name} successfully resolved")
-                    if 'ra' in returned_resolved_obj:
-                        resolved_obj['RA'] = float(returned_resolved_obj['ra'])
-                    if 'dec' in returned_resolved_obj:
-                        resolved_obj['DEC'] = float(returned_resolved_obj['dec'])
-                    if 'object_ids' in returned_resolved_obj:
-                        resolved_obj['object_ids'] = returned_resolved_obj['object_ids']
-                    if 'object_type' in returned_resolved_obj:
-                        resolved_obj['object_type'] = returned_resolved_obj['object_type']
-                    resolved_obj['entity_portal_link'] = entities_portal_url.format(quoted_name)
-                    resolved_obj['message'] = f'{name} successfully resolved'
-                elif not returned_resolved_obj['success']:
-                    logger.info(f"resolution of the object {name} unsuccessful")
+            root = ET.fromstring(res.text)
+            resolved_obj['name'] = name.replace('_', ' ')
+            resolver_tag = root.find('.//Resolver')
+            if resolver_tag is not None:
+                ra_tag = resolver_tag.find('.//jradeg')
+                dec_tag = resolver_tag.find('.//jdedeg')
+                if ra_tag is None or dec_tag is None:
+                    info_tag = root.find('.//INFO')
                     resolved_obj['message'] = f'{name} could not be resolved'
+                    if info_tag is not None:
+                        message_info = info_tag.text
+                        resolved_obj['message'] += f': {message_info}'
+                else:
+                    resolved_obj['RA'] = float(ra_tag.text)
+                    resolved_obj['DEC'] = float(dec_tag.text)
+                    resolved_obj['entity_portal_link'] = entities_portal_url.format(quoted_name)
+            else:
+                warning_msg = ("There seems to be some problem in completing the request for the resolution of the object"
+                               f" \"{name}\" using the external resolver.")
+                resolved_obj['message'] = f'{name} could not be resolved'
+                info_tag = root.find('.//INFO')
+                if info_tag is not None:
+                    warning_msg += (f"The request lead to the error {info_tag.text}, "
+                                   "this might be due to an error in the name of the object that ha been provided.")
+                    resolved_obj['message'] += f': {info_tag.text}'
+                logger.warning(warning_msg)
+                if sentry_dsn is not None:
+                    sentry.capture_message(f'Failed to resolve object "{name}" using the remote resolver. '
+                                           f'URL: {local_name_resolver_url.format(quoted_name)} '
+                                           f'Status Code: {res.status_code} '
+                                           f'Response: {res.text}'
+                                           f"Info returned from the resolver: {resolved_obj['message']}")
         else:
-            logger.warning(f"there seems to be some problem in completing the request for the resolution of the object: {name}\n"
-                           f"the request lead to the error {res.text}, "
+            logger.warning("There seems to be some problem in completing the request for the resolution of the object"
+                           f" \"{name}\" using the external resolver.\n"
+                           f"The request lead to the error {res.text}, "
                            "this might be due to an error in the url or the service "
-                           "requested is currently not available, "
-                           "please check your request and try to issue it again")
-            raise InternalError('issue when performing a request to the local resolver',
-                                status_code=500,
-                                payload={'drupal_helper_error_message': res.text})
+                           "requested is currently not available. The object could not be resolved.")
+            if sentry_dsn is not None:
+                sentry.capture_message(f'Failed to resolve object "{name}" using the remote resolver. '
+                                       f'URL: {local_name_resolver_url.format(quoted_name)} '
+                                       f'Status Code: {res.status_code} '
+                                       f'Response: {res.text}')
+            resolved_obj['message'] = f'{name} could not be resolved: {res.text}'
     return resolved_obj
 
 

cdci_data_analysis/config_dir/conf_env.yml.example

@@ -115,8 +115,10 @@ dispatcher:
         product_gallery_secret_key: PRODUCT_GALLERY_SECRET_KEY
         # timezone used within the drupal configuration, these two values have to be always aligned
         product_gallery_timezone: PRODUCT_GALLERY_SECRET_KEY
-        # url of the name resolver
-        name_resolver_url: NAME_RESOLVER_URL
+        # url of the local name resolver
+        local_name_resolver_url: NAME_RESOLVER_URL
+        # url of the external name resolver
+        external_name_resolver_url: NAME_RESOLVER_URL
         # url of the online catalog for astrophysical entities
         entities_portal_url: ENTITIES_PORTAL_URL
         # url for the conversion of a given time, in UTC format, to the correspondent REVNUM

cdci_data_analysis/configurer.py

@@ -260,7 +260,10 @@ def __init__(self, cfg_dict, origin=None):
                                      disp_dict.get('product_gallery_options', {}).get('product_gallery_secret_key', None),
                                      disp_dict.get('product_gallery_options', {}).get('product_gallery_timezone',
                                                                                       "Europe/Zurich"),
-                                     disp_dict.get('product_gallery_options', {}).get('name_resolver_url', 'https://resolver-prod.obsuks1.unige.ch/api/v1.1/byname/{}'),
+                                     disp_dict.get('product_gallery_options', {}).get('local_name_resolver_url',
+                                                                                      'https://resolver-prod.obsuks1.unige.ch/api/v1.1/byname/{}'),
+                                     disp_dict.get('product_gallery_options', {}).get('external_name_resolver_url',
+                                                                                      'http://cdsweb.u-strasbg.fr/cgi-bin/nph-sesame/-oxp/NSV?{}'),
                                      disp_dict.get('product_gallery_options', {}).get('entities_portal_url', 'http://cdsportal.u-strasbg.fr/?target={}'),
                                      disp_dict.get('product_gallery_options', {}).get('converttime_revnum_service_url', 'https://www.astro.unige.ch/mmoda/dispatch-data/gw/timesystem/api/v1.0/converttime/UTC/{}/REVNUM'),
                                      disp_dict.get('renku_options', {}).get('renku_gitlab_repository_url', None),
@@ -338,7 +341,8 @@ def set_conf_dispatcher(self,
                             product_gallery_url,
                             product_gallery_secret_key,
                             product_gallery_timezone,
-                            name_resolver_url,
+                            local_name_resolver_url,
+                            external_name_resolver_url,
                             entities_portal_url,
                             converttime_revnum_service_url,
                             renku_gitlab_repository_url,
@@ -389,7 +393,8 @@ def set_conf_dispatcher(self,
         self.product_gallery_url = product_gallery_url
         self.product_gallery_secret_key = product_gallery_secret_key
         self.product_gallery_timezone = product_gallery_timezone
-        self.name_resolver_url = name_resolver_url
+        self.local_name_resolver_url = local_name_resolver_url
+        self.external_name_resolver_url = external_name_resolver_url
         self.entities_portal_url = entities_portal_url
         self.converttime_revnum_service_url = converttime_revnum_service_url
         self.renku_gitlab_repository_url = renku_gitlab_repository_url

cdci_data_analysis/flask_app/app.py

@@ -650,12 +650,17 @@ def resolve_name():
 
     name = par_dic.get('name', None)
 
-    name_resolver_url = app_config.name_resolver_url
+    local_name_resolver_url = app_config.local_name_resolver_url
+    external_name_resolver_url = app_config.external_name_resolver_url
     entities_portal_url = app_config.entities_portal_url
 
-    resolve_object = drupal_helper.resolve_name(name_resolver_url=name_resolver_url,
+    sentry_dsn = sentry.sentry_url
+
+    resolve_object = drupal_helper.resolve_name(local_name_resolver_url=local_name_resolver_url,
+                                                external_name_resolver_url=external_name_resolver_url,
                                                 entities_portal_url=entities_portal_url,
-                                                name=name)
+                                                name=name,
+                                                sentry_dsn=sentry_dsn)
 
     return resolve_object
 

cdci_data_analysis/pytest_fixtures.py

@@ -1,6 +1,7 @@
 # this could be a separate package or/and a pytest plugin
 from json import JSONDecodeError
 
+import responses
 import sentry_sdk
 import yaml
 
@@ -604,13 +605,55 @@
                 '\n        product_gallery_url: "http://cdciweb02.astro.unige.ch/mmoda/galleryd"'
                 f'\n        product_gallery_secret_key: "{os.getenv("DISPATCHER_PRODUCT_GALLERY_SECRET_KEY", "secret_key")}"'
                 '\n        product_gallery_timezone: "Europe/Zurich"'
-                '\n        name_resolver_url: "https://resolver-prod.obsuks1.unige.ch/api/v1.1/byname/{}"'
+                '\n        local_name_resolver_url: "https://resolver-prod.obsuks1.unige.ch/api/v1.1/byname/{}"'
+                '\n        external_name_resolver_url: "http://cdsweb.u-strasbg.fr/cgi-bin/nph-sesame/-oxp/NSV?{}"'
                 '\n        entities_portal_url: "http://cdsportal.u-strasbg.fr/?target={}"'
                 '\n        converttime_revnum_service_url: "https://www.astro.unige.ch/mmoda/dispatch-data/gw/timesystem/api/v1.0/converttime/UTC/{}/REVNUM"')
 
     yield fn
 
 
+@pytest.fixture
+def dispatcher_test_conf_with_gallery_invalid_local_resolver_fn(dispatcher_test_conf_fn):
+    fn = "test-dispatcher-conf-with-gallery.yaml"
+
+    with open(fn, "w") as f:
+        with open(dispatcher_test_conf_fn) as f_default:
+            f.write(f_default.read())
+
+        f.write('\n    product_gallery_options:'
+                '\n        product_gallery_url: "http://cdciweb02.astro.unige.ch/mmoda/galleryd"'
+                f'\n        product_gallery_secret_key: "{os.getenv("DISPATCHER_PRODUCT_GALLERY_SECRET_KEY", "secret_key")}"'
+                '\n        product_gallery_timezone: "Europe/Zurich"'
+                '\n        local_name_resolver_url: "http://invalid_url/"'
+                '\n        external_name_resolver_url: "http://cdsweb.u-strasbg.fr/cgi-bin/nph-sesame/-oxp/NSV?{}"'
+                '\n        entities_portal_url: "http://cdsportal.u-strasbg.fr/?target={}"'
+                '\n        converttime_revnum_service_url: "https://www.astro.unige.ch/mmoda/dispatch-data/gw/timesystem/api/v1.0/converttime/UTC/{}/REVNUM"')
+
+    yield fn
+
+
+@pytest.fixture
+def dispatcher_test_conf_with_vo_options_fn(dispatcher_test_conf_fn):
+    fn = "test-dispatcher-conf-with-vo-options.yaml"
+
+    with open(fn, "w") as f:
+        with open(dispatcher_test_conf_fn) as f_default:
+            f.write(f_default.read())
+
+        f.write('\n    vo_options:'
+                '\n         vo_mysql_pg_host: "localhost"'
+                '\n         vo_mysql_pg_user: "user"'
+                '\n         vo_mysql_pg_password: "password"'
+                '\n         vo_mysql_pg_db: "database"'
+                '\n         vo_psql_pg_host: "localhost"'
+                '\n         vo_psql_pg_user: "user"'
+                '\n         vo_psql_pg_password: "password"'
+                '\n         vo_psql_pg_db: "database"')
+
+    yield fn
+
+
 @pytest.fixture
 def dispatcher_test_conf_with_matrix_options_fn(dispatcher_test_conf_fn):
     fn = "test-dispatcher-conf-with-matrix-options.yaml"
@@ -708,10 +751,21 @@
     yield yaml.load(open(dispatcher_test_conf_with_gallery_fn), Loader=yaml.SafeLoader)['dispatcher']
 
 
+@pytest.fixture
+def dispatcher_test_conf_with_gallery_invalid_local_resolver(dispatcher_test_conf_with_gallery_invalid_local_resolver_fn):
+    yield yaml.load(open(dispatcher_test_conf_with_gallery_invalid_local_resolver_fn), Loader=yaml.SafeLoader)['dispatcher']
+
+
+@pytest.fixture
+def dispatcher_test_conf_with_vo_options(dispatcher_test_conf_with_vo_options_fn):
+    yield yaml.load(open(dispatcher_test_conf_with_vo_options_fn), Loader=yaml.SafeLoader)['dispatcher']
+
+
 @pytest.fixture
 def dispatcher_test_conf_with_matrix_options(dispatcher_test_conf_with_matrix_options_fn):
     yield yaml.load(open(dispatcher_test_conf_with_matrix_options_fn), Loader=yaml.SafeLoader)['dispatcher']
 
+
 @pytest.fixture
 def dispatcher_test_conf_with_gallery_no_resolver(dispatcher_test_conf_with_gallery_no_resolver_fn):
     yield yaml.load(open(dispatcher_test_conf_with_gallery_no_resolver_fn), Loader=yaml.SafeLoader)['dispatcher']
@@ -1120,6 +1174,20 @@
     os.kill(pid, signal.SIGINT)
 
 
+@pytest.fixture
+def dispatcher_live_fixture_with_gallery_invalid_local_resolver(pytestconfig, dispatcher_test_conf_with_gallery_invalid_local_resolver_fn,
+                                                                dispatcher_debug):
+    dispatcher_state = start_dispatcher(pytestconfig.rootdir, dispatcher_test_conf_with_gallery_invalid_local_resolver_fn)
+
+    service = dispatcher_state['url']
+    pid = dispatcher_state['pid']
+
+    yield service
+
+    kill_child_processes(pid, signal.SIGINT)
+    os.kill(pid, signal.SIGINT)
+
+
 @pytest.fixture
 def dispatcher_live_fixture_no_products_url(pytestconfig, dispatcher_test_conf_no_products_url_fn, dispatcher_debug):
     dispatcher_state = start_dispatcher(pytestconfig.rootdir, dispatcher_test_conf_no_products_url_fn)