From 840cc1c51ee4208bf04520a3916b39a086a04557 Mon Sep 17 00:00:00 2001 From: snacktavish Date: Tue, 9 May 2023 12:04:38 -0700 Subject: [PATCH 01/33] syntax error fix --- phylesystem_api/phylesystem_api/api_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phylesystem_api/phylesystem_api/api_utils.py b/phylesystem_api/phylesystem_api/api_utils.py index fda76e7d..15b5819c 100644 --- a/phylesystem_api/phylesystem_api/api_utils.py +++ b/phylesystem_api/phylesystem_api/api_utils.py @@ -669,7 +669,7 @@ def find_in_request(request, property_name, default_value=None, return_all_value # N.B. HTML comments are stripped by default. Non-allowed tags will appear # "naked" in output, so we can identify any bad actors. allowed_curation_comment_tags = ['p', 'br', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'pre', 'code'] # any others? -ot_markdown_tags = list(set( bleach.sanitizer.ALLOWED_TAGS + allowed_curation_comment_tags)) +ot_markdown_tags = list(set(bleach.sanitizer.ALLOWED_TAGS)) + list(set(allowed_curation_comment_tags)) # allow hyperlinks with target="_blank" ot_markdown_attributes = {} ot_markdown_attributes.update(bleach.sanitizer.ALLOWED_ATTRIBUTES) From c0e25a978c4e5c69e60fd93f670b460f387869c1 Mon Sep 17 00:00:00 2001 From: snacktavish Date: Tue, 23 May 2023 12:17:04 -0700 Subject: [PATCH 02/33] test hardcoded caching fix --- phylesystem_api/phylesystem_api/views/default.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py index c1ad9e24..6a63bfb3 100644 --- a/phylesystem_api/phylesystem_api/views/default.py +++ b/phylesystem_api/phylesystem_api/views/default.py @@ -84,7 +84,11 @@ def fetch_and_cache(url): #import pdb; pdb.set_trace() root_relative_url = "/{}".format(url) _LOG.warn(">> root_relative_url: {}".format(root_relative_url)) - fetch_url = request.relative_url(root_relative_url) +# fetch_url = request.relative_url(root_relative_url) + ## NEEDS correct domain.... + config_dict = phylesystem.get_configuration_dict() + _LOG.warn(config_dict) + fetch_url = "https://devapi.opentreeoflife.org" + root_relative_url _LOG.warn("NOT CACHED, FETCHING THIS URL: {}".format(fetch_url)) _LOG.warn(" request.method = {}".format(request.method)) try: From 2206459cfe541c6bbc1f3a02528e3a1db3fb33cb Mon Sep 17 00:00:00 2001 From: snacktavish Date: Tue, 23 May 2023 12:32:46 -0700 Subject: [PATCH 03/33] instantiate phylesystem --- phylesystem_api/phylesystem_api/views/default.py | 1 + 1 file changed, 1 insertion(+) diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py index 6a63bfb3..15d164d3 100644 --- a/phylesystem_api/phylesystem_api/views/default.py +++ b/phylesystem_api/phylesystem_api/views/default.py @@ -86,6 +86,7 @@ def fetch_and_cache(url): _LOG.warn(">> root_relative_url: {}".format(root_relative_url)) # fetch_url = request.relative_url(root_relative_url) ## NEEDS correct domain.... + phylesystem = api_utils.get_phylesystem(request) config_dict = phylesystem.get_configuration_dict() _LOG.warn(config_dict) fetch_url = "https://devapi.opentreeoflife.org" + root_relative_url From cf7fff8c5fe4b95d92c8014452b31ea094f3e4cc Mon Sep 17 00:00:00 2001 From: snacktavish Date: Tue, 23 May 2023 13:30:35 -0700 Subject: [PATCH 04/33] try keepalive-no in requests --- phylesystem_api/phylesystem_api/views/default.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py index 15d164d3..2c1ccdcd 100644 --- a/phylesystem_api/phylesystem_api/views/default.py +++ b/phylesystem_api/phylesystem_api/views/default.py @@ -85,13 +85,12 @@ def fetch_and_cache(url): root_relative_url = "/{}".format(url) _LOG.warn(">> root_relative_url: {}".format(root_relative_url)) # fetch_url = request.relative_url(root_relative_url) - ## NEEDS correct domain.... - phylesystem = api_utils.get_phylesystem(request) - config_dict = phylesystem.get_configuration_dict() _LOG.warn(config_dict) fetch_url = "https://devapi.opentreeoflife.org" + root_relative_url _LOG.warn("NOT CACHED, FETCHING THIS URL: {}".format(fetch_url)) _LOG.warn(" request.method = {}".format(request.method)) + s = requests.session() + s.config['keep_alive'] = False try: if request.method == 'POST': # assume a typical API request with JSON payload From d5ac28a56211f510918b6a5efe3fea8e6c324b3a Mon Sep 17 00:00:00 2001 From: snacktavish Date: Tue, 23 May 2023 13:34:08 -0700 Subject: [PATCH 05/33] remove bad log statement --- phylesystem_api/phylesystem_api/views/default.py | 1 - 1 file changed, 1 deletion(-) diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py index 2c1ccdcd..9069e828 100644 --- a/phylesystem_api/phylesystem_api/views/default.py +++ b/phylesystem_api/phylesystem_api/views/default.py @@ -85,7 +85,6 @@ def fetch_and_cache(url): root_relative_url = "/{}".format(url) _LOG.warn(">> root_relative_url: {}".format(root_relative_url)) # fetch_url = request.relative_url(root_relative_url) - _LOG.warn(config_dict) fetch_url = "https://devapi.opentreeoflife.org" + root_relative_url _LOG.warn("NOT CACHED, FETCHING THIS URL: {}".format(fetch_url)) _LOG.warn(" request.method = {}".format(request.method)) From 7a139af0af1ff1ec39303be610d22970f1ef15d1 Mon Sep 17 00:00:00 2001 From: snacktavish Date: Tue, 23 May 2023 13:37:14 -0700 Subject: [PATCH 06/33] remove unsuccesful attempt to not keep alive via requests --- phylesystem_api/phylesystem_api/views/default.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py index 9069e828..1a53a06a 100644 --- a/phylesystem_api/phylesystem_api/views/default.py +++ b/phylesystem_api/phylesystem_api/views/default.py @@ -88,8 +88,6 @@ def fetch_and_cache(url): fetch_url = "https://devapi.opentreeoflife.org" + root_relative_url _LOG.warn("NOT CACHED, FETCHING THIS URL: {}".format(fetch_url)) _LOG.warn(" request.method = {}".format(request.method)) - s = requests.session() - s.config['keep_alive'] = False try: if request.method == 'POST': # assume a typical API request with JSON payload From 9bce36a27c842a5a15f831712d0673f861f107be Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Wed, 24 May 2023 00:45:02 -0400 Subject: [PATCH 07/33] Update example config with newer expected vars Still need to sweep the code for possible uses of these. --- api.config.example | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/api.config.example b/api.config.example index ac7181fc..3a476e0e 100644 --- a/api.config.example +++ b/api.config.example @@ -1,6 +1,9 @@ # Rename this file to "api.config" to make it active [apis] +api_version = 3 +default_apis_base_url = {{ apis_common_base_URL }} +production_apis_base_url = {{ apis_production_base_URL }} # if true, blocks all writing behavior read_only = READ_ONLY_MODE @@ -37,11 +40,12 @@ following_repo_remote = FOLLOWING_REPO_REMOTE git_ssh = GIT_SSH pkey = PKEY -oti_base_url = OTI_BASE_URL -otindex_base_url = OTINDEX_BASE_URL -collections_api_base_url = COLLECTIONS_API_BASE_URL -amendments_api_base_url = AMENDMENTS_API_BASE_URL -favorites_api_base_url = FAVORITES_API_BASE_URL +# deprecate these overly-specific base URLs? or copy same value to all? +##oti_base_url = OTI_BASE_URL +##otindex_base_url = OTINDEX_BASE_URL +##collections_api_base_url = COLLECTIONS_API_BASE_URL +##amendments_api_base_url = AMENDMENTS_API_BASE_URL +##favorites_api_base_url = FAVORITES_API_BASE_URL opentree_docstore_url = OPENTREE_DOCSTORE_URL # Push scripts will substitute the OTI_BASE_URL server-config variables, shared From f6cbf29280e577488f1e8bfe0bb40d23c1032b81 Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Wed, 24 May 2023 00:46:13 -0400 Subject: [PATCH 08/33] Test some approaches to modified Connection header When fetching stuff for phylesystemapi's RAM cache, so far we have simply repeated all request headers in the proxied request. But our Pyramid implementation will not support Keep-Alive or some other headers because we're running it under a WSGI server (waitress). --- phylesystem_api/phylesystem_api/views/default.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py index 15d164d3..20585932 100644 --- a/phylesystem_api/phylesystem_api/views/default.py +++ b/phylesystem_api/phylesystem_api/views/default.py @@ -92,6 +92,13 @@ def fetch_and_cache(url): fetch_url = "https://devapi.opentreeoflife.org" + root_relative_url _LOG.warn("NOT CACHED, FETCHING THIS URL: {}".format(fetch_url)) _LOG.warn(" request.method = {}".format(request.method)) + _LOG.warn(" STARTING request Connection header: {}".format( request.headers.get('Connection', ''))) + proxy_safe_headers = request.headers.set('Connection', 'Close') + _LOG.warn(" MODIFIED request Connection header: {}".format( request.headers.get('Connection', ''))) + _LOG.warn(" proxy_safe_headers:") + _LOG.warn( proxy_safe_headers ) + _LOG.warn(" proxy_safe_headers Connection header: {}".format( proxy_safe_headers.get('Connection', ''))) + try: if request.method == 'POST': # assume a typical API request with JSON payload From 2995bba1de6abc68bb41bb3c8e5b9b91eb4a303c Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Wed, 24 May 2023 01:11:20 -0400 Subject: [PATCH 09/33] Revise testing of modified headers --- phylesystem_api/phylesystem_api/views/default.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py index b73dc4bb..e8105e86 100644 --- a/phylesystem_api/phylesystem_api/views/default.py +++ b/phylesystem_api/phylesystem_api/views/default.py @@ -89,11 +89,12 @@ def fetch_and_cache(url): _LOG.warn("NOT CACHED, FETCHING THIS URL: {}".format(fetch_url)) _LOG.warn(" request.method = {}".format(request.method)) _LOG.warn(" STARTING request Connection header: {}".format( request.headers.get('Connection', ''))) - proxy_safe_headers = request.headers.set('Connection', 'Close') + #proxy_safe_headers = ??? + request.headers.setdefault('Connection', 'Close') _LOG.warn(" MODIFIED request Connection header: {}".format( request.headers.get('Connection', ''))) - _LOG.warn(" proxy_safe_headers:") - _LOG.warn( proxy_safe_headers ) - _LOG.warn(" proxy_safe_headers Connection header: {}".format( proxy_safe_headers.get('Connection', ''))) + #_LOG.warn(" proxy_safe_headers:") + #_LOG.warn( proxy_safe_headers ) + #_LOG.warn(" proxy_safe_headers Connection header: {}".format( proxy_safe_headers.get('Connection', ''))) try: if request.method == 'POST': From b3bbbe279f9a3f40d2109add735ec62156980d33 Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Wed, 24 May 2023 14:11:30 -0400 Subject: [PATCH 10/33] Simpler fix to Connection header --- phylesystem_api/phylesystem_api/views/default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py index e8105e86..b24863f3 100644 --- a/phylesystem_api/phylesystem_api/views/default.py +++ b/phylesystem_api/phylesystem_api/views/default.py @@ -90,7 +90,7 @@ def fetch_and_cache(url): _LOG.warn(" request.method = {}".format(request.method)) _LOG.warn(" STARTING request Connection header: {}".format( request.headers.get('Connection', ''))) #proxy_safe_headers = ??? - request.headers.setdefault('Connection', 'Close') + request.headers['Connection'] = 'Close' _LOG.warn(" MODIFIED request Connection header: {}".format( request.headers.get('Connection', ''))) #_LOG.warn(" proxy_safe_headers:") #_LOG.warn( proxy_safe_headers ) From 2bd29b4c148213d0540e83e1f206376081c6a4d3 Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Wed, 24 May 2023 14:18:21 -0400 Subject: [PATCH 11/33] Show all headers (something's weird) --- phylesystem_api/phylesystem_api/views/default.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py index b24863f3..3743a6e3 100644 --- a/phylesystem_api/phylesystem_api/views/default.py +++ b/phylesystem_api/phylesystem_api/views/default.py @@ -88,6 +88,8 @@ def fetch_and_cache(url): fetch_url = "https://devapi.opentreeoflife.org" + root_relative_url _LOG.warn("NOT CACHED, FETCHING THIS URL: {}".format(fetch_url)) _LOG.warn(" request.method = {}".format(request.method)) + _LOG.warn(" request.headers:") + _LOG.warn( request.headers ) _LOG.warn(" STARTING request Connection header: {}".format( request.headers.get('Connection', ''))) #proxy_safe_headers = ??? request.headers['Connection'] = 'Close' From db88ff878fd0da0f1682799117287ed952b93809 Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Wed, 24 May 2023 14:28:06 -0400 Subject: [PATCH 12/33] Argh, trying again for a full header listing --- phylesystem_api/phylesystem_api/views/default.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py index 3743a6e3..8275c0e6 100644 --- a/phylesystem_api/phylesystem_api/views/default.py +++ b/phylesystem_api/phylesystem_api/views/default.py @@ -88,8 +88,10 @@ def fetch_and_cache(url): fetch_url = "https://devapi.opentreeoflife.org" + root_relative_url _LOG.warn("NOT CACHED, FETCHING THIS URL: {}".format(fetch_url)) _LOG.warn(" request.method = {}".format(request.method)) - _LOG.warn(" request.headers:") - _LOG.warn( request.headers ) + _LOG.warn(" request.headers.items() :") + _LOG.warn( request.headers.items() ) + _LOG.warn(" dict(request.headers) :") + _LOG.warn( dict(request.headers) ) _LOG.warn(" STARTING request Connection header: {}".format( request.headers.get('Connection', ''))) #proxy_safe_headers = ??? request.headers['Connection'] = 'Close' From 5d93b2764c51aa1ecadea099ee12eec42408fcfe Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Wed, 24 May 2023 14:44:40 -0400 Subject: [PATCH 13/33] One more time, trying to remove Connection header --- .../phylesystem_api/views/default.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py index 8275c0e6..8f022262 100644 --- a/phylesystem_api/phylesystem_api/views/default.py +++ b/phylesystem_api/phylesystem_api/views/default.py @@ -88,17 +88,14 @@ def fetch_and_cache(url): fetch_url = "https://devapi.opentreeoflife.org" + root_relative_url _LOG.warn("NOT CACHED, FETCHING THIS URL: {}".format(fetch_url)) _LOG.warn(" request.method = {}".format(request.method)) - _LOG.warn(" request.headers.items() :") - _LOG.warn( request.headers.items() ) - _LOG.warn(" dict(request.headers) :") + + # modify or discard "hop-by-hop" headers + _LOG.warn(" STARTING request.headers:") + _LOG.warn( dict(request.headers) ) + _LOG.warn(" STARTING request headers: {}".format( request.headers.get('Connection', ''))) + request.headers.pop('Connection') + _LOG.warn(" MODIFIED request.headers:") _LOG.warn( dict(request.headers) ) - _LOG.warn(" STARTING request Connection header: {}".format( request.headers.get('Connection', ''))) - #proxy_safe_headers = ??? - request.headers['Connection'] = 'Close' - _LOG.warn(" MODIFIED request Connection header: {}".format( request.headers.get('Connection', ''))) - #_LOG.warn(" proxy_safe_headers:") - #_LOG.warn( proxy_safe_headers ) - #_LOG.warn(" proxy_safe_headers Connection header: {}".format( proxy_safe_headers.get('Connection', ''))) try: if request.method == 'POST': From e2e25fe5853f4ac203cb6a0ad8a6c94419012002 Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Wed, 24 May 2023 15:01:21 -0400 Subject: [PATCH 14/33] More test chatter --- phylesystem_api/phylesystem_api/views/default.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py index 8f022262..f2b86f5b 100644 --- a/phylesystem_api/phylesystem_api/views/default.py +++ b/phylesystem_api/phylesystem_api/views/default.py @@ -94,6 +94,7 @@ def fetch_and_cache(url): _LOG.warn( dict(request.headers) ) _LOG.warn(" STARTING request headers: {}".format( request.headers.get('Connection', ''))) request.headers.pop('Connection') + request.headers.pop('Keep-Alive') _LOG.warn(" MODIFIED request.headers:") _LOG.warn( dict(request.headers) ) @@ -102,6 +103,7 @@ def fetch_and_cache(url): # assume a typical API request with JSON payload # (pass this along unchanged) _LOG.warn(" treating as POST") + _LOG.warn(" headers: {}".format(request.headers)) fetched = requests.post(url=fetch_url, data=request.body, headers=request.headers) @@ -117,9 +119,11 @@ def fetch_and_cache(url): # TODO: For more flexibility, we might examine and mimic the original request (headers, etc) _LOG.warn("... and now we're back with fetched, which is a {}".format( type(fetched) )) fetched.raise_for_status() + _LOG.warn("... AFTER fetched.raise_for_status()") fetched.encoding = 'utf-8' # Optional: requests infers this internally try: test_for_json = fetched.json() # missing JSON payload will raise an error + _LOG.warn("... AFTER test_for_json") return Response( headers=fetched.headers, body=fetched.text, # missing JSON payload will raise an error @@ -127,6 +131,7 @@ def fetch_and_cache(url): charset='UTF-8', content_type='application/json') except requests.exceptions.JSONDecodeError: + _LOG.warn("... ERROR in test_for_json") return Response( headers=fetched.headers, body=response.text, From d28f5df0b05c2de1251d3e6ede0b42310eeac4fe Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Wed, 24 May 2023 15:04:12 -0400 Subject: [PATCH 15/33] Remove RESPONSE hop-by-hop headers --- phylesystem_api/phylesystem_api/views/default.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py index f2b86f5b..7e4e728e 100644 --- a/phylesystem_api/phylesystem_api/views/default.py +++ b/phylesystem_api/phylesystem_api/views/default.py @@ -121,6 +121,15 @@ def fetch_and_cache(url): fetched.raise_for_status() _LOG.warn("... AFTER fetched.raise_for_status()") fetched.encoding = 'utf-8' # Optional: requests infers this internally + + # modify or discard "hop-by-hop" headers + _LOG.warn(" STARTING fetched.headers:") + _LOG.warn( dict(fetched.headers) ) + fetched.headers.pop('Connection') + fetched.headers.pop('Keep-Alive') + _LOG.warn(" MODIFIED fetched.headers:") + _LOG.warn( dict(fetched.headers) ) + try: test_for_json = fetched.json() # missing JSON payload will raise an error _LOG.warn("... AFTER test_for_json") From a70ecfe245e5bdf4e813fed4dd2b97cf433a7989 Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Wed, 24 May 2023 15:36:07 -0400 Subject: [PATCH 16/33] Safer removal of optional keys --- phylesystem_api/phylesystem_api/views/default.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py index 7e4e728e..6d3841e7 100644 --- a/phylesystem_api/phylesystem_api/views/default.py +++ b/phylesystem_api/phylesystem_api/views/default.py @@ -93,8 +93,8 @@ def fetch_and_cache(url): _LOG.warn(" STARTING request.headers:") _LOG.warn( dict(request.headers) ) _LOG.warn(" STARTING request headers: {}".format( request.headers.get('Connection', ''))) - request.headers.pop('Connection') - request.headers.pop('Keep-Alive') + request.headers.pop('Connection', None) + request.headers.pop('Keep-Alive', None) _LOG.warn(" MODIFIED request.headers:") _LOG.warn( dict(request.headers) ) @@ -125,8 +125,8 @@ def fetch_and_cache(url): # modify or discard "hop-by-hop" headers _LOG.warn(" STARTING fetched.headers:") _LOG.warn( dict(fetched.headers) ) - fetched.headers.pop('Connection') - fetched.headers.pop('Keep-Alive') + fetched.headers.pop('Connection', None) + fetched.headers.pop('Keep-Alive', None) _LOG.warn(" MODIFIED fetched.headers:") _LOG.warn( dict(fetched.headers) ) From f0412e106cbe7870005be038d4f985395c60ff25 Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Wed, 24 May 2023 15:54:23 -0400 Subject: [PATCH 17/33] Clean up code, add more hop-by-hop headers --- .../phylesystem_api/views/default.py | 41 ++++++++++++------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py index 6d3841e7..798388f7 100644 --- a/phylesystem_api/phylesystem_api/views/default.py +++ b/phylesystem_api/phylesystem_api/views/default.py @@ -90,13 +90,19 @@ def fetch_and_cache(url): _LOG.warn(" request.method = {}".format(request.method)) # modify or discard "hop-by-hop" headers - _LOG.warn(" STARTING request.headers:") - _LOG.warn( dict(request.headers) ) - _LOG.warn(" STARTING request headers: {}".format( request.headers.get('Connection', ''))) - request.headers.pop('Connection', None) - request.headers.pop('Keep-Alive', None) - _LOG.warn(" MODIFIED request.headers:") - _LOG.warn( dict(request.headers) ) + hop_by_hop_headers = ['Keep-Alive', + 'Transfer-Encoding', + 'TE', + 'Connection', + 'Trailer', + 'Upgrade', + 'Proxy-Authorization', + 'Proxy-Authenticate', + ] + for bad_header in hop_by_hop_headers: + request.headers.pop(bad_header, None) + #_LOG.warn(" MODIFIED request.headers:") + #_LOG.warn( dict(request.headers) ) try: if request.method == 'POST': @@ -123,16 +129,22 @@ def fetch_and_cache(url): fetched.encoding = 'utf-8' # Optional: requests infers this internally # modify or discard "hop-by-hop" headers - _LOG.warn(" STARTING fetched.headers:") - _LOG.warn( dict(fetched.headers) ) - fetched.headers.pop('Connection', None) - fetched.headers.pop('Keep-Alive', None) - _LOG.warn(" MODIFIED fetched.headers:") - _LOG.warn( dict(fetched.headers) ) + hop_by_hop_headers = ['Keep-Alive', + 'Transfer-Encoding', + 'TE', + 'Connection', + 'Trailer', + 'Upgrade', + 'Proxy-Authorization', + 'Proxy-Authenticate', + ] + for bad_header in hop_by_hop_headers: + fetched.headers.pop(bad_header, None) + #_LOG.warn(" MODIFIED fetched.headers:") + #_LOG.warn( dict(fetched.headers) ) try: test_for_json = fetched.json() # missing JSON payload will raise an error - _LOG.warn("... AFTER test_for_json") return Response( headers=fetched.headers, body=fetched.text, # missing JSON payload will raise an error @@ -140,7 +152,6 @@ def fetch_and_cache(url): charset='UTF-8', content_type='application/json') except requests.exceptions.JSONDecodeError: - _LOG.warn("... ERROR in test_for_json") return Response( headers=fetched.headers, body=response.text, From d5ed564125f992048704de53615ef0f8d1b568ed Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Wed, 24 May 2023 20:41:25 -0400 Subject: [PATCH 18/33] Cleanup and consolidate hop-by-hop definitions --- .../phylesystem_api/views/default.py | 30 +++++++------------ 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py index 798388f7..b6a9f558 100644 --- a/phylesystem_api/phylesystem_api/views/default.py +++ b/phylesystem_api/phylesystem_api/views/default.py @@ -78,6 +78,17 @@ def pull_through_cache(request): target_url = request.matchdict.get('target_url') _LOG.warn(">> target_url: {}".format(target_url)) + # Some headers should not be used when adding to our RAM cache + hop_by_hop_headers = ['Keep-Alive', + 'Transfer-Encoding', + 'TE', + 'Connection', + 'Trailer', + 'Upgrade', + 'Proxy-Authorization', + 'Proxy-Authenticate', + ] + @cache_region('short_term', 'pull-through') def fetch_and_cache(url): # let's restrict this to URLs on this api server, to avoid shenanigans @@ -90,15 +101,6 @@ def fetch_and_cache(url): _LOG.warn(" request.method = {}".format(request.method)) # modify or discard "hop-by-hop" headers - hop_by_hop_headers = ['Keep-Alive', - 'Transfer-Encoding', - 'TE', - 'Connection', - 'Trailer', - 'Upgrade', - 'Proxy-Authorization', - 'Proxy-Authenticate', - ] for bad_header in hop_by_hop_headers: request.headers.pop(bad_header, None) #_LOG.warn(" MODIFIED request.headers:") @@ -125,19 +127,9 @@ def fetch_and_cache(url): # TODO: For more flexibility, we might examine and mimic the original request (headers, etc) _LOG.warn("... and now we're back with fetched, which is a {}".format( type(fetched) )) fetched.raise_for_status() - _LOG.warn("... AFTER fetched.raise_for_status()") fetched.encoding = 'utf-8' # Optional: requests infers this internally # modify or discard "hop-by-hop" headers - hop_by_hop_headers = ['Keep-Alive', - 'Transfer-Encoding', - 'TE', - 'Connection', - 'Trailer', - 'Upgrade', - 'Proxy-Authorization', - 'Proxy-Authenticate', - ] for bad_header in hop_by_hop_headers: fetched.headers.pop(bad_header, None) #_LOG.warn(" MODIFIED fetched.headers:") From 6b9a6ca8aa018503cc4c628b3562d97bdc504329 Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Tue, 27 Jun 2023 03:31:11 -0400 Subject: [PATCH 19/33] Add POST payload (request body) to our cache keys! This should address the problem of getting bogus cached values when the URL is an exact match (useless for POST requests). --- phylesystem_api/phylesystem_api/views/default.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py index a4f73533..1ffea8b3 100644 --- a/phylesystem_api/phylesystem_api/views/default.py +++ b/phylesystem_api/phylesystem_api/views/default.py @@ -77,8 +77,12 @@ def pull_through_cache(request): """ # _LOG = api_utils.get_logger(request, 'ot_api') api_utils.raise_on_CORS_preflight(request) + + # gather any request elements used to build a unique cache key target_url = request.matchdict.get('target_url') _LOG.warn(">> target_url: {}".format(target_url)) + post_payload = request.body + _LOG.warn(">> post_payload: {}".format(post_payload)) # Some headers should not be used when adding to our RAM cache hop_by_hop_headers = ['Keep-Alive', @@ -92,7 +96,7 @@ def pull_through_cache(request): ] @cache_region('short_term', 'pull-through') - def fetch_and_cache(url): + def fetch_and_cache(url, post_payload): # let's restrict this to URLs on this api server, to avoid shenanigans #import pdb; pdb.set_trace() root_relative_url = "/{}".format(url) @@ -162,7 +166,7 @@ def fetch_and_cache(url): raise HTTPBadRequest(body='Unknown exception in cached call!') _LOG.warn("...trying to fetch-and-cache...") - return fetch_and_cache(target_url) + return fetch_and_cache(target_url, post_payload) @view_config(route_name='render_markdown') From 5fdd74ff73d9e355f8c29c9637276a31e035edfc Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Tue, 27 Jun 2023 12:15:57 -0400 Subject: [PATCH 20/33] Emulate stronger cache keys from legacy web2py This should eliminate goofy cache hits that return, e.g., a synth-subtree from a prior/different request. --- phylesystem_api/phylesystem_api/views/default.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py index 1ffea8b3..078c9ddd 100644 --- a/phylesystem_api/phylesystem_api/views/default.py +++ b/phylesystem_api/phylesystem_api/views/default.py @@ -81,8 +81,6 @@ def pull_through_cache(request): # gather any request elements used to build a unique cache key target_url = request.matchdict.get('target_url') _LOG.warn(">> target_url: {}".format(target_url)) - post_payload = request.body - _LOG.warn(">> post_payload: {}".format(post_payload)) # Some headers should not be used when adding to our RAM cache hop_by_hop_headers = ['Keep-Alive', @@ -95,8 +93,10 @@ def pull_through_cache(request): 'Proxy-Authenticate', ] - @cache_region('short_term', 'pull-through') - def fetch_and_cache(url, post_payload): + # Create a unique cache key with the URL and any vars (GET *and* POST) to its "query string" + # ALSO include the request method (HTTP verb) to respond to OPTIONS requests + @cache_region('short_term', "cached:{}:{}:{}".format(request.method, target_url, request.body)) + def fetch_and_cache(url): # let's restrict this to URLs on this api server, to avoid shenanigans #import pdb; pdb.set_trace() root_relative_url = "/{}".format(url) @@ -166,7 +166,7 @@ def fetch_and_cache(url, post_payload): raise HTTPBadRequest(body='Unknown exception in cached call!') _LOG.warn("...trying to fetch-and-cache...") - return fetch_and_cache(target_url, post_payload) + return fetch_and_cache(target_url) @view_config(route_name='render_markdown') From 5c7220b66e77497f4bffef09d57fe250a24efddd Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Tue, 27 Jun 2023 12:51:17 -0400 Subject: [PATCH 21/33] Log unique cache keys for review --- phylesystem_api/phylesystem_api/views/default.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py index 078c9ddd..d4ec38dc 100644 --- a/phylesystem_api/phylesystem_api/views/default.py +++ b/phylesystem_api/phylesystem_api/views/default.py @@ -67,6 +67,15 @@ def base_API_view(request): "source_url": "https://github.com/OpenTreeOfLife/phylesystem-api" } + + +# Create a unique cache key with the URL and any vars (GET *and* POST) to its "query string" +# ALSO include the request method (HTTP verb) to respond to OPTIONS requests +def create_unique_cache_key(target_url, request): + unique_key = "cached:{}:{}:{}".format(request.method, target_url, request.body) + _LOG.warn(">> unique cache key: {}".format(unique_key)) + return unique_key + @view_config(route_name='pull_through_cache') def pull_through_cache(request): """ @@ -93,9 +102,7 @@ def pull_through_cache(request): 'Proxy-Authenticate', ] - # Create a unique cache key with the URL and any vars (GET *and* POST) to its "query string" - # ALSO include the request method (HTTP verb) to respond to OPTIONS requests - @cache_region('short_term', "cached:{}:{}:{}".format(request.method, target_url, request.body)) + @cache_region('short_term', create_unique_cache_key(target_url, request)) def fetch_and_cache(url): # let's restrict this to URLs on this api server, to avoid shenanigans #import pdb; pdb.set_trace() From b05a69729117925a2510fc38880eaf28906305cb Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Tue, 27 Jun 2023 13:01:54 -0400 Subject: [PATCH 22/33] Encode request body for simpler cache keys --- phylesystem_api/phylesystem_api/views/default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py index d4ec38dc..09944b8b 100644 --- a/phylesystem_api/phylesystem_api/views/default.py +++ b/phylesystem_api/phylesystem_api/views/default.py @@ -72,7 +72,7 @@ def base_API_view(request): # Create a unique cache key with the URL and any vars (GET *and* POST) to its "query string" # ALSO include the request method (HTTP verb) to respond to OPTIONS requests def create_unique_cache_key(target_url, request): - unique_key = "cached:{}:{}:{}".format(request.method, target_url, request.body) + unique_key = "cached:{}:{}:{}".format(request.method, target_url, request.body.decode('utf-8')) _LOG.warn(">> unique cache key: {}".format(unique_key)) return unique_key From a1ba35c31ab6b238b787f314c7e87987cc5db30c Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Tue, 27 Jun 2023 13:13:52 -0400 Subject: [PATCH 23/33] Remove log chatter --- phylesystem_api/phylesystem_api/views/default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py index 09944b8b..4a98a945 100644 --- a/phylesystem_api/phylesystem_api/views/default.py +++ b/phylesystem_api/phylesystem_api/views/default.py @@ -73,7 +73,7 @@ def base_API_view(request): # ALSO include the request method (HTTP verb) to respond to OPTIONS requests def create_unique_cache_key(target_url, request): unique_key = "cached:{}:{}:{}".format(request.method, target_url, request.body.decode('utf-8')) - _LOG.warn(">> unique cache key: {}".format(unique_key)) + #_LOG.warn(">> unique cache key: {}".format(unique_key)) return unique_key @view_config(route_name='pull_through_cache') From a8a0c74b99efd6421703349c3f62813023f1ec0d Mon Sep 17 00:00:00 2001 From: snacktavish Date: Fri, 7 Jul 2023 11:04:51 -0700 Subject: [PATCH 24/33] get base_url from config --- phylesystem_api/phylesystem_api/views/default.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py index 4a98a945..b6546755 100644 --- a/phylesystem_api/phylesystem_api/views/default.py +++ b/phylesystem_api/phylesystem_api/views/default.py @@ -108,8 +108,9 @@ def fetch_and_cache(url): #import pdb; pdb.set_trace() root_relative_url = "/{}".format(url) _LOG.warn(">> root_relative_url: {}".format(root_relative_url)) -# fetch_url = request.relative_url(root_relative_url) - fetch_url = "https://devapi.opentreeoflife.org" + root_relative_url + conf_obj = api_utils.get_conf_object + base_url = conf.get("apis", "default_apis_base_url") + fetch_url = base_url + root_relative_url _LOG.warn("NOT CACHED, FETCHING THIS URL: {}".format(fetch_url)) _LOG.warn(" request.method = {}".format(request.method)) From 917d3ba11fd9ca9338d35f8e22fc718fd2d8ac5e Mon Sep 17 00:00:00 2001 From: snacktavish Date: Fri, 7 Jul 2023 11:07:37 -0700 Subject: [PATCH 25/33] fix config syntax --- phylesystem_api/phylesystem_api/views/default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py index b6546755..e1fa7faa 100644 --- a/phylesystem_api/phylesystem_api/views/default.py +++ b/phylesystem_api/phylesystem_api/views/default.py @@ -108,7 +108,7 @@ def fetch_and_cache(url): #import pdb; pdb.set_trace() root_relative_url = "/{}".format(url) _LOG.warn(">> root_relative_url: {}".format(root_relative_url)) - conf_obj = api_utils.get_conf_object + conf = api_utils.get_conf_object(request) base_url = conf.get("apis", "default_apis_base_url") fetch_url = base_url + root_relative_url _LOG.warn("NOT CACHED, FETCHING THIS URL: {}".format(fetch_url)) From 0a110c20a65288084673803890cd129cb7b36dee Mon Sep 17 00:00:00 2001 From: snacktavish Date: Tue, 11 Jul 2023 10:17:12 -0700 Subject: [PATCH 26/33] git failure log still not working --- phylesystem_api/phylesystem_api/api_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phylesystem_api/phylesystem_api/api_utils.py b/phylesystem_api/phylesystem_api/api_utils.py index 15b5819c..a6e04368 100644 --- a/phylesystem_api/phylesystem_api/api_utils.py +++ b/phylesystem_api/phylesystem_api/api_utils.py @@ -43,7 +43,7 @@ READ_ONLY_MODE = True def get_private_dir(request): - _LOG.debug("WHY PROVATE DIR") + _LOG.debug("WHY PRIVATE DIR") return "~/private/" def atomic_write_json_if_not_found(obj, dest, request): From a07d554db739f2646e8572269692239fb428eae7 Mon Sep 17 00:00:00 2001 From: snacktavish Date: Tue, 11 Jul 2023 11:35:55 -0700 Subject: [PATCH 27/33] attempt to fix trailing slash on get --- phylesystem_api/phylesystem_api/routes.py | 4 +++- phylesystem_api/phylesystem_api/views/study.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/phylesystem_api/phylesystem_api/routes.py b/phylesystem_api/phylesystem_api/routes.py index 2ccc0326..62d77907 100644 --- a/phylesystem_api/phylesystem_api/routes.py +++ b/phylesystem_api/phylesystem_api/routes.py @@ -15,7 +15,6 @@ def includeme(config): config.add_route('pull_through_cache', '/cached/{target_url:.*}') config.add_route('merge_docstore_changes', '/{api_version}/merge_docstore_changes/{doc_id}/{starting_commit_SHA}') config.add_route('push_docstore_changes', '/{api_version}/push_docstore_changes/{doc_type}/{doc_id}') - config.add_route('push_docstore_changes_bare', '/{api_version}/push_docstore_changes', request_method='PUT') # # STUDY/TREE ROUTES @@ -29,12 +28,15 @@ def includeme(config): config.add_route('create_study', '/{api_version}/study/') config.add_route('study_CORS_preflight', '/{api_version}/study/{study_id}', request_method='OPTIONS') config.add_route('fetch_study', '/{api_version}/study/{study_id}', request_method='GET') + config.add_route('fetch_study_label', '/{api_version}/study/{study_id}/', request_method='GET') config.add_route('update_study', '/{api_version}/study/{study_id}', request_method='PUT') config.add_route('delete_study', '/{api_version}/study/{study_id}', request_method='DELETE') config.add_route('get_study_file_list', '/{api_version}/study/{study_id}/file') config.add_route('get_study_single_file', '/{api_version}/study/{study_id}/file/{file_id}') config.add_route('get_study_external_url', '/{api_version}/study/external_url/{study_id}') config.add_route('get_study_tree', '/{api_version}/study/{study_id}/tree/{tree_id_with_extension}') + config.add_route('get_study_tree_label', '/{api_version}/study/{study_id}/tree/{tree_id_with_extension}/') + # # TREE COLLECTION ROUTES # diff --git a/phylesystem_api/phylesystem_api/views/study.py b/phylesystem_api/phylesystem_api/views/study.py index c8294a34..704b4cb0 100644 --- a/phylesystem_api/phylesystem_api/views/study.py +++ b/phylesystem_api/phylesystem_api/views/study.py @@ -162,7 +162,7 @@ def __finish_write_verb(phylesystem, - +@view_config(route_name='fetch_study_label', renderer=None) ## IS this allowed??? @view_config(route_name='fetch_study', renderer=None) def fetch_study(request): repo_parent, repo_remote, git_ssh, pkey, git_hub_remote, max_filesize, max_num_trees, read_only_mode = api_utils.read_phylesystem_config(request) @@ -693,6 +693,7 @@ def get_study_external_url(request): raise HTTPNotFound(body='{"error": 1, "description": "study not found"}') @view_config(route_name='get_study_tree', renderer='json') +@view_config(route_name='get_study_tree_label', renderer='json') def get_study_tree(request): api_utils.raise_on_CORS_preflight(request) From 7f83503e9cceb9a79c9bbdeadb264d3694246c21 Mon Sep 17 00:00:00 2001 From: snacktavish Date: Tue, 11 Jul 2023 11:42:10 -0700 Subject: [PATCH 28/33] restore accidentally deleted route --- phylesystem_api/phylesystem_api/routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phylesystem_api/phylesystem_api/routes.py b/phylesystem_api/phylesystem_api/routes.py index 62d77907..918c59cf 100644 --- a/phylesystem_api/phylesystem_api/routes.py +++ b/phylesystem_api/phylesystem_api/routes.py @@ -15,7 +15,7 @@ def includeme(config): config.add_route('pull_through_cache', '/cached/{target_url:.*}') config.add_route('merge_docstore_changes', '/{api_version}/merge_docstore_changes/{doc_id}/{starting_commit_SHA}') config.add_route('push_docstore_changes', '/{api_version}/push_docstore_changes/{doc_type}/{doc_id}') - + config.add_route('push_docstore_changes_bare', '/{api_version}/push_docstore_changes', request_method='PUT') # # STUDY/TREE ROUTES # From 6fd64f4d72c357d6615b438d78b756005b42fa0a Mon Sep 17 00:00:00 2001 From: snacktavish Date: Tue, 11 Jul 2023 11:55:08 -0700 Subject: [PATCH 29/33] optional trailing slash on all study fetch --- phylesystem_api/phylesystem_api/routes.py | 3 +-- phylesystem_api/phylesystem_api/views/study.py | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/phylesystem_api/phylesystem_api/routes.py b/phylesystem_api/phylesystem_api/routes.py index 918c59cf..3944e500 100644 --- a/phylesystem_api/phylesystem_api/routes.py +++ b/phylesystem_api/phylesystem_api/routes.py @@ -27,8 +27,7 @@ def includeme(config): config.add_route('exclude_tree_from_synth', '/{api_version}/exclude_tree_from_synth') config.add_route('create_study', '/{api_version}/study/') config.add_route('study_CORS_preflight', '/{api_version}/study/{study_id}', request_method='OPTIONS') - config.add_route('fetch_study', '/{api_version}/study/{study_id}', request_method='GET') - config.add_route('fetch_study_label', '/{api_version}/study/{study_id}/', request_method='GET') + config.add_route('fetch_study', '/{api_version}/study/{study_id}/', request_method='GET') config.add_route('update_study', '/{api_version}/study/{study_id}', request_method='PUT') config.add_route('delete_study', '/{api_version}/study/{study_id}', request_method='DELETE') config.add_route('get_study_file_list', '/{api_version}/study/{study_id}/file') diff --git a/phylesystem_api/phylesystem_api/views/study.py b/phylesystem_api/phylesystem_api/views/study.py index 704b4cb0..c6ec55b3 100644 --- a/phylesystem_api/phylesystem_api/views/study.py +++ b/phylesystem_api/phylesystem_api/views/study.py @@ -162,7 +162,6 @@ def __finish_write_verb(phylesystem, -@view_config(route_name='fetch_study_label', renderer=None) ## IS this allowed??? @view_config(route_name='fetch_study', renderer=None) def fetch_study(request): repo_parent, repo_remote, git_ssh, pkey, git_hub_remote, max_filesize, max_num_trees, read_only_mode = api_utils.read_phylesystem_config(request) From 8e9c1a8476c05d4723d3efccc7f10e4ee93eb844 Mon Sep 17 00:00:00 2001 From: snacktavish Date: Tue, 11 Jul 2023 12:04:35 -0700 Subject: [PATCH 30/33] optional return to split fetch --- phylesystem_api/phylesystem_api/routes.py | 3 ++- phylesystem_api/phylesystem_api/views/study.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/phylesystem_api/phylesystem_api/routes.py b/phylesystem_api/phylesystem_api/routes.py index 3944e500..918c59cf 100644 --- a/phylesystem_api/phylesystem_api/routes.py +++ b/phylesystem_api/phylesystem_api/routes.py @@ -27,7 +27,8 @@ def includeme(config): config.add_route('exclude_tree_from_synth', '/{api_version}/exclude_tree_from_synth') config.add_route('create_study', '/{api_version}/study/') config.add_route('study_CORS_preflight', '/{api_version}/study/{study_id}', request_method='OPTIONS') - config.add_route('fetch_study', '/{api_version}/study/{study_id}/', request_method='GET') + config.add_route('fetch_study', '/{api_version}/study/{study_id}', request_method='GET') + config.add_route('fetch_study_label', '/{api_version}/study/{study_id}/', request_method='GET') config.add_route('update_study', '/{api_version}/study/{study_id}', request_method='PUT') config.add_route('delete_study', '/{api_version}/study/{study_id}', request_method='DELETE') config.add_route('get_study_file_list', '/{api_version}/study/{study_id}/file') diff --git a/phylesystem_api/phylesystem_api/views/study.py b/phylesystem_api/phylesystem_api/views/study.py index c6ec55b3..12dda82e 100644 --- a/phylesystem_api/phylesystem_api/views/study.py +++ b/phylesystem_api/phylesystem_api/views/study.py @@ -162,7 +162,8 @@ def __finish_write_verb(phylesystem, -@view_config(route_name='fetch_study', renderer=None) +@view_config(route_name='fetch_study', renderer=None, request_method='GET') +@view_config(route_name='fetch_study_label', renderer=None, request_method='GET') def fetch_study(request): repo_parent, repo_remote, git_ssh, pkey, git_hub_remote, max_filesize, max_num_trees, read_only_mode = api_utils.read_phylesystem_config(request) #_LOG = api_utils.get_logger(request, 'ot_api.default.v1') From 5d7f5f9b1001f0799dccb91ff7ac1e881591ef3d Mon Sep 17 00:00:00 2001 From: snacktavish Date: Tue, 11 Jul 2023 12:24:32 -0700 Subject: [PATCH 31/33] debug extention --- phylesystem_api/phylesystem_api/views/study.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/phylesystem_api/phylesystem_api/views/study.py b/phylesystem_api/phylesystem_api/views/study.py index 12dda82e..264665ae 100644 --- a/phylesystem_api/phylesystem_api/views/study.py +++ b/phylesystem_api/phylesystem_api/views/study.py @@ -170,14 +170,15 @@ def fetch_study(request): _LOG.debug("Fetching study") api_version = request.matchdict['api_version'] study_id = request.matchdict['study_id'] + _LOG.debug('study_id = {}'.format(study_id)) content_id = None version_history = None comment_html = None - final_path_part = request.path.split('/')[-1] ##TODO What if there are other parts... # does this look like a filename? if so, grab its extension request_extension = None - fpps = final_path_part.split('.') + fpps = study_id.split('.') if len(fpps) > 1: + _LOG.debug('len(fpps) > 1') request_extension = fpps[-1] study_id = '.'.join(fpps[:-1]) _LOG.debug("Request extension is {}".format) From a6f8aa42af1635a92dc0093bd95c2a88b33ee195 Mon Sep 17 00:00:00 2001 From: "Mark T. Holder" Date: Wed, 12 Jul 2023 12:37:50 -0500 Subject: [PATCH 32/33] only render to JSON if outformat is json --- phylesystem_api/phylesystem_api/views/study.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/phylesystem_api/phylesystem_api/views/study.py b/phylesystem_api/phylesystem_api/views/study.py index 264665ae..f42f3321 100644 --- a/phylesystem_api/phylesystem_api/views/study.py +++ b/phylesystem_api/phylesystem_api/views/study.py @@ -693,8 +693,8 @@ def get_study_external_url(request): except: raise HTTPNotFound(body='{"error": 1, "description": "study not found"}') -@view_config(route_name='get_study_tree', renderer='json') -@view_config(route_name='get_study_tree_label', renderer='json') +@view_config(route_name='get_study_tree', renderer=None) +@view_config(route_name='get_study_tree_label', renderer=None) def get_study_tree(request): api_utils.raise_on_CORS_preflight(request) @@ -751,4 +751,8 @@ def get_study_tree(request): if result_data is None: raise HTTPNotFound(body='subresource "tree/{t}" not found in study "{s}"'.format(t=tree_id, s=study_id)) - return result_data + if out_schema.is_json(): + return render_to_response('json', result_data, request) + else: + # _LOG.debug(result_data) + return render_to_response('string', result_data, request) From 71b6014f46da9fe67f456bd90faaa4ee72c5de97 Mon Sep 17 00:00:00 2001 From: "Mark T. Holder" Date: Tue, 18 Jul 2023 12:37:10 -0500 Subject: [PATCH 33/33] log exceptions generated when fetching a study from the docstore --- phylesystem_api/phylesystem_api/views/study.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phylesystem_api/phylesystem_api/views/study.py b/phylesystem_api/phylesystem_api/views/study.py index f42f3321..c34b7c59 100644 --- a/phylesystem_api/phylesystem_api/views/study.py +++ b/phylesystem_api/phylesystem_api/views/study.py @@ -209,7 +209,7 @@ def fetch_study(request): except: comment_html = '' except: - # _LOG.exception('GET failed') + _LOG.exception('GET failed') e = sys.exc_info()[0] raise HTTPBadRequest(e)