From ba3cba5c39239e0b17b20c18392b8eee2094e411 Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Thu, 27 Jul 2023 09:27:31 -0500
Subject: [PATCH 01/33] avoiding deprecated functions

---
 phylesystem_api/phylesystem_api/views/default.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py
index 08a5576b..b3445dfe 100644
--- a/phylesystem_api/phylesystem_api/views/default.py
+++ b/phylesystem_api/phylesystem_api/views/default.py
@@ -204,7 +204,7 @@ def clear_cache_keys(request):
     """
     api_utils.raise_on_CORS_preflight(request)
     key_pattern = request.matchdict.get("key_pattern")
-    _LOG.warn(">> key_pattern: {}".format(key_pattern))
+    _LOG.warning(">> key_pattern: {}".format(key_pattern))
 
     """
     # TODO: decode this from URL-encoding??
@@ -465,7 +465,7 @@ def _get_synth_input_collection_ids():
         )
     cfg = ConfigParser()
     try:
-        cfg.readfp(conf_fo)
+        cfg.read_file(conf_fo)
     except:
         raise HTTPInternalServerError(
             body="Could not parse file from {}".format(url_of_synth_config)
@@ -528,7 +528,7 @@ def merge_docstore_changes(request):
         m = traceback.format_exc()
         raise HTTPConflict(
             detail=json.dumps(
-                {"error": 1, "description": "Could not merge! Details: %s" % (m)}
+                {"error": 1, "description": "Could not merge! Details: {}".format(m)}
             )
         )
 

From e69eb7b6dbfd229844307d3a82368c20c945d7fe Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Thu, 27 Jul 2023 09:28:11 -0500
Subject: [PATCH 02/33] avoiding deprecated logging.warn function

---
 .../phylesystem_api/views/study.py            | 34 +++++++++----------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/phylesystem_api/phylesystem_api/views/study.py b/phylesystem_api/phylesystem_api/views/study.py
index 9d039bfc..10c8df20 100644
--- a/phylesystem_api/phylesystem_api/views/study.py
+++ b/phylesystem_api/phylesystem_api/views/study.py
@@ -685,15 +685,15 @@ def update_study(request):
 @view_config(route_name="delete_study", renderer="json")
 def delete_study(request):
     # _LOG = api_utils.get_logger(request, 'delete_study')
-    _LOG.warn("delete_study STARTING...")
+    _LOG.warninging("delete_study STARTING...")
     api_version = request.matchdict["api_version"]
     study_id = request.matchdict["study_id"]
-    _LOG.warn("api_version={}".format(api_version))
-    _LOG.warn("study_id={}".format(study_id))
+    _LOG.warning("api_version={}".format(api_version))
+    _LOG.warning("study_id={}".format(study_id))
 
     # this method requires authentication
     auth_info = api_utils.authenticate(request)
-    _LOG.warn("auth_info={}".format(auth_info))
+    _LOG.warning("auth_info={}".format(auth_info))
 
     # gather any user-provided git-commit message
     try:
@@ -703,39 +703,39 @@ def delete_study(request):
             commit_msg = None
     except:
         commit_msg = None
-    _LOG.warn("commit_msg={}".format(commit_msg))
+    _LOG.warning("commit_msg={}".format(commit_msg))
 
     parent_sha = find_in_request(request, "starting_commit_SHA", None)
-    _LOG.warn("parent_sha={}".format(parent_sha))
+    _LOG.warning("parent_sha={}".format(parent_sha))
 
     phylesystem = api_utils.get_phylesystem(
         request
     )  # set READONLY flag before testing!
     api_utils.raise_if_read_only()
-    _LOG.warn("passed the read-only test")
+    _LOG.warning("passed the read-only test")
 
-    _LOG.warn("trying to delete now... via:")
-    _LOG.warn(phylesystem.delete_study)
+    _LOG.warning("trying to delete now... via:")
+    _LOG.warning(phylesystem.delete_study)
     try:
         x = phylesystem.delete_study(
             study_id, auth_info, parent_sha, commit_msg=commit_msg
         )
-        _LOG.warn("initial return x:")
-        _LOG.warn(x)
+        _LOG.warning("initial return x:")
+        _LOG.warning(x)
         if x.get("error") == 0:
-            _LOG.warn("calling deferred push...")
+            _LOG.warning("calling deferred push...")
             api_utils.deferred_push_to_gh_call(
                 request, None, doc_type="nexson", auth_token=auth_info["auth_token"]
             )
-            _LOG.warn("back from deferred push")
+            _LOG.warning("back from deferred push")
         return x
     except GitWorkflowError as err:
-        _LOG.warn("got a GitWorkflowError:")
-        _LOG.warn(err)
+        _LOG.warning("got a GitWorkflowError:")
+        _LOG.warning(err)
         raise HTTPBadRequest(err.msg)
     except Exception as err:
-        _LOG.warn("some other kind of error")
-        _LOG.warn(err)
+        _LOG.warning("some other kind of error")
+        _LOG.warning(err)
         # _LOG.exception('Exception getting nexson content in phylesystem.delete_study')
         raise HTTPBadRequest(
             json.dumps({"error": 1, "description": "Unknown error in study deletion"})

From 80fac7c9555642058a861f929fdfb65798bd21f1 Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Thu, 27 Jul 2023 09:30:03 -0500
Subject: [PATCH 03/33] removed unused and poorly-coded function

---
 phylesystem_api/phylesystem_api/gitdata.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/phylesystem_api/phylesystem_api/gitdata.py b/phylesystem_api/phylesystem_api/gitdata.py
index e107561d..8e2c9d1c 100644
--- a/phylesystem_api/phylesystem_api/gitdata.py
+++ b/phylesystem_api/phylesystem_api/gitdata.py
@@ -7,12 +7,6 @@ class GitData(PhylesystemGitAction):
     def __init__(self, repo, **kwargs):
         PhylesystemGitAction.__init__(self, repo, **kwargs)
 
-    def delete_remote_branch(self, remote, branch, env={}):
-        "Delete a remote branch"
-        # deleting a branch is the same as
-        # git push remote :branch
-        self.push(remote, env, ":%s" % branch)
-
     def pull(self, remote, env={}, branch=None):
         """
         Pull a branch from a given remote

From 4d4085780924dbe8a0bfa1c564949f89156a5516 Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Thu, 27 Jul 2023 14:07:22 -0500
Subject: [PATCH 04/33] non functional test

---
 phylesystem_api/tests/test_views.py | 14 --------------
 1 file changed, 14 deletions(-)
 delete mode 100644 phylesystem_api/tests/test_views.py

diff --git a/phylesystem_api/tests/test_views.py b/phylesystem_api/tests/test_views.py
deleted file mode 100644
index 8e9e701f..00000000
--- a/phylesystem_api/tests/test_views.py
+++ /dev/null
@@ -1,14 +0,0 @@
-from phylesystem_api.views.default import my_view
-from phylesystem_api.views.notfound import notfound_view
-
-
-def test_my_view(app_request):
-    info = my_view(app_request)
-    assert app_request.response.status_int == 200
-    assert info["project"] == "phylesystem API"
-
-
-def test_notfound_view(app_request):
-    info = notfound_view(app_request)
-    assert app_request.response.status_int == 404
-    assert info == {}

From 41f33be1ee6ed222e60727382fb5c45530fb347e Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Thu, 27 Jul 2023 15:17:20 -0500
Subject: [PATCH 05/33] refactoring based on pycharm warnings

---
 phylesystem_api/phylesystem_api/api_utils.py  |   7 +-
 phylesystem_api/phylesystem_api/cors.py       |   1 -
 phylesystem_api/phylesystem_api/gitdata.py    |   6 +-
 .../phylesystem_api/static/theme.css          | 447 +++++++++---------
 .../phylesystem_api/templates/404.jinja2      |   8 +-
 .../phylesystem_api/templates/home.jinja2     |  31 +-
 .../phylesystem_api/templates/layout.jinja2   | 133 +++---
 .../phylesystem_api/views/amendment.py        |  34 +-
 .../phylesystem_api/views/amendments.py       |  18 +-
 .../phylesystem_api/views/collection.py       |  39 +-
 .../phylesystem_api/views/collections.py      |  14 -
 .../phylesystem_api/views/default.py          |  23 +-
 .../phylesystem_api/views/notfound.py         |  17 +-
 .../phylesystem_api/views/studies.py          |  28 +-
 .../phylesystem_api/views/study.py            | 184 +++----
 phylesystem_api/tests/conftest.py             |   1 -
 phylesystem_api/tests/test_gitdata.py         | 225 ---------
 17 files changed, 443 insertions(+), 773 deletions(-)
 delete mode 100644 phylesystem_api/tests/test_gitdata.py

diff --git a/phylesystem_api/phylesystem_api/api_utils.py b/phylesystem_api/phylesystem_api/api_utils.py
index 9a37c42e..f984f87a 100644
--- a/phylesystem_api/phylesystem_api/api_utils.py
+++ b/phylesystem_api/phylesystem_api/api_utils.py
@@ -6,6 +6,7 @@
 from peyotl.utility import read_config as read_peyotl_config
 from configparser import ConfigParser
 from datetime import datetime
+from peyotl.api import OTI
 
 # see exception subclasses at https://docs.pylonsproject.org/projects/pyramid/en/latest/api/httpexceptions.html
 from pyramid.request import Request
@@ -859,7 +860,7 @@ def remove_tags(markup):
     except (UnicodeDecodeError, AttributeError):
         pass
     try:
-        markup = u"".join(ElementTree.fromstring(markup).itertext())
+        markup = "".join(ElementTree.fromstring(markup).itertext())
     except ElementTree.ParseError:
         # if it won't parse (badly-formed XML/HTML, or plaintext), return unchanged
         pass
@@ -893,3 +894,7 @@ def extract_json_from_http_call(request, data_field_name="data", request_params=
             )
         )
     return json_obj
+
+
+def get_oti_wrapper(request):
+    return OTI(oti=get_oti_domain(request))
diff --git a/phylesystem_api/phylesystem_api/cors.py b/phylesystem_api/phylesystem_api/cors.py
index 7c30e52a..651aada6 100644
--- a/phylesystem_api/phylesystem_api/cors.py
+++ b/phylesystem_api/phylesystem_api/cors.py
@@ -4,7 +4,6 @@
 def includeme(config):
     config.add_directive("add_cors_preflight_handler", add_cors_preflight_handler)
     config.add_route_predicate("cors_preflight", CorsPreflightPredicate)
-
     config.add_subscriber(add_cors_to_response, "pyramid.events.NewResponse")
 
 
diff --git a/phylesystem_api/phylesystem_api/gitdata.py b/phylesystem_api/phylesystem_api/gitdata.py
index 8e2c9d1c..87dbbc28 100644
--- a/phylesystem_api/phylesystem_api/gitdata.py
+++ b/phylesystem_api/phylesystem_api/gitdata.py
@@ -7,7 +7,7 @@ class GitData(PhylesystemGitAction):
     def __init__(self, repo, **kwargs):
         PhylesystemGitAction.__init__(self, repo, **kwargs)
 
-    def pull(self, remote, env={}, branch=None):
+    def pull(self, remote, env=None, branch=None):
         """
         Pull a branch from a given remote
 
@@ -23,11 +23,11 @@ def pull(self, remote, env={}, branch=None):
             branch_to_pull = branch
         else:
             branch_to_pull = self.current_branch()
-
+        env = {} if env is None else env
         # if there is no PKEY, we don't need to override env
         # We are explicit about what we are pushing, since the default behavior
         # is different in different versions of Git and/or by configuration
-        if env["PKEY"]:
+        if "PKEY" in env:
             new_env = os.environ.copy()
             new_env.update(env)
             git(
diff --git a/phylesystem_api/phylesystem_api/static/theme.css b/phylesystem_api/phylesystem_api/static/theme.css
index 8ad40b7a..f60c0857 100644
--- a/phylesystem_api/phylesystem_api/static/theme.css
+++ b/phylesystem_api/phylesystem_api/static/theme.css
@@ -13,601 +13,614 @@
 /* SYNTAX HIGHLIGHTING */
 /***********************/
 .highlight {
-  background-color: #222;
-  padding: 5px 10px;
-  margin: 20px 0;
+    background-color: #222;
+    padding: 5px 10px;
+    margin: 20px 0;
 }
 
 .highlight pre {
-  /* overflow: scroll; Prefer no word wrap? Uncomment this line and comment out the 2 lines below. */
-  word-break: break-all;
-  word-wrap: break-word;
+    /* overflow: scroll; Prefer no word wrap? Uncomment this line and comment out the 2 lines below. */
+    word-break: break-all;
+    word-wrap: break-word;
 }
 
 code {
-  font-family: 'Courier', monospace;
-  font-size: 14px;
-  color: #999;
+    font-family: 'Courier', monospace;
+    font-size: 14px;
+    color: #999;
 }
 
 /* Comment */
 .highlight .c, .highlight .c1 {
-  color: #586E75;
+    color: #586E75;
 }
 
 /* Error */
 .highlight .err {
-  color: #93A1A1;
+    color: #93A1A1;
 }
 
 /* Generic */
 .highlight .g {
-  color: #93A1A1;
+    color: #93A1A1;
 }
 
 /* Keyword */
 .highlight .k {
-  color: #859900;
+    color: #859900;
 }
 
 /* Literal */
 .highlight .l {
-  color: #93A1A1;
+    color: #93A1A1;
 }
 
 /* Name */
 .highlight .n {
-  color: #93A1A1;
+    color: #93A1A1;
 }
 
 /* Operator */
 .highlight .o {
-  color: #859900;
+    color: #859900;
 }
 
 /* Other */
 .highlight .x {
-  color: #CB4B16;
+    color: #CB4B16;
 }
 
 /* Punctuation */
 .highlight .p {
-  color: #93A1A1;
+    color: #93A1A1;
 }
 
 /* Comment.Multiline */
 .highlight .cm {
-  color: #586E75;
+    color: #586E75;
 }
 
 /* Comment.Preproc */
 .highlight .cp {
-  color: #859900;
+    color: #859900;
 }
 
 /* Comment.Single */
 .highlight .c1 {
-  color: #586E75;
+    color: #586E75;
 }
 
 /* Comment.Special */
 .highlight .cs {
-  color: #859900;
+    color: #859900;
 }
 
 /* Generic.Deleted */
 .highlight .gd {
-  color: #2AA198;
+    color: #2AA198;
 }
 
 /* Generic.Emph */
 .highlight .ge {
-  color: #93A1A1;
-  font-style: italic;
+    color: #93A1A1;
+    font-style: italic;
 }
 
 /* Generic.Error */
 .highlight .gr {
-  color: #DC322F;
+    color: #DC322F;
 }
 
 /* Generic.Heading */
 .highlight .gh {
-  color: #CB4B16;
+    color: #CB4B16;
 }
 
 /* Generic.Inserted */
 .highlight .gi {
-  color: #859900;
+    color: #859900;
 }
 
 /* Generic.Output */
 .highlight .go {
-  color: #93A1A1;
+    color: #93A1A1;
 }
 
 /* Generic.Prompt */
 .highlight .gp {
-  color: #93A1A1;
+    color: #93A1A1;
 }
 
 /* Generic.Strong */
 .highlight .gs {
-  color: #93A1A1;
-  font-weight: bold;
+    color: #93A1A1;
+    font-weight: bold;
 }
 
 /* Generic.Subheading */
 .highlight .gu {
-  color: #CB4B16;
+    color: #CB4B16;
 }
 
 /* Generic.Traceback */
 .highlight .gt {
-  color: #93A1A1;
+    color: #93A1A1;
 }
 
 /* Keyword.Constant */
 .highlight .kc {
-  color: #CB4B16;
+    color: #CB4B16;
 }
 
 /* Keyword.Declaration */
 .highlight .kd {
-  color: #268BD2;
+    color: #268BD2;
 }
 
 /* Keyword.Namespace */
 .highlight .kn {
-  color: #859900;
+    color: #859900;
 }
 
 /* Keyword.Pseudo */
 .highlight .kp {
-  color: #859900;
+    color: #859900;
 }
 
 /* Keyword.Reserved */
 .highlight .kr {
-  color: #268BD2;
+    color: #268BD2;
 }
 
 /* Keyword.Type */
 .highlight .kt {
-  color: #DC322F;
+    color: #DC322F;
 }
 
 /* Literal.Date */
 .highlight .ld {
-  color: #93A1A1;
+    color: #93A1A1;
 }
 
 /* Literal.Number */
 .highlight .m {
-  color: #2AA198;
+    color: #2AA198;
 }
 
 /* Literal.String */
 .highlight .s {
-  color: #2AA198;
+    color: #2AA198;
 }
 
 /* Name.Attribute */
 .highlight .na {
-  color: #93A1A1;
+    color: #93A1A1;
 }
 
 /* Name.Builtin */
 .highlight .nb {
-  color: #B58900;
+    color: #B58900;
 }
 
 /* Name.Class */
 .highlight .nc {
-  color: #268BD2;
+    color: #268BD2;
 }
 
 /* Name.Constant */
 .highlight .no {
-  color: #CB4B16;
+    color: #CB4B16;
 }
 
 /* Name.Decorator */
 .highlight .nd {
-  color: #268BD2;
+    color: #268BD2;
 }
 
 /* Name.Entity */
 .highlight .ni {
-  color: #CB4B16;
+    color: #CB4B16;
 }
 
 /* Name.Exception */
 .highlight .ne {
-  color: #CB4B16;
+    color: #CB4B16;
 }
 
 /* Name.Function */
 .highlight .nf {
-  color: #268BD2;
+    color: #268BD2;
 }
 
 /* Name.Label */
 .highlight .nl {
-  color: #93A1A1;
+    color: #93A1A1;
 }
 
 /* Name.Namespace */
 .highlight .nn {
-  color: #93A1A1;
+    color: #93A1A1;
 }
 
 /* Name.Other */
 .highlight .nx {
-  color: #555;
+    color: #555;
 }
 
 /* Name.Property */
 .highlight .py {
-  color: #93A1A1;
+    color: #93A1A1;
 }
 
 /* Name.Tag */
 .highlight .nt {
-  color: #268BD2;
+    color: #268BD2;
 }
 
 /* Name.Variable */
 .highlight .nv {
-  color: #268BD2;
+    color: #268BD2;
 }
 
 /* Operator.Word */
 .highlight .ow {
-  color: #859900;
+    color: #859900;
 }
 
 /* Text.Whitespace */
 .highlight .w {
-  color: #93A1A1;
+    color: #93A1A1;
 }
 
 /* Literal.Number.Float */
 .highlight .mf {
-  color: #2AA198;
+    color: #2AA198;
 }
 
 /* Literal.Number.Hex */
 .highlight .mh {
-  color: #2AA198;
+    color: #2AA198;
 }
 
 /* Literal.Number.Integer */
 .highlight .mi {
-  color: #2AA198;
+    color: #2AA198;
 }
 
 /* Literal.Number.Oct */
 .highlight .mo {
-  color: #2AA198;
+    color: #2AA198;
 }
 
 /* Literal.String.Backtick */
 .highlight .sb {
-  color: #586E75;
+    color: #586E75;
 }
 
 /* Literal.String.Char */
 .highlight .sc {
-  color: #2AA198;
+    color: #2AA198;
 }
 
 /* Literal.String.Doc */
 .highlight .sd {
-  color: #93A1A1;
+    color: #93A1A1;
 }
 
 /* Literal.String.Double */
 .highlight .s2 {
-  color: #2AA198;
+    color: #2AA198;
 }
 
 /* Literal.String.Escape */
 .highlight .se {
-  color: #CB4B16;
+    color: #CB4B16;
 }
 
 /* Literal.String.Heredoc */
 .highlight .sh {
-  color: #93A1A1;
+    color: #93A1A1;
 }
 
 /* Literal.String.Interpol */
 .highlight .si {
-  color: #2AA198;
+    color: #2AA198;
 }
 
 /* Literal.String.Other */
 .highlight .sx {
-  color: #2AA198;
+    color: #2AA198;
 }
 
 /* Literal.String.Regex */
 .highlight .sr {
-  color: #DC322F;
+    color: #DC322F;
 }
 
 /* Literal.String.Single */
 .highlight .s1 {
-  color: #2AA198;
+    color: #2AA198;
 }
 
 /* Literal.String.Symbol */
 .highlight .ss {
-  color: #2AA198;
+    color: #2AA198;
 }
 
 /* Name.Builtin.Pseudo */
 .highlight .bp {
-  color: #268BD2;
+    color: #268BD2;
 }
 
 /* Name.Variable.Class */
 .highlight .vc {
-  color: #268BD2;
+    color: #268BD2;
 }
 
 /* Name.Variable.Global */
 .highlight .vg {
-  color: #268BD2;
+    color: #268BD2;
 }
 
 /* Name.Variable.Instance */
 .highlight .vi {
-  color: #268BD2;
+    color: #268BD2;
 }
 
 /* Literal.Number.Integer.Long */
 .highlight .il {
-  color: #2AA198;
+    color: #2AA198;
 }
 
 html {
-  padding: auto;
-  background-color: #f3f3f3;
+    padding: auto;
+    background-color: #f3f3f3;
 }
 
 body {
-  /* margin: 0; */
-  font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
-  font-size: 14px;
-  line-height: 20px;
-  color: #333333;
-  background-color: #ffffff;
-  margin: 0 auto;
-  padding: 1em 20px;
-  border-left: 1px solid #ddd;
-  border-right: 1px solid #ddd;
-  overflow-y: scroll;
-  /* for consistent page width */
+    /* margin: 0; */
+    font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
+    font-size: 14px;
+    line-height: 20px;
+    color: #333333;
+    background-color: #ffffff;
+    margin: 0 auto;
+    padding: 1em 20px;
+    border-left: 1px solid #ddd;
+    border-right: 1px solid #ddd;
+    overflow-y: scroll;
+    /* for consistent page width */
 }
 
 /* responsive page width (but not too wide) */
 @media (min-width: 768px) {
-  body {
-    width: 724px;
-  }
+    body {
+        width: 724px;
+    }
 }
+
 @media (max-width: 767px) {
-  body {
-    width: auto;
-  }
+    body {
+        width: auto;
+    }
 }
+
 h1 {
-  color: #666;
-  height: 32px;
-  margin-bottom: 0px;
+    color: #666;
+    height: 32px;
+    margin-bottom: 0px;
 }
 
 .maintenance-notice {
-  margin: 30px 40px 20px;
-  color: #900;
-  font-size: 170%;
-  font-weight: bold;
-  line-height: normal;
+    margin: 30px 40px 20px;
+    color: #900;
+    font-size: 170%;
+    font-weight: bold;
+    line-height: normal;
 }
 
 .maintenance-details {
-  margin: 0px 40px;
-  font-size: 130%;
-  line-height: 1.4em;
+    margin: 0px 40px;
+    font-size: 130%;
+    line-height: 1.4em;
 }
 
 .wrapper-masthead {
-  overflow: hidden;
-  /* prevent collapse */
+    overflow: hidden;
+    /* prevent collapse */
 }
 
 .masthead {
-  overflow: hidden;
-  /* prevent collapse */
-  border-bottom: 1px solid #f3f3f3;
+    overflow: hidden;
+    /* prevent collapse */
+    border-bottom: 1px solid #f3f3f3;
 }
+
 @media screen and (max-width: 640px) {
-  .masthead {
-    text-align: center;
-  }
+    .masthead {
+        text-align: center;
+    }
 }
 
 .site-avatar {
-  float: left;
-  width: 70px;
-  height: 70px;
-  margin-right: 15px;
+    float: left;
+    width: 70px;
+    height: 70px;
+    margin-right: 15px;
 }
+
 @media screen and (max-width: 640px) {
-  .site-avatar {
-    float: none;
-    display: block;
-    margin: 0 auto;
-  }
+    .site-avatar {
+        float: none;
+        display: block;
+        margin: 0 auto;
+    }
 }
+
 .site-avatar img {
-  border-radius: 5px;
+    border-radius: 5px;
 }
 
 .site-info {
-  float: left;
-  padding-bottom: 4px;
+    float: left;
+    padding-bottom: 4px;
 }
+
 @media screen and (max-width: 640px) {
-  .site-info {
-    float: none;
-    display: block;
-    margin: 0 auto;
-  }
+    .site-info {
+        float: none;
+        display: block;
+        margin: 0 auto;
+    }
 }
 
 .site-name {
-  background: url(/static/mini-opentree-logo.png) no-repeat center left;
-  padding: 12px 0 0 28px;
-  height: 32px;
-  margin: 0;
+    background: url(/static/mini-opentree-logo.png) no-repeat center left;
+    padding: 12px 0 0 28px;
+    height: 32px;
+    margin: 0;
 }
 
 .site-name a {
-  color: #666;
-  cursor: pointer;
-  font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
-  font-weight: 300;
-  font-size: 28px;
-  letter-spacing: 1px;
-  text-decoration: none;
+    color: #666;
+    cursor: pointer;
+    font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
+    font-weight: 300;
+    font-size: 28px;
+    letter-spacing: 1px;
+    text-decoration: none;
 }
 
 .site-description {
-  margin: -5px 0 0 0;
-  color: #666;
-  font-size: 16px;
+    margin: -5px 0 0 0;
+    color: #666;
+    font-size: 16px;
 }
+
 @media screen and (max-width: 640px) {
-  .site-description {
-    margin: 3px 0;
-  }
+    .site-description {
+        margin: 3px 0;
+    }
 }
 
 .masthead {
-  position: relative;
-  /* support positioning within this */
+    position: relative;
+    /* support positioning within this */
 }
 
 nav {
-  display: inline-block;
-  position: absolute;
-  right: 0px;
-  bottom: 0px;
-  border-top: 1px solid #f3f3f3;
-  font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
-  font-size: 18px;
+    display: inline-block;
+    position: absolute;
+    right: 0px;
+    bottom: 0px;
+    border-top: 1px solid #f3f3f3;
+    font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
+    font-size: 18px;
 }
+
 @media screen and (max-width: 640px) {
-  nav {
-    float: none;
-    margin-top: 9px;
-    display: block;
-    font-size: 16px;
-  }
+    nav {
+        float: none;
+        margin-top: 9px;
+        display: block;
+        font-size: 16px;
+    }
 }
+
 nav a {
-  margin-left: 20px;
-  color: #333;
-  text-align: right;
-  font-weight: 300;
-  letter-spacing: 1px;
-  text-decoration: none;
-  cursor: pointer;
+    margin-left: 20px;
+    color: #333;
+    text-align: right;
+    font-weight: 300;
+    letter-spacing: 1px;
+    text-decoration: none;
+    cursor: pointer;
 }
+
 nav a:hover, nav a:active {
-  color: #4183C4;
+    color: #4183C4;
 }
+
 @media screen and (max-width: 640px) {
-  nav a {
-    margin: 0 10px;
-    color: #4183C4;
-  }
+    nav a {
+        margin: 0 10px;
+        color: #4183C4;
+    }
 }
 
 .boxlink {
-  display: block;
-  overflow: hidden;
-  /* to prevent collapse */
-  text-decoration: none;
-  color: black;
+    display: block;
+    overflow: hidden;
+    /* to prevent collapse */
+    text-decoration: none;
+    color: black;
 }
 
 .container {
-  clear: both;
+    clear: both;
 }
 
 .boxcontainer {
-  margin-top: 30px;
-  margin-bottom: 30px;
-  border: 1px solid #ddd;
-  border-radius: 5px;
-  padding: 0.7em 0.5em;
-  clear: both;
+    margin-top: 30px;
+    margin-bottom: 30px;
+    border: 1px solid #ddd;
+    border-radius: 5px;
+    padding: 0.7em 0.5em;
+    clear: both;
 }
 
 .boxcontainer:hover {
-  background-color: #eef;
-  color: green;
-  border: 1px solid #559;
+    background-color: #eef;
+    color: green;
+    border: 1px solid #559;
 }
 
 .boxcontainer .featured-image {
-  float: left;
-  margin-right: 1em;
-  border-radius: 5px;
-  width: 50%;
+    float: left;
+    margin-right: 1em;
+    border-radius: 5px;
+    width: 50%;
 }
 
 .boxcontainer p {
-  margin: 0;
+    margin: 0;
 }
 
 .posts > .post {
-  padding-bottom: 2em;
-  border-bottom: 1px solid #f3f3f3;
+    padding-bottom: 2em;
+    border-bottom: 1px solid #f3f3f3;
 }
 
 .posts > .post:last-child {
-  padding-bottom: 1em;
-  border-bottom: none;
+    padding-bottom: 1em;
+    border-bottom: none;
 }
 
 .post blockquote {
-  margin: 1.8em .8em;
-  border-left: 2px solid #666;
-  padding: 0.1em 1em;
-  color: #666;
-  font-size: 22px;
-  font-style: italic;
+    margin: 1.8em .8em;
+    border-left: 2px solid #666;
+    padding: 0.1em 1em;
+    color: #666;
+    font-size: 22px;
+    font-style: italic;
 }
+
 .post .comments {
-  margin-top: 10px;
+    margin-top: 10px;
 }
+
 .post .read-more {
-  text-transform: uppercase;
-  font-size: 15px;
+    text-transform: uppercase;
+    font-size: 15px;
 }
 
 .wrapper-footer {
-  margin: 50px -21px -15px;
-  border-top: 1px solid #ddd;
-  background-color: #f3f3f3;
+    margin: 50px -21px -15px;
+    border-top: 1px solid #ddd;
+    background-color: #f3f3f3;
 }
 
 footer {
-  padding: 20px 0;
-  text-align: center;
+    padding: 20px 0;
+    text-align: center;
 }
diff --git a/phylesystem_api/phylesystem_api/templates/404.jinja2 b/phylesystem_api/phylesystem_api/templates/404.jinja2
index aaf12413..3e8b23e9 100644
--- a/phylesystem_api/phylesystem_api/templates/404.jinja2
+++ b/phylesystem_api/phylesystem_api/templates/404.jinja2
@@ -1,8 +1,8 @@
 {% extends "layout.jinja2" %}
 
 {% block content %}
-<div class="content">
-  <h1><span class="font-semi-bold">Pyramid</span> <span class="smaller">Starter project</span></h1>
-  <p class="lead"><span class="font-semi-bold">404</span> Page Not Found</p>
-</div>
+    <div class="content">
+        <h1><span class="font-semi-bold">Pyramid</span> <span class="smaller">Starter project</span></h1>
+        <p class="lead"><span class="font-semi-bold">404</span> Page Not Found</p>
+    </div>
 {% endblock content %}
diff --git a/phylesystem_api/phylesystem_api/templates/home.jinja2 b/phylesystem_api/phylesystem_api/templates/home.jinja2
index 1a1fc28f..387148f6 100644
--- a/phylesystem_api/phylesystem_api/templates/home.jinja2
+++ b/phylesystem_api/phylesystem_api/templates/home.jinja2
@@ -1,19 +1,20 @@
 {% extends "layout.jinja2" %}
 
 {% block content %}
-  <h1>phylesystem API</h1>
-  <p class="lead">
-      Welcome to the phylesystem API server! This API is used to explore and modify
-      studies and trees in the
-      <a href="https://github.com/OpenTreeOfLife/phylesystem"
-         target="_blank">phylesystem repository on
-      GitHub</a>, as well as smaller documents like tree collections and
-      taxonomic annotations.
-  </p>
-  <p>
-      The phylesystem API is
-      <a href="https://github.com/OpenTreeOfLife/phylesystem-api/tree/master/docs" target="_blank">documented here</a>,
-      with helpful examples. There's also
-      <a href="https://github.com/OpenTreeOfLife/germinator/wiki/Open-Tree-of-Life-Web-APIs" target="_blank">a more complete reference to all OpenTree APIs</a> beyond the phylesystem.
-  </p>
+    <h1>phylesystem API</h1>
+    <p class="lead">
+        Welcome to the phylesystem API server! This API is used to explore and modify
+        studies and trees in the
+        <a href="https://github.com/OpenTreeOfLife/phylesystem"
+           target="_blank">phylesystem repository on
+            GitHub</a>, as well as smaller documents like tree collections and
+        taxonomic annotations.
+    </p>
+    <p>
+        The phylesystem API is
+        <a href="https://github.com/OpenTreeOfLife/phylesystem-api/tree/master/docs" target="_blank">documented here</a>,
+        with helpful examples. There's also
+        <a href="https://github.com/OpenTreeOfLife/germinator/wiki/Open-Tree-of-Life-Web-APIs" target="_blank">a more
+            complete reference to all OpenTree APIs</a> beyond the phylesystem.
+    </p>
 {% endblock content %}
diff --git a/phylesystem_api/phylesystem_api/templates/layout.jinja2 b/phylesystem_api/phylesystem_api/templates/layout.jinja2
index 308b2fc6..2a3fc24c 100644
--- a/phylesystem_api/phylesystem_api/templates/layout.jinja2
+++ b/phylesystem_api/phylesystem_api/templates/layout.jinja2
@@ -1,90 +1,103 @@
 <!DOCTYPE html>
 <html>
-  <head>
-    <meta charset="utf-8" />
+<head>
+    <meta charset="utf-8"/>
     <meta http-equiv='X-UA-Compatible' content='IE=edge'>
     <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1.0'>
 
     <title>Open Tree of Life – {{ title }}</title>
 
-    <meta name="author" content="Open Tree of Life" />
+    <meta name="author" content="Open Tree of Life"/>
     <meta name="description" content="">
 
     <!-- Custom styles for this scaffold -->
-    <link href="{{request.static_url('phylesystem_api:static/theme.css')}}" rel="stylesheet">
+    <link href="{{ request.static_url('phylesystem_api:static/theme.css') }}" rel="stylesheet">
 
     <!-- HTML5 shiv and Respond.js IE8 support of HTML5 elements and media queries -->
     <!--[if lt IE 9]>
-      <script src="//oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js" integrity="sha384-0s5Pv64cNZJieYFkXYOTId2HMA2Lfb6q2nAcx2n0RTLUnCAoTTsS0nKEO27XyKcY" crossorigin="anonymous"></script>
-      <script src="//oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js" integrity="sha384-ZoaMbDF+4LeFxg6WdScQ9nnR1QC2MIRxA1O9KWEXQwns1G8UNyIEZIQidzb0T1fo" crossorigin="anonymous"></script>
+    <script src="//oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"
+            integrity="sha384-0s5Pv64cNZJieYFkXYOTId2HMA2Lfb6q2nAcx2n0RTLUnCAoTTsS0nKEO27XyKcY"
+            crossorigin="anonymous"></script>
+    <script src="//oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js"
+            integrity="sha384-ZoaMbDF+4LeFxg6WdScQ9nnR1QC2MIRxA1O9KWEXQwns1G8UNyIEZIQidzb0T1fo"
+            crossorigin="anonymous"></script>
     <![endif]-->
-  </head>
+</head>
 
-  <body>
-    <div class="wrapper-masthead">
-      <div class="container">
+<body>
+<div class="wrapper-masthead">
+    <div class="container">
         <header class="masthead clearfix">
-          <div class="site-info">
-            <h1 class="site-name"><a href="/">Open Tree of Life</a></h1>
-            <p class="site-description"></p>
-          </div>
+            <div class="site-info">
+                <h1 class="site-name"><a href="/">Open Tree of Life</a></h1>
+                <p class="site-description"></p>
+            </div>
         </header>
-      </div>
     </div>
+</div>
 
 
-    <div id="main" role="main" class="container">
-     <article class="page">
+<div id="main" role="main" class="container">
+    <article class="page">
 
-      <div class="entry">
-        {% block content %}
-            <em>No content!</em>
-        {% endblock content %}
-      </div>
+        <div class="entry">
+            {% block content %}
+                <em>No content!</em>
+            {% endblock content %}
+        </div>
 
-     </article>
-    </div>
+    </article>
+</div>
 
 
-    <div class="wrapper-footer">
-      <div class="container">
+<div class="wrapper-footer">
+    <div class="container">
         <footer class="footer">
 
-          <a href="http://github.com/OpenTreeOfLife"><svg width="40px" height="40px" viewBox="0 0 60 60" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:sketch="http://www.bohemiancoding.com/sketch/ns">
-    <path d="M0.336871032,30 C0.336871032,13.4314567 13.5672313,0 29.8877097,0 C46.208188,0 59.4385483,13.4314567 59.4385483,30 C59.4385483,46.5685433 46.208188,60 29.8877097,60 C13.5672313,60 0.336871032,46.5685433 0.336871032,30 Z M0.336871032,30" id="Github" fill="#333333" sketch:type="MSShapeGroup"></path>
-    <path d="M18.2184245,31.9355566 C19.6068506,34.4507902 22.2845295,36.0156764 26.8007287,36.4485173 C26.1561023,36.9365335 25.3817877,37.8630984 25.2749857,38.9342607 C24.4644348,39.4574749 22.8347506,39.62966 21.5674303,39.2310659 C19.7918469,38.6717023 19.1119377,35.1642642 16.4533306,35.6636959 C15.8773626,35.772144 15.9917933,36.1507609 16.489567,36.4722998 C17.3001179,36.9955141 18.0629894,37.6500075 18.6513541,39.04366 C19.1033554,40.113871 20.0531304,42.0259813 23.0569369,42.0259813 C24.2489236,42.0259813 25.0842679,41.8832865 25.0842679,41.8832865 C25.0842679,41.8832865 25.107154,44.6144649 25.107154,45.6761142 C25.107154,46.9004355 23.4507693,47.2457569 23.4507693,47.8346108 C23.4507693,48.067679 23.9990832,48.0895588 24.4396415,48.0895588 C25.3102685,48.0895588 27.1220883,47.3646693 27.1220883,46.0918317 C27.1220883,45.0806012 27.1382993,41.6806599 27.1382993,41.0860982 C27.1382993,39.785673 27.8372803,39.3737607 27.8372803,39.3737607 C27.8372803,39.3737607 27.924057,46.3153869 27.6704022,47.2457569 C27.3728823,48.3397504 26.8360115,48.1846887 26.8360115,48.6727049 C26.8360115,49.3985458 29.0168704,48.8505978 29.7396911,47.2571725 C30.2984945,46.0166791 30.0543756,39.2072834 30.0543756,39.2072834 L30.650369,39.1949165 C30.650369,39.1949165 30.6837446,42.3123222 30.6637192,43.7373675 C30.6427402,45.2128317 30.5426134,47.0792797 31.4208692,47.9592309 C31.9977907,48.5376205 33.868733,49.5526562 33.868733,48.62514 C33.868733,48.0857536 32.8436245,47.6424485 32.8436245,46.1831564 L32.8436245,39.4688905 C33.6618042,39.4688905 33.5387911,41.6768547 33.5387911,41.6768547 L33.5988673,45.7788544 C33.5988673,45.7788544 33.4186389,47.2733446 35.2190156,47.8992991 C35.8541061,48.1209517 37.2139245,48.1808835 37.277815,47.8089257 C37.3417055,47.4360167 35.6405021,46.8814096 35.6252446,45.7236791 C35.6157088,45.0178155 35.6567131,44.6059032 35.6567131,41.5379651 C35.6567131,38.470027 35.2438089,37.336079 33.8048426,36.4323453 C38.2457082,35.9766732 40.9939527,34.880682 42.3337458,31.9450695 C42.4383619,31.9484966 42.8791491,30.5737742 42.8219835,30.5742482 C43.1223642,29.4659853 43.2844744,28.1550957 43.3168964,26.6025764 C43.3092677,22.3930799 41.2895654,20.9042975 40.9014546,20.205093 C41.4736082,17.0182425 40.8060956,15.5675121 40.4961791,15.0699829 C39.3518719,14.6637784 36.5149435,16.1145088 34.9653608,17.1371548 C32.438349,16.3998984 27.0982486,16.4712458 25.0957109,17.3274146 C21.4005522,14.6875608 19.445694,15.0918628 19.445694,15.0918628 C19.445694,15.0918628 18.1821881,17.351197 19.1119377,20.6569598 C17.8961113,22.2028201 16.9902014,23.2968136 16.9902014,26.1963718 C16.9902014,27.8297516 17.1828264,29.2918976 17.6176632,30.5685404 C17.5643577,30.5684093 18.2008493,31.9359777 18.2184245,31.9355566 Z M18.2184245,31.9355566" id="Path" fill="#FFFFFF" sketch:type="MSShapeGroup"></path>
-    <path d="M59.4385483,30 C59.4385483,46.5685433 46.208188,60 29.8877097,60 C23.8348308,60 18.2069954,58.1525134 13.5216148,54.9827754 L47.3818361,5.81941103 C54.6937341,11.2806503 59.4385483,20.0777973 59.4385483,30 Z M59.4385483,30" id="reflec" fill-opacity="0.08" fill="#000000" sketch:type="MSShapeGroup"></path>
-</svg></a>
-
-
-
-
-          <a href="http://twitter.com/opentreeoflife"><svg width="40px" height="40px" viewBox="0 0 60 60" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:sketch="http://www.bohemiancoding.com/sketch/ns">
-    <path d="M0,30 C0,13.4314567 13.4508663,0 30.0433526,0 C46.6358389,0 60.0867052,13.4314567 60.0867052,30 C60.0867052,46.5685433 46.6358389,60 30.0433526,60 C13.4508663,60 0,46.5685433 0,30 Z M0,30" fill="#4099FF" sketch:type="MSShapeGroup"></path>
-    <path d="M29.2997675,23.8879776 L29.3627206,24.9260453 L28.3135016,24.798935 C24.4943445,24.3116787 21.1578281,22.6592444 18.3249368,19.8840023 L16.9399677,18.5069737 L16.5832333,19.5238563 C15.8277956,21.7906572 16.3104363,24.1845684 17.8842648,25.7946325 C18.72364,26.6844048 18.5347806,26.8115152 17.0868584,26.2818888 C16.5832333,26.1124083 16.1425613,25.985298 16.1005925,26.0488532 C15.9537019,26.1971486 16.457327,28.1249885 16.8560302,28.8876505 C17.4016241,29.9469033 18.5137962,30.9849709 19.7308902,31.5993375 L20.7591248,32.0865938 L19.5420308,32.1077788 C18.3669055,32.1077788 18.3249368,32.1289639 18.4508431,32.57385 C18.8705307,33.9508786 20.5282967,35.4126474 22.3749221,36.048199 L23.6759536,36.4930852 L22.5427971,37.1710069 C20.8640467,38.1455194 18.891515,38.6963309 16.9189833,38.738701 C15.9746862,38.759886 15.1982642,38.8446262 15.1982642,38.9081814 C15.1982642,39.1200319 17.7583585,40.306395 19.2482495,40.7724662 C23.7179224,42.1494948 29.0269705,41.5563132 33.0140027,39.2047722 C35.846894,37.5311528 38.6797853,34.2050993 40.0018012,30.9849709 C40.7152701,29.2689815 41.428739,26.1335934 41.428739,24.6294545 C41.428739,23.654942 41.4916922,23.5278317 42.6668174,22.3626537 C43.359302,21.6847319 44.0098178,20.943255 44.135724,20.7314044 C44.3455678,20.3288884 44.3245835,20.3288884 43.2543801,20.6890343 C41.4707078,21.324586 41.2188952,21.2398458 42.1002392,20.2865183 C42.750755,19.6085965 43.527177,18.3798634 43.527177,18.0197174 C43.527177,17.9561623 43.2124113,18.0620876 42.8556769,18.252753 C42.477958,18.4646036 41.6385828,18.7823794 41.0090514,18.9730449 L39.8758949,19.3331908 L38.8476603,18.634084 C38.281082,18.252753 37.4836756,17.829052 37.063988,17.7019416 C35.9937846,17.4053509 34.357003,17.447721 33.3917215,17.7866818 C30.768674,18.7400093 29.110908,21.1974757 29.2997675,23.8879776 Z M29.2997675,23.8879776" id="Path" fill="#FFFFFF" sketch:type="MSShapeGroup"></path>
-    <path d="M60.0867052,30 C60.0867052,46.5685433 46.6358389,60 30.0433526,60 C23.8895925,60 18.1679598,58.1525134 13.4044895,54.9827754 L47.8290478,5.81941103 C55.2628108,11.2806503 60.0867052,20.0777973 60.0867052,30 Z M60.0867052,30" id="reflec" fill-opacity="0.08" fill="#000000" sketch:type="MSShapeGroup"></path>
-</svg></a>
-
-
-
-<footer>
-  <hr/>
-  <div class="row">
-    <div class="col-md-6" align="center">
-       <h4>
-  Open Tree is funded by NSF ABI #1759838, ABI 1759846, and AVAToL #1208809.
-      </h4>
-    </div>
-    <div class="col-md-6" align="center">
-      <h4>
-  <a href="https://gitter.im/OpenTreeOfLife/public">Contact OpenTree </a>
-      </h4>
-    </div>
-  </div>
-</footer>
+            <a href="https://github.com/OpenTreeOfLife">
+                <svg width="40px" height="40px" viewBox="0 0 60 60" version="1.1" xmlns="http://www.w3.org/2000/svg"
+                     xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:sketch="http://www.bohemiancoding.com/sketch/ns">
+                    <path d="M0.336871032,30 C0.336871032,13.4314567 13.5672313,0 29.8877097,0 C46.208188,0 59.4385483,13.4314567 59.4385483,30 C59.4385483,46.5685433 46.208188,60 29.8877097,60 C13.5672313,60 0.336871032,46.5685433 0.336871032,30 Z M0.336871032,30"
+                          id="Github" fill="#333333" sketch:type="MSShapeGroup"></path>
+                    <path d="M18.2184245,31.9355566 C19.6068506,34.4507902 22.2845295,36.0156764 26.8007287,36.4485173 C26.1561023,36.9365335 25.3817877,37.8630984 25.2749857,38.9342607 C24.4644348,39.4574749 22.8347506,39.62966 21.5674303,39.2310659 C19.7918469,38.6717023 19.1119377,35.1642642 16.4533306,35.6636959 C15.8773626,35.772144 15.9917933,36.1507609 16.489567,36.4722998 C17.3001179,36.9955141 18.0629894,37.6500075 18.6513541,39.04366 C19.1033554,40.113871 20.0531304,42.0259813 23.0569369,42.0259813 C24.2489236,42.0259813 25.0842679,41.8832865 25.0842679,41.8832865 C25.0842679,41.8832865 25.107154,44.6144649 25.107154,45.6761142 C25.107154,46.9004355 23.4507693,47.2457569 23.4507693,47.8346108 C23.4507693,48.067679 23.9990832,48.0895588 24.4396415,48.0895588 C25.3102685,48.0895588 27.1220883,47.3646693 27.1220883,46.0918317 C27.1220883,45.0806012 27.1382993,41.6806599 27.1382993,41.0860982 C27.1382993,39.785673 27.8372803,39.3737607 27.8372803,39.3737607 C27.8372803,39.3737607 27.924057,46.3153869 27.6704022,47.2457569 C27.3728823,48.3397504 26.8360115,48.1846887 26.8360115,48.6727049 C26.8360115,49.3985458 29.0168704,48.8505978 29.7396911,47.2571725 C30.2984945,46.0166791 30.0543756,39.2072834 30.0543756,39.2072834 L30.650369,39.1949165 C30.650369,39.1949165 30.6837446,42.3123222 30.6637192,43.7373675 C30.6427402,45.2128317 30.5426134,47.0792797 31.4208692,47.9592309 C31.9977907,48.5376205 33.868733,49.5526562 33.868733,48.62514 C33.868733,48.0857536 32.8436245,47.6424485 32.8436245,46.1831564 L32.8436245,39.4688905 C33.6618042,39.4688905 33.5387911,41.6768547 33.5387911,41.6768547 L33.5988673,45.7788544 C33.5988673,45.7788544 33.4186389,47.2733446 35.2190156,47.8992991 C35.8541061,48.1209517 37.2139245,48.1808835 37.277815,47.8089257 C37.3417055,47.4360167 35.6405021,46.8814096 35.6252446,45.7236791 C35.6157088,45.0178155 35.6567131,44.6059032 35.6567131,41.5379651 C35.6567131,38.470027 35.2438089,37.336079 33.8048426,36.4323453 C38.2457082,35.9766732 40.9939527,34.880682 42.3337458,31.9450695 C42.4383619,31.9484966 42.8791491,30.5737742 42.8219835,30.5742482 C43.1223642,29.4659853 43.2844744,28.1550957 43.3168964,26.6025764 C43.3092677,22.3930799 41.2895654,20.9042975 40.9014546,20.205093 C41.4736082,17.0182425 40.8060956,15.5675121 40.4961791,15.0699829 C39.3518719,14.6637784 36.5149435,16.1145088 34.9653608,17.1371548 C32.438349,16.3998984 27.0982486,16.4712458 25.0957109,17.3274146 C21.4005522,14.6875608 19.445694,15.0918628 19.445694,15.0918628 C19.445694,15.0918628 18.1821881,17.351197 19.1119377,20.6569598 C17.8961113,22.2028201 16.9902014,23.2968136 16.9902014,26.1963718 C16.9902014,27.8297516 17.1828264,29.2918976 17.6176632,30.5685404 C17.5643577,30.5684093 18.2008493,31.9359777 18.2184245,31.9355566 Z M18.2184245,31.9355566"
+                          id="Path" fill="#FFFFFF" sketch:type="MSShapeGroup"></path>
+                    <path d="M59.4385483,30 C59.4385483,46.5685433 46.208188,60 29.8877097,60 C23.8348308,60 18.2069954,58.1525134 13.5216148,54.9827754 L47.3818361,5.81941103 C54.6937341,11.2806503 59.4385483,20.0777973 59.4385483,30 Z M59.4385483,30"
+                          id="reflec" fill-opacity="0.08" fill="#000000" sketch:type="MSShapeGroup"></path>
+                </svg>
+            </a>
+
+
+            <a href="https://twitter.com/opentreeoflife">
+                <svg width="40px" height="40px" viewBox="0 0 60 60" version="1.1" xmlns="http://www.w3.org/2000/svg"
+                     xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:sketch="http://www.bohemiancoding.com/sketch/ns">
+                    <path d="M0,30 C0,13.4314567 13.4508663,0 30.0433526,0 C46.6358389,0 60.0867052,13.4314567 60.0867052,30 C60.0867052,46.5685433 46.6358389,60 30.0433526,60 C13.4508663,60 0,46.5685433 0,30 Z M0,30"
+                          fill="#4099FF" sketch:type="MSShapeGroup"></path>
+                    <path d="M29.2997675,23.8879776 L29.3627206,24.9260453 L28.3135016,24.798935 C24.4943445,24.3116787 21.1578281,22.6592444 18.3249368,19.8840023 L16.9399677,18.5069737 L16.5832333,19.5238563 C15.8277956,21.7906572 16.3104363,24.1845684 17.8842648,25.7946325 C18.72364,26.6844048 18.5347806,26.8115152 17.0868584,26.2818888 C16.5832333,26.1124083 16.1425613,25.985298 16.1005925,26.0488532 C15.9537019,26.1971486 16.457327,28.1249885 16.8560302,28.8876505 C17.4016241,29.9469033 18.5137962,30.9849709 19.7308902,31.5993375 L20.7591248,32.0865938 L19.5420308,32.1077788 C18.3669055,32.1077788 18.3249368,32.1289639 18.4508431,32.57385 C18.8705307,33.9508786 20.5282967,35.4126474 22.3749221,36.048199 L23.6759536,36.4930852 L22.5427971,37.1710069 C20.8640467,38.1455194 18.891515,38.6963309 16.9189833,38.738701 C15.9746862,38.759886 15.1982642,38.8446262 15.1982642,38.9081814 C15.1982642,39.1200319 17.7583585,40.306395 19.2482495,40.7724662 C23.7179224,42.1494948 29.0269705,41.5563132 33.0140027,39.2047722 C35.846894,37.5311528 38.6797853,34.2050993 40.0018012,30.9849709 C40.7152701,29.2689815 41.428739,26.1335934 41.428739,24.6294545 C41.428739,23.654942 41.4916922,23.5278317 42.6668174,22.3626537 C43.359302,21.6847319 44.0098178,20.943255 44.135724,20.7314044 C44.3455678,20.3288884 44.3245835,20.3288884 43.2543801,20.6890343 C41.4707078,21.324586 41.2188952,21.2398458 42.1002392,20.2865183 C42.750755,19.6085965 43.527177,18.3798634 43.527177,18.0197174 C43.527177,17.9561623 43.2124113,18.0620876 42.8556769,18.252753 C42.477958,18.4646036 41.6385828,18.7823794 41.0090514,18.9730449 L39.8758949,19.3331908 L38.8476603,18.634084 C38.281082,18.252753 37.4836756,17.829052 37.063988,17.7019416 C35.9937846,17.4053509 34.357003,17.447721 33.3917215,17.7866818 C30.768674,18.7400093 29.110908,21.1974757 29.2997675,23.8879776 Z M29.2997675,23.8879776"
+                          id="Path" fill="#FFFFFF" sketch:type="MSShapeGroup"></path>
+                    <path d="M60.0867052,30 C60.0867052,46.5685433 46.6358389,60 30.0433526,60 C23.8895925,60 18.1679598,58.1525134 13.4044895,54.9827754 L47.8290478,5.81941103 C55.2628108,11.2806503 60.0867052,20.0777973 60.0867052,30 Z M60.0867052,30"
+                          id="reflec" fill-opacity="0.08" fill="#000000" sketch:type="MSShapeGroup"></path>
+                </svg>
+            </a>
+
+
+            <footer>
+                <hr/>
+                <div class="row">
+                    <div class="col-md-6" align="center">
+                        <h4>
+                            Open Tree is funded by NSF ABI #1759838, ABI 1759846, and AVAToL #1208809.
+                        </h4>
+                    </div>
+                    <div class="col-md-6" align="center">
+                        <h4>
+                            <a href="https://gitter.im/OpenTreeOfLife/public">Contact OpenTree </a>
+                        </h4>
+                    </div>
+                </div>
+            </footer>
 
         </footer>
-      </div>
     </div>
-  </body>
+</div>
+</body>
 </html>
diff --git a/phylesystem_api/phylesystem_api/views/amendment.py b/phylesystem_api/phylesystem_api/views/amendment.py
index 0733b9c0..256c6ad9 100644
--- a/phylesystem_api/phylesystem_api/views/amendment.py
+++ b/phylesystem_api/phylesystem_api/views/amendment.py
@@ -27,20 +27,18 @@ def __extract_and_validate_amendment(request, kwargs):
         amendment_obj = extract_json_from_http_call(
             request, data_field_name="json", request_params=kwargs
         )
-    except HTTPException as err:
+    except HTTPException:
         # payload not found
         return None, None, None
     try:
         errors, amendment_adaptor = validate_amendment(amendment_obj)
-    except HTTPException as err:
-        # _LOG.exception('JSON payload failed validation (raising HTTP response)')
-        # pprint(err)
-        raise err
+    except HTTPException:
+        raise
     except Exception as err:
         # _LOG.exception('JSON payload failed validation (reporting err.msg)')
         # pprint(err)
         try:
-            msg = err.get("msg", "No message found")
+            msg = getattr(err, "msg", "No message found")
         except:
             msg = str(err)
         raise HTTPBadRequest(body=msg)
@@ -77,9 +75,6 @@ def create_amendment(request, **kwargs):
     except:
         commit_msg = None
 
-    phylesystem = api_utils.get_phylesystem(
-        request
-    )  # set READONLY flag before testing!
     api_utils.raise_if_read_only()
     # fetch and parse the JSON payload, if any
     (
@@ -130,7 +125,6 @@ def create_amendment(request, **kwargs):
 def fetch_amendment(request):
     # NB - This method does not require authentication!
     # _LOG = api_utils.get_logger(request, 'ot_api.amendment')
-    api_version = request.matchdict["api_version"]
     amendment_id = request.matchdict["amendment_id"]
     if not AMENDMENT_ID_PATTERN.match(amendment_id):
         raise HTTPBadRequest(
@@ -146,8 +140,6 @@ def fetch_amendment(request):
 
     # fetch the current amendment JSON
     # _LOG.debug('GET /v2/amendment/{}'.format(str(amendment_id)))
-    version_history = None
-    comment_html = None
     try:
         parent_sha = find_in_request(request, "starting_commit_SHA", None)
     except:
@@ -201,7 +193,6 @@ def fetch_amendment(request):
 @view_config(route_name="update_amendment", renderer="json")
 def update_amendment(request):
     # _LOG = api_utils.get_logger(request, 'ot_api.amendment')
-    api_version = request.matchdict["api_version"]
     amendment_id = request.matchdict["amendment_id"]
     if not AMENDMENT_ID_PATTERN.match(amendment_id):
         raise HTTPBadRequest(
@@ -283,19 +274,14 @@ def update_amendment(request):
 @view_config(route_name="delete_amendment", renderer="json")
 def delete_amendment(request):
     # _LOG = api_utils.get_logger(request, 'ot_api.amendment')
-    api_version = request.matchdict["api_version"]
     amendment_id = request.matchdict.get["amendment_id"]
     if not AMENDMENT_ID_PATTERN.match(amendment_id):
-        raise HTTPBadRequest(
-            body=json.dumps(
-                {
-                    "error": 1,
-                    "description": "invalid amendment ID ({}) provided".format(
-                        amendment_id
-                    ),
-                }
-            )
-        )
+        msg = "invalid amendment ID ({}) provided".format(amendment_id)
+        body = {
+            "error": 1,
+            "description": msg,
+        }
+        raise HTTPBadRequest(body=json.dumps(body))
 
     # this method requires authentication
     auth_info = api_utils.authenticate(request)
diff --git a/phylesystem_api/phylesystem_api/views/amendments.py b/phylesystem_api/phylesystem_api/views/amendments.py
index c675dd11..9cab85c1 100644
--- a/phylesystem_api/phylesystem_api/views/amendments.py
+++ b/phylesystem_api/phylesystem_api/views/amendments.py
@@ -1,30 +1,20 @@
 import json
+import logging
 import os
 
 import phylesystem_api.api_utils as api_utils
-from peyotl.api import OTI
 from peyotl.nexson_syntax import read_as_json
 
 # see exception subclasses at https://docs.pylonsproject.org/projects/pyramid/en/latest/api/httpexceptions.html
 from pyramid.httpexceptions import (
     HTTPException,
-    HTTPError,
     HTTPInternalServerError,
 )
 from pyramid.view import view_config
-import logging
 
 _LOG = logging.getLogger("phylesystem_api")
 
 
-def _init(request, response):
-    response.view = "generic.json"
-    # CORS support for cross-domain API requests (from anywhere)
-    response.headers["Access-Control-Allow-Origin"] = "*"
-    response.headers["Access-Control-Allow-Credentials"] = "true"
-    return OTI(oti=api_utils.get_oti_domain(request))
-
-
 def _bool_arg(v):
     if isinstance(v, str):
         u = v.upper()
@@ -49,8 +39,6 @@ def list_all_amendment_ids(request):
 @view_config(route_name="list_all_amendments", renderer="json")
 def list_all(request):
     api_utils.raise_on_CORS_preflight(request)
-    # if behavior varies based on /v1/, /v2/, ...
-    api_version = request.matchdict["api_version"]
     # TODO: proxy to oti for a filtered list?
     # For now, let's just return all collections (complete JSON)
     amendment_list = []
@@ -75,8 +63,6 @@ def list_all(request):
             amendment_list.append(props)
     except HTTPException:
         raise
-    except HTTPError:
-        raise
     except Exception as x:
         msg = ",".join(x.args)
         raise HTTPInternalServerError(
@@ -101,8 +87,6 @@ def get_amendments_config(request):
 @view_config(route_name="amendments_push_failure", renderer="json")
 def amendments_push_failure(request):
     api_utils.raise_on_CORS_preflight(request)
-    # if behavior varies based on /v1/, /v2/, ...
-    api_version = request.matchdict["api_version"]
     # this should find a type-specific push_failure file
     request.matchdict["doc_type"] = "amendment"
     fail_file = api_utils.get_failed_push_filepath(request)
diff --git a/phylesystem_api/phylesystem_api/views/collection.py b/phylesystem_api/phylesystem_api/views/collection.py
index b65f1f39..b5a10270 100644
--- a/phylesystem_api/phylesystem_api/views/collection.py
+++ b/phylesystem_api/phylesystem_api/views/collection.py
@@ -3,11 +3,9 @@
 import sys
 import traceback
 
-import phylesystem_api.api_utils as api_utils
 from peyotl.collections_store import COLLECTION_ID_PATTERN
 from peyotl.collections_store.validation import validate_collection
 from peyotl.phylesystem.git_workflows import GitWorkflowError
-from phylesystem_api.api_utils import find_in_request, extract_json_from_http_call
 
 # see exception subclasses at https://docs.pylonsproject.org/projects/pyramid/en/latest/api/httpexceptions.html
 from pyramid.httpexceptions import (
@@ -17,30 +15,29 @@
 )
 from pyramid.view import view_config
 
+import phylesystem_api.api_utils as api_utils
+from phylesystem_api.api_utils import find_in_request, extract_json_from_http_call
+
 _LOG = logging.getLogger("phylesystem_api")
 
 
 def __extract_and_validate_collection(request, request_params):
-    from pprint import pprint
-
     try:
         collection_obj = extract_json_from_http_call(
             request, data_field_name="json", request_params=request_params
         )
-    except HTTPException as err:
+    except HTTPException:
         # payload not found
         return None, None, None
     try:
         errors, collection_adaptor = validate_collection(collection_obj)
-    except HTTPException as err:
-        # _LOG.exception('JSON payload failed validation (raising HTTP response)')
-        pprint(err)
-        raise err
+    except HTTPException:
+        _LOG.exception("JSON payload failed validation (raising HTTP response)")
+        raise
     except Exception as err:
-        # _LOG.exception('JSON payload failed validation (reporting err.msg)')
-        pprint(err)
+        _LOG.exception("JSON payload failed validation (reporting err.msg)")
         try:
-            msg = err.get("msg", "No message found")
+            msg = getattr(err, "msg", "No message found")
         except:
             msg = str(err)
         raise HTTPBadRequest(msg)
@@ -70,10 +67,6 @@ def create_collection(request):
             commit_msg = None
     except:
         commit_msg = None
-
-    phylesystem = api_utils.get_phylesystem(
-        request
-    )  # set READONLY flag before testing!
     api_utils.raise_if_read_only()
 
     # fetch and parse the JSON payload, if any
@@ -160,7 +153,7 @@ def create_collection(request):
         )
         new_collection_id, commit_return = r
     except GitWorkflowError as err:
-        raise HTTPBadRequest(anyjson.dumps({"error": 1, "description": msg}))
+        raise HTTPBadRequest(json.dumps({"error": 1, "description": str(err)}))
     except:
         raise HTTPBadRequest(traceback.format_exc())
     if commit_return["error"] != 0:
@@ -179,7 +172,6 @@ def create_collection(request):
 def fetch_collection(request):
     # NB - This method does not require authentication!
     # _LOG = api_utils.get_logger(request, 'ot_api.collection')
-    api_version = request.matchdict["api_version"]
     collection_id = request.matchdict["collection_id"]
     if not COLLECTION_ID_PATTERN.match(collection_id):
         raise HTTPBadRequest(
@@ -194,8 +186,6 @@ def fetch_collection(request):
         )
 
     # gather details to return with the JSON core document
-    version_history = None
-    comment_html = None
     parent_sha = find_in_request(request, "starting_commit_SHA", None)
     # _LOG.debug('parent_sha = {}'.format(parent_sha))
     # return the correct nexson of study_id, using the specified view
@@ -274,7 +264,6 @@ def update_collection(request):
     _LOG.debug("COLLECTION: auth_info {}".format(auth_info))
     owner_id = auth_info.get("login", None)
 
-    api_version = request.matchdict["api_version"]
     collection_id = request.matchdict["collection_id"]
     if not COLLECTION_ID_PATTERN.match(collection_id):
         raise HTTPBadRequest(
@@ -296,9 +285,6 @@ def update_collection(request):
     except:
         commit_msg = None
 
-    phylesystem = api_utils.get_phylesystem(
-        request
-    )  # set READONLY flag before testing!
     api_utils.raise_if_read_only()
 
     # fetch and parse the JSON payload, if any
@@ -361,7 +347,6 @@ def delete_collection(request):
     # NB - This method requires authentication!
     auth_info = api_utils.authenticate(request)
 
-    api_version = request.matchdict["api_version"]
     collection_id = request.matchdict["collection_id"]
     if not COLLECTION_ID_PATTERN.match(collection_id):
         raise HTTPBadRequest(
@@ -383,13 +368,9 @@ def delete_collection(request):
     except:
         commit_msg = None
 
-    phylesystem = api_utils.get_phylesystem(
-        request
-    )  # set READONLY flag before testing!
     api_utils.raise_if_read_only()
 
     # remove this collection from the docstore
-    owner_id = auth_info.get("login", None)
     docstore = api_utils.get_tree_collection_store(request)
     parent_sha = find_in_request(request, "starting_commit_SHA", None)
     if parent_sha is None:
diff --git a/phylesystem_api/phylesystem_api/views/collections.py b/phylesystem_api/phylesystem_api/views/collections.py
index 6c1ba73e..a64f11a6 100644
--- a/phylesystem_api/phylesystem_api/views/collections.py
+++ b/phylesystem_api/phylesystem_api/views/collections.py
@@ -2,27 +2,17 @@
 import os
 
 import phylesystem_api.api_utils as api_utils
-from peyotl.api import OTI
 from peyotl.nexson_syntax import read_as_json
 
 # see exception subclasses at https://docs.pylonsproject.org/projects/pyramid/en/latest/api/httpexceptions.html
 from pyramid.httpexceptions import (
     HTTPException,
-    HTTPError,
     HTTPInternalServerError,
     HTTPNotImplemented,
 )
 from pyramid.view import view_config
 
 
-def _init(request, response):
-    response.view = "generic.json"
-    # CORS support for cross-domain API requests (from anywhere)
-    response.headers["Access-Control-Allow-Origin"] = "*"
-    response.headers["Access-Control-Allow-Credentials"] = "true"
-    return OTI(oti=api_utils.get_oti_domain(request))
-
-
 def _bool_arg(v):
     if isinstance(v, str):
         u = v.upper()
@@ -83,8 +73,6 @@ def find_trees_in_collections(request):
 @view_config(route_name="find_collections", renderer="json")
 def find_collections(request):
     api_utils.raise_on_CORS_preflight(request)
-    # if behavior varies based on /v1/, /v2/, ...
-    api_version = request.matchdict["api_version"]
     # TODO: proxy to oti for a filtered list?
     # For now, let's just return all collections (complete JSON)
     try:
@@ -109,8 +97,6 @@ def find_collections(request):
             collection_list.append(props)
     except HTTPException:
         raise
-    except HTTPError:
-        raise
     except Exception as x:
         msg = ",".join(x.args)
         raise HTTPInternalServerError(
diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py
index b3445dfe..092372c8 100644
--- a/phylesystem_api/phylesystem_api/views/default.py
+++ b/phylesystem_api/phylesystem_api/views/default.py
@@ -37,7 +37,7 @@
     call_http_json = None
     _LOG.debug("call_http_json was not imported from api_utils")
 
-from beaker.cache import cache_region, region_invalidate
+from beaker.cache import cache_region
 
 
 @view_config(route_name="index", renderer="phylesystem_api:templates/home.jinja2")
@@ -154,9 +154,7 @@ def fetch_and_cache(url):
             # _LOG.warning( dict(fetched.headers) )
 
             try:
-                test_for_json = (
-                    fetched.json()
-                )  # missing JSON payload will raise an error
+                fetched.json()  # missing JSON payload will raise an error
                 return Response(
                     headers=fetched.headers,
                     body=fetched.text,  # missing JSON payload will raise an error
@@ -288,7 +286,6 @@ def include_tree_in_synth(request):
         )
     # examine this study and tree, to confirm it exists *and* to capture its name
     sds = api_utils.get_phylesystem(request)
-    found_study = None
     try:
         found_study = sds.return_doc(study_id, commit_sha=None, return_WIP_map=False)[0]
         tree_collections_by_id = found_study.get("nexml").get("treesById")
@@ -381,7 +378,7 @@ def include_tree_in_synth(request):
             )
 
     # fetch and return the updated list of synth-input trees
-    return trees_in_synth(kwargs)
+    return trees_in_synth(request)
 
 
 @view_config(route_name="exclude_tree_from_synth", renderer="json")
@@ -445,9 +442,8 @@ def exclude_tree_from_synth(request):
                     doc_type="collection",
                     auth_token=auth_info["auth_token"],
                 )
-
     # fetch and return the updated list of synth-input trees
-    return trees_in_synth(kwargs)
+    return trees_in_synth(request)
 
 
 def _get_synth_input_collection_ids():
@@ -509,8 +505,6 @@ def merge_docstore_changes(request):
             "description": "Could not merge master into WIP! Details: ..."
         }
     """
-    # if behavior varies based on /v1/, /v2/, ...
-    api_version = request.matchdict["api_version"]
     resource_id = request.matchdict["doc_id"]
     starting_commit_SHA = request.matchdict["starting_commit_SHA"]
     api_utils.raise_if_read_only()
@@ -538,14 +532,11 @@ def merge_docstore_changes(request):
 def push_docstore_changes(request):
     """OpenTree API method to update branch on master
 
-    curl -X POST http://devapi.opentreeoflife.org/v3/push_docstore_changes/nexson/ot_999
+    curl -X POST https://devapi.opentreeoflife.org/v3/push_docstore_changes/nexson/ot_999
     """
-    # if behavior varies based on /v1/, /v2/, ...
-    api_version = request.matchdict["api_version"]
     _LOG.debug("push_docstore_changes")
     #    _LOG.debug(request.__dict__)
     _LOG.debug(request.matchdict)
-    api_version = request.matchdict["api_version"]
     doc_type = request.matchdict.get("doc_type", None)
     resource_id = request.matchdict.get("doc_id", None)  # Whyyyyy doc id here??
     #    resource_id = request.matchdict.get('resource_id', None)
@@ -553,7 +544,7 @@ def push_docstore_changes(request):
     data = request.json_body
     if doc_type is None:
         doc_type = data.get("doc_type")
-    if resource_id == None:
+    if resource_id is None:
         resource_id = data.get("resource_id")
 
     api_utils.raise_if_read_only()
@@ -564,7 +555,7 @@ def push_docstore_changes(request):
     _LOG.debug("Going to authenicateeee")
 
     # this method requires authentication
-    auth_info = api_utils.authenticate(request)
+    api_utils.authenticate(request)
     _LOG.debug("Made it past auth")
 
     # TODO
diff --git a/phylesystem_api/phylesystem_api/views/notfound.py b/phylesystem_api/phylesystem_api/views/notfound.py
index a83683a5..660aedae 100644
--- a/phylesystem_api/phylesystem_api/views/notfound.py
+++ b/phylesystem_api/phylesystem_api/views/notfound.py
@@ -1,19 +1,6 @@
 from pyramid.response import Response
 from pyramid.view import notfound_view_config
-
-try:
-    import anyjson
-except:
-    import json
-
-    class Wrapper(object):
-        pass
-
-    anyjson = Wrapper()
-    anyjson.loads = json.loads
-
-
-# _LOG = logging.getLogger('phylesystem_api')
+import json
 
 # most API pages should be JSON, so here's a suitable 404 response
 @notfound_view_config(renderer="json", accept="application/json", append_slash=True)
@@ -21,7 +8,7 @@ def notfound(request):
     # _LOG.debug("Request not found")
     # _LOG.debug(request)
     return Response(
-        body=anyjson.dumps({"message": "Nothing found at this URL"}),
+        body=json.dumps({"message": "Nothing found at this URL"}),
         status="404 Not Found",
         charset="UTF-8",
         content_type="application/json",
diff --git a/phylesystem_api/phylesystem_api/views/studies.py b/phylesystem_api/phylesystem_api/views/studies.py
index 0fa4068a..53d6b8bc 100644
--- a/phylesystem_api/phylesystem_api/views/studies.py
+++ b/phylesystem_api/phylesystem_api/views/studies.py
@@ -1,13 +1,11 @@
 import json
 
 import phylesystem_api.api_utils as api_utils
-from peyotl.api import OTI
 from phylesystem_api.api_utils import find_in_request
 
 # see exception subclasses at https://docs.pylonsproject.org/projects/pyramid/en/latest/api/httpexceptions.html
 from pyramid.httpexceptions import (
     HTTPException,
-    HTTPError,
     HTTPBadRequest,
     HTTPInternalServerError,
 )
@@ -18,12 +16,11 @@ def _raise400(msg):
     raise HTTPBadRequest(body=json.dumps({"error": 1, "description": msg}))
 
 
-def _init(request, response):
+def _configure_response(response):
     response.view = "generic.json"
     # CORS support for cross-domain API requests (from anywhere)
     response.headers["Access-Control-Allow-Origin"] = "*"
     response.headers["Access-Control-Allow-Credentials"] = "true"
-    return OTI(oti=api_utils.get_oti_domain(request))
 
 
 def _bool_arg(v):
@@ -38,7 +35,8 @@ def _bool_arg(v):
 
 @view_config(route_name="study_properties", renderer="json")
 def properties(request):
-    oti = _init(request, request.response)
+    _configure_response(request.response)
+    oti = api_utils.get_oti_wrapper(request)
     n = list(oti.node_search_term_set)
     n.sort()
     t = list(oti.tree_search_term_set)
@@ -50,9 +48,7 @@ def properties(request):
 
 @view_config(route_name="find_studies", renderer="json")
 def find_studies(request):
-    # if behavior varies based on /v1/, /v2/, ...
-    api_version = request.matchdict["api_version"]
-    oti = _init(request, request.response)
+    oti = api_utils.get_oti_wrapper(request)
     verbose = _bool_arg(find_in_request(request, "verbose", False))
     if (verbose is not True) and (verbose is not False):
         _raise400('"verbose" setting must be a boolean')
@@ -60,7 +56,6 @@ def find_studies(request):
     try:
         if field is None:
             match_list = oti.find_all_studies(verbose=verbose)
-            resp = {"matched_studies": match_list}
         else:
             value = find_in_request(request, "value", None)
             if value is None:
@@ -72,14 +67,11 @@ def find_studies(request):
                 match_list = oti.find_studies(
                     {field: value}, verbose=verbose, exact=exact
                 )
-                resp = {"matched_studies": match_list}
             except ValueError as x:
-                _raise400(x.message)
-
+                _raise400(str(x))
+        resp = {"matched_studies": match_list}
     except HTTPException:
         raise
-    except HTTPError:
-        raise
     except Exception as x:
         msg = ",".join(x.args)
         raise HTTPInternalServerError(
@@ -90,18 +82,18 @@ def find_studies(request):
                 }
             )
         )
+    _configure_response(request.response)
     return resp
 
 
 @view_config(route_name="find_trees", renderer="json")
 def find_trees(request):
-    # if behavior varies based on /v1/, /v2/, ...
-    api_version = request.matchdict["api_version"]
     try:
         msg = request.json_body
     except:
         _raise400("missing or invalid request JSON")
-    oti = _init(request, request.response)
+    _configure_response(request.response)
+    oti = api_utils.get_oti_wrapper(request)
     verbose = _bool_arg(msg.get("verbose", False))
     if (verbose is not True) and (verbose is not False):
         _raise400('"verbose" setting must be a boolean')
@@ -120,7 +112,7 @@ def find_trees(request):
     except HTTPException:
         raise
     except ValueError as x:
-        _raise400(x.message)
+        _raise400(str(x))
     except Exception as x:
         msg = ",".join(x.args)
         raise HTTPInternalServerError(
diff --git a/phylesystem_api/phylesystem_api/views/study.py b/phylesystem_api/phylesystem_api/views/study.py
index 10c8df20..cd2ffdf1 100644
--- a/phylesystem_api/phylesystem_api/views/study.py
+++ b/phylesystem_api/phylesystem_api/views/study.py
@@ -186,26 +186,38 @@ def __finish_write_verb(
     return annotated_commit
 
 
+def _fetch_duplicate_study_ids(request, study_DOI=None, study_ID=None):
+    # Use the oti (docstore index) service to see if there are other studies in
+    # the collection with the same DOI; return the IDs of any duplicate studies
+    # found, or an empty list if there are no dupes.
+    if not study_DOI:
+        return []
+    oti = api_utils.get_oti_wrapper(request)
+    duplicate_study_ids = oti.find_studies(
+        {"ot:studyPublication": study_DOI}, verbose=False, exact=True
+    )
+    try:
+        duplicate_study_ids.remove(study_ID)
+    except ValueError:
+        # ignore error, if oti is lagging and doesn't have this study yet
+        pass
+        pass
+    return duplicate_study_ids
+
+
+def _fetch_shard_name(phylesystem, study_id):
+    try:
+        return phylesystem.get_repo_and_path_fragment(study_id)[0]
+    except:
+        _LOG.debug("_fetch_shard_name failed for study {}".format(study_id))
+        return None
+
+
 @view_config(route_name="fetch_study", renderer=None, request_method="GET")
 @view_config(route_name="fetch_study_label", renderer=None, request_method="GET")
 def fetch_study(request):
-    (
-        repo_parent,
-        repo_remote,
-        git_ssh,
-        pkey,
-        git_hub_remote,
-        max_filesize,
-        max_num_trees,
-        read_only_mode,
-    ) = api_utils.read_phylesystem_config(request)
-    # _LOG = api_utils.get_logger(request, 'ot_api.default.v1')
-    _LOG.debug("Fetching study")
-    api_version = request.matchdict["api_version"]
     study_id = request.matchdict["study_id"]
     _LOG.debug("study_id = {}".format(study_id))
-    version_history = None
-    comment_html = None
     # does this look like a filename? if so, grab its extension
     request_extension = None
     fpps = study_id.split(".")
@@ -277,13 +289,15 @@ def fetch_study(request):
         except KeyError:
             study_DOI = None
         try:
-            duplicate_study_ids = _fetch_duplicate_study_ids(study_DOI, study_id)
+            duplicate_study_ids = _fetch_duplicate_study_ids(
+                request, study_DOI, study_id
+            )
         except:
             # _LOG.exception('call to OTI check for duplicate DOIs failed')
             duplicate_study_ids = None
 
         try:
-            shard_name = _fetch_shard_name(study_id)
+            shard_name = _fetch_shard_name(phylesystem, study_id)
         except:
             # _LOG.exception('check for shard name failed')
             shard_name = None
@@ -317,9 +331,8 @@ def _new_nexson_with_crossref_metadata(doi, ref_string, include_cc0=False):
     # N.B. The recommended API method is very different for DOI vs.
     # a reference string, but we can also use the filter option to get
     # consistent behavior and response.
-    best_match = None
     no_match_found = False
-    meta_publication_reference = u""
+    meta_publication_reference = ""
     try:
         if doi:
             # use the supplied DOI to fetch study metadata
@@ -335,10 +348,10 @@ def _new_nexson_with_crossref_metadata(doi, ref_string, include_cc0=False):
             ).text  # always Unicode
         try:
             response_json = json.loads(lookup_response)
-            response_status = response_json.get("status", u"")
+            response_status = response_json.get("status", "")
         except:
-            response_status = u"missing or malformed response text"
-        if response_status == u"ok":
+            response_status = "missing or malformed response text"
+        if response_status == "ok":
             matching_records = response_json.get("message", {}).get("items", [])
             if len(matching_records) == 0:
                 no_match_found = True
@@ -346,29 +359,29 @@ def _new_nexson_with_crossref_metadata(doi, ref_string, include_cc0=False):
             # Something went wrong (TODO: Capture and log any error message?)
             no_match_found = True
 
-    except requests.RequestException as e:
+    except requests.RequestException:
         # Both calls above should return a 200 status even if there's no match.
         # So apparently the CrossRef service is down for some reason.
         no_match_found = True
 
     if no_match_found:
         # Add a bogus reference string to signal the lack of results
-        meta_publication_url = u""
-        meta_year = u""
+        meta_publication_url = ""
+        meta_year = ""
         if doi:
-            meta_publication_reference = u"No matching publication found for this DOI!"
+            meta_publication_reference = "No matching publication found for this DOI!"
         else:
             meta_publication_reference = (
-                u"No matching publication found for this reference string"
+                "No matching publication found for this reference string"
             )
     else:
         # We got a match! These are sorted by score, so we should assume the
         # first (possibly only) record is the best match.
         # See https://github.com/CrossRef/rest-api-doc#sort-order
         match = matching_records[0]
-        meta_publication_url = match.get("URL", u"")  # should be its DOI in URL form
+        meta_publication_url = match.get("URL", "")  # should be its DOI in URL form
         # Try to capture the publication year (print first, or online)
-        meta_year = u""
+        meta_year = ""
         date_parts = match.get("published-print", {}).get("date-parts", [])
         if len(date_parts) == 0:
             # try again using online publication
@@ -381,7 +394,7 @@ def _new_nexson_with_crossref_metadata(doi, ref_string, include_cc0=False):
                 meta_year = inner_date_parts[0]
 
         # capture the raw DOI so we can try to retrieve a reference string below
-        doi = match.get("DOI", u"")
+        doi = match.get("DOI", "")
 
     # We need another API call to fetch a plain-text reference string.
     # NB - this is probabl APA style (based on conversation with CrossRef API team)
@@ -398,47 +411,30 @@ def _new_nexson_with_crossref_metadata(doi, ref_string, include_cc0=False):
         except requests.RequestException as e:
             # Any response but 200 means no match found, or the CrossRef
             # service is down for some reason.
-            _GLOG.debug("URLError fetching ref-text!!")
-            _GLOG.debug(e)
-            meta_publication_reference = u"No matching publication found for this DOI!"
+            _LOG.debug("URLError fetching ref-text!!")
+            _LOG.debug(e)
+            meta_publication_reference = "No matching publication found for this DOI!"
 
     # add any found values to a fresh NexSON template
     nexson = get_empty_nexson(BY_ID_HONEY_BADGERFISH, include_cc0=include_cc0)
     nexml_el = nexson["nexml"]
-    nexml_el[u"^ot:studyPublicationReference"] = meta_publication_reference
+    nexml_el["^ot:studyPublicationReference"] = meta_publication_reference
     if meta_publication_url:
-        nexml_el[u"^ot:studyPublication"] = {"@href": meta_publication_url}
+        nexml_el["^ot:studyPublication"] = {"@href": meta_publication_url}
     if meta_year:
-        nexml_el[u"^ot:studyYear"] = meta_year
+        nexml_el["^ot:studyYear"] = meta_year
     return nexson
 
 
 @view_config(route_name="create_study", renderer="json", request_method="POST")
 def create_study(request):
-    api_version = request.matchdict["api_version"]
-
     # this method requires authentication
     auth_info = api_utils.authenticate(request)
-
-    # gather any user-provided git-commit message
-    try:
-        commit_msg = find_in_request(request, "commit_msg", "")
-        if commit_msg.strip() == "":
-            # git rejects empty commit messages
-            commit_msg = None
-    except:
-        commit_msg = None
-
-    phylesystem = api_utils.get_phylesystem(
-        request
-    )  # set READONLY flag before testing!
+    phylesystem = api_utils.get_phylesystem(request)
     api_utils.raise_if_read_only()
 
     # we're creating a new study (possibly with import instructions in the payload)
-    import_from_location = find_in_request(request, "import_from_location", "")
     treebase_id = find_in_request(request, "treebase_id", "")
-    nexml_fetch_url = find_in_request(request, "nexml_fetch_url", "")
-    nexml_pasted_string = find_in_request(request, "nexml_pasted_string", "")
     publication_doi = find_in_request(request, "publication_DOI", "")
     # if a URL or something other than a valid DOI was entered, don't submit it to crossref API
     publication_doi_for_crossref = __make_valid_DOI(publication_doi) or None
@@ -461,13 +457,7 @@ def create_study(request):
     importing_from_treebase_id = (
         import_method == "import-method-TREEBASE_ID" and treebase_id != ""
     )
-    importing_from_nexml_fetch = (
-        import_method == "import-method-NEXML" and nexml_fetch_url
-    )
     importing_from_post_arg = import_method == "import-method-POST"
-    importing_from_nexml_string = (
-        import_method == "import-method-NEXML" and nexml_pasted_string
-    )
     importing_from_crossref_API = (
         import_method == "import-method-PUBLICATION_DOI"
         and publication_doi_for_crossref
@@ -489,28 +479,22 @@ def create_study(request):
         try:
             treebase_id = int(treebase_id)
         except ValueError as e:
-            raise HTTPBadRequest(
-                json.dumps(
-                    {
-                        "error": 1,
-                        "description": "TreeBASE ID should be a simple integer, not '%s'! Details:\n%s"
-                        % (treebase_id, e.message),
-                    }
-                )
-            )
+            msg = "TreeBASE ID should be a simple integer, not '{}'! Details:\n{}"
+            msg = msg.format(treebase_id, str(e))
+            raise HTTPBadRequest(json.dumps({"error": 1, "description": msg}))
         try:
             new_study_nexson = import_nexson_from_treebase(
                 treebase_id, nexson_syntax_version=BY_ID_HONEY_BADGERFISH
             )
-        except Exception as e:
-            raise HTTPInternalServerError(
-                json.dumps(
-                    {
-                        "error": 1,
-                        "description": "Unexpected error parsing the file obtained from TreeBASE. Please report this bug to the Open Tree of Life developers.",
-                    }
-                )
+        except Exception:
+            msg = "Unexpected error parsing the file obtained from TreeBASE. Please report this bug to the Open Tree of Life developers."
+            bodys = json.dumps(
+                {
+                    "error": 1,
+                    "description": msg,
+                }
             )
+            raise HTTPInternalServerError(bodys)
     elif importing_from_crossref_API:
         new_study_nexson = _new_nexson_with_crossref_metadata(
             doi=publication_doi_for_crossref,
@@ -529,7 +513,7 @@ def create_study(request):
         )
         if publication_doi:
             # submitter entered an invalid DOI (or other URL); add it now
-            new_study_nexson["nexml"][u"^ot:studyPublication"] = {
+            new_study_nexson["nexml"]["^ot:studyPublication"] = {
                 "@href": publication_doi
             }
 
@@ -539,32 +523,32 @@ def create_study(request):
         # If submitter requested the CC0 waiver or other waiver/license, make sure it's here
         if cc0_agreement:
             nexml["^xhtml:license"] = {
-                "@href": "http://creativecommons.org/publicdomain/zero/1.0/"
+                "@href": "https://creativecommons.org/publicdomain/zero/1.0/"
             }
         elif using_existing_license:
             existing_license = find_in_request(request, "alternate_license", "")
             if existing_license == "CC-0":
                 nexml["^xhtml:license"] = {
                     "@name": "CC0",
-                    "@href": "http://creativecommons.org/publicdomain/zero/1.0/",
+                    "@href": "https://creativecommons.org/publicdomain/zero/1.0/",
                 }
                 pass
             elif existing_license == "CC-BY-2.0":
                 nexml["^xhtml:license"] = {
                     "@name": "CC-BY 2.0",
-                    "@href": "http://creativecommons.org/licenses/by/2.0/",
+                    "@href": "https://creativecommons.org/licenses/by/2.0/",
                 }
                 pass
             elif existing_license == "CC-BY-2.5":
                 nexml["^xhtml:license"] = {
                     "@name": "CC-BY 2.5",
-                    "@href": "http://creativecommons.org/licenses/by/2.5/",
+                    "@href": "https://creativecommons.org/licenses/by/2.5/",
                 }
                 pass
             elif existing_license == "CC-BY-3.0":
                 nexml["^xhtml:license"] = {
                     "@name": "CC-BY 3.0",
-                    "@href": "http://creativecommons.org/licenses/by/3.0/",
+                    "@href": "https://creativecommons.org/licenses/by/3.0/",
                 }
                 pass
             # NOTE that we don't offer CC-BY 4.0, which is problematic for data
@@ -572,7 +556,7 @@ def create_study(request):
                 # default to version 3, if not specified.
                 nexml["^xhtml:license"] = {
                     "@name": "CC-BY 3.0",
-                    "@href": "http://creativecommons.org/licenses/by/3.0/",
+                    "@href": "https://creativecommons.org/licenses/by/3.0/",
                 }
                 pass
             else:  # assume it's something else
@@ -608,7 +592,6 @@ def create_study(request):
 
 @view_config(route_name="update_study", renderer="json")
 def update_study(request):
-    api_version = request.matchdict["api_version"]
     study_id = request.matchdict["study_id"]
 
     # this method requires authentication
@@ -638,21 +621,11 @@ def update_study(request):
     api_utils.raise_if_read_only()
 
     repo_nexml2json = phylesystem.repo_nexml2json
-    (
-        repo_parent,
-        repo_remote,
-        git_ssh,
-        pkey,
-        git_hub_remote,
-        max_filesize,
-        max_num_trees,
-        read_only_mode,
-    ) = api_utils.read_phylesystem_config(request)
     bundle = __extract_and_validate_nexson(request, repo_nexml2json, request.json_body)
     nexson, annotation, nexson_adaptor = bundle
     try:
         gd = phylesystem.create_git_action(study_id)
-    except KeyError as err:
+    except KeyError:
         # _LOG.debug('PUT failed in create_git_action (probably a bad study ID)')
         raise HTTPBadRequest("invalid study ID, please check the URL")
     try:
@@ -685,7 +658,7 @@ def update_study(request):
 @view_config(route_name="delete_study", renderer="json")
 def delete_study(request):
     # _LOG = api_utils.get_logger(request, 'delete_study')
-    _LOG.warninging("delete_study STARTING...")
+    _LOG.warning("delete_study STARTING...")
     api_version = request.matchdict["api_version"]
     study_id = request.matchdict["study_id"]
     _LOG.warning("api_version={}".format(api_version))
@@ -746,15 +719,11 @@ def delete_study(request):
 @view_config(route_name="get_study_single_file", renderer="json")
 def get_study_file(request):
     api_utils.raise_on_CORS_preflight(request)
-
-    api_version = request.matchdict["api_version"]
     study_id = request.matchdict["study_id"]
     file_id = request.matchdict.get("file_id", None)
-
-    result_data = None
     phylesystem = api_utils.get_phylesystem(request)
     repo_nexml2json = phylesystem.repo_nexml2json
-    out_schema = __validate_output_nexml2json(
+    __validate_output_nexml2json(
         repo_nexml2json,
         request.params,  # combined GET and POST
         "file",
@@ -835,10 +804,7 @@ def get_study_file(request):
 @view_config(route_name="get_study_external_url", renderer="json")
 def get_study_external_url(request):
     api_utils.raise_on_CORS_preflight(request)
-
-    api_version = request.matchdict["api_version"]
     study_id = request.matchdict["study_id"]
-
     phylesystem = api_utils.get_phylesystem(request)
     try:
         u = phylesystem.get_public_url(study_id)
@@ -894,7 +860,6 @@ def get_study_ometa(request):
 def _fine_grained_get(request, subresource, content_id=None, file_ext=None):
     api_utils.raise_on_CORS_preflight(request)
     study_id = request.matchdict["study_id"]
-    result_data = None
     phylesystem = api_utils.get_phylesystem(request)
     repo_nexml2json = phylesystem.repo_nexml2json
     out_schema = __validate_output_nexml2json(
@@ -921,13 +886,6 @@ def _fine_grained_get(request, subresource, content_id=None, file_ext=None):
         study_nexson, head_sha, wip_map = r
         blob_sha = phylesystem.get_blob_sha_for_study_id(study_id, head_sha)
         phylesystem.add_validation_annotation(study_nexson, blob_sha)
-        version_history = phylesystem.get_version_history_for_study_id(study_id)
-        try:
-            comment_html = api_utils.markdown_to_html(
-                study_nexson["nexml"]["^ot:comment"], open_links_in_new_window=True
-            )
-        except:
-            comment_html = ""
     except:
         _LOG.exception("GET failed")
         e = sys.exc_info()[0]
diff --git a/phylesystem_api/tests/conftest.py b/phylesystem_api/tests/conftest.py
index 987dc704..dca95985 100644
--- a/phylesystem_api/tests/conftest.py
+++ b/phylesystem_api/tests/conftest.py
@@ -36,7 +36,6 @@ def testapp(app):
             "HTTP_HOST": "example.com",
         },
     )
-
     return testapp
 
 
diff --git a/phylesystem_api/tests/test_gitdata.py b/phylesystem_api/tests/test_gitdata.py
deleted file mode 100644
index 143f9e64..00000000
--- a/phylesystem_api/tests/test_gitdata.py
+++ /dev/null
@@ -1,225 +0,0 @@
-import unittest
-import os
-import sys
-import time
-from phylesystem_api.gitdata import GitData, MergeException
-import simplejson as json
-from sh import git
-
-try:
-    # Python 2 only:
-    from ConfigParser import SafeConfigParser
-except ImportError:
-    # Python 2 and 3 (after ``pip install configparser``)
-    from configparser import SafeConfigParser
-
-
-class TestGitData(unittest.TestCase):
-    @classmethod
-    def setUpClass(self):
-        conf = SafeConfigParser(allow_no_value=True)
-        if os.path.isfile("../private/localconfig"):
-            conf.read("../private/localconfig")
-        else:
-            conf.read("../private/config")
-
-        self.repo = conf.get("apis", "repo_path")
-        self.gd = GitData(repo=self.repo)
-        self.orig_cwd = os.getcwd()
-
-        # go into our data repo
-        os.chdir(self.repo)
-
-        self.testing_branch_name = "testing_%d" % int(time.time())
-        # create the branch
-        git.checkout("-b", self.testing_branch_name)
-
-        # start all tests on the master branch
-        git.checkout("master")
-
-    @classmethod
-    def tearDownClass(self):
-        git.branch("-D", self.testing_branch_name)
-
-    def test_merge(self):
-        def cleanup_merge():
-            git.checkout("master")
-            # if something failed, the branch might still exist
-            if self.gd.branch_exists("to_merge_1"):
-                git.branch("-D", "to_merge_1")
-
-            git.branch("-D", "base_branch")
-
-        self.addCleanup(cleanup_merge)
-
-        git.checkout("-b", "to_merge_1")
-        git.checkout("-b", "base_branch")
-
-        file = open("foo.txt", "w")
-        file.write("ABC\n")
-        file.close()
-
-        git.add("foo.txt")
-
-        git.commit("-m", "Test commit")
-
-        new_sha = self.gd.merge("to_merge_1", "base_branch")
-
-        self.assertTrue(new_sha != "", "new_sha=%s is non-empty" % new_sha)
-        self.assertEqual(len(new_sha), 40, "SHA is 40 chars")
-
-        self.assertTrue(True, "Merge succeeded")
-
-    def test_merge_conflict(self):
-        def cleanup_merge_conflict():
-            git.checkout("master")
-            # if something failed, the branch might still exist
-            if self.gd.branch_exists("to_merge_1"):
-                git.branch("-D", "to_merge_1")
-
-            if self.gd.branch_exists("to_merge_2"):
-                git.branch("-D", "to_merge_2")
-
-        self.addCleanup(cleanup_merge_conflict)
-
-        git.checkout("master")
-        git.checkout("-b", "to_merge_1")
-
-        file = open("foo.txt", "w")
-        file.write("ABC\n")
-        file.close()
-
-        git.add("foo.txt")
-        git.commit("-m", "Test commit")
-
-        git.checkout("master")
-
-        git.checkout("-b", "to_merge_2")
-
-        file = open("foo.txt", "w")
-        file.write("XYZ\n")
-        file.close()
-
-        git.add("foo.txt")
-        git.commit("-m", "Test commit")
-
-        self.assertRaises(
-            MergeException, lambda: self.gd.merge("to_merge_1", "to_merge_2")
-        )
-
-    def test_current_branch(self):
-        git.checkout(self.testing_branch_name)
-        branch_name = self.gd.current_branch()
-        self.assertEqual(branch_name, self.testing_branch_name)
-
-    def test_fetch(self):
-        study_id = 438
-        study_nexson, head_sha = self.gd.fetch_study(study_id)
-        valid = 1
-        try:
-            json.loads(study_nexson)
-        except:
-            valid = 0
-        self.assertTrue(valid, "fetch_study(%s) returned valid JSON" % study_id)
-
-    def test_write(self):
-        def cleanup_write():
-            git.checkout("master")
-            git.branch("-D", "johndoe_study_9998")
-            git.branch("-D", "johndoe_study_9999")
-
-        self.addCleanup(cleanup_write)
-
-        author = "John Doe <john@doe.com>"
-        content = '{"foo":"bar"}'
-        study_id = 9999
-        branch = "johndoe_study_%s" % study_id
-        new_sha = self.gd.write_study(study_id, content, branch, author)
-        self.assertTrue(new_sha != "", "new_sha is non-empty")
-        self.assertEqual(len(new_sha), 40, "SHA is 40 chars")
-        fetched_content, head_sha = self.gd.fetch_study(9999)
-        self.assertEqual(
-            content, fetched_content, "correct content found via fetch_study"
-        )
-
-        author = "John Doe <john@doe.com>"
-        content = '{"foo2":"bar2"}'
-        study_id = 9998
-        branch = "johndoe_study_%s" % study_id
-        new_sha = self.gd.write_study(study_id, content, branch, author)
-
-        merge_base_sha1 = git(
-            "merge-base", "johndoe_study_9999", "johndoe_study_9998"
-        ).strip()
-
-        master_sha1 = git("rev-parse", "master").strip()
-
-        self.assertEqual(
-            master_sha1,
-            merge_base_sha1,
-            "Verify that writing new study branches from master and not the current branch",
-        )
-
-    def test_remove(self):
-        def cleanup_remove():
-            git.checkout("master")
-            git.branch("-D", "johndoe_study_777")
-
-        self.addCleanup(cleanup_remove)
-
-        author = "John Doe <john@doe.com>"
-        content = '{"foo2":"bar3"}'
-        study_id = 777
-        branch = "johndoe_study_%s" % study_id
-
-        new_sha = self.gd.remove_study(study_id, branch, author)
-        self.assertTrue(new_sha != "", "new_sha is non-empty")
-        self.assertEqual(len(new_sha), 40, "SHA is 40 chars")
-
-        deleted_study_dir = "%s/study/%s" % (self.repo, study_id)
-        self.assertFalse(
-            os.path.exists(deleted_study_dir),
-            "%s should no longer exist" % deleted_study_dir,
-        )
-
-    def test_branch_exists(self):
-        exists = self.gd.branch_exists("nothisdoesnotexist")
-        self.assertTrue(exists == 0, "branch does not exist")
-
-        branch_name = self.testing_branch_name
-
-        exists = self.gd.branch_exists(branch_name)
-        self.assertTrue(exists, "%s branch exists" % branch_name)
-
-    def test_newest_study_id(self):
-        def cleanup_newest():
-            git.checkout("master")
-            git.branch("-D", "leto_study_o9999")
-
-        self.addCleanup(cleanup_newest)
-
-        git.checkout("master")
-        newest_id = self.gd.newest_study_id()
-
-        self.assertGreaterEqual(newest_id, 2600)
-
-        git.checkout("-b", "leto_study_o9999")
-
-        newest_id = self.gd.newest_study_id()
-        self.assertGreaterEqual(newest_id, 9999)
-
-
-def suite():
-    loader = unittest.TestLoader()
-    testsuite = loader.loadTestsFromTestCase(TestGitData)
-    return testsuite
-
-
-def test_main():
-    testsuite = suite()
-    runner = unittest.TextTestRunner(sys.stdout, verbosity=2)
-    result = runner.run(testsuite)
-
-
-if __name__ == "__main__":
-    test_main()

From a6559836f61bb19114fa032c8f5b361f2056b139 Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Thu, 27 Jul 2023 15:34:46 -0500
Subject: [PATCH 06/33] helper function for raising internal server errors

---
 phylesystem_api/phylesystem_api/api_utils.py  |  9 ++++++
 .../phylesystem_api/views/amendments.py       | 12 ++------
 .../phylesystem_api/views/collections.py      | 11 +-------
 .../phylesystem_api/views/studies.py          | 28 ++++---------------
 .../phylesystem_api/views/study.py            |  9 +-----
 5 files changed, 18 insertions(+), 51 deletions(-)

diff --git a/phylesystem_api/phylesystem_api/api_utils.py b/phylesystem_api/phylesystem_api/api_utils.py
index f984f87a..5731d119 100644
--- a/phylesystem_api/phylesystem_api/api_utils.py
+++ b/phylesystem_api/phylesystem_api/api_utils.py
@@ -15,6 +15,7 @@
     HTTPServerError,
     HTTPBadRequest,
     HTTPForbidden,
+    HTTPInternalServerError,
 )
 from beaker.cache import cache_managers
 import tempfile
@@ -38,6 +39,14 @@
 READ_ONLY_MODE = True
 
 
+def raise_int_server_err(msg):
+    body = {
+        "error": 1,
+        "description": msg,
+    }
+    raise HTTPInternalServerError(body=json.dumps(body))
+
+
 def get_private_dir(request):
     _LOG.debug("WHY PRIVATE DIR")
     return "~/private/"
diff --git a/phylesystem_api/phylesystem_api/views/amendments.py b/phylesystem_api/phylesystem_api/views/amendments.py
index 9cab85c1..b8cf784c 100644
--- a/phylesystem_api/phylesystem_api/views/amendments.py
+++ b/phylesystem_api/phylesystem_api/views/amendments.py
@@ -8,7 +8,6 @@
 # see exception subclasses at https://docs.pylonsproject.org/projects/pyramid/en/latest/api/httpexceptions.html
 from pyramid.httpexceptions import (
     HTTPException,
-    HTTPInternalServerError,
 )
 from pyramid.view import view_config
 
@@ -64,15 +63,8 @@ def list_all(request):
     except HTTPException:
         raise
     except Exception as x:
-        msg = ",".join(x.args)
-        raise HTTPInternalServerError(
-            body=json.dumps(
-                {
-                    "error": 1,
-                    "description": "Unexpected error calling oti: {}".format(msg),
-                }
-            )
-        )
+        msg = "Unexpected error calling oti: {}".format(",".join(x.args))
+        api_utils.raise_int_server_err(msg)
     return amendment_list
 
 
diff --git a/phylesystem_api/phylesystem_api/views/collections.py b/phylesystem_api/phylesystem_api/views/collections.py
index a64f11a6..6696fa24 100644
--- a/phylesystem_api/phylesystem_api/views/collections.py
+++ b/phylesystem_api/phylesystem_api/views/collections.py
@@ -7,7 +7,6 @@
 # see exception subclasses at https://docs.pylonsproject.org/projects/pyramid/en/latest/api/httpexceptions.html
 from pyramid.httpexceptions import (
     HTTPException,
-    HTTPInternalServerError,
     HTTPNotImplemented,
 )
 from pyramid.view import view_config
@@ -98,13 +97,5 @@ def find_collections(request):
     except HTTPException:
         raise
     except Exception as x:
-        msg = ",".join(x.args)
-        raise HTTPInternalServerError(
-            body=json.dumps(
-                {
-                    "error": 1,
-                    "description": "Unexpected error calling oti: {}".format(msg),
-                }
-            )
-        )
+        api_utils.raise_int_server_err("Unexpected error gathering collections")
     return collection_list
diff --git a/phylesystem_api/phylesystem_api/views/studies.py b/phylesystem_api/phylesystem_api/views/studies.py
index 53d6b8bc..b0672e2c 100644
--- a/phylesystem_api/phylesystem_api/views/studies.py
+++ b/phylesystem_api/phylesystem_api/views/studies.py
@@ -4,11 +4,7 @@
 from phylesystem_api.api_utils import find_in_request
 
 # see exception subclasses at https://docs.pylonsproject.org/projects/pyramid/en/latest/api/httpexceptions.html
-from pyramid.httpexceptions import (
-    HTTPException,
-    HTTPBadRequest,
-    HTTPInternalServerError,
-)
+from pyramid.httpexceptions import HTTPException, HTTPBadRequest
 from pyramid.view import view_config
 
 
@@ -73,15 +69,8 @@ def find_studies(request):
     except HTTPException:
         raise
     except Exception as x:
-        msg = ",".join(x.args)
-        raise HTTPInternalServerError(
-            body=json.dumps(
-                {
-                    "error": 1,
-                    "description": "Unexpected error calling oti: {}".format(msg),
-                }
-            )
-        )
+        msg = "Unexpected error calling oti: {}".format(",".join(x.args))
+        api_utils.raise_int_server_err(msg)
     _configure_response(request.response)
     return resp
 
@@ -114,13 +103,6 @@ def find_trees(request):
     except ValueError as x:
         _raise400(str(x))
     except Exception as x:
-        msg = ",".join(x.args)
-        raise HTTPInternalServerError(
-            body=json.dumps(
-                {
-                    "error": 1,
-                    "description": "Unexpected error calling oti: {}".format(msg),
-                }
-            )
-        )
+        msg = "Unexpected error calling oti: {}".format(",".join(x.args))
+        api_utils.raise_int_server_err(msg)
     return resp
diff --git a/phylesystem_api/phylesystem_api/views/study.py b/phylesystem_api/phylesystem_api/views/study.py
index cd2ffdf1..8d830557 100644
--- a/phylesystem_api/phylesystem_api/views/study.py
+++ b/phylesystem_api/phylesystem_api/views/study.py
@@ -23,7 +23,6 @@
 from pyramid.httpexceptions import (
     HTTPNotFound,
     HTTPBadRequest,
-    HTTPInternalServerError,
 )
 from pyramid.renderers import render_to_response
 from pyramid.view import view_config
@@ -488,13 +487,7 @@ def create_study(request):
             )
         except Exception:
             msg = "Unexpected error parsing the file obtained from TreeBASE. Please report this bug to the Open Tree of Life developers."
-            bodys = json.dumps(
-                {
-                    "error": 1,
-                    "description": msg,
-                }
-            )
-            raise HTTPInternalServerError(bodys)
+            api_utils.raise_int_server_err(msg)
     elif importing_from_crossref_API:
         new_study_nexson = _new_nexson_with_crossref_metadata(
             doi=publication_doi_for_crossref,

From f28875a5d2d9582522546492f659e9303c40b35c Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Thu, 27 Jul 2023 15:55:58 -0500
Subject: [PATCH 07/33] raise400 util moved to api_utils

---
 phylesystem_api/phylesystem_api/api_utils.py  | 28 ++----
 .../phylesystem_api/views/amendment.py        | 54 +++-------
 .../phylesystem_api/views/amendments.py       |  5 -
 .../phylesystem_api/views/collection.py       | 98 ++++---------------
 .../phylesystem_api/views/default.py          | 28 ++----
 .../phylesystem_api/views/studies.py          | 26 +++--
 .../phylesystem_api/views/study.py            | 23 ++---
 7 files changed, 68 insertions(+), 194 deletions(-)

diff --git a/phylesystem_api/phylesystem_api/api_utils.py b/phylesystem_api/phylesystem_api/api_utils.py
index 5731d119..f3eb5e48 100644
--- a/phylesystem_api/phylesystem_api/api_utils.py
+++ b/phylesystem_api/phylesystem_api/api_utils.py
@@ -47,6 +47,10 @@ def raise_int_server_err(msg):
     raise HTTPInternalServerError(body=json.dumps(body))
 
 
+def raise400(msg):
+    raise HTTPBadRequest(body=json.dumps({"error": 1, "description": msg}))
+
+
 def get_private_dir(request):
     _LOG.debug("WHY PRIVATE DIR")
     return "~/private/"
@@ -307,14 +311,7 @@ def _read_phylesystem_from_conf_obj(conf):
     try:
         max_num_trees = int(max_num_trees)
     except ValueError:
-        raise HTTPBadRequest(
-            json.dumps(
-                {
-                    "error": 1,
-                    "description": "max number of trees per study in config is not an integer",
-                }
-            )
-        )
+        raise400("max number of trees per study in config is not an integer")
     try:
         read_only = conf.get("apis", "read_only") == "true"
     except:
@@ -471,14 +468,7 @@ def authenticate(request):
     try:
         auth_info["login"] = gh_user.login
     except BadCredentialsException:
-        raise HTTPBadRequest(
-            json.dumps(
-                {
-                    "error": 1,
-                    "description": "You have provided an invalid or expired authentication token",
-                }
-            )
-        )
+        raise400("You have provided an invalid or expired authentication token")
     # use the Github Oauth token to get a name/email if not specified
     # we don't provide these as default values above because they would
     # generate API calls regardless of author_name/author_email being specifed
@@ -897,11 +887,7 @@ def extract_json_from_http_call(request, data_field_name="data", request_params=
     except:
         # _LOG = api_utils.get_logger(request, 'ot_api.default.v1')
         # _LOG.exception('Exception getting JSON content in extract_json_from_http_call')
-        raise HTTPBadRequest(
-            body=json.dumps(
-                {"error": 1, "description": "no collection JSON found in request"}
-            )
-        )
+        raise400("no collection JSON found in request")
     return json_obj
 
 
diff --git a/phylesystem_api/phylesystem_api/views/amendment.py b/phylesystem_api/phylesystem_api/views/amendment.py
index 256c6ad9..13630230 100644
--- a/phylesystem_api/phylesystem_api/views/amendment.py
+++ b/phylesystem_api/phylesystem_api/views/amendment.py
@@ -10,7 +10,11 @@
 )
 from peyotl.phylesystem.git_workflows import GitWorkflowError
 import phylesystem_api.api_utils as api_utils
-from phylesystem_api.api_utils import find_in_request, extract_json_from_http_call
+from phylesystem_api.api_utils import (
+    find_in_request,
+    extract_json_from_http_call,
+    raise400,
+)
 import json
 import logging
 
@@ -83,16 +87,8 @@ def create_amendment(request, **kwargs):
         amendment_adapter,
     ) = __extract_and_validate_amendment(request, kwargs)
     if amendment_obj is None:
-        raise HTTPBadRequest(
-            body=json.dumps(
-                {
-                    "error": 1,
-                    "description": "amendment JSON expected for HTTP method {}".format(
-                        request.method
-                    ),
-                }
-            )
-        )
+        msg = "amendment JSON expected for HTTP method {}".format(request.method)
+        raise400(msg)
 
     # Create a new amendment with the data provided
     # _LOG = api_utils.get_logger(request, 'ot_api.default.amendments.POST')
@@ -127,16 +123,8 @@ def fetch_amendment(request):
     # _LOG = api_utils.get_logger(request, 'ot_api.amendment')
     amendment_id = request.matchdict["amendment_id"]
     if not AMENDMENT_ID_PATTERN.match(amendment_id):
-        raise HTTPBadRequest(
-            body=json.dumps(
-                {
-                    "error": 1,
-                    "description": "invalid amendment ID ({}) provided".format(
-                        amendment_id
-                    ),
-                }
-            )
-        )
+        msg = "invalid amendment ID ({}) provided".format(amendment_id)
+        raise400(msg)
 
     # fetch the current amendment JSON
     # _LOG.debug('GET /v2/amendment/{}'.format(str(amendment_id)))
@@ -195,16 +183,8 @@ def update_amendment(request):
     # _LOG = api_utils.get_logger(request, 'ot_api.amendment')
     amendment_id = request.matchdict["amendment_id"]
     if not AMENDMENT_ID_PATTERN.match(amendment_id):
-        raise HTTPBadRequest(
-            body=json.dumps(
-                {
-                    "error": 1,
-                    "description": "invalid amendment ID ({}) provided".format(
-                        amendment_id
-                    ),
-                }
-            )
-        )
+        msg = "invalid amendment ID ({}) provided".format(amendment_id)
+        raise400(msg)
 
     # this method requires authentication
     auth_info = api_utils.authenticate(request)
@@ -225,16 +205,8 @@ def update_amendment(request):
         amendment_adapter,
     ) = __extract_and_validate_amendment(request, request.params)
     if amendment_obj is None:
-        raise HTTPBadRequest(
-            body=json.dumps(
-                {
-                    "error": 1,
-                    "description": "amendment JSON expected for HTTP method {}".format(
-                        request.method
-                    ),
-                }
-            )
-        )
+        msg = "amendment JSON expected for HTTP method {}".format(request.method)
+        raise400(msg)
 
     api_utils.raise_if_read_only()
 
diff --git a/phylesystem_api/phylesystem_api/views/amendments.py b/phylesystem_api/phylesystem_api/views/amendments.py
index b8cf784c..c768e5f9 100644
--- a/phylesystem_api/phylesystem_api/views/amendments.py
+++ b/phylesystem_api/phylesystem_api/views/amendments.py
@@ -24,11 +24,6 @@ def _bool_arg(v):
     return v
 
 
-"""
-    raise HTTPBadRequest(body=json.dumps({"error": 1, "description": msg}))
-"""
-
-
 @view_config(route_name="list_all_amendment_ids", renderer="json")
 def list_all_amendment_ids(request):
     docstore = api_utils.get_taxonomic_amendment_store(request)
diff --git a/phylesystem_api/phylesystem_api/views/collection.py b/phylesystem_api/phylesystem_api/views/collection.py
index b5a10270..1d5b6f1e 100644
--- a/phylesystem_api/phylesystem_api/views/collection.py
+++ b/phylesystem_api/phylesystem_api/views/collection.py
@@ -16,7 +16,11 @@
 from pyramid.view import view_config
 
 import phylesystem_api.api_utils as api_utils
-from phylesystem_api.api_utils import find_in_request, extract_json_from_http_call
+from phylesystem_api.api_utils import (
+    find_in_request,
+    extract_json_from_http_call,
+    raise400,
+)
 
 _LOG = logging.getLogger("phylesystem_api")
 
@@ -76,42 +80,21 @@ def create_collection(request):
         collection_adapter,
     ) = __extract_and_validate_collection(request, request.params)
     if collection_obj is None:
-        raise HTTPBadRequest(
-            body=json.dumps(
-                {
-                    "error": 1,
-                    "description": "collection JSON expected for HTTP method {}".format(
-                        request.method
-                    ),
-                }
-            )
-        )
+        msg = "collection JSON expected for HTTP method {}".format(request.method)
+        raise400(msg)
 
     auth_info = api_utils.authenticate(request)
     owner_id = auth_info.get("login", None)
     if owner_id is None:
-        raise HTTPBadRequest(
-            json.dumps(
-                {
-                    "error": 1,
-                    "description": "no GitHub userid found for HTTP method {}".format(
-                        request.env.request_method
-                    ),
-                }
-            )
+        msg = "no GitHub userid found for HTTP method {}".format(
+            request.env.request_method
         )
+        raise400(msg)
 
     # try to extract a usable collection ID from the JSON payload (confirm owner_id against above)
     url = collection_obj.get("url", None)
     if url is None:
-        raise HTTPBadRequest(
-            json.dumps(
-                {
-                    "error": 1,
-                    "description": "no collection URL provided in query string or JSON payload",
-                }
-            )
-        )
+        raise400("no collection URL provided in query string or JSON payload")
     try:
         collection_id = url.split("/collection/")[1]
     except:
@@ -153,7 +136,7 @@ def create_collection(request):
         )
         new_collection_id, commit_return = r
     except GitWorkflowError as err:
-        raise HTTPBadRequest(json.dumps({"error": 1, "description": str(err)}))
+        raise400(str(err))
     except:
         raise HTTPBadRequest(traceback.format_exc())
     if commit_return["error"] != 0:
@@ -174,16 +157,8 @@ def fetch_collection(request):
     # _LOG = api_utils.get_logger(request, 'ot_api.collection')
     collection_id = request.matchdict["collection_id"]
     if not COLLECTION_ID_PATTERN.match(collection_id):
-        raise HTTPBadRequest(
-            body=json.dumps(
-                {
-                    "error": 1,
-                    "description": "invalid collection ID ({}) provided".format(
-                        collection_id
-                    ),
-                }
-            )
-        )
+        msg = "invalid collection ID ({}) provided".format(collection_id)
+        raise400(msg)
 
     # gather details to return with the JSON core document
     parent_sha = find_in_request(request, "starting_commit_SHA", None)
@@ -266,17 +241,8 @@ def update_collection(request):
 
     collection_id = request.matchdict["collection_id"]
     if not COLLECTION_ID_PATTERN.match(collection_id):
-        raise HTTPBadRequest(
-            body=json.dumps(
-                {
-                    "error": 1,
-                    "description": "invalid collection ID ({}) provided".format(
-                        collection_id
-                    ),
-                }
-            )
-        )
-
+        msg = "invalid collection ID ({}) provided".format(collection_id)
+        raise400(msg)
     try:
         commit_msg = find_in_request(request, "commit_msg", "")
         if commit_msg.strip() == "":
@@ -294,16 +260,8 @@ def update_collection(request):
         collection_adapter,
     ) = __extract_and_validate_collection(request, request.params)
     if collection_obj is None:
-        raise HTTPBadRequest(
-            body=json.dumps(
-                {
-                    "error": 1,
-                    "description": "collection JSON expected for HTTP method {}".format(
-                        request.method
-                    ),
-                }
-            )
-        )
+        msg = "collection JSON expected for HTTP method {}".format(request.method)
+        raise400(msg)
 
     # submit new json for this id, and read the results
     parent_sha = find_in_request(request, "starting_commit_SHA", None)
@@ -349,16 +307,8 @@ def delete_collection(request):
 
     collection_id = request.matchdict["collection_id"]
     if not COLLECTION_ID_PATTERN.match(collection_id):
-        raise HTTPBadRequest(
-            body=json.dumps(
-                {
-                    "error": 1,
-                    "description": "invalid collection ID ({}) provided".format(
-                        collection_id
-                    ),
-                }
-            )
-        )
+        msg = "invalid collection ID ({}) provided".format(collection_id)
+        raise400(msg)
 
     try:
         commit_msg = find_in_request(request, "commit_msg", "")
@@ -389,10 +339,4 @@ def delete_collection(request):
     except GitWorkflowError as err:
         raise HTTPBadRequest(err.msg)
     except:
-        # _LOG.exception('Unknown error in collection deletion')
-        # raise HTTPBadRequest(traceback.format_exc())
-        raise HTTPBadRequest(
-            json.dumps(
-                {"error": 1, "description": "Unknown error in collection deletion"}
-            )
-        )
+        raise400("Unknown error in collection deletion")
diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py
index 092372c8..27852bdc 100644
--- a/phylesystem_api/phylesystem_api/views/default.py
+++ b/phylesystem_api/phylesystem_api/views/default.py
@@ -14,6 +14,7 @@
     GitWorkflowError,
     merge_from_master,
 )
+from phylesystem_api.api_utils import raise400
 from pyramid.httpexceptions import (
     HTTPException,
     HTTPNotFound,
@@ -281,9 +282,7 @@ def include_tree_in_synth(request):
     tree_id = request.params.get("tree_id", "").strip()
     # check for empty/missing ids
     if (study_id == "") or (tree_id == ""):
-        raise HTTPBadRequest(
-            body='{"error": 1, "description": "Expecting study_id and tree_id arguments"}'
-        )
+        raise400("Expecting study_id and tree_id arguments")
     # examine this study and tree, to confirm it exists *and* to capture its name
     sds = api_utils.get_phylesystem(request)
     try:
@@ -387,9 +386,7 @@ def exclude_tree_from_synth(request):
     tree_id = request.params.get("tree_id", "").strip()
     # check for empty/missing ids
     if (study_id == "") or (tree_id == ""):
-        raise HTTPBadRequest(
-            body='{"error": 1, "description": "Expecting study_id and tree_id arguments"}'
-        )
+        raise400("Expecting study_id and tree_id arguments")
     # find this tree in ANY synth-input collection; if found, remove it and update the collection
     coll_id_list = _get_synth_input_collection_ids()
     cds = api_utils.get_tree_collection_store(request)
@@ -517,7 +514,7 @@ def merge_docstore_changes(request):
     try:
         return merge_from_master(gd, resource_id, auth_info, starting_commit_SHA)
     except GitWorkflowError as err:
-        raise HTTPBadRequest(body=json.dumps({"error": 1, "description": err.msg}))
+        raise400(err.msg)
     except:
         m = traceback.format_exc()
         raise HTTPConflict(
@@ -583,10 +580,7 @@ def push_docstore_changes(request):
                         'Could not create an adaptor for git actions on study ID "{}". '
                         "If you are confident that this is a valid study ID, please report this as a bug."
                     )
-                    m = m.format(resource_id)
-                    raise HTTPBadRequest(
-                        body=json.dumps({"error": 1, "description": m})
-                    )
+                    raise400(m.format(resource_id))
                 master_sha = ga.get_master_sha()
                 obj = {
                     "date": timestamp,
@@ -632,10 +626,7 @@ def push_docstore_changes(request):
                         'Could not create an adaptor for git actions on collection ID "{}". '
                         "If you are confident that this is a valid collection ID, please report this as a bug."
                     )
-                    m = m.format(resource_id)
-                    raise HTTPBadRequest(
-                        body=json.dumps({"error": 1, "description": m})
-                    )
+                    raise400(m.format(resource_id))
                 master_sha = ga.get_master_sha()
                 obj = {
                     "date": timestamp,
@@ -678,10 +669,7 @@ def push_docstore_changes(request):
                         'Could not create an adaptor for git actions on amendment ID "{}". '
                         "If you are confident that this is a valid amendment ID, please report this as a bug."
                     )
-                    m = m.format(resource_id)
-                    raise HTTPBadRequest(
-                        body=json.dumps({"error": 1, "description": m})
-                    )
+                    raise400(m.format(resource_id))
                 master_sha = ga.get_master_sha()
                 obj = {
                     "date": timestamp,
@@ -701,7 +689,7 @@ def push_docstore_changes(request):
             )
 
     else:
-        raise HTTPBadRequest(body="Can't push unknown doc_type '{}'".format(doc_type))
+        raise400("Can't push unknown doc_type '{}'".format(doc_type))
 
     if os.path.exists(fail_file):
         # log any old fail_file, and remove it because the pushes are working
diff --git a/phylesystem_api/phylesystem_api/views/studies.py b/phylesystem_api/phylesystem_api/views/studies.py
index b0672e2c..4c61d66d 100644
--- a/phylesystem_api/phylesystem_api/views/studies.py
+++ b/phylesystem_api/phylesystem_api/views/studies.py
@@ -1,17 +1,13 @@
 import json
 
 import phylesystem_api.api_utils as api_utils
-from phylesystem_api.api_utils import find_in_request
+from phylesystem_api.api_utils import find_in_request, raise400
 
 # see exception subclasses at https://docs.pylonsproject.org/projects/pyramid/en/latest/api/httpexceptions.html
 from pyramid.httpexceptions import HTTPException, HTTPBadRequest
 from pyramid.view import view_config
 
 
-def _raise400(msg):
-    raise HTTPBadRequest(body=json.dumps({"error": 1, "description": msg}))
-
-
 def _configure_response(response):
     response.view = "generic.json"
     # CORS support for cross-domain API requests (from anywhere)
@@ -47,7 +43,7 @@ def find_studies(request):
     oti = api_utils.get_oti_wrapper(request)
     verbose = _bool_arg(find_in_request(request, "verbose", False))
     if (verbose is not True) and (verbose is not False):
-        _raise400('"verbose" setting must be a boolean')
+        raise400('"verbose" setting must be a boolean')
     field = find_in_request(request, "property", None)
     try:
         if field is None:
@@ -55,16 +51,16 @@ def find_studies(request):
         else:
             value = find_in_request(request, "value", None)
             if value is None:
-                _raise400('If "property" is sent, a "value" argument must be used.')
+                raise400('If "property" is sent, a "value" argument must be used.')
             exact = _bool_arg(find_in_request(request, "exact", False))
             if (exact is not True) and (exact is not False):
-                _raise400('"exact" setting must be a boolean')
+                raise400('"exact" setting must be a boolean')
             try:
                 match_list = oti.find_studies(
                     {field: value}, verbose=verbose, exact=exact
                 )
             except ValueError as x:
-                _raise400(str(x))
+                raise400(str(x))
         resp = {"matched_studies": match_list}
     except HTTPException:
         raise
@@ -80,28 +76,28 @@ def find_trees(request):
     try:
         msg = request.json_body
     except:
-        _raise400("missing or invalid request JSON")
+        raise400("missing or invalid request JSON")
     _configure_response(request.response)
     oti = api_utils.get_oti_wrapper(request)
     verbose = _bool_arg(msg.get("verbose", False))
     if (verbose is not True) and (verbose is not False):
-        _raise400('"verbose" setting must be a boolean')
+        raise400('"verbose" setting must be a boolean')
     field = msg.get("property")
     if field is None:
-        _raise400('A "property" argument must be used.')
+        raise400('A "property" argument must be used.')
     value = msg.get("value")
     if value is None:
-        _raise400('A "value" argument must be used.')
+        raise400('A "value" argument must be used.')
     exact = _bool_arg(msg.get("exact", False))
     if (exact is not True) and (exact is not False):
-        _raise400('"exact" setting must be a boolean')
+        raise400('"exact" setting must be a boolean')
     try:
         match_list = oti.find_trees({field: value}, verbose=verbose, exact=exact)
         resp = {"matched_studies": match_list}
     except HTTPException:
         raise
     except ValueError as x:
-        _raise400(str(x))
+        raise400(str(x))
     except Exception as x:
         msg = "Unexpected error calling oti: {}".format(",".join(x.args))
         api_utils.raise_int_server_err(msg)
diff --git a/phylesystem_api/phylesystem_api/views/study.py b/phylesystem_api/phylesystem_api/views/study.py
index 8d830557..370b8e5a 100644
--- a/phylesystem_api/phylesystem_api/views/study.py
+++ b/phylesystem_api/phylesystem_api/views/study.py
@@ -16,7 +16,7 @@
     GitWorkflowError,
     validate_and_convert_nexson,
 )
-from phylesystem_api.api_utils import find_in_request
+from phylesystem_api.api_utils import find_in_request, raise400
 from pyramid.encode import quote_plus, urlencode
 
 # see exception subclasses at https://docs.pylonsproject.org/projects/pyramid/en/latest/api/httpexceptions.html
@@ -45,9 +45,7 @@ def __extract_nexson_from_http_call(request, **kwargs):
     except:
         # _LOG = api_utils.get_logger(request, 'ot_api.default.v1')
         # _LOG.exception('Exception getting nexson content in __extract_nexson_from_http_call')
-        raise HTTPBadRequest(
-            json.dumps({"error": 1, "description": "NexSON must be valid JSON"})
-        )
+        raise400("NexSON must be valid JSON")
     return nexson
 
 
@@ -78,7 +76,7 @@ def __extract_and_validate_nexson(request, repo_nexml2json, kwargs):
     except GitWorkflowError as err:
         # _LOG = api_utils.get_logger(request, 'ot_api.default.v1')
         # _LOG.exception('PUT failed in validation')
-        raise HTTPBadRequest(err.msg or "No message found")
+        raise400(err.msg or "No message found")
     return nexson, annotation, nexson_adaptor
 
 
@@ -146,7 +144,7 @@ def __validate_output_nexml2json(
     if msg:
         # _LOG = api_utils.get_logger(request, 'ot_api.default.v1')
         # _LOG.debug('output sniffing err msg = ' + msg)
-        raise HTTPBadRequest(json.dumps({"error": 1, "description": msg}))
+        raise400(msg)
     return schema
 
 
@@ -276,11 +274,9 @@ def fetch_study(request):
                 study_nexson, serialize=serialize, src_schema=src_schema
             )
         except:
-            msg = (
-                "Exception in coercing to the required NexSON version for validation. "
+            raise400(
+                "Exception in coercing to the required NexSON version for validation."
             )
-            # _LOG.exception(msg)
-            raise HTTPBadRequest(msg)
 
     if out_schema.is_json():
         try:
@@ -479,8 +475,7 @@ def create_study(request):
             treebase_id = int(treebase_id)
         except ValueError as e:
             msg = "TreeBASE ID should be a simple integer, not '{}'! Details:\n{}"
-            msg = msg.format(treebase_id, str(e))
-            raise HTTPBadRequest(json.dumps({"error": 1, "description": msg}))
+            raise400(msg.format(treebase_id, str(e)))
         try:
             new_study_nexson = import_nexson_from_treebase(
                 treebase_id, nexson_syntax_version=BY_ID_HONEY_BADGERFISH
@@ -703,9 +698,7 @@ def delete_study(request):
         _LOG.warning("some other kind of error")
         _LOG.warning(err)
         # _LOG.exception('Exception getting nexson content in phylesystem.delete_study')
-        raise HTTPBadRequest(
-            json.dumps({"error": 1, "description": "Unknown error in study deletion"})
-        )
+        raise400("Unknown error in study deletion")
 
 
 @view_config(route_name="get_study_file_list", renderer="json")

From 3bedfa633b8c2b848de6fd11e87d13d5ca56af51 Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Thu, 27 Jul 2023 16:13:05 -0500
Subject: [PATCH 08/33] stripping out some dead code and raise404 helper

---
 phylesystem_api/phylesystem_api/api_utils.py  | 136 +-----------------
 .../phylesystem_api/views/amendment.py        |  14 +-
 .../phylesystem_api/views/collection.py       |  39 +----
 .../phylesystem_api/views/default.py          |  31 ++--
 .../phylesystem_api/views/study.py            |  45 ++----
 5 files changed, 34 insertions(+), 231 deletions(-)

diff --git a/phylesystem_api/phylesystem_api/api_utils.py b/phylesystem_api/phylesystem_api/api_utils.py
index f3eb5e48..12c8a4a4 100644
--- a/phylesystem_api/phylesystem_api/api_utils.py
+++ b/phylesystem_api/phylesystem_api/api_utils.py
@@ -16,6 +16,7 @@
     HTTPBadRequest,
     HTTPForbidden,
     HTTPInternalServerError,
+    HTTPNotFound,
 )
 from beaker.cache import cache_managers
 import tempfile
@@ -51,6 +52,10 @@ def raise400(msg):
     raise HTTPBadRequest(body=json.dumps({"error": 1, "description": msg}))
 
 
+def raise404(msg):
+    raise HTTPNotFound(body=json.dumps({"error": 1, "description": msg}))
+
+
 def get_private_dir(request):
     _LOG.debug("WHY PRIVATE DIR")
     return "~/private/"
@@ -410,29 +415,6 @@ def read_favorites_config(request):
     return favorites_repo_parent, favorites_repo_remote, git_ssh, pkey, git_hub_remote
 
 
-def read_logging_config(request):
-    conf = get_conf_object(request)
-    try:
-        level = conf.get("logging", "level")
-        if not level.strip():
-            level = "WARNING"
-    except:
-        level = "WARNING"
-    try:
-        logging_format_name = conf.get("logging", "formatter")
-        if not logging_format_name.strip():
-            logging_format_name = "NONE"
-    except:
-        logging_format_name = "NONE"
-    try:
-        logging_filepath = conf.get("logging", "filepath")
-        if not logging_filepath.strip():
-            logging_filepath = None
-    except:
-        logging_filepath = None
-    return level, logging_format_name, logging_filepath
-
-
 def _raise_missing_auth_token():
     raise HTTPBadRequest(
         json.dumps(
@@ -478,90 +460,6 @@ def authenticate(request):
     return auth_info
 
 
-''' ## using logging module directly
-_LOGGING_LEVEL_ENVAR="OT_API_LOGGING_LEVEL"
-_LOGGING_FORMAT_ENVAR="OT_API_LOGGING_FORMAT"
-_LOGGING_FILE_PATH_ENVAR = 'OT_API_LOG_FILE_PATH'
-
-def _get_logging_level(s=None):
-    if s is None:
-        return logging.NOTSET
-    supper = s.upper()
-    if supper == "NOTSET":
-        level = logging.NOTSET
-    elif supper == "DEBUG":
-        level = logging.DEBUG
-    elif supper == "INFO":
-        level = logging.INFO
-    elif supper == "WARNING":
-        level = logging.WARNING
-    elif supper == "ERROR":
-        level = logging.ERROR
-    elif supper == "CRITICAL":
-        level = logging.CRITICAL
-    else:
-        level = logging.NOTSET
-    return level
-
-def _get_logging_formatter(s=None):
-    if s is None:
-        s == 'NONE'
-    else:
-        s = s.upper()
-    rich_formatter = logging.Formatter("[%(asctime)s] %(filename)s (%(lineno)d): %(levelname) 8s: %(message)s")
-    simple_formatter = logging.Formatter("%(levelname) 8s: %(message)s")
-    raw_formatter = logging.Formatter("%(message)s")
-    default_formatter = None
-    logging_formatter = default_formatter
-    if s == "RICH":
-        logging_formatter = rich_formatter
-    elif s == "SIMPLE":
-        logging_formatter = simple_formatter
-    else:
-        logging_formatter = None
-    if logging_formatter is not None:
-        logging_formatter.datefmt='%H:%M:%S'
-    return logging_formatter
-
-def get_logger(request, name="ot_api"):
-    """
-    Returns a logger with name set as given, and configured
-    to the level given by the environment variable _LOGGING_LEVEL_ENVAR.
-    """
-#     package_dir = os.path.dirname(module_path)
-#     config_filepath = os.path.join(package_dir, _LOGGING_CONFIG_FILE)
-#     if os.path.exists(config_filepath):
-#         try:
-#             logging.config.fileConfig(config_filepath)
-#             logger_set = True
-#         except:
-#             logger_set = False
-    logger = logging.getLogger(name)
-    if len(logger.handlers) == 0:
-        if request is None:
-            level = _get_logging_level(os.environ.get(_LOGGING_LEVEL_ENVAR))
-            logging_formatter = _get_logging_formatter(os.environ.get(_LOGGING_FORMAT_ENVAR))
-            logging_filepath = os.environ.get(_LOGGING_FILE_PATH_ENVAR)
-        else:
-            level_str, logging_format_name, logging_filepath = read_logging_config(request)
-            logging_formatter = _get_logging_formatter(logging_format_name)
-            level = _get_logging_level(level_str)
-
-        logger.setLevel(level)
-        if logging_filepath is not None:
-            log_dir = os.path.split(logging_filepath)[0]
-            if log_dir and not os.path.exists(log_dir):
-                os.makedirs(log_dir)
-            ch = logging.FileHandler(logging_filepath)
-        else:
-            ch = logging.StreamHandler()
-        ch.setLevel(level)
-        ch.setFormatter(logging_formatter)
-        logger.addHandler(ch)
-    return logger
-'''
-
-
 def log_time_diff(log_obj, operation="", prev_time=None):
     """If prev_time is not None, logs (at debug level) to
     log_obj the difference between now and the naive datetime
@@ -585,12 +483,6 @@ def log_time_diff(log_obj, operation="", prev_time=None):
     return n
 
 
-def get_otindex_base_url(request):
-    conf = get_conf_object(request)
-    otindex_base_url = conf.get("apis", "otindex_base_url")
-    return otindex_base_url
-
-
 def get_oti_base_url(request):
     conf = get_conf_object(request)
     oti_base_url = conf.get("apis", "oti_base_url")
@@ -616,24 +508,6 @@ def get_collections_api_base_url(request):
     return base_url
 
 
-def get_amendments_api_base_url(request):
-    conf = get_conf_object(request)
-    base_url = conf.get("apis", "amendments_api_base_url")
-    if base_url.startswith("//"):
-        # Prepend scheme to a scheme-relative URL
-        base_url = "https:" + base_url
-    return base_url
-
-
-def get_favorites_api_base_url(request):
-    conf = get_conf_object(request)
-    base_url = conf.get("apis", "favorites_api_base_url")
-    if base_url.startswith("//"):
-        # Prepend scheme to a scheme-relative URL
-        base_url = "http:" + base_url
-    return base_url
-
-
 def clear_matching_cache_keys(key_pattern):
     # emulate our regex-powered cache clearing, as used in web2py
     """
diff --git a/phylesystem_api/phylesystem_api/views/amendment.py b/phylesystem_api/phylesystem_api/views/amendment.py
index 13630230..16bea1a8 100644
--- a/phylesystem_api/phylesystem_api/views/amendment.py
+++ b/phylesystem_api/phylesystem_api/views/amendment.py
@@ -5,7 +5,6 @@
 # see exception subclasses at https://docs.pylonsproject.org/projects/pyramid/en/latest/api/httpexceptions.html
 from pyramid.httpexceptions import (
     HTTPException,
-    HTTPNotFound,
     HTTPBadRequest,
 )
 from peyotl.phylesystem.git_workflows import GitWorkflowError
@@ -14,6 +13,7 @@
     find_in_request,
     extract_json_from_http_call,
     raise400,
+    raise404,
 )
 import json
 import logging
@@ -141,15 +141,7 @@ def fetch_amendment(request):
             amendment_id, commit_sha=parent_sha, return_WIP_map=True
         )
     except:
-        # _LOG.exception('GET failed')
-        raise HTTPNotFound(
-            json.dumps(
-                {
-                    "error": 1,
-                    "description": "Amendment '{}' GET failure".format(amendment_id),
-                }
-            )
-        )
+        raise404("Amendment '{}' GET failure".format(amendment_id))
     try:
         amendment_json, head_sha, wip_map = r
         ## if returning_full_study:  # TODO: offer bare vs. full output (w/ history, etc)
@@ -159,7 +151,7 @@ def fetch_amendment(request):
         e = sys.exc_info()[0]
         raise HTTPBadRequest(body=e)
     if not amendment_json:
-        raise HTTPNotFound("Amendment '{s}' has no JSON data!".format(s=amendment_id))
+        raise404("Amendment '{s}' has no JSON data!".format(s=amendment_id))
 
     try:
         external_url = amendments.get_public_url(amendment_id)
diff --git a/phylesystem_api/phylesystem_api/views/collection.py b/phylesystem_api/phylesystem_api/views/collection.py
index 1d5b6f1e..e4b4f9f5 100644
--- a/phylesystem_api/phylesystem_api/views/collection.py
+++ b/phylesystem_api/phylesystem_api/views/collection.py
@@ -10,7 +10,6 @@
 # see exception subclasses at https://docs.pylonsproject.org/projects/pyramid/en/latest/api/httpexceptions.html
 from pyramid.httpexceptions import (
     HTTPException,
-    HTTPNotFound,
     HTTPBadRequest,
 )
 from pyramid.view import view_config
@@ -20,6 +19,7 @@
     find_in_request,
     extract_json_from_http_call,
     raise400,
+    raise404,
 )
 
 _LOG = logging.getLogger("phylesystem_api")
@@ -98,31 +98,12 @@ def create_collection(request):
     try:
         collection_id = url.split("/collection/")[1]
     except:
-        # _LOG.exception('{} failed'.format(request.env.request_method))
-        raise HTTPNotFound(
-            json.dumps(
-                {
-                    "error": 1,
-                    "description": "invalid URL, no collection id found: {}".format(
-                        url
-                    ),
-                }
-            )
-        )
+        raise404("invalid URL, no collection id found: {}".format(url))
     try:
         assert collection_id.split("/")[0] == owner_id
     except:
         # _LOG.exception('{} failed'.format(request.env.request_method))
-        raise HTTPNotFound(
-            body=json.dumps(
-                {
-                    "error": 1,
-                    "description": "collection URL in JSON doesn't match logged-in user: {}".format(
-                        url
-                    ),
-                }
-            )
-        )
+        raise404("collection URL in JSON doesn't match logged-in user: {}".format(url))
 
     # Create a new collection with the data provided
     auth_info = auth_info or api_utils.authenticate(request)
@@ -170,15 +151,7 @@ def fetch_collection(request):
             collection_id, commit_sha=parent_sha, return_WIP_map=True
         )
     except:
-        # _LOG.exception('GET failed')
-        raise HTTPNotFound(
-            body=json.dumps(
-                {
-                    "error": 1,
-                    "description": "Collection '{}' GET failure".format(collection_id),
-                }
-            )
-        )
+        raise404("Collection '{}' GET failure".format(collection_id))
     try:
         collection_json, head_sha, wip_map = r
         ## if returning_full_study:  # TODO: offer bare vs. full output (w/ history, etc)
@@ -195,9 +168,7 @@ def fetch_collection(request):
         e = sys.exc_info()[0]
         raise HTTPBadRequest(e)
     if not collection_json:
-        raise HTTPNotFound(
-            body="Collection '{s}' has no JSON data!".format(s=collection_id)
-        )
+        raise404("Collection '{s}' has no JSON data!".format(s=collection_id))
     # add/restore the url field (using the visible fetch URL)
     base_url = api_utils.get_collections_api_base_url(request)
     collection_json["url"] = "{b}/v2/collection/{i}".format(b=base_url, i=collection_id)
diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py
index 27852bdc..067e0a8d 100644
--- a/phylesystem_api/phylesystem_api/views/default.py
+++ b/phylesystem_api/phylesystem_api/views/default.py
@@ -14,10 +14,9 @@
     GitWorkflowError,
     merge_from_master,
 )
-from phylesystem_api.api_utils import raise400
+from phylesystem_api.api_utils import raise400, raise404
 from pyramid.httpexceptions import (
     HTTPException,
-    HTTPNotFound,
     HTTPConflict,
     HTTPBadRequest,
     HTTPInternalServerError,
@@ -264,9 +263,7 @@ def trees_in_synth(request):
                 cds.return_doc(coll_id, commit_sha=None, return_WIP_map=False)[0]
             )
         except:
-            msg = "GET of collection {} failed".format(coll_id)
-            # _LOG.exception(msg)
-            raise HTTPNotFound(body=json.dumps({"error": 1, "description": msg}))
+            raise404("GET of collection {} failed".format(coll_id))
     try:
         result = concatenate_collections(coll_list)
     except:
@@ -297,11 +294,9 @@ def include_tree_in_synth(request):
         # _LOG.exception('*** FOUND IT: {}'.format(found_tree_name))
     except:  # report a missing/misidentified tree
         # _LOG.exception('problem finding tree')
-        raise HTTPNotFound(
-            body='{{"error": 1, "description": "Specified tree \'{t}\' in study \'{s}\' not found! Save this study and try again?"}}'.format(
-                s=study_id, t=tree_id
-            )
-        )
+        msg = "Specified tree '{t}' in study '{s}' not found! Save this study and try again?"
+        msg = msg.format(s=study_id, t=tree_id)
+        raise404(msg)
     already_included_in_synth_input_collections = False
     # Look ahead to see if it's already in an included collection; if so, skip
     # adding it again.
@@ -311,9 +306,7 @@ def include_tree_in_synth(request):
         try:
             coll = cds.return_doc(coll_id, commit_sha=None, return_WIP_map=False)[0]
         except:
-            msg = "GET of collection {} failed".format(coll_id)
-            # _LOG.exception(msg)
-            raise HTTPNotFound(body=json.dumps({"error": 1, "description": msg}))
+            raise404("GET of collection {} failed".format(coll_id))
         if tree_is_in_collection(coll, study_id, tree_id):
             already_included_in_synth_input_collections = True
     if not already_included_in_synth_input_collections:
@@ -342,14 +335,12 @@ def include_tree_in_synth(request):
             auth_info = api_utils.authenticate(request)
             owner_id = auth_info.get("login", None)
         except:
-            msg = "include_tree_in_synth(): Authentication failed"
-            raise HTTPNotFound(body=json.dumps({"error": 1, "description": msg}))
+            raise404("include_tree_in_synth(): Authentication failed")
         try:
             parent_sha = request.params.get("starting_commit_SHA", None)
             merged_sha = None  # TODO: request.params.get('???', None)
         except:
-            msg = "include_tree_in_synth(): fetch of starting_commit_SHA failed"
-            raise HTTPNotFound(body=json.dumps({"error": 1, "description": msg}))
+            raise404("include_tree_in_synth(): fetch of starting_commit_SHA failed")
         try:
             r = cds.update_existing_collection(
                 owner_id,
@@ -394,14 +385,12 @@ def exclude_tree_from_synth(request):
         auth_info = api_utils.authenticate(request)
         owner_id = auth_info.get("login", None)
     except:
-        msg = "include_tree_in_synth(): Authentication failed"
-        raise HTTPNotFound(body=json.dumps({"error": 1, "description": msg}))
+        raise404("include_tree_in_synth(): Authentication failed")
     for coll_id in coll_id_list:
         try:
             coll = cds.return_doc(coll_id, commit_sha=None, return_WIP_map=False)[0]
         except:
-            msg = "GET of collection {} failed".format(coll_id)
-            raise HTTPNotFound(body=json.dumps({"error": 1, "description": msg}))
+            raise404("GET of collection {} failed".format(coll_id))
         if tree_is_in_collection(coll, study_id, tree_id):
             # remove it and update the collection
             decision_list = coll.get("decisions", [])
diff --git a/phylesystem_api/phylesystem_api/views/study.py b/phylesystem_api/phylesystem_api/views/study.py
index 370b8e5a..9b79251b 100644
--- a/phylesystem_api/phylesystem_api/views/study.py
+++ b/phylesystem_api/phylesystem_api/views/study.py
@@ -16,7 +16,7 @@
     GitWorkflowError,
     validate_and_convert_nexson,
 )
-from phylesystem_api.api_utils import find_in_request, raise400
+from phylesystem_api.api_utils import find_in_request, raise400, raise404
 from pyramid.encode import quote_plus, urlencode
 
 # see exception subclasses at https://docs.pylonsproject.org/projects/pyramid/en/latest/api/httpexceptions.html
@@ -242,12 +242,7 @@ def fetch_study(request):
             study_id, commit_sha=parent_sha, return_WIP_map=True
         )
     except:
-        # _LOG.exception('GET failed')
-        raise HTTPNotFound(
-            body=json.dumps(
-                {"error": 1, "description": "Study #%s GET failure" % study_id}
-            )
-        )
+        raise404("Study #{} GET failure".format(study_id))
     try:
         study_nexson, head_sha, wip_map = r
         blob_sha = phylesystem.get_blob_sha_for_study_id(study_id, head_sha)
@@ -722,12 +717,7 @@ def get_study_file(request):
             study_id, commit_sha=parent_sha, return_WIP_map=True
         )
     except:
-        # _LOG.exception('GET failed')
-        raise HTTPNotFound(
-            body=json.dumps(
-                {"error": 1, "description": "Study #%s GET failure" % study_id}
-            )
-        )
+        raise404("Study #{} GET failure".format(study_id))
     study_nexson, head_sha, wip_map = r
     # TODO: return a description of the requested file, or a list of all files
     m_list = extract_supporting_file_messages(study_nexson)
@@ -754,11 +744,8 @@ def get_study_file(request):
                     matching = m
                     break
             if matching is None:
-                raise HTTPNotFound(
-                    body='No file with id="{f}" found in study="{s}"'.format(
-                        f=file_id, s=study_id
-                    )
-                )
+                msg = 'No file with id="{f}" found in study="{s}"'
+                raise404(msg.format(f=file_id, s=study_id))
             u = None
             files = m.get("data", {}).get("files", {}).get("file", [])
             for f in files:
@@ -766,11 +753,9 @@ def get_study_file(request):
                     u = f["@url"]
                     break
             if u is None:
-                raise HTTPNotFound(
-                    body='No @url found in the message with id="{f}" found in study="{s}"'.format(
-                        f=file_id, s=study_id
-                    )
-                )
+                msg = 'No @url found in the message with id="{f}" found in study="{s}"'
+                raise404(msg.format(f=file_id, s=study_id))
+
             # TEMPORARY HACK TODO
             u = u.replace("uploadid=", "uploadId=")
             # TODO: should not hard-code this, I suppose... (but not doing so requires more config...)
@@ -782,9 +767,7 @@ def get_study_file(request):
             return fetched.text
         except Exception as x:
             # _LOG.exception('file_get failed')
-            raise HTTPNotFound(
-                body='Could not retrieve file. Exception: "{}"'.format(str(x))
-            )
+            raise404('Could not retrieve file. Exception: "{}"'.format(str(x)))
 
 
 @view_config(route_name="get_study_external_url", renderer="json")
@@ -796,7 +779,7 @@ def get_study_external_url(request):
         u = phylesystem.get_public_url(study_id)
         return json.dumps({"url": u, "study_id": study_id})
     except:
-        raise HTTPNotFound(body='{"error": 1, "description": "study not found"}')
+        raise404("study not found")
 
 
 @view_config(route_name="get_study_tree", renderer=None)
@@ -861,13 +844,7 @@ def _fine_grained_get(request, subresource, content_id=None, file_ext=None):
             study_id, commit_sha=parent_sha, return_WIP_map=True
         )
     except:
-        # _LOG.exception('GET failed')
-        raise HTTPNotFound(
-            body=json.dumps(
-                {"error": 1, "description": "Study #%s GET failure" % study_id}
-            )
-        )
-
+        raise404("Study #{} GET failure".format(study_id))
     try:
         study_nexson, head_sha, wip_map = r
         blob_sha = phylesystem.get_blob_sha_for_study_id(study_id, head_sha)

From f6591029e0947dc321b8199125f94e3620d41776 Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Fri, 28 Jul 2023 09:43:16 -0500
Subject: [PATCH 09/33] minor

---
 phylesystem_api/phylesystem_api/views/amendment.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/phylesystem_api/phylesystem_api/views/amendment.py b/phylesystem_api/phylesystem_api/views/amendment.py
index 16bea1a8..35d57c83 100644
--- a/phylesystem_api/phylesystem_api/views/amendment.py
+++ b/phylesystem_api/phylesystem_api/views/amendment.py
@@ -283,4 +283,3 @@ def delete_amendment(request):
     except:
         # _LOG.exception('Unknown error in amendment deletion')
         raise HTTPBadRequest(body=traceback.format_exc())
-        # raise HTTPBadRequest(body=json.dumps({"error": 1, "description": 'Unknown error in amendment deletion'}))

From c959a48c4a93e294128e095aa747a28cd9a36a8d Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Fri, 28 Jul 2023 10:43:49 -0500
Subject: [PATCH 10/33] minor refactoring

---
 phylesystem_api/phylesystem_api/api_utils.py  |  53 +++---
 .../phylesystem_api/views/amendment.py        |  14 +-
 .../phylesystem_api/views/amendments.py       |  10 -
 .../phylesystem_api/views/collection.py       |  14 +-
 .../phylesystem_api/views/collections.py      |  10 -
 .../phylesystem_api/views/default.py          | 180 ++++++++----------
 .../phylesystem_api/views/studies.py          |  78 ++++----
 .../phylesystem_api/views/study.py            |  51 ++---
 8 files changed, 173 insertions(+), 237 deletions(-)

diff --git a/phylesystem_api/phylesystem_api/api_utils.py b/phylesystem_api/phylesystem_api/api_utils.py
index 12c8a4a4..273fe565 100644
--- a/phylesystem_api/phylesystem_api/api_utils.py
+++ b/phylesystem_api/phylesystem_api/api_utils.py
@@ -40,6 +40,16 @@
 READ_ONLY_MODE = True
 
 
+def bool_arg(v):
+    if isinstance(v, str):
+        u = v.upper()
+        if u in ["TRUE", "YES"]:
+            return True
+        if u in ["FALSE", "NO"]:
+            return False
+    return v
+
+
 def raise_int_server_err(msg):
     body = {
         "error": 1,
@@ -460,6 +470,11 @@ def authenticate(request):
     return auth_info
 
 
+def auth_and_not_read_only(request):
+    raise_if_read_only()
+    return authenticate(request)
+
+
 def log_time_diff(log_obj, operation="", prev_time=None):
     """If prev_time is not None, logs (at debug level) to
     log_obj the difference between now and the naive datetime
@@ -639,33 +654,25 @@ def find_in_request(
     request, property_name, default_value=None, return_all_values=False
 ):
     """Search JSON body (if any), then try GET/POST keys"""
-    try:
-        assert isinstance(request, Request)
-    except AssertionError:
-        raise HTTPServerError(
-            json.dumps(
-                {
-                    "error": 1,
-                    "description": "request not provided in call to api_utils.find_in_request!",
-                }
-            )
-        )
+    if not isinstance(request, Request):
+        msg = "request not provided in call to api_utils.find_in_request!"
+        raise_int_server_err(msg)
     try:
         # recommended practice is all vars in the JSON body
         found_value = request.json_body.get(property_name)
+        if found_value is not None:
+            return found_value
     except:
-        found_value = None
-    if found_value is None:
-        # sometimes we allow vars from the query-string or form values
-        if return_all_values:
-            # NB - request.params combines GET and POST into a shared MultiDict
-            found_value = request.params.getall(property_name)
-        else:
-            # NB this returns None if default not specified!
-            found_value = request.params.get(property_name, default_value)
-    if found_value is None:
-        return default_value
-    return found_value
+        pass  # happens if we have no json_body
+    # sometimes we allow vars from the query-string or form values
+    if return_all_values:
+        # NB - request.params combines GET and POST into a shared MultiDict
+        found_value = request.params.getall(property_name)
+        if found_value is None:
+            return default_value
+        return found_value
+    # NB this returns None if default not specified!
+    return request.params.get(property_name, default_value)
 
 
 # Define a consistent cleaner to sanitize user input. We need a few
diff --git a/phylesystem_api/phylesystem_api/views/amendment.py b/phylesystem_api/phylesystem_api/views/amendment.py
index 35d57c83..dbcebdc2 100644
--- a/phylesystem_api/phylesystem_api/views/amendment.py
+++ b/phylesystem_api/phylesystem_api/views/amendment.py
@@ -67,8 +67,7 @@ def create_amendment(request, **kwargs):
     _LOG.debug("Begin create ammendemt")
     # _LOG = api_utils.get_logger(request, 'ot_api.amendment')
 
-    # this method requires authentication
-    auth_info = api_utils.authenticate(request)
+    auth_info = api_utils.auth_and_not_read_only(request)
 
     # gather any user-provided git-commit message
     try:
@@ -79,7 +78,6 @@ def create_amendment(request, **kwargs):
     except:
         commit_msg = None
 
-    api_utils.raise_if_read_only()
     # fetch and parse the JSON payload, if any
     (
         amendment_obj,
@@ -178,8 +176,7 @@ def update_amendment(request):
         msg = "invalid amendment ID ({}) provided".format(amendment_id)
         raise400(msg)
 
-    # this method requires authentication
-    auth_info = api_utils.authenticate(request)
+    auth_info = api_utils.auth_and_not_read_only(request)
 
     # gather any user-provided git-commit message
     try:
@@ -200,8 +197,6 @@ def update_amendment(request):
         msg = "amendment JSON expected for HTTP method {}".format(request.method)
         raise400(msg)
 
-    api_utils.raise_if_read_only()
-
     # update an existing amendment with the data provided
     # _LOG = api_utils.get_logger(request, 'ot_api.default.amendments.PUT')
     # submit new json for this id, and read the results
@@ -247,8 +242,7 @@ def delete_amendment(request):
         }
         raise HTTPBadRequest(body=json.dumps(body))
 
-    # this method requires authentication
-    auth_info = api_utils.authenticate(request)
+    auth_info = api_utils.auth_and_not_read_only(request)
 
     # gather any user-provided git-commit message
     try:
@@ -259,8 +253,6 @@ def delete_amendment(request):
     except:
         commit_msg = None
 
-    api_utils.raise_if_read_only()
-
     # remove this amendment from the docstore
     # _LOG = api_utils.get_logger(request, 'ot_api.default.amendments.POST')
     docstore = api_utils.get_taxonomic_amendment_store(request)
diff --git a/phylesystem_api/phylesystem_api/views/amendments.py b/phylesystem_api/phylesystem_api/views/amendments.py
index c768e5f9..f58f8aa5 100644
--- a/phylesystem_api/phylesystem_api/views/amendments.py
+++ b/phylesystem_api/phylesystem_api/views/amendments.py
@@ -14,16 +14,6 @@
 _LOG = logging.getLogger("phylesystem_api")
 
 
-def _bool_arg(v):
-    if isinstance(v, str):
-        u = v.upper()
-        if u in ["TRUE", "YES"]:
-            return True
-        if u in ["FALSE", "NO"]:
-            return False
-    return v
-
-
 @view_config(route_name="list_all_amendment_ids", renderer="json")
 def list_all_amendment_ids(request):
     docstore = api_utils.get_taxonomic_amendment_store(request)
diff --git a/phylesystem_api/phylesystem_api/views/collection.py b/phylesystem_api/phylesystem_api/views/collection.py
index e4b4f9f5..256484d3 100644
--- a/phylesystem_api/phylesystem_api/views/collection.py
+++ b/phylesystem_api/phylesystem_api/views/collection.py
@@ -71,7 +71,7 @@ def create_collection(request):
             commit_msg = None
     except:
         commit_msg = None
-    api_utils.raise_if_read_only()
+    auth_info = api_utils.auth_and_not_read_only(request)
 
     # fetch and parse the JSON payload, if any
     (
@@ -83,7 +83,6 @@ def create_collection(request):
         msg = "collection JSON expected for HTTP method {}".format(request.method)
         raise400(msg)
 
-    auth_info = api_utils.authenticate(request)
     owner_id = auth_info.get("login", None)
     if owner_id is None:
         msg = "no GitHub userid found for HTTP method {}".format(
@@ -106,9 +105,6 @@ def create_collection(request):
         raise404("collection URL in JSON doesn't match logged-in user: {}".format(url))
 
     # Create a new collection with the data provided
-    auth_info = auth_info or api_utils.authenticate(request)
-    _LOG.debug("COLLECTIONS: auth_info is {}".format(auth_info))
-
     # submit the json and proposed id (if any), and read the results
     docstore = api_utils.get_tree_collection_store(request)
     try:
@@ -205,7 +201,7 @@ def fetch_collection(request):
 def update_collection(request):
     # _LOG = api_utils.get_logger(request, 'ot_api.collection')
     # NB - This method requires authentication!
-    auth_info = api_utils.authenticate(request)
+    auth_info = api_utils.auth_and_not_read_only(request)
     _LOG.debug("COLLECTION: update_collection")
     _LOG.debug("COLLECTION: auth_info {}".format(auth_info))
     owner_id = auth_info.get("login", None)
@@ -222,8 +218,6 @@ def update_collection(request):
     except:
         commit_msg = None
 
-    api_utils.raise_if_read_only()
-
     # fetch and parse the JSON payload, if any
     (
         collection_obj,
@@ -274,7 +268,7 @@ def update_collection(request):
 def delete_collection(request):
     # _LOG = api_utils.get_logger(request, 'ot_api.collection')
     # NB - This method requires authentication!
-    auth_info = api_utils.authenticate(request)
+    auth_info = api_utils.auth_and_not_read_only(request)
 
     collection_id = request.matchdict["collection_id"]
     if not COLLECTION_ID_PATTERN.match(collection_id):
@@ -289,8 +283,6 @@ def delete_collection(request):
     except:
         commit_msg = None
 
-    api_utils.raise_if_read_only()
-
     # remove this collection from the docstore
     docstore = api_utils.get_tree_collection_store(request)
     parent_sha = find_in_request(request, "starting_commit_SHA", None)
diff --git a/phylesystem_api/phylesystem_api/views/collections.py b/phylesystem_api/phylesystem_api/views/collections.py
index 6696fa24..2762090d 100644
--- a/phylesystem_api/phylesystem_api/views/collections.py
+++ b/phylesystem_api/phylesystem_api/views/collections.py
@@ -12,16 +12,6 @@
 from pyramid.view import view_config
 
 
-def _bool_arg(v):
-    if isinstance(v, str):
-        u = v.upper()
-        if u in ["TRUE", "YES"]:
-            return True
-        if u in ["FALSE", "NO"]:
-            return False
-    return v
-
-
 @view_config(route_name="get_collections_config", renderer="json")
 def get_collections_config(request):
     api_utils.raise_on_CORS_preflight(request)
diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py
index 067e0a8d..e786e4c2 100644
--- a/phylesystem_api/phylesystem_api/views/default.py
+++ b/phylesystem_api/phylesystem_api/views/default.py
@@ -24,6 +24,7 @@
 )
 from pyramid.response import Response
 from pyramid.view import view_config
+from phylesystem_api.api_utils import raise_int_server_err, find_in_request
 
 import json
 
@@ -259,18 +260,16 @@ def trees_in_synth(request):
     cds = api_utils.get_tree_collection_store(request)
     for coll_id in coll_id_list:
         try:
-            coll_list.append(
-                cds.return_doc(coll_id, commit_sha=None, return_WIP_map=False)[0]
-            )
+            el = cds.return_doc(coll_id, commit_sha=None, return_WIP_map=False)[0]
         except:
             raise404("GET of collection {} failed".format(coll_id))
+        coll_list.append(el)
     try:
-        result = concatenate_collections(coll_list)
+        return concatenate_collections(coll_list)
     except:
         # _LOG.exception('concatenation of collections failed')
         e = sys.exc_info()[0]
         raise HTTPBadRequest(body=e)
-    return result
 
 
 @view_config(route_name="include_tree_in_synth", renderer="json")
@@ -281,23 +280,30 @@ def include_tree_in_synth(request):
     if (study_id == "") or (tree_id == ""):
         raise400("Expecting study_id and tree_id arguments")
     # examine this study and tree, to confirm it exists *and* to capture its name
+    auth_info = api_utils.auth_and_not_read_only(request)
+    owner_id = auth_info.get("login", None)
     sds = api_utils.get_phylesystem(request)
     try:
         found_study = sds.return_doc(study_id, commit_sha=None, return_WIP_map=False)[0]
-        tree_collections_by_id = found_study.get("nexml").get("treesById")
+    except:
+        raise404("Study {} not found".format(study_id))
+    found_tree = None
+    try:
+        tree_collections_by_id = found_study.get("nexml", {}).get("treesById", {})
         for trees_id, trees_collection in list(tree_collections_by_id.items()):
-            trees_by_id = trees_collection.get("treeById")
+            trees_by_id = trees_collection.get("treeById", {})
             if tree_id in list(trees_by_id.keys()):
-                # _LOG.exception('*** FOUND IT ***')
                 found_tree = trees_by_id.get(tree_id)
-        found_tree_name = found_tree["@label"] or tree_id
-        # _LOG.exception('*** FOUND IT: {}'.format(found_tree_name))
-    except:  # report a missing/misidentified tree
-        # _LOG.exception('problem finding tree')
+    except Exception as x:
+        raise_int_server_err(str(x))
+    if found_tree is None:
         msg = "Specified tree '{t}' in study '{s}' not found! Save this study and try again?"
         msg = msg.format(s=study_id, t=tree_id)
         raise404(msg)
-    already_included_in_synth_input_collections = False
+    try:
+        found_tree_name = found_tree.get("@label") or tree_id
+    except Exception as x:
+        raise_int_server_err(str(x))
     # Look ahead to see if it's already in an included collection; if so, skip
     # adding it again.
     coll_id_list = _get_synth_input_collection_ids()
@@ -308,65 +314,59 @@ def include_tree_in_synth(request):
         except:
             raise404("GET of collection {} failed".format(coll_id))
         if tree_is_in_collection(coll, study_id, tree_id):
-            already_included_in_synth_input_collections = True
-    if not already_included_in_synth_input_collections:
-        # find the default synth-input collection and parse its JSON
-        default_collection_id = coll_id_list[-1]
-        # N.B. For now, we assume that the last listed synth-input collection
-        # is the sensible default, so we already have it in coll
-        decision_list = coll.get("decisions", [])
-        # construct and add a sensible decision entry for this tree
-        decision_list.append(
-            {
-                "name": found_tree_name or "",
-                "treeID": tree_id,
-                "studyID": study_id,
-                "SHA": "",
-                "decision": "INCLUDED",
-                "comments": "Added via API (include_tree_in_synth) from {p}".format(
-                    p=found_study.get("nexml")["^ot:studyPublicationReference"]
-                ),
-            }
+            return trees_in_synth(request)
+
+    # find the default synth-input collection and parse its JSON
+    default_collection_id = coll_id_list[-1]
+    # N.B. For now, we assume that the last listed synth-input collection
+    # is the sensible default, so we already have it in coll
+    decision_list = coll.get("decisions", [])
+    # construct and add a sensible decision entry for this tree
+    decision_list.append(
+        {
+            "name": found_tree_name or "",
+            "treeID": tree_id,
+            "studyID": study_id,
+            "SHA": "",
+            "decision": "INCLUDED",
+            "comments": "Added via API (include_tree_in_synth) from {p}".format(
+                p=found_study.get("nexml")["^ot:studyPublicationReference"]
+            ),
+        }
+    )
+    # update (or add) the decision list for this collection
+    coll["decisions"] = decision_list
+    # update the default collection (forces re-indexing)
+    try:
+        parent_sha = request.params.get("starting_commit_SHA", None)
+        merged_sha = None  # TODO: request.params.get('???', None)
+    except:
+        raise404("include_tree_in_synth(): fetch of starting_commit_SHA failed")
+    try:
+        r = cds.update_existing_collection(
+            owner_id,
+            default_collection_id,
+            coll,
+            auth_info,
+            parent_sha,
+            merged_sha,
+            commit_msg="Updated via API (include_tree_in_synth)",
+        )
+        commit_return = r
+    except GitWorkflowError as err:
+        raise HTTPBadRequest(body=err.msg)
+    except:
+        raise HTTPBadRequest(body=traceback.format_exc())
+
+    # check for 'merge needed'?
+    mn = commit_return.get("merge_needed")
+    if (mn is not None) and (not mn):
+        api_utils.deferred_push_to_gh_call(
+            request,
+            default_collection_id,
+            doc_type="collection",
+            auth_token=auth_info["auth_token"],
         )
-        # update (or add) the decision list for this collection
-        coll["decisions"] = decision_list
-        # update the default collection (forces re-indexing)
-        try:
-            auth_info = api_utils.authenticate(request)
-            owner_id = auth_info.get("login", None)
-        except:
-            raise404("include_tree_in_synth(): Authentication failed")
-        try:
-            parent_sha = request.params.get("starting_commit_SHA", None)
-            merged_sha = None  # TODO: request.params.get('???', None)
-        except:
-            raise404("include_tree_in_synth(): fetch of starting_commit_SHA failed")
-        try:
-            r = cds.update_existing_collection(
-                owner_id,
-                default_collection_id,
-                coll,
-                auth_info,
-                parent_sha,
-                merged_sha,
-                commit_msg="Updated via API (include_tree_in_synth)",
-            )
-            commit_return = r
-        except GitWorkflowError as err:
-            raise HTTPBadRequest(body=err.msg)
-        except:
-            raise HTTPBadRequest(body=traceback.format_exc())
-
-        # check for 'merge needed'?
-        mn = commit_return.get("merge_needed")
-        if (mn is not None) and (not mn):
-            api_utils.deferred_push_to_gh_call(
-                request,
-                default_collection_id,
-                doc_type="collection",
-                auth_token=auth_info["auth_token"],
-            )
-
     # fetch and return the updated list of synth-input trees
     return trees_in_synth(request)
 
@@ -381,11 +381,8 @@ def exclude_tree_from_synth(request):
     # find this tree in ANY synth-input collection; if found, remove it and update the collection
     coll_id_list = _get_synth_input_collection_ids()
     cds = api_utils.get_tree_collection_store(request)
-    try:
-        auth_info = api_utils.authenticate(request)
-        owner_id = auth_info.get("login", None)
-    except:
-        raise404("include_tree_in_synth(): Authentication failed")
+    auth_info = api_utils.auth_and_not_read_only(request)
+    owner_id = auth_info.get("login", None)
     for coll_id in coll_id_list:
         try:
             coll = cds.return_doc(coll_id, commit_sha=None, return_WIP_map=False)[0]
@@ -493,11 +490,7 @@ def merge_docstore_changes(request):
     """
     resource_id = request.matchdict["doc_id"]
     starting_commit_SHA = request.matchdict["starting_commit_SHA"]
-    api_utils.raise_if_read_only()
-
-    # this method requires authentication
-    auth_info = api_utils.authenticate(request)
-
+    auth_info = api_utils.auth_and_not_read_only(request)
     phylesystem = api_utils.get_phylesystem(request)
     gd = phylesystem.create_git_action(resource_id)
     try:
@@ -521,29 +514,10 @@ def push_docstore_changes(request):
     curl -X POST https://devapi.opentreeoflife.org/v3/push_docstore_changes/nexson/ot_999
     """
     _LOG.debug("push_docstore_changes")
-    #    _LOG.debug(request.__dict__)
-    _LOG.debug(request.matchdict)
-    doc_type = request.matchdict.get("doc_type", None)
-    resource_id = request.matchdict.get("doc_id", None)  # Whyyyyy doc id here??
-    #    resource_id = request.matchdict.get('resource_id', None)
-
-    data = request.json_body
-    if doc_type is None:
-        doc_type = data.get("doc_type")
-    if resource_id is None:
-        resource_id = data.get("resource_id")
-
-    api_utils.raise_if_read_only()
-
-    #    _LOG = api_utils.get_logger(request, 'ot_api.push.v3.PUT')
+    doc_type = find_in_request(request, "doc_type", None)
+    resource_id = find_in_request(request, "doc_id", None)
+    api_utils.auth_and_not_read_only(request)
     fail_file = api_utils.get_failed_push_filepath(request, doc_type=doc_type)
-    _LOG.debug(">> fail_file for type '{t}': {f}".format(t=doc_type, f=fail_file))
-    _LOG.debug("Going to authenicateeee")
-
-    # this method requires authentication
-    api_utils.authenticate(request)
-    _LOG.debug("Made it past auth")
-
     # TODO
     if doc_type.lower() == "nexson":
         phylesystem = api_utils.get_phylesystem(request)
diff --git a/phylesystem_api/phylesystem_api/views/studies.py b/phylesystem_api/phylesystem_api/views/studies.py
index 4c61d66d..82c4b1d1 100644
--- a/phylesystem_api/phylesystem_api/views/studies.py
+++ b/phylesystem_api/phylesystem_api/views/studies.py
@@ -1,7 +1,8 @@
 import json
+import threading
 
 import phylesystem_api.api_utils as api_utils
-from phylesystem_api.api_utils import find_in_request, raise400
+from phylesystem_api.api_utils import find_in_request, raise400, bool_arg
 
 # see exception subclasses at https://docs.pylonsproject.org/projects/pyramid/en/latest/api/httpexceptions.html
 from pyramid.httpexceptions import HTTPException, HTTPBadRequest
@@ -15,35 +16,50 @@ def _configure_response(response):
     response.headers["Access-Control-Allow-Credentials"] = "true"
 
 
-def _bool_arg(v):
-    if isinstance(v, str):
-        u = v.upper()
-        if u in ["TRUE", "YES"]:
-            return True
-        if u in ["FALSE", "NO"]:
-            return False
-    return v
-
-
 @view_config(route_name="study_properties", renderer="json")
 def properties(request):
     _configure_response(request.response)
-    oti = api_utils.get_oti_wrapper(request)
-    n = list(oti.node_search_term_set)
-    n.sort()
-    t = list(oti.tree_search_term_set)
-    t.sort()
-    s = list(oti.study_search_term_set)
-    s.sort()
-    return {"node_properties": n, "tree_properties": t, "study_properties": s}
+    return _get_study_properties(request)
+
+
+# global dict so that we don't have to fetch the same info from otindex repeatedly
+_study_prop_dict = None
+_study_prop_dict_lock = threading.Lock()
+
+
+def _get_study_properties(request):
+    global _study_prop_dict
+    if _study_prop_dict is None:
+        with _study_prop_dict_lock:
+            if _study_prop_dict is not None:
+                return _study_prop_dict
+
+            oti = api_utils.get_oti_wrapper(request)
+            n = list(oti.node_search_term_set)
+            n.sort()
+            t = list(oti.tree_search_term_set)
+            t.sort()
+            s = list(oti.study_search_term_set)
+            s.sort()
+            _study_prop_dict = {
+                "node_properties": n,
+                "tree_properties": t,
+                "study_properties": s,
+            }
+    return _study_prop_dict
+
+
+def validate_bool_setting(request, prop_name, default):
+    bs = bool_arg(find_in_request(request, prop_name, default))
+    if (bs is not True) and (bs is not False):
+        raise400('"{}" setting must be a boolean'.format(prop_name))
+    return bs
 
 
 @view_config(route_name="find_studies", renderer="json")
 def find_studies(request):
     oti = api_utils.get_oti_wrapper(request)
-    verbose = _bool_arg(find_in_request(request, "verbose", False))
-    if (verbose is not True) and (verbose is not False):
-        raise400('"verbose" setting must be a boolean')
+    verbose = validate_bool_setting(request, "verbose", False)
     field = find_in_request(request, "property", None)
     try:
         if field is None:
@@ -52,23 +68,20 @@ def find_studies(request):
             value = find_in_request(request, "value", None)
             if value is None:
                 raise400('If "property" is sent, a "value" argument must be used.')
-            exact = _bool_arg(find_in_request(request, "exact", False))
-            if (exact is not True) and (exact is not False):
-                raise400('"exact" setting must be a boolean')
+            exact = validate_bool_setting(request, "exact", False)
             try:
                 match_list = oti.find_studies(
                     {field: value}, verbose=verbose, exact=exact
                 )
             except ValueError as x:
                 raise400(str(x))
-        resp = {"matched_studies": match_list}
     except HTTPException:
         raise
     except Exception as x:
         msg = "Unexpected error calling oti: {}".format(",".join(x.args))
         api_utils.raise_int_server_err(msg)
     _configure_response(request.response)
-    return resp
+    return {"matched_studies": match_list}
 
 
 @view_config(route_name="find_trees", renderer="json")
@@ -79,21 +92,16 @@ def find_trees(request):
         raise400("missing or invalid request JSON")
     _configure_response(request.response)
     oti = api_utils.get_oti_wrapper(request)
-    verbose = _bool_arg(msg.get("verbose", False))
-    if (verbose is not True) and (verbose is not False):
-        raise400('"verbose" setting must be a boolean')
+    verbose = validate_bool_setting(request, "verbose", False)
     field = msg.get("property")
     if field is None:
         raise400('A "property" argument must be used.')
     value = msg.get("value")
     if value is None:
         raise400('A "value" argument must be used.')
-    exact = _bool_arg(msg.get("exact", False))
-    if (exact is not True) and (exact is not False):
-        raise400('"exact" setting must be a boolean')
+    exact = validate_bool_setting(request, "exact", False)
     try:
         match_list = oti.find_trees({field: value}, verbose=verbose, exact=exact)
-        resp = {"matched_studies": match_list}
     except HTTPException:
         raise
     except ValueError as x:
@@ -101,4 +109,4 @@ def find_trees(request):
     except Exception as x:
         msg = "Unexpected error calling oti: {}".format(",".join(x.args))
         api_utils.raise_int_server_err(msg)
-    return resp
+    return {"matched_studies": match_list}
diff --git a/phylesystem_api/phylesystem_api/views/study.py b/phylesystem_api/phylesystem_api/views/study.py
index 9b79251b..47b0914d 100644
--- a/phylesystem_api/phylesystem_api/views/study.py
+++ b/phylesystem_api/phylesystem_api/views/study.py
@@ -419,9 +419,8 @@ def _new_nexson_with_crossref_metadata(doi, ref_string, include_cc0=False):
 @view_config(route_name="create_study", renderer="json", request_method="POST")
 def create_study(request):
     # this method requires authentication
-    auth_info = api_utils.authenticate(request)
+    auth_info = api_utils.auth_and_not_read_only(request)
     phylesystem = api_utils.get_phylesystem(request)
-    api_utils.raise_if_read_only()
 
     # we're creating a new study (possibly with import instructions in the payload)
     treebase_id = find_in_request(request, "treebase_id", "")
@@ -576,9 +575,7 @@ def create_study(request):
 @view_config(route_name="update_study", renderer="json")
 def update_study(request):
     study_id = request.matchdict["study_id"]
-
-    # this method requires authentication
-    auth_info = api_utils.authenticate(request)
+    auth_info = api_utils.auth_and_not_read_only(request)
     _LOG.debug("STUDY: update_study")
     # _LOG.debug("STUDY: auth_info {}".format(auth_info))
 
@@ -598,11 +595,7 @@ def update_study(request):
     except:
         commit_msg = None
 
-    phylesystem = api_utils.get_phylesystem(
-        request
-    )  # set READONLY flag before testing!
-    api_utils.raise_if_read_only()
-
+    phylesystem = api_utils.get_phylesystem(request)
     repo_nexml2json = phylesystem.repo_nexml2json
     bundle = __extract_and_validate_nexson(request, repo_nexml2json, request.json_body)
     nexson, annotation, nexson_adaptor = bundle
@@ -641,15 +634,12 @@ def update_study(request):
 @view_config(route_name="delete_study", renderer="json")
 def delete_study(request):
     # _LOG = api_utils.get_logger(request, 'delete_study')
-    _LOG.warning("delete_study STARTING...")
-    api_version = request.matchdict["api_version"]
+    _LOG.debug("delete_study STARTING...")
     study_id = request.matchdict["study_id"]
-    _LOG.warning("api_version={}".format(api_version))
-    _LOG.warning("study_id={}".format(study_id))
+    _LOG.debug("study_id={}".format(study_id))
 
     # this method requires authentication
-    auth_info = api_utils.authenticate(request)
-    _LOG.warning("auth_info={}".format(auth_info))
+    auth_info = api_utils.auth_and_not_read_only(request)
 
     # gather any user-provided git-commit message
     try:
@@ -659,40 +649,33 @@ def delete_study(request):
             commit_msg = None
     except:
         commit_msg = None
-    _LOG.warning("commit_msg={}".format(commit_msg))
+    _LOG.debug("commit_msg={}".format(commit_msg))
 
     parent_sha = find_in_request(request, "starting_commit_SHA", None)
-    _LOG.warning("parent_sha={}".format(parent_sha))
+    _LOG.debug("parent_sha={}".format(parent_sha))
 
-    phylesystem = api_utils.get_phylesystem(
-        request
-    )  # set READONLY flag before testing!
-    api_utils.raise_if_read_only()
-    _LOG.warning("passed the read-only test")
+    phylesystem = api_utils.get_phylesystem(request)
 
-    _LOG.warning("trying to delete now... via:")
-    _LOG.warning(phylesystem.delete_study)
+    _LOG.debug("trying to delete now... via:")
+    _LOG.debug(phylesystem.delete_study)
     try:
         x = phylesystem.delete_study(
             study_id, auth_info, parent_sha, commit_msg=commit_msg
         )
-        _LOG.warning("initial return x:")
-        _LOG.warning(x)
+        _LOG.debug("initial return x:")
+        _LOG.debug(x)
         if x.get("error") == 0:
-            _LOG.warning("calling deferred push...")
+            _LOG.debug("calling deferred push...")
             api_utils.deferred_push_to_gh_call(
                 request, None, doc_type="nexson", auth_token=auth_info["auth_token"]
             )
-            _LOG.warning("back from deferred push")
+            _LOG.debug("back from deferred push")
         return x
     except GitWorkflowError as err:
-        _LOG.warning("got a GitWorkflowError:")
-        _LOG.warning(err)
+        _LOG.exception("got a GitWorkflowError:")
         raise HTTPBadRequest(err.msg)
     except Exception as err:
-        _LOG.warning("some other kind of error")
-        _LOG.warning(err)
-        # _LOG.exception('Exception getting nexson content in phylesystem.delete_study')
+        _LOG.exception("Exception getting nexson content in phylesystem.delete_study")
         raise400("Unknown error in study deletion")
 
 

From 35e1fb554b15d6eeb3fc0992bc63a2c3aa2d3360 Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Fri, 28 Jul 2023 11:19:40 -0500
Subject: [PATCH 11/33] bug fix from previous refactoring

---
 phylesystem_api/phylesystem_api/api_utils.py  | 25 ++++++-------------
 phylesystem_api/phylesystem_api/routes.py     |  2 +-
 .../phylesystem_api/views/default.py          |  6 +++--
 3 files changed, 12 insertions(+), 21 deletions(-)

diff --git a/phylesystem_api/phylesystem_api/api_utils.py b/phylesystem_api/phylesystem_api/api_utils.py
index 273fe565..27c41ec8 100644
--- a/phylesystem_api/phylesystem_api/api_utils.py
+++ b/phylesystem_api/phylesystem_api/api_utils.py
@@ -88,19 +88,12 @@ def atomic_write_json_if_not_found(obj, dest, request):
 
 def compose_push_to_github_url(request, resource_id, doc_type):
     if resource_id is None:
-        call = "{p}://{d}/v3/push_docstore_changes".format(
-            p=request.environ["wsgi.url_scheme"], d=request.environ["HTTP_HOST"]
+        return request.route_url(
+            "push_docstore_changes_bare", api_version="v3", doc_type=doc_type
         )
-    else:
-        call = "{p}://{d}/v3/push_docstore_changes/{dt}/{r}".format(
-            p=request.environ["wsgi.url_scheme"],
-            d=request.environ["HTTP_HOST"],
-            dt=doc_type,
-            r=resource_id,
-        )
-    # _LOG.debug("Logging push to github call")
-    # _LOG.debug(call)
-    return call
+    return request.route_url(
+        "push_docstore_changes", api_version="v3", doc_type=doc_type, doc_id=resource_id
+    )
 
 
 # this allows us to raise HTTP(...)
@@ -598,6 +591,7 @@ def raise_if_read_only():
 
 
 def call_http_json(url, verb="GET", data=None, headers=None):
+    _LOG.debug("call_http_json data={}".format(str(data)))
     if headers is None:
         headers = {
             "content-type": "application/json",
@@ -622,12 +616,7 @@ def call_http_json(url, verb="GET", data=None, headers=None):
 def deferred_push_to_gh_call(request, resource_id, doc_type="nexson", auth_token=None):
     ##TODO Thius needs to create a bare URL for collections, and pass in the resource id etc as data
     # _LOG.debug("deferred_push_to_gh_call")
-    if READ_ONLY_MODE:
-        raise HTTPForbidden(
-            json.dumps(
-                {"error": 1, "description": "phylesystem-api running in read-only mode"}
-            )
-        )
+    raise_if_read_only()
     # Pass the resource_id in data, so that two-part collection IDs will be recognized
     # (else the second part will trigger an unwanted JSONP response from the push)
     if doc_type == "collection" or doc_type == "amendment":
diff --git a/phylesystem_api/phylesystem_api/routes.py b/phylesystem_api/phylesystem_api/routes.py
index 0e834f74..48ddcd35 100644
--- a/phylesystem_api/phylesystem_api/routes.py
+++ b/phylesystem_api/phylesystem_api/routes.py
@@ -27,7 +27,7 @@ def includeme(config):
     )
     config.add_route(
         "push_docstore_changes_bare",
-        "/{api_version}/push_docstore_changes",
+        "/{api_version}/push_docstore_changes/{doc_type}",
         request_method="PUT",
     )
     #
diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py
index e786e4c2..fbddc0dd 100644
--- a/phylesystem_api/phylesystem_api/views/default.py
+++ b/phylesystem_api/phylesystem_api/views/default.py
@@ -514,8 +514,10 @@ def push_docstore_changes(request):
     curl -X POST https://devapi.opentreeoflife.org/v3/push_docstore_changes/nexson/ot_999
     """
     _LOG.debug("push_docstore_changes")
-    doc_type = find_in_request(request, "doc_type", None)
-    resource_id = find_in_request(request, "doc_id", None)
+    doc_type = request.matchdict["doc_type"]
+    resource_id = request.matchdict.get("doc_id")
+    if resource_id is None:
+        resource_id = find_in_request(request, "doc_id", None)
     api_utils.auth_and_not_read_only(request)
     fail_file = api_utils.get_failed_push_filepath(request, doc_type=doc_type)
     # TODO

From d16d652c7c25df923c101413543d1108c5053295 Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Fri, 28 Jul 2023 16:48:30 -0500
Subject: [PATCH 12/33] new test

---
 ws-tests/test_study_verbose_list.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 ws-tests/test_study_verbose_list.py

diff --git a/ws-tests/test_study_verbose_list.py b/ws-tests/test_study_verbose_list.py
new file mode 100644
index 00000000..bb875eef
--- /dev/null
+++ b/ws-tests/test_study_verbose_list.py
@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+import sys, os
+from opentreetesting import test_http_json_method, config
+
+DOMAIN = config("host", "apihost")
+SUBMIT_URI = DOMAIN + "/v3/studies/find_studies"
+# NB this is not a duplicate of /v3/studies/find_studies !
+# It's a rarely-used method to fetch all studies prior to building the oti index!
+# See this helpful issue from @jar398:
+#   https://github.com/OpenTreeOfLife/germinator/issues/94#issuecomment-227027174
+data = {"verbose": True}
+r = test_http_json_method(SUBMIT_URI, "GET", data=data, expected_status=200, return_bool_data=True)
+if not r[0]:
+    sys.exit(1)

From 16446917994c2048b0e9f225d966989da2de8192 Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Sat, 29 Jul 2023 13:25:55 -0500
Subject: [PATCH 13/33] minor

---
 phylesystem_api/phylesystem_api/api_utils.py  | 179 ++++++++++--------
 .../phylesystem_api/views/amendments.py       |   4 +-
 .../phylesystem_api/views/collection.py       |  11 +-
 .../phylesystem_api/views/collections.py      |   4 +-
 .../phylesystem_api/views/studies.py          |   9 +-
 .../phylesystem_api/views/study.py            |  27 +--
 6 files changed, 117 insertions(+), 117 deletions(-)

diff --git a/phylesystem_api/phylesystem_api/api_utils.py b/phylesystem_api/phylesystem_api/api_utils.py
index 27c41ec8..ab0a4e83 100644
--- a/phylesystem_api/phylesystem_api/api_utils.py
+++ b/phylesystem_api/phylesystem_api/api_utils.py
@@ -1,30 +1,30 @@
+import json
+import logging
+import os
+import tempfile
+import threading
+from configparser import ConfigParser
+from datetime import datetime
+
+import requests
+from beaker.cache import cache_managers
 from github import Github, BadCredentialsException
+from peyotl.amendments import TaxonomicAmendmentStore
+from peyotl.api import OTI
+from peyotl.collections_store import TreeCollectionStore
 from peyotl.nexson_syntax import write_as_json
 from peyotl.phylesystem import Phylesystem
-from peyotl.collections_store import TreeCollectionStore
-from peyotl.amendments import TaxonomicAmendmentStore
 from peyotl.utility import read_config as read_peyotl_config
-from configparser import ConfigParser
-from datetime import datetime
-from peyotl.api import OTI
-
-# see exception subclasses at https://docs.pylonsproject.org/projects/pyramid/en/latest/api/httpexceptions.html
-from pyramid.request import Request
 from pyramid.httpexceptions import (
     HTTPOk,
-    HTTPServerError,
     HTTPBadRequest,
     HTTPForbidden,
     HTTPInternalServerError,
     HTTPNotFound,
 )
-from beaker.cache import cache_managers
-import tempfile
-import logging
-import json
-import requests
-import os
-import threading
+
+# see exception subclasses at https://docs.pylonsproject.org/projects/pyramid/en/latest/api/httpexceptions.html
+from pyramid.request import Request
 
 try:
     import xml.etree.cElementTree as ElementTree
@@ -112,27 +112,18 @@ def get_phylesystem(request, conf_obj=None):
 
     if conf_obj is None:
         conf_obj = get_conf_object(request)
-    (
-        repo_parent,
-        repo_remote,
-        git_ssh,
-        pkey,
-        git_hub_remote,
-        max_filesize,
-        max_num_trees,
-        READ_ONLY_MODE,
-    ) = read_phylesystem_config(request, conf_obj=conf_obj)
+    pc = read_phylesystem_config(request, conf_obj=conf_obj)
     peyotl_config, cfg_filename = read_peyotl_config()
     if "phylesystem" not in peyotl_config.sections():
         peyotl_config.add_section("phylesystem")
     peyotl_config.set(
-        "phylesystem", "max_file_size", max_filesize
+        "phylesystem", "max_file_size", pc.max_filesize
     )  # overrides peyotl config with max phylesytem-api filesize
-    push_mirror = os.path.join(repo_parent, "mirror")
+    push_mirror = os.path.join(pc.repo_parent, "mirror")
     pmi = {
         "parent_dir": push_mirror,
         "remote_map": {
-            "GitHubRemote": git_hub_remote,
+            "GitHubRemote": pc.git_hub_remote,
         },
     }
     mirror_info = {"push": pmi}
@@ -143,9 +134,9 @@ def get_phylesystem(request, conf_obj=None):
     except:
         pass
     _PHYLESYSTEM = Phylesystem(
-        repos_par=repo_parent,
-        git_ssh=git_ssh,
-        pkey=pkey,
+        repos_par=pc.repo_parent,
+        git_ssh=pc.git_ssh,
+        pkey=pc.pkey,
         git_action_class=GitData,
         mirror_info=mirror_info,
         **a
@@ -286,54 +277,86 @@ def get_conf_object(request=None, localconfig_filename=None):
     return conf
 
 
-def read_phylesystem_config(request, conf_obj=None):
-    """Load settings for managing the main Nexson docstore"""
-    if conf_obj is None:
-        conf_obj = get_conf_object(request)
-    return _read_phylesystem_from_conf_obj(conf_obj)
+_phylesystem_config = None
+_phylesystem_config_lock = threading.Lock()
 
 
-def _read_phylesystem_from_conf_obj(conf):
-    repo_parent = conf.get("apis", "repo_parent")
-    repo_remote = conf.get("apis", "repo_remote")
-    try:
-        git_ssh = conf.get("apis", "git_ssh")
-    except:
-        git_ssh = "ssh"
-    try:
-        pkey = conf.get("apis", "pkey")
-    except:
-        pkey = None
-    try:
-        git_hub_remote = conf.get("apis", "git_hub_remote")
-    except:
-        git_hub_remote = "git@github.com:OpenTreeOfLife"
-    try:
-        max_filesize = conf.get("filesize", "peyotl_max_file_size")
-    except:
-        max_filesize = "20000000"
-    try:
-        max_num_trees = conf.get("filesize", "validation_max_num_trees")
-    except:
-        max_num_trees = 65
-    try:
-        max_num_trees = int(max_num_trees)
-    except ValueError:
-        raise400("max number of trees per study in config is not an integer")
-    try:
-        read_only = conf.get("apis", "read_only") == "true"
-    except:
-        read_only = False
-    return (
-        repo_parent,
-        repo_remote,
-        git_ssh,
-        pkey,
-        git_hub_remote,
-        max_filesize,
-        max_num_trees,
-        read_only,
-    )
+def read_phylesystem_config(request, conf_obj=None):
+    """Load settings for managing the main Nexson docstore"""
+    global _phylesystem_config
+    if _phylesystem_config is None:
+        if conf_obj is None:
+            conf_obj = get_conf_object(request)
+        with _phylesystem_config_lock:
+            if _phylesystem_config is None:
+                _phylesystem_config = PhylesystemConfig(conf_obj)
+    return _phylesystem_config
+
+
+class PhylesystemConfig(object):
+    def __init__(self, conf):
+        self._repo_parent = conf.get("apis", "repo_parent")
+        self._repo_remote = conf.get("apis", "repo_remote")
+        try:
+            self._git_ssh = conf.get("apis", "git_ssh")
+        except:
+            self._git_ssh = "ssh"
+        try:
+            self._pkey = conf.get("apis", "pkey")
+        except:
+            self._pkey = None
+        try:
+            self._git_hub_remote = conf.get("apis", "git_hub_remote")
+        except:
+            self._git_hub_remote = "git@github.com:OpenTreeOfLife"
+        try:
+            self._max_filesize = conf.get("filesize", "peyotl_max_file_size")
+        except:
+            self._max_filesize = "20000000"
+        try:
+            self._max_num_trees = conf.get("filesize", "validation_max_num_trees")
+        except:
+            self._max_num_trees = 65
+        try:
+            self._max_num_trees = int(self._max_num_trees)
+        except ValueError:
+            raise400("max number of trees per study in config is not an integer")
+        try:
+            self._read_only = conf.get("apis", "read_only") == "true"
+        except:
+            self._read_only = False
+
+    @property
+    def repo_parent(self):
+        return self._repo_parent
+
+    @property
+    def repo_remote(self):
+        return self._repo_remote
+
+    @property
+    def git_ssh(self):
+        return self._git_ssh
+
+    @property
+    def pkey(self):
+        return self._pkey
+
+    @property
+    def git_hub_remote(self):
+        return self._git_hub_remote
+
+    @property
+    def max_filesize(self):
+        return self._max_filesize
+
+    @property
+    def max_num_trees(self):
+        return self._max_num_trees
+
+    @property
+    def read_only(self):
+        return self._read_only
 
 
 def read_collections_config(request):
diff --git a/phylesystem_api/phylesystem_api/views/amendments.py b/phylesystem_api/phylesystem_api/views/amendments.py
index f58f8aa5..115e0ff1 100644
--- a/phylesystem_api/phylesystem_api/views/amendments.py
+++ b/phylesystem_api/phylesystem_api/views/amendments.py
@@ -1,8 +1,6 @@
-import json
 import logging
 import os
 
-import phylesystem_api.api_utils as api_utils
 from peyotl.nexson_syntax import read_as_json
 
 # see exception subclasses at https://docs.pylonsproject.org/projects/pyramid/en/latest/api/httpexceptions.html
@@ -11,6 +9,8 @@
 )
 from pyramid.view import view_config
 
+import phylesystem_api.api_utils as api_utils
+
 _LOG = logging.getLogger("phylesystem_api")
 
 
diff --git a/phylesystem_api/phylesystem_api/views/collection.py b/phylesystem_api/phylesystem_api/views/collection.py
index 256484d3..1d5693cb 100644
--- a/phylesystem_api/phylesystem_api/views/collection.py
+++ b/phylesystem_api/phylesystem_api/views/collection.py
@@ -44,14 +44,11 @@ def __extract_and_validate_collection(request, request_params):
             msg = getattr(err, "msg", "No message found")
         except:
             msg = str(err)
-        raise HTTPBadRequest(msg)
+        raise400(msg)
     if len(errors) > 0:
-        # _LOG = api_utils.get_logger(request, 'ot_api.default.v1')
-        msg = "JSON payload failed validation with {nerrors} errors:\n{errors}".format(
-            nerrors=len(errors), errors="\n  ".join(errors)
-        )
-        # _LOG.exception(msg)
-        raise HTTPBadRequest(msg)
+        msg = "JSON payload failed validation with {nerrors} errors:\n{errors}"
+        msg = msg.format(nerrors=len(errors), errors="\n  ".join(errors))
+        raise400(msg)
     return collection_obj, errors, collection_adaptor
 
 
diff --git a/phylesystem_api/phylesystem_api/views/collections.py b/phylesystem_api/phylesystem_api/views/collections.py
index 2762090d..d79a0eb2 100644
--- a/phylesystem_api/phylesystem_api/views/collections.py
+++ b/phylesystem_api/phylesystem_api/views/collections.py
@@ -1,7 +1,5 @@
-import json
 import os
 
-import phylesystem_api.api_utils as api_utils
 from peyotl.nexson_syntax import read_as_json
 
 # see exception subclasses at https://docs.pylonsproject.org/projects/pyramid/en/latest/api/httpexceptions.html
@@ -11,6 +9,8 @@
 )
 from pyramid.view import view_config
 
+import phylesystem_api.api_utils as api_utils
+
 
 @view_config(route_name="get_collections_config", renderer="json")
 def get_collections_config(request):
diff --git a/phylesystem_api/phylesystem_api/views/studies.py b/phylesystem_api/phylesystem_api/views/studies.py
index 82c4b1d1..67ed26c8 100644
--- a/phylesystem_api/phylesystem_api/views/studies.py
+++ b/phylesystem_api/phylesystem_api/views/studies.py
@@ -1,13 +1,12 @@
-import json
 import threading
 
-import phylesystem_api.api_utils as api_utils
-from phylesystem_api.api_utils import find_in_request, raise400, bool_arg
-
 # see exception subclasses at https://docs.pylonsproject.org/projects/pyramid/en/latest/api/httpexceptions.html
-from pyramid.httpexceptions import HTTPException, HTTPBadRequest
+from pyramid.httpexceptions import HTTPException
 from pyramid.view import view_config
 
+import phylesystem_api.api_utils as api_utils
+from phylesystem_api.api_utils import find_in_request, raise400, bool_arg
+
 
 def _configure_response(response):
     response.view = "generic.json"
diff --git a/phylesystem_api/phylesystem_api/views/study.py b/phylesystem_api/phylesystem_api/views/study.py
index 47b0914d..df18c2fc 100644
--- a/phylesystem_api/phylesystem_api/views/study.py
+++ b/phylesystem_api/phylesystem_api/views/study.py
@@ -42,42 +42,27 @@ def __extract_nexson_from_http_call(request, **kwargs):
             nexson = json.loads(nexson)
         if "nexson" in nexson:
             nexson = nexson["nexson"]
+        return nexson
     except:
-        # _LOG = api_utils.get_logger(request, 'ot_api.default.v1')
-        # _LOG.exception('Exception getting nexson content in __extract_nexson_from_http_call')
         raise400("NexSON must be valid JSON")
-    return nexson
 
 
 def __extract_and_validate_nexson(request, repo_nexml2json, kwargs):
     try:
         nexson = __extract_nexson_from_http_call(request, **kwargs)
-        # from peyotl.manip import count_num_trees
-        # numtrees=count_num_trees(nexson,repo_nexml2json)
-        # _LOG = api_utils.get_logger(request, 'ot_api.default.v1')
-        # _LOG.debug('number of trees in nexson is {}, max number of trees is {}'.format(numtrees,max_num_trees))
-        (
-            repo_parent,
-            repo_remote,
-            git_ssh,
-            pkey,
-            git_hub_remote,
-            max_filesize,
-            max_num_trees,
-            read_only_mode,
-        ) = api_utils.read_phylesystem_config(request)
+        pc = api_utils.read_phylesystem_config(request)
         bundle = validate_and_convert_nexson(
             nexson,
             repo_nexml2json,
             allow_invalid=False,
-            max_num_trees_per_study=max_num_trees,
+            max_num_trees_per_study=pc.max_num_trees,
         )
         nexson, annotation, validation_log, nexson_adaptor = bundle
+        return nexson, annotation, nexson_adaptor
     except GitWorkflowError as err:
         # _LOG = api_utils.get_logger(request, 'ot_api.default.v1')
         # _LOG.exception('PUT failed in validation')
         raise400(err.msg or "No message found")
-    return nexson, annotation, nexson_adaptor
 
 
 def __make_valid_DOI(candidate):
@@ -161,7 +146,6 @@ def __finish_write_verb(
     master_file_blob_included=None,
 ):
     """Called by PUT and POST handlers to avoid code repetition."""
-    # global TIMING
     # TODO, need to make this spawn a thread to do the second commit rather than block
     a = phylesystem.annotate_and_write(
         git_data,
@@ -175,10 +159,7 @@ def __finish_write_verb(
         master_file_blob_included,
     )
     annotated_commit = a
-    # TIMING = api_utils.log_time_diff(_LOG, 'annotated commit', TIMING)
     if annotated_commit["error"] != 0:
-        # _LOG = api_utils.get_logger(request, 'ot_api.default.v1')
-        # _LOG.debug('annotated_commit failed')
         raise HTTPBadRequest(body=json.dumps(annotated_commit))
     return annotated_commit
 

From c2f310a8d20322254e324715407868ff3dc79e51 Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Sat, 29 Jul 2023 13:51:33 -0500
Subject: [PATCH 14/33] minor refactoring

---
 .../phylesystem_api/views/collection.py       | 131 +++++++++---------
 .../phylesystem_api/views/study.py            |  98 +++++--------
 2 files changed, 98 insertions(+), 131 deletions(-)

diff --git a/phylesystem_api/phylesystem_api/views/collection.py b/phylesystem_api/phylesystem_api/views/collection.py
index 1d5693cb..fed6c9b4 100644
--- a/phylesystem_api/phylesystem_api/views/collection.py
+++ b/phylesystem_api/phylesystem_api/views/collection.py
@@ -58,6 +58,68 @@ def collection_CORS_preflight(request):
     api_utils.raise_on_CORS_preflight(request)
 
 
+@view_config(route_name="fetch_collection", renderer="json")
+def fetch_collection(request):
+    # NB - This method does not require authentication!
+    # _LOG = api_utils.get_logger(request, 'ot_api.collection')
+    collection_id = request.matchdict["collection_id"]
+    if not COLLECTION_ID_PATTERN.match(collection_id):
+        msg = "invalid collection ID ({}) provided".format(collection_id)
+        raise400(msg)
+
+    # gather details to return with the JSON core document
+    parent_sha = find_in_request(request, "starting_commit_SHA", None)
+    collections = api_utils.get_tree_collection_store(request)
+    try:
+        r = collections.return_doc(
+            collection_id, commit_sha=parent_sha, return_WIP_map=True
+        )
+    except:
+        raise404("Collection '{}' GET failure".format(collection_id))
+    try:
+        collection_json, head_sha, wip_map = r
+        version_history = collections.get_version_history_for_doc_id(collection_id)
+        try:
+            # pre-render internal description (assumes markdown!)
+            comment_html = api_utils.markdown_to_html(
+                collection_json["description"], open_links_in_new_window=True
+            )
+        except:
+            comment_html = ""
+    except:
+        _LOG.exception("GET failed")
+        e = sys.exc_info()[0]
+        raise HTTPBadRequest(e)
+    if not collection_json:
+        raise404("Collection '{s}' has no JSON data!".format(s=collection_id))
+    # add/restore the url field (using the visible fetch URL)
+    base_url = api_utils.get_collections_api_base_url(request)
+    collection_json["url"] = "{b}/v2/collection/{i}".format(b=base_url, i=collection_id)
+    try:
+        external_url = collections.get_public_url(collection_id)
+    except:
+        external_url = "NOT FOUND"
+    result = {
+        "sha": head_sha,
+        "data": collection_json,
+        "branch2sha": wip_map,
+        "commentHTML": comment_html,
+        "external_url": external_url,
+    }
+    if version_history:
+        result["versionHistory"] = version_history
+        latest_commit = version_history[0]
+        last_modified = {
+            "author_name": latest_commit.get("author_name"),
+            "relative_date": latest_commit.get("relative_date"),
+            "display_date": latest_commit.get("date"),
+            "ISO_date": latest_commit.get("date_ISO_8601"),
+            "sha": latest_commit.get("id"),  # this is the commit hash
+        }
+        result["lastModified"] = last_modified
+    return result
+
+
 @view_config(route_name="create_collection", renderer="json", request_method="POST")
 def create_collection(request):
     # gather any user-provided git-commit message
@@ -125,75 +187,6 @@ def create_collection(request):
     return commit_return
 
 
-@view_config(route_name="fetch_collection", renderer="json")
-def fetch_collection(request):
-    # NB - This method does not require authentication!
-    # _LOG = api_utils.get_logger(request, 'ot_api.collection')
-    collection_id = request.matchdict["collection_id"]
-    if not COLLECTION_ID_PATTERN.match(collection_id):
-        msg = "invalid collection ID ({}) provided".format(collection_id)
-        raise400(msg)
-
-    # gather details to return with the JSON core document
-    parent_sha = find_in_request(request, "starting_commit_SHA", None)
-    # _LOG.debug('parent_sha = {}'.format(parent_sha))
-    # return the correct nexson of study_id, using the specified view
-    collections = api_utils.get_tree_collection_store(request)
-    try:
-        r = collections.return_doc(
-            collection_id, commit_sha=parent_sha, return_WIP_map=True
-        )
-    except:
-        raise404("Collection '{}' GET failure".format(collection_id))
-    try:
-        collection_json, head_sha, wip_map = r
-        ## if returning_full_study:  # TODO: offer bare vs. full output (w/ history, etc)
-        version_history = collections.get_version_history_for_doc_id(collection_id)
-        try:
-            # pre-render internal description (assumes markdown!)
-            comment_html = api_utils.markdown_to_html(
-                collection_json["description"], open_links_in_new_window=True
-            )
-        except:
-            comment_html = ""
-    except:
-        # _LOG.exception('GET failed')
-        e = sys.exc_info()[0]
-        raise HTTPBadRequest(e)
-    if not collection_json:
-        raise404("Collection '{s}' has no JSON data!".format(s=collection_id))
-    # add/restore the url field (using the visible fetch URL)
-    base_url = api_utils.get_collections_api_base_url(request)
-    collection_json["url"] = "{b}/v2/collection/{i}".format(b=base_url, i=collection_id)
-    try:
-        external_url = collections.get_public_url(collection_id)
-    except:
-        # _LOG = api_utils.get_logger(request, 'ot_api.default.v1')
-        # _LOG.exception('collection {} not found in external_url'.format(collection))
-        external_url = "NOT FOUND"
-    result = {
-        "sha": head_sha,
-        "data": collection_json,
-        "branch2sha": wip_map,
-        "commentHTML": comment_html,
-        "external_url": external_url,
-    }
-    if version_history:
-        result["versionHistory"] = version_history
-
-        # reckon and add 'lastModified' property, based on commit history?
-        latest_commit = version_history[0]
-        last_modified = {
-            "author_name": latest_commit.get("author_name"),
-            "relative_date": latest_commit.get("relative_date"),
-            "display_date": latest_commit.get("date"),
-            "ISO_date": latest_commit.get("date_ISO_8601"),
-            "sha": latest_commit.get("id"),  # this is the commit hash
-        }
-        result["lastModified"] = last_modified
-    return result
-
-
 @view_config(route_name="update_collection", renderer="json")
 def update_collection(request):
     # _LOG = api_utils.get_logger(request, 'ot_api.collection')
diff --git a/phylesystem_api/phylesystem_api/views/study.py b/phylesystem_api/phylesystem_api/views/study.py
index df18c2fc..5a608e3f 100644
--- a/phylesystem_api/phylesystem_api/views/study.py
+++ b/phylesystem_api/phylesystem_api/views/study.py
@@ -191,6 +191,12 @@ def _fetch_shard_name(phylesystem, study_id):
         return None
 
 
+@view_config(route_name="create_study", renderer="json", request_method="OPTIONS")
+@view_config(route_name="study_CORS_preflight", renderer="json")
+def study_CORS_preflight(request):
+    api_utils.raise_on_CORS_preflight(request)
+
+
 @view_config(route_name="fetch_study", renderer=None, request_method="GET")
 @view_config(route_name="fetch_study_label", renderer=None, request_method="GET")
 def fetch_study(request):
@@ -216,8 +222,6 @@ def fetch_study(request):
         content_id=None,
     )
     parent_sha = find_in_request(request, "starting_commit_SHA", None)
-    # _LOG.debug('parent_sha = {}'.format(parent_sha))
-    # return the correct nexson of study_id, using the specified view
     try:
         r = phylesystem.return_study(
             study_id, commit_sha=parent_sha, return_WIP_map=True
@@ -291,12 +295,6 @@ def fetch_study(request):
         return render_to_response("string", result_data, request)
 
 
-@view_config(route_name="create_study", renderer="json", request_method="OPTIONS")
-@view_config(route_name="study_CORS_preflight", renderer="json")
-def study_CORS_preflight(request):
-    api_utils.raise_on_CORS_preflight(request)
-
-
 def _new_nexson_with_crossref_metadata(doi, ref_string, include_cc0=False):
     # look for matching studies via CrossRef.org API
     # N.B. The recommended API method is very different for DOI vs.
@@ -397,6 +395,35 @@ def _new_nexson_with_crossref_metadata(doi, ref_string, include_cc0=False):
     return nexson
 
 
+# NOTE that we don't offer CC-BY 4.0, which is problematic for data
+_license2url = {
+    "CC=0": ("CC0", "https://creativecommons.org/publicdomain/zero/1.0/"),
+    "CC-BY-2.0": ("CC-BY 2.0", "https://creativecommons.org/licenses/by/2.0/"),
+    "CC-BY-2.5": ("CC-BY 2.5", "https://creativecommons.org/licenses/by/2.5/"),
+    "CC-BY-3.0": ("CC-BY 3.0", "https://creativecommons.org/licenses/by/3.0/"),
+    "CC-BY": ("CC-BY 3.0", "https://creativecommons.org/licenses/by/3.0/"),
+}
+
+
+def _add_license_info(cc0_agreement, nexml, request):
+    # If submitter requested the CC0 waiver or other waiver/license, make sure it's here
+    if cc0_agreement:
+        existing_license = "CC-0"
+    else:
+        # Are they using an existing license or waiver (CC0, CC-BY, something else?)
+        chosen_license = find_in_request(request, "chosen_license", "")
+        if chosen_license != "study-data-has-existing-license":
+            return
+        existing_license = find_in_request(request, "alternate_license", "")
+    lic_name_url = _license2url.get(existing_license)
+    if lic_name_url is None:
+        license_name = find_in_request(request, "alt_license_name", "")
+        license_url = find_in_request(request, "alt_license_URL", "")
+    else:
+        license_name, license_url = lic_name_url
+    nexml["^xhtml:license"] = {"@name": license_name, "@href": license_url}
+
+
 @view_config(route_name="create_study", renderer="json", request_method="POST")
 def create_study(request):
     # this method requires authentication
@@ -433,12 +460,6 @@ def create_study(request):
         and publication_doi_for_crossref
     ) or (import_method == "import-method-PUBLICATION_REFERENCE" and publication_ref)
 
-    # Are they using an existing license or waiver (CC0, CC-BY, something else?)
-    using_existing_license = (
-        find_in_request(request, "chosen_license", "")
-        == "study-data-has-existing-license"
-    )
-
     # any of these methods should returna parsed NexSON dict (vs. string)
     if importing_from_treebase_id:
         # make sure the treebase ID is an integer
@@ -483,54 +504,7 @@ def create_study(request):
     nexml = new_study_nexson["nexml"]
 
     if not importing_from_post_arg:
-        # If submitter requested the CC0 waiver or other waiver/license, make sure it's here
-        if cc0_agreement:
-            nexml["^xhtml:license"] = {
-                "@href": "https://creativecommons.org/publicdomain/zero/1.0/"
-            }
-        elif using_existing_license:
-            existing_license = find_in_request(request, "alternate_license", "")
-            if existing_license == "CC-0":
-                nexml["^xhtml:license"] = {
-                    "@name": "CC0",
-                    "@href": "https://creativecommons.org/publicdomain/zero/1.0/",
-                }
-                pass
-            elif existing_license == "CC-BY-2.0":
-                nexml["^xhtml:license"] = {
-                    "@name": "CC-BY 2.0",
-                    "@href": "https://creativecommons.org/licenses/by/2.0/",
-                }
-                pass
-            elif existing_license == "CC-BY-2.5":
-                nexml["^xhtml:license"] = {
-                    "@name": "CC-BY 2.5",
-                    "@href": "https://creativecommons.org/licenses/by/2.5/",
-                }
-                pass
-            elif existing_license == "CC-BY-3.0":
-                nexml["^xhtml:license"] = {
-                    "@name": "CC-BY 3.0",
-                    "@href": "https://creativecommons.org/licenses/by/3.0/",
-                }
-                pass
-            # NOTE that we don't offer CC-BY 4.0, which is problematic for data
-            elif existing_license == "CC-BY":
-                # default to version 3, if not specified.
-                nexml["^xhtml:license"] = {
-                    "@name": "CC-BY 3.0",
-                    "@href": "https://creativecommons.org/licenses/by/3.0/",
-                }
-                pass
-            else:  # assume it's something else
-                alt_license_name = find_in_request(request, "alt_license_name", "")
-                alt_license_url = find_in_request(request, "alt_license_URL", "")
-                # OK to add a name here? mainly to capture submitter's intent
-                nexml["^xhtml:license"] = {
-                    "@name": alt_license_name,
-                    "@href": alt_license_url,
-                }
-
+        _add_license_info(cc0_agreement, nexml, request)
     nexml["^ot:curatorName"] = auth_info.get("name", "")
 
     repo_nexml2json = phylesystem.repo_nexml2json

From d768389da521950af2119d05b0d60a965a576128 Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Sat, 29 Jul 2023 15:04:12 -0500
Subject: [PATCH 15/33] untested, but more use of repeated code

---
 phylesystem_api/phylesystem_api/api_utils.py  | 42 +++++++++
 .../phylesystem_api/views/amendment.py        | 90 ++++++-------------
 .../phylesystem_api/views/collection.py       | 67 ++++++--------
 .../phylesystem_api/views/study.py            | 58 ++++++------
 4 files changed, 127 insertions(+), 130 deletions(-)

diff --git a/phylesystem_api/phylesystem_api/api_utils.py b/phylesystem_api/phylesystem_api/api_utils.py
index ab0a4e83..028535f4 100644
--- a/phylesystem_api/phylesystem_api/api_utils.py
+++ b/phylesystem_api/phylesystem_api/api_utils.py
@@ -66,6 +66,48 @@ def raise404(msg):
     raise HTTPNotFound(body=json.dumps({"error": 1, "description": msg}))
 
 
+def fetch_doc(
+    request,
+    doc_id,
+    doc_store,
+    doc_type_name,
+    doc_id_validator=None,
+    add_version_history=True,
+):
+    if (doc_id_validator is not None) and (not doc_id_validator(doc_id)):
+        msg = "invalid {n} ID ({i}) provided".format(n=doc_type_name, i=doc_id)
+        raise400(msg)
+    parent_sha = find_in_request(request, "starting_commit_SHA", None)
+    try:
+        r = doc_store.return_doc(doc_id, commit_sha=parent_sha, return_WIP_map=True)
+    except:
+        raise404("{n} '{i}' GET failure".format(n=doc_type_name, i=doc_id))
+    document, head_sha, wip_map = r
+    if not document:
+        raise404("{n} '{i}' has no JSON data!".format(n=doc_type_name, i=doc_id))
+    version_history = None
+    if add_version_history:
+        try:
+            version_history = doc_store.get_version_history_for_doc_id(doc_id)
+        except:
+            m = "fetching version history failed"
+            _LOG.exception(m)
+            raise_int_server_err(m)
+    try:
+        external_url = doc_store.get_public_url(doc_id)
+    except:
+        external_url = "NOT FOUND"
+    result = {
+        "sha": head_sha,
+        "data": document,
+        "branch2sha": wip_map,
+        "external_url": external_url,
+    }
+    if version_history:
+        result["versionHistory"] = version_history
+    return result
+
+
 def get_private_dir(request):
     _LOG.debug("WHY PRIVATE DIR")
     return "~/private/"
diff --git a/phylesystem_api/phylesystem_api/views/amendment.py b/phylesystem_api/phylesystem_api/views/amendment.py
index dbcebdc2..177c83d5 100644
--- a/phylesystem_api/phylesystem_api/views/amendment.py
+++ b/phylesystem_api/phylesystem_api/views/amendment.py
@@ -1,26 +1,25 @@
+import json
+import logging
 import traceback
-import sys
-from pyramid.view import view_config
+
+from peyotl.amendments import AMENDMENT_ID_PATTERN
+from peyotl.amendments.validation import validate_amendment
+from peyotl.phylesystem.git_workflows import GitWorkflowError
 
 # see exception subclasses at https://docs.pylonsproject.org/projects/pyramid/en/latest/api/httpexceptions.html
 from pyramid.httpexceptions import (
     HTTPException,
     HTTPBadRequest,
 )
-from peyotl.phylesystem.git_workflows import GitWorkflowError
+from pyramid.view import view_config
+
 import phylesystem_api.api_utils as api_utils
 from phylesystem_api.api_utils import (
     find_in_request,
     extract_json_from_http_call,
     raise400,
-    raise404,
+    fetch_doc,
 )
-import json
-import logging
-
-from peyotl.amendments import AMENDMENT_ID_PATTERN
-from peyotl.amendments.validation import validate_amendment
-
 
 _LOG = logging.getLogger("phylesystem_api")
 
@@ -62,6 +61,24 @@ def amendment_CORS_preflight(request):
     api_utils.raise_on_CORS_preflight(request)
 
 
+def is_valid_amendment_id(doc_id):
+    return bool(AMENDMENT_ID_PATTERN.match(doc_id))
+
+
+@view_config(route_name="fetch_amendment", renderer="json")
+def fetch_amendment(request):
+    # NB - This method does not require authentication!
+    amendment_id = request.matchdict["amendment_id"]
+    return fetch_doc(
+        request,
+        doc_id=amendment_id,
+        doc_store=api_utils.get_taxonomic_amendment_store(request),
+        doc_type_name="amendment",
+        doc_id_validator=is_valid_amendment_id,
+        add_version_history=True,
+    )
+
+
 @view_config(route_name="create_amendment", renderer="json", request_method="POST")
 def create_amendment(request, **kwargs):
     _LOG.debug("Begin create ammendemt")
@@ -115,59 +132,6 @@ def create_amendment(request, **kwargs):
     return commit_return
 
 
-@view_config(route_name="fetch_amendment", renderer="json")
-def fetch_amendment(request):
-    # NB - This method does not require authentication!
-    # _LOG = api_utils.get_logger(request, 'ot_api.amendment')
-    amendment_id = request.matchdict["amendment_id"]
-    if not AMENDMENT_ID_PATTERN.match(amendment_id):
-        msg = "invalid amendment ID ({}) provided".format(amendment_id)
-        raise400(msg)
-
-    # fetch the current amendment JSON
-    # _LOG.debug('GET /v2/amendment/{}'.format(str(amendment_id)))
-    try:
-        parent_sha = find_in_request(request, "starting_commit_SHA", None)
-    except:
-        # probably a simple request w/o JSON payload
-        parent_sha = None
-    # _LOG.debug('parent_sha = {}'.format(parent_sha))
-    # return the correct nexson of study_id, using the specified view
-    amendments = api_utils.get_taxonomic_amendment_store(request)
-    try:
-        r = amendments.return_doc(
-            amendment_id, commit_sha=parent_sha, return_WIP_map=True
-        )
-    except:
-        raise404("Amendment '{}' GET failure".format(amendment_id))
-    try:
-        amendment_json, head_sha, wip_map = r
-        ## if returning_full_study:  # TODO: offer bare vs. full output (w/ history, etc)
-        version_history = amendments.get_version_history_for_doc_id(amendment_id)
-    except:
-        # _LOG.exception('GET failed')
-        e = sys.exc_info()[0]
-        raise HTTPBadRequest(body=e)
-    if not amendment_json:
-        raise404("Amendment '{s}' has no JSON data!".format(s=amendment_id))
-
-    try:
-        external_url = amendments.get_public_url(amendment_id)
-    except:
-        # _LOG = api_utils.get_logger(request, 'ot_api.default.v1')
-        # _LOG.exception('amendment {} not found in external_url'.format(amendment))
-        external_url = "NOT FOUND"
-    result = {
-        "sha": head_sha,
-        "data": amendment_json,
-        "branch2sha": wip_map,
-        "external_url": external_url,
-    }
-    if version_history:
-        result["versionHistory"] = version_history
-    return result
-
-
 @view_config(route_name="update_amendment", renderer="json")
 def update_amendment(request):
     # _LOG = api_utils.get_logger(request, 'ot_api.amendment')
diff --git a/phylesystem_api/phylesystem_api/views/collection.py b/phylesystem_api/phylesystem_api/views/collection.py
index fed6c9b4..4e11a048 100644
--- a/phylesystem_api/phylesystem_api/views/collection.py
+++ b/phylesystem_api/phylesystem_api/views/collection.py
@@ -1,6 +1,5 @@
 import json
 import logging
-import sys
 import traceback
 
 from peyotl.collections_store import COLLECTION_ID_PATTERN
@@ -20,6 +19,7 @@
     extract_json_from_http_call,
     raise400,
     raise404,
+    fetch_doc,
 )
 
 _LOG = logging.getLogger("phylesystem_api")
@@ -58,56 +58,43 @@ def collection_CORS_preflight(request):
     api_utils.raise_on_CORS_preflight(request)
 
 
+def is_valid_collection_id(doc_id):
+    return bool(COLLECTION_ID_PATTERN.match(doc_id))
+
+
 @view_config(route_name="fetch_collection", renderer="json")
 def fetch_collection(request):
     # NB - This method does not require authentication!
-    # _LOG = api_utils.get_logger(request, 'ot_api.collection')
     collection_id = request.matchdict["collection_id"]
-    if not COLLECTION_ID_PATTERN.match(collection_id):
-        msg = "invalid collection ID ({}) provided".format(collection_id)
-        raise400(msg)
+    result = fetch_doc(
+        request,
+        doc_id=collection_id,
+        doc_store=api_utils.get_tree_collection_store(request),
+        doc_type_name="collection",
+        doc_id_validator=is_valid_collection_id,
+        add_version_history=True,
+    )
+    return add_collection_specific_fields(request, collection_id, result)
 
-    # gather details to return with the JSON core document
-    parent_sha = find_in_request(request, "starting_commit_SHA", None)
-    collections = api_utils.get_tree_collection_store(request)
+
+def add_collection_specific_fields(request, collection_id, result):
+    collection_json = result["data"]
+    # Add commentHTML to result JSON
     try:
-        r = collections.return_doc(
-            collection_id, commit_sha=parent_sha, return_WIP_map=True
+        comment_html = api_utils.markdown_to_html(
+            collection_json["description"], open_links_in_new_window=True
         )
     except:
-        raise404("Collection '{}' GET failure".format(collection_id))
-    try:
-        collection_json, head_sha, wip_map = r
-        version_history = collections.get_version_history_for_doc_id(collection_id)
-        try:
-            # pre-render internal description (assumes markdown!)
-            comment_html = api_utils.markdown_to_html(
-                collection_json["description"], open_links_in_new_window=True
-            )
-        except:
-            comment_html = ""
-    except:
-        _LOG.exception("GET failed")
-        e = sys.exc_info()[0]
-        raise HTTPBadRequest(e)
-    if not collection_json:
-        raise404("Collection '{s}' has no JSON data!".format(s=collection_id))
-    # add/restore the url field (using the visible fetch URL)
+        comment_html = ""
+    result["commentHTML"] = comment_html
+
+    # Add the url field to the stored JSON
     base_url = api_utils.get_collections_api_base_url(request)
     collection_json["url"] = "{b}/v2/collection/{i}".format(b=base_url, i=collection_id)
-    try:
-        external_url = collections.get_public_url(collection_id)
-    except:
-        external_url = "NOT FOUND"
-    result = {
-        "sha": head_sha,
-        "data": collection_json,
-        "branch2sha": wip_map,
-        "commentHTML": comment_html,
-        "external_url": external_url,
-    }
+
+    # Add the lastModified field to the result JSON
+    version_history = result.get("versionHistory")
     if version_history:
-        result["versionHistory"] = version_history
         latest_commit = version_history[0]
         last_modified = {
             "author_name": latest_commit.get("author_name"),
diff --git a/phylesystem_api/phylesystem_api/views/study.py b/phylesystem_api/phylesystem_api/views/study.py
index 5a608e3f..35e547fd 100644
--- a/phylesystem_api/phylesystem_api/views/study.py
+++ b/phylesystem_api/phylesystem_api/views/study.py
@@ -16,7 +16,7 @@
     GitWorkflowError,
     validate_and_convert_nexson,
 )
-from phylesystem_api.api_utils import find_in_request, raise400, raise404
+from phylesystem_api.api_utils import find_in_request, raise400, raise404, fetch_doc
 from pyramid.encode import quote_plus, urlencode
 
 # see exception subclasses at https://docs.pylonsproject.org/projects/pyramid/en/latest/api/httpexceptions.html
@@ -221,29 +221,33 @@ def fetch_study(request):
         type_ext=request_extension,
         content_id=None,
     )
-    parent_sha = find_in_request(request, "starting_commit_SHA", None)
+    result = fetch_doc(
+        request,
+        doc_id=study_id,
+        doc_store=phylesystem,
+        doc_type_name="study",
+        add_version_history=out_schema.is_json(),
+    )
+    return add_study_specific_fields(
+        request, study_id, out_schema, phylesystem, repo_nexml2json, result
+    )
+
+
+def add_study_specific_fields(
+    request, study_id, out_schema, phylesystem, repo_nexml2json, result
+):
+    # TODO: is phylesystem.return_study == doc_store.return_doc
+    head_sha = result["sha"]
+    study_nexson = result["data"]
+    blob_sha = phylesystem.get_blob_sha_for_study_id(study_id, head_sha)
+    phylesystem.add_validation_annotation(study_nexson, blob_sha)
+
     try:
-        r = phylesystem.return_study(
-            study_id, commit_sha=parent_sha, return_WIP_map=True
+        comment_html = api_utils.markdown_to_html(
+            study_nexson["nexml"]["^ot:comment"], open_links_in_new_window=True
         )
     except:
-        raise404("Study #{} GET failure".format(study_id))
-    try:
-        study_nexson, head_sha, wip_map = r
-        blob_sha = phylesystem.get_blob_sha_for_study_id(study_id, head_sha)
-        phylesystem.add_validation_annotation(study_nexson, blob_sha)
-        version_history = phylesystem.get_version_history_for_study_id(study_id)
-        try:
-            comment_html = api_utils.markdown_to_html(
-                study_nexson["nexml"]["^ot:comment"], open_links_in_new_window=True
-            )
-        except:
-            comment_html = ""
-    except:
-        _LOG.exception("GET failed")
-        e = sys.exc_info()[0]
-        raise HTTPBadRequest(e)
-
+        comment_html = ""
     if out_schema.format_str == "nexson" and out_schema.version == repo_nexml2json:
         result_data = study_nexson
     else:
@@ -254,10 +258,9 @@ def fetch_study(request):
                 study_nexson, serialize=serialize, src_schema=src_schema
             )
         except:
-            raise400(
-                "Exception in coercing to the required NexSON version for validation."
-            )
-
+            m = "Exception while coercing NexSON to output schema."
+            _LOG.exception(m)
+            raise400(m)
     if out_schema.is_json():
         try:
             study_DOI = study_nexson["nexml"]["^ot:studyPublication"]["@href"]
@@ -276,7 +279,8 @@ def fetch_study(request):
         except:
             # _LOG.exception('check for shard name failed')
             shard_name = None
-
+        wip_map = result["branch2sha"]
+        version_history = result.get("versionHistory")
         result = {
             "sha": head_sha,
             "data": result_data,
@@ -629,7 +633,7 @@ def delete_study(request):
     except GitWorkflowError as err:
         _LOG.exception("got a GitWorkflowError:")
         raise HTTPBadRequest(err.msg)
-    except Exception as err:
+    except Exception:
         _LOG.exception("Exception getting nexson content in phylesystem.delete_study")
         raise400("Unknown error in study deletion")
 

From 1cbd6915c62a903dd66cdbbf8ee2a29b51d01122 Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Sun, 30 Jul 2023 11:49:06 -0500
Subject: [PATCH 16/33] fix of bug introduced when refactoring
 get_phylesystem_config

---
 phylesystem_api/phylesystem_api/api_utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/phylesystem_api/phylesystem_api/api_utils.py b/phylesystem_api/phylesystem_api/api_utils.py
index 028535f4..bdbe7969 100644
--- a/phylesystem_api/phylesystem_api/api_utils.py
+++ b/phylesystem_api/phylesystem_api/api_utils.py
@@ -155,6 +155,7 @@ def get_phylesystem(request, conf_obj=None):
     if conf_obj is None:
         conf_obj = get_conf_object(request)
     pc = read_phylesystem_config(request, conf_obj=conf_obj)
+    READ_ONLY_MODE = pc.read_only
     peyotl_config, cfg_filename = read_peyotl_config()
     if "phylesystem" not in peyotl_config.sections():
         peyotl_config.add_section("phylesystem")

From 6a7fdd1604090d385a0776a500369ea0ba834684 Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Sun, 30 Jul 2023 13:33:38 -0500
Subject: [PATCH 17/33] more refacotring to increase code reuse

---
 phylesystem_api/phylesystem_api/api_utils.py  |  78 ++++++++++
 .../phylesystem_api/views/amendment.py        | 137 ++++++------------
 .../phylesystem_api/views/collection.py       | 129 ++++++-----------
 .../phylesystem_api/views/study.py            | 104 ++++++-------
 4 files changed, 217 insertions(+), 231 deletions(-)

diff --git a/phylesystem_api/phylesystem_api/api_utils.py b/phylesystem_api/phylesystem_api/api_utils.py
index bdbe7969..95b10c25 100644
--- a/phylesystem_api/phylesystem_api/api_utils.py
+++ b/phylesystem_api/phylesystem_api/api_utils.py
@@ -15,6 +15,7 @@
 from peyotl.nexson_syntax import write_as_json
 from peyotl.phylesystem import Phylesystem
 from peyotl.utility import read_config as read_peyotl_config
+from peyotl.phylesystem.git_workflows import GitWorkflowError
 from pyramid.httpexceptions import (
     HTTPOk,
     HTTPBadRequest,
@@ -66,6 +67,83 @@ def raise404(msg):
     raise HTTPNotFound(body=json.dumps({"error": 1, "description": msg}))
 
 
+def get_owner_id(request, auth_info):
+    owner_id = auth_info.get("login")
+    if owner_id is None:
+        raise400("no GitHub userid obtained from auth token")
+    return owner_id
+
+
+def get_commit_message(request):
+    try:
+        commit_msg = find_in_request(request, "commit_msg", "")
+        if commit_msg.strip() == "":
+            return None
+        return commit_msg
+    except:
+        return None
+
+
+def get_parent_sha(request):
+    parent_sha = find_in_request(request, "starting_commit_SHA", None)
+    if parent_sha is None:
+        msg = 'Expecting a "starting_commit_SHA" argument with the SHA of the parent'
+        raise400(msg)
+    return parent_sha
+
+
+def commit_doc_and_trigger_push(
+    request,
+    commit_fn,
+    doc,
+    doc_id,
+    doc_type_name,
+    auth_info,
+    parent_sha=None,
+    merged_sha=None,
+    commit_msg=None,
+):
+    """Tries to commit doc, raises HTTP errors as needed. Triggers push thread.
+
+    `request` the Request obj,
+    `commit_fn` a function that takes (doc, doc_id, auth_inf, commit_msg)
+       and return the new_doc_id and a commit_return object.
+    `doc` the document as an object
+    `doc_id` or None if the docstore mints the ID.
+    `doc_type_name` "amendment", "nexson", or "collection"
+    `auth_info` dict
+    `commit_msg` string
+    """
+    try:
+        r = commit_fn(
+            doc,
+            doc_id,
+            auth_info,
+            parent_sha=parent_sha,
+            merged_sha=merged_sha,
+            commit_msg=commit_msg,
+        )
+        new_doc_id, commit_return = r
+    except GitWorkflowError as err:
+        raise400(err.msg)
+    except Exception as x:
+        raise400(str(x))
+    if commit_return["error"] != 0:
+        _LOG.debug("commit of {} failed with error code".format(doc_type_name))
+        raise HTTPBadRequest(body=json.dumps(commit_return))
+    # check for 'merge needed'
+    mn = commit_return.get("merge_needed")
+    if (mn is not None) and (not mn):
+        _LOG.debug("commit of {} deferred_push_to_gh_call".format(doc_type_name))
+        deferred_push_to_gh_call(
+            request,
+            new_doc_id,
+            doc_type=doc_type_name,
+            auth_token=auth_info["auth_token"],
+        )
+    return commit_return
+
+
 def fetch_doc(
     request,
     doc_id,
diff --git a/phylesystem_api/phylesystem_api/views/amendment.py b/phylesystem_api/phylesystem_api/views/amendment.py
index 177c83d5..22324309 100644
--- a/phylesystem_api/phylesystem_api/views/amendment.py
+++ b/phylesystem_api/phylesystem_api/views/amendment.py
@@ -19,6 +19,9 @@
     extract_json_from_http_call,
     raise400,
     fetch_doc,
+    commit_doc_and_trigger_push,
+    get_parent_sha,
+    get_commit_message,
 )
 
 _LOG = logging.getLogger("phylesystem_api")
@@ -82,19 +85,8 @@ def fetch_amendment(request):
 @view_config(route_name="create_amendment", renderer="json", request_method="POST")
 def create_amendment(request, **kwargs):
     _LOG.debug("Begin create ammendemt")
-    # _LOG = api_utils.get_logger(request, 'ot_api.amendment')
-
     auth_info = api_utils.auth_and_not_read_only(request)
-
-    # gather any user-provided git-commit message
-    try:
-        commit_msg = find_in_request(request, "commit_msg", "")
-        if commit_msg.strip() == "":
-            # git rejects empty commit messages
-            commit_msg = None
-    except:
-        commit_msg = None
-
+    commit_msg = get_commit_message(request)
     # fetch and parse the JSON payload, if any
     (
         amendment_obj,
@@ -105,52 +97,35 @@ def create_amendment(request, **kwargs):
         msg = "amendment JSON expected for HTTP method {}".format(request.method)
         raise400(msg)
 
-    # Create a new amendment with the data provided
-    # _LOG = api_utils.get_logger(request, 'ot_api.default.amendments.POST')
-    # submit the json and proposed id (if any), and read the results
-    docstore = api_utils.get_taxonomic_amendment_store(request)
     # N.B. add_new_amendment below takes care of minting new ottids,
     # assigning them to new taxa, and returning a per-taxon mapping to the
     # caller. It will assign the new amendment id accordingly!
-    try:
-        r = docstore.add_new_amendment(amendment_obj, auth_info, commit_msg=commit_msg)
-        new_amendment_id, commit_return = r
-    except GitWorkflowError as err:
-        raise HTTPBadRequest(body=err.msg)
-    except:
-        raise HTTPBadRequest(body=traceback.format_exc())
-    if commit_return["error"] != 0:
-        # _LOG.debug('add_new_amendment failed with error code')
-        raise HTTPBadRequest(body=json.dumps(commit_return))
-    _LOG.debug("create ammendemt deferred_push_to_gh_call")
-    api_utils.deferred_push_to_gh_call(
+    docstore = api_utils.get_taxonomic_amendment_store(request)
+
+    def amend_commit_fn(doc, doc_id, auth_info, commit_msg):
+        return docstore.add_new_amendment(
+            doc, auth_info=auth_info, commit_msg=commit_msg
+        )
+
+    return commit_doc_and_trigger_push(
         request,
-        new_amendment_id,
-        doc_type="amendment",
-        auth_token=auth_info["auth_token"],
+        commit_fn=amend_commit_fn,
+        doc=amendment_obj,
+        doc_id=None,
+        doc_type_name="amendment",
+        auth_info=auth_info,
+        commit_msg=commit_msg,
     )
-    return commit_return
 
 
 @view_config(route_name="update_amendment", renderer="json")
 def update_amendment(request):
     # _LOG = api_utils.get_logger(request, 'ot_api.amendment')
     amendment_id = request.matchdict["amendment_id"]
-    if not AMENDMENT_ID_PATTERN.match(amendment_id):
-        msg = "invalid amendment ID ({}) provided".format(amendment_id)
-        raise400(msg)
-
-    auth_info = api_utils.auth_and_not_read_only(request)
-
-    # gather any user-provided git-commit message
-    try:
-        commit_msg = find_in_request(request, "commit_msg", "")
-        if commit_msg.strip() == "":
-            # git rejects empty commit messages
-            commit_msg = None
-    except:
-        commit_msg = None
-
+    if not is_valid_amendment_id(amendment_id):
+        raise400("invalid amendment ID ({}) provided".format(amendment_id))
+    r_auth_info = api_utils.auth_and_not_read_only(request)
+    r_commit_msg = get_commit_message(request)
     # fetch and parse the JSON payload, if any
     (
         amendment_obj,
@@ -164,67 +139,45 @@ def update_amendment(request):
     # update an existing amendment with the data provided
     # _LOG = api_utils.get_logger(request, 'ot_api.default.amendments.PUT')
     # submit new json for this id, and read the results
-    parent_sha = find_in_request(request, "starting_commit_SHA", None)
-    merged_sha = None  # TODO: find_in_request(request, '???', None)
+    r_parent_sha = get_parent_sha(request)
+    r_merged_sha = None  # TODO: find_in_request(request, '???', None)
     docstore = api_utils.get_taxonomic_amendment_store(request)
-    try:
-        r = docstore.update_existing_amendment(
-            amendment_id,
-            amendment_obj,
+
+    def update_amendment_fn(doc, doc_id, auth_info, parent_sha, merged_sha, commit_msg):
+        return doc_id, docstore.update_existing_amendment(
+            doc_id,
+            doc,
             auth_info,
             parent_sha,
             merged_sha,
             commit_msg=commit_msg,
         )
-        commit_return = r
-    except GitWorkflowError as err:
-        raise HTTPBadRequest(body=err.msg)
-    except:
-        raise HTTPBadRequest(body=traceback.format_exc())
 
-    # check for 'merge needed'?
-    mn = commit_return.get("merge_needed")
-    if (mn is not None) and (not mn):
-        api_utils.deferred_push_to_gh_call(
-            request,
-            amendment_id,
-            doc_type="amendment",
-            auth_token=auth_info["auth_token"],
-        )
-    return commit_return
+    blob = commit_doc_and_trigger_push(
+        request,
+        commit_fn=update_amendment_fn,
+        doc=amendment_obj,
+        doc_id=amendment_id,
+        doc_type_name="amendment",
+        auth_info=r_auth_info,
+        parent_sha=r_parent_sha,
+        merged_sha=r_merged_sha,
+        commit_msg=r_commit_msg,
+    )
+    return blob
 
 
 @view_config(route_name="delete_amendment", renderer="json")
 def delete_amendment(request):
     # _LOG = api_utils.get_logger(request, 'ot_api.amendment')
     amendment_id = request.matchdict.get["amendment_id"]
-    if not AMENDMENT_ID_PATTERN.match(amendment_id):
-        msg = "invalid amendment ID ({}) provided".format(amendment_id)
-        body = {
-            "error": 1,
-            "description": msg,
-        }
-        raise HTTPBadRequest(body=json.dumps(body))
+    if not is_valid_amendment_id(amendment_id):
+        raise400("invalid amendment ID ({}) provided".format(amendment_id))
 
     auth_info = api_utils.auth_and_not_read_only(request)
-
-    # gather any user-provided git-commit message
-    try:
-        commit_msg = find_in_request(request, "commit_msg", "")
-        if commit_msg.strip() == "":
-            # git rejects empty commit messages
-            commit_msg = None
-    except:
-        commit_msg = None
-
-    # remove this amendment from the docstore
-    # _LOG = api_utils.get_logger(request, 'ot_api.default.amendments.POST')
+    parent_sha = get_parent_sha(request)
+    commit_msg = get_commit_message(request)
     docstore = api_utils.get_taxonomic_amendment_store(request)
-    parent_sha = find_in_request(request, "starting_commit_SHA")
-    if parent_sha is None:
-        raise HTTPBadRequest(
-            body='Expecting a "starting_commit_SHA" argument with the SHA of the parent'
-        )
     try:
         x = docstore.delete_amendment(
             amendment_id, auth_info, parent_sha, commit_msg=commit_msg
diff --git a/phylesystem_api/phylesystem_api/views/collection.py b/phylesystem_api/phylesystem_api/views/collection.py
index 4e11a048..df99e2a0 100644
--- a/phylesystem_api/phylesystem_api/views/collection.py
+++ b/phylesystem_api/phylesystem_api/views/collection.py
@@ -20,6 +20,10 @@
     raise400,
     raise404,
     fetch_doc,
+    commit_doc_and_trigger_push,
+    get_parent_sha,
+    get_commit_message,
+    get_owner_id,
 )
 
 _LOG = logging.getLogger("phylesystem_api")
@@ -109,16 +113,8 @@ def add_collection_specific_fields(request, collection_id, result):
 
 @view_config(route_name="create_collection", renderer="json", request_method="POST")
 def create_collection(request):
-    # gather any user-provided git-commit message
-    try:
-        commit_msg = find_in_request(request, "commit_msg", "")
-        if commit_msg.strip() == "":
-            # git rejects empty commit messages
-            commit_msg = None
-    except:
-        commit_msg = None
     auth_info = api_utils.auth_and_not_read_only(request)
-
+    commit_msg = get_commit_message(request)
     # fetch and parse the JSON payload, if any
     (
         collection_obj,
@@ -129,12 +125,7 @@ def create_collection(request):
         msg = "collection JSON expected for HTTP method {}".format(request.method)
         raise400(msg)
 
-    owner_id = auth_info.get("login", None)
-    if owner_id is None:
-        msg = "no GitHub userid found for HTTP method {}".format(
-            request.env.request_method
-        )
-        raise400(msg)
+    owner_id = get_owner_id(request, auth_info)
 
     # try to extract a usable collection ID from the JSON payload (confirm owner_id against above)
     url = collection_obj.get("url", None)
@@ -153,47 +144,35 @@ def create_collection(request):
     # Create a new collection with the data provided
     # submit the json and proposed id (if any), and read the results
     docstore = api_utils.get_tree_collection_store(request)
-    try:
-        r = docstore.add_new_collection(
-            owner_id, collection_obj, auth_info, collection_id, commit_msg=commit_msg
+
+    def coll_commit_fn(doc, doc_id, auth_info, parent_sha, merged_sha, commit_msg):
+        return docstore.add_new_collection(
+            owner_id, doc, auth_info, doc_id, commit_msg=commit_msg
         )
-        new_collection_id, commit_return = r
-    except GitWorkflowError as err:
-        raise400(str(err))
-    except:
-        raise HTTPBadRequest(traceback.format_exc())
-    if commit_return["error"] != 0:
-        # _LOG.debug('add_new_collection failed with error code')
-        raise HTTPBadRequest(json.dumps(commit_return))
-    api_utils.deferred_push_to_gh_call(
+
+    return commit_doc_and_trigger_push(
         request,
-        new_collection_id,
-        doc_type="collection",
-        auth_token=auth_info["auth_token"],
+        commit_fn=coll_commit_fn,
+        doc=collection_obj,
+        doc_id=collection_id,
+        doc_type_name="collection",
+        auth_info=auth_info,
+        commit_msg=commit_msg,
     )
-    return commit_return
 
 
 @view_config(route_name="update_collection", renderer="json")
 def update_collection(request):
     # _LOG = api_utils.get_logger(request, 'ot_api.collection')
     # NB - This method requires authentication!
-    auth_info = api_utils.auth_and_not_read_only(request)
+    r_auth_info = api_utils.auth_and_not_read_only(request)
     _LOG.debug("COLLECTION: update_collection")
-    _LOG.debug("COLLECTION: auth_info {}".format(auth_info))
-    owner_id = auth_info.get("login", None)
+    owner_id = get_owner_id(request, r_auth_info)
 
     collection_id = request.matchdict["collection_id"]
-    if not COLLECTION_ID_PATTERN.match(collection_id):
+    if not is_valid_collection_id(collection_id):
         msg = "invalid collection ID ({}) provided".format(collection_id)
         raise400(msg)
-    try:
-        commit_msg = find_in_request(request, "commit_msg", "")
-        if commit_msg.strip() == "":
-            # git rejects empty commit messages
-            commit_msg = None
-    except:
-        commit_msg = None
 
     # fetch and parse the JSON payload, if any
     (
@@ -206,39 +185,37 @@ def update_collection(request):
         raise400(msg)
 
     # submit new json for this id, and read the results
-    parent_sha = find_in_request(request, "starting_commit_SHA", None)
-    merged_sha = None  # TODO: find_in_request(request, '???', None)
     docstore = api_utils.get_tree_collection_store(request)
-    try:
-        r = docstore.update_existing_collection(
+
+    def update_collection_fn(
+        doc, doc_id, auth_info, parent_sha, merged_sha, commit_msg
+    ):
+        return doc_id, docstore.update_existing_collection(
             owner_id,
-            collection_id,
-            collection_obj,
+            doc_id,
+            doc,
             auth_info,
             parent_sha,
             merged_sha,
             commit_msg=commit_msg,
         )
-        commit_return = r
-    except GitWorkflowError as err:
-        raise HTTPBadRequest(err.msg)
-    except:
-        raise HTTPBadRequest(traceback.format_exc())
-
-    # check for 'merge needed'?
-    mn = commit_return.get("merge_needed")
-    if (mn is not None) and (not mn):
-        api_utils.deferred_push_to_gh_call(
-            request,
-            collection_id,
-            doc_type="collection",
-            auth_token=auth_info["auth_token"],
-        )
-    # Add updated commit history to the blob
-    commit_return["versionHistory"] = docstore.get_version_history_for_doc_id(
-        collection_id
+
+    r_parent_sha = find_in_request(request, "starting_commit_SHA", None)
+    r_merged_sha = None  # TODO: find_in_request(request, '???', None)
+    r_commit_msg = get_commit_message(request)
+    blob = commit_doc_and_trigger_push(
+        request,
+        commit_fn=update_collection_fn,
+        doc=collection_obj,
+        doc_id=collection_id,
+        doc_type_name="collection",
+        auth_info=r_auth_info,
+        parent_sha=r_parent_sha,
+        merged_sha=r_merged_sha,
+        commit_msg=r_commit_msg,
     )
-    return commit_return
+    blob["versionHistory"] = docstore.get_version_history_for_doc_id(collection_id)
+    return blob
 
 
 @view_config(route_name="delete_collection", renderer="json")
@@ -246,27 +223,15 @@ def delete_collection(request):
     # _LOG = api_utils.get_logger(request, 'ot_api.collection')
     # NB - This method requires authentication!
     auth_info = api_utils.auth_and_not_read_only(request)
-
     collection_id = request.matchdict["collection_id"]
-    if not COLLECTION_ID_PATTERN.match(collection_id):
+    if not is_valid_collection_id(collection_id):
         msg = "invalid collection ID ({}) provided".format(collection_id)
         raise400(msg)
-
-    try:
-        commit_msg = find_in_request(request, "commit_msg", "")
-        if commit_msg.strip() == "":
-            # git rejects empty commit messages
-            commit_msg = None
-    except:
-        commit_msg = None
+    commit_msg = get_commit_message(request)
 
     # remove this collection from the docstore
     docstore = api_utils.get_tree_collection_store(request)
-    parent_sha = find_in_request(request, "starting_commit_SHA", None)
-    if parent_sha is None:
-        raise HTTPBadRequest(
-            'Expecting a "starting_commit_SHA" argument with the SHA of the parent'
-        )
+    parent_sha = get_parent_sha(request)
     try:
         x = docstore.delete_collection(
             collection_id, auth_info, parent_sha, commit_msg=commit_msg
diff --git a/phylesystem_api/phylesystem_api/views/study.py b/phylesystem_api/phylesystem_api/views/study.py
index 35e547fd..a229e3e3 100644
--- a/phylesystem_api/phylesystem_api/views/study.py
+++ b/phylesystem_api/phylesystem_api/views/study.py
@@ -16,7 +16,15 @@
     GitWorkflowError,
     validate_and_convert_nexson,
 )
-from phylesystem_api.api_utils import find_in_request, raise400, raise404, fetch_doc
+from phylesystem_api.api_utils import (
+    find_in_request,
+    raise400,
+    raise404,
+    fetch_doc,
+    commit_doc_and_trigger_push,
+    get_parent_sha,
+    get_commit_message,
+)
 from pyramid.encode import quote_plus, urlencode
 
 # see exception subclasses at https://docs.pylonsproject.org/projects/pyramid/en/latest/api/httpexceptions.html
@@ -236,7 +244,6 @@ def fetch_study(request):
 def add_study_specific_fields(
     request, study_id, out_schema, phylesystem, repo_nexml2json, result
 ):
-    # TODO: is phylesystem.return_study == doc_store.return_doc
     head_sha = result["sha"]
     study_nexson = result["data"]
     blob_sha = phylesystem.get_blob_sha_for_study_id(study_id, head_sha)
@@ -431,7 +438,7 @@ def _add_license_info(cc0_agreement, nexml, request):
 @view_config(route_name="create_study", renderer="json", request_method="POST")
 def create_study(request):
     # this method requires authentication
-    auth_info = api_utils.auth_and_not_read_only(request)
+    r_auth_info = api_utils.auth_and_not_read_only(request)
     phylesystem = api_utils.get_phylesystem(request)
 
     # we're creating a new study (possibly with import instructions in the payload)
@@ -504,56 +511,32 @@ def create_study(request):
             new_study_nexson["nexml"]["^ot:studyPublication"] = {
                 "@href": publication_doi
             }
-
     nexml = new_study_nexson["nexml"]
-
     if not importing_from_post_arg:
         _add_license_info(cc0_agreement, nexml, request)
-    nexml["^ot:curatorName"] = auth_info.get("name", "")
+    nexml["^ot:curatorName"] = r_auth_info.get("name", "")
 
-    repo_nexml2json = phylesystem.repo_nexml2json
-    new_study_id = None  # TODO: should we ever specify an ID here?
-    try:
-        r = phylesystem.ingest_new_study(
-            new_study_nexson, repo_nexml2json, auth_info, new_study_id
+    def study_commit_fn(doc, doc_id, auth_info, **kwargs):
+        return phylesystem.ingest_new_study(
+            doc, phylesystem.repo_nexml2json, auth_info, doc_id
         )
-        new_resource_id, commit_return = r
-    except GitWorkflowError as err:
-        raise HTTPBadRequest(err.msg)
-    except:
-        raise HTTPBadRequest(traceback.format_exc())
-    if commit_return["error"] != 0:
-        # _LOG.debug('ingest_new_study failed with error code')
-        raise HTTPBadRequest(json.dumps(commit_return))
-    api_utils.deferred_push_to_gh_call(
-        request, new_resource_id, doc_type="nexson", auth_token=auth_info["auth_token"]
+
+    return commit_doc_and_trigger_push(
+        request,
+        commit_fn=study_commit_fn,
+        doc=new_study_nexson,
+        doc_id=None,
+        doc_type_name="nexson",
+        auth_info=r_auth_info,
+        commit_msg=None,
     )
-    return commit_return
 
 
 @view_config(route_name="update_study", renderer="json")
 def update_study(request):
     study_id = request.matchdict["study_id"]
-    auth_info = api_utils.auth_and_not_read_only(request)
+    r_auth_info = api_utils.auth_and_not_read_only(request)
     _LOG.debug("STUDY: update_study")
-    # _LOG.debug("STUDY: auth_info {}".format(auth_info))
-
-    parent_sha = find_in_request(request, "starting_commit_SHA")
-    if parent_sha is None:
-        raise HTTPBadRequest(
-            'Expecting a "starting_commit_SHA" argument with the SHA of the parent'
-        )
-    master_file_blob_included = find_in_request(request, "merged_SHA")
-
-    # gather any user-provided git-commit message
-    try:
-        commit_msg = find_in_request(request, "commit_msg", "")
-        if commit_msg.strip() == "":
-            # git rejects empty commit messages
-            commit_msg = None
-    except:
-        commit_msg = None
-
     phylesystem = api_utils.get_phylesystem(request)
     repo_nexml2json = phylesystem.repo_nexml2json
     bundle = __extract_and_validate_nexson(request, repo_nexml2json, request.json_body)
@@ -562,30 +545,37 @@ def update_study(request):
         gd = phylesystem.create_git_action(study_id)
     except KeyError:
         # _LOG.debug('PUT failed in create_git_action (probably a bad study ID)')
-        raise HTTPBadRequest("invalid study ID, please check the URL")
-    try:
-        blob = __finish_write_verb(
+        raise400("invalid study ID, please check the URL")
+
+    def update_study_fn(doc, doc_id, auth_info, parent_sha, merged_sha, commit_msg):
+        return doc_id, __finish_write_verb(
             phylesystem,
             gd,
-            nexson=nexson,
-            resource_id=study_id,
+            nexson=doc,
+            resource_id=doc_id,
             auth_info=auth_info,
             adaptor=nexson_adaptor,
             annotation=annotation,
             parent_sha=parent_sha,
             commit_msg=commit_msg,
-            master_file_blob_included=master_file_blob_included,
+            master_file_blob_included=merged_sha,
         )
-    except GitWorkflowError as err:
-        # _LOG.exception('PUT failed in __finish_write_verb')
-        raise HTTPBadRequest(err.msg)
-    # TIMING = api_utils.log_time_diff(_LOG, 'blob creation', TIMING)
-    mn = blob.get("merge_needed")
-    if (mn is not None) and (not mn):
-        api_utils.deferred_push_to_gh_call(
-            request, study_id, doc_type="nexson", auth_token=auth_info["auth_token"]
-        )
-    # Add updated commit history to the blob
+
+    r_parent_sha = get_parent_sha(request)
+    r_commit_msg = get_commit_message(request)
+    r_merged_sha = find_in_request(request, "merged_SHA")
+
+    blob = commit_doc_and_trigger_push(
+        request,
+        commit_fn=update_study_fn,
+        doc=nexson,
+        doc_id=study_id,
+        doc_type_name="nexson",
+        auth_info=r_auth_info,
+        parent_sha=r_parent_sha,
+        merged_sha=r_merged_sha,
+        commit_msg=r_commit_msg,
+    )
     blob["versionHistory"] = phylesystem.get_version_history_for_study_id(study_id)
     return blob
 

From 33b7295d0ae7a599603605fbbb2ef591c0f0fedc Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Sun, 30 Jul 2023 13:47:25 -0500
Subject: [PATCH 18/33] still more refacotring to increase code reuse

---
 .../phylesystem_api/views/amendment.py        | 40 +++++++------
 .../phylesystem_api/views/collection.py       | 43 +++++++------
 .../phylesystem_api/views/study.py            | 60 +++++++------------
 3 files changed, 68 insertions(+), 75 deletions(-)

diff --git a/phylesystem_api/phylesystem_api/views/amendment.py b/phylesystem_api/phylesystem_api/views/amendment.py
index 22324309..147a7167 100644
--- a/phylesystem_api/phylesystem_api/views/amendment.py
+++ b/phylesystem_api/phylesystem_api/views/amendment.py
@@ -173,22 +173,28 @@ def delete_amendment(request):
     amendment_id = request.matchdict.get["amendment_id"]
     if not is_valid_amendment_id(amendment_id):
         raise400("invalid amendment ID ({}) provided".format(amendment_id))
-
-    auth_info = api_utils.auth_and_not_read_only(request)
-    parent_sha = get_parent_sha(request)
-    commit_msg = get_commit_message(request)
+    r_auth_info = api_utils.auth_and_not_read_only(request)
+    r_parent_sha = get_parent_sha(request)
+    r_commit_msg = get_commit_message(request)
     docstore = api_utils.get_taxonomic_amendment_store(request)
-    try:
-        x = docstore.delete_amendment(
-            amendment_id, auth_info, parent_sha, commit_msg=commit_msg
+
+    def del_amendment_fn(doc, doc_id, auth_info, parent_sha, merged_sha, commit_msg):
+        return doc_id, docstore.delete_amendment(
+            doc_id,
+            auth_info,
+            parent_sha,
+            commit_msg=commit_msg,
         )
-        if x.get("error") == 0:
-            api_utils.deferred_push_to_gh_call(
-                request, None, doc_type="amendment", auth_token=auth_info["auth_token"]
-            )
-        return x
-    except GitWorkflowError as err:
-        raise HTTPBadRequest(body=err.msg)
-    except:
-        # _LOG.exception('Unknown error in amendment deletion')
-        raise HTTPBadRequest(body=traceback.format_exc())
+
+    blob = commit_doc_and_trigger_push(
+        request,
+        commit_fn=del_amendment_fn,
+        doc=None,
+        doc_id=amendment_id,
+        doc_type_name="amendment",
+        auth_info=r_auth_info,
+        parent_sha=r_parent_sha,
+        merged_sha=None,
+        commit_msg=r_commit_msg,
+    )
+    return blob
diff --git a/phylesystem_api/phylesystem_api/views/collection.py b/phylesystem_api/phylesystem_api/views/collection.py
index df99e2a0..552a0650 100644
--- a/phylesystem_api/phylesystem_api/views/collection.py
+++ b/phylesystem_api/phylesystem_api/views/collection.py
@@ -222,26 +222,31 @@ def update_collection_fn(
 def delete_collection(request):
     # _LOG = api_utils.get_logger(request, 'ot_api.collection')
     # NB - This method requires authentication!
-    auth_info = api_utils.auth_and_not_read_only(request)
+    r_auth_info = api_utils.auth_and_not_read_only(request)
+    r_parent_sha = get_parent_sha(request)
     collection_id = request.matchdict["collection_id"]
     if not is_valid_collection_id(collection_id):
-        msg = "invalid collection ID ({}) provided".format(collection_id)
-        raise400(msg)
-    commit_msg = get_commit_message(request)
-
-    # remove this collection from the docstore
+        raise400("invalid collection ID ({}) provided".format(collection_id))
+    r_commit_msg = get_commit_message(request)
     docstore = api_utils.get_tree_collection_store(request)
-    parent_sha = get_parent_sha(request)
-    try:
-        x = docstore.delete_collection(
-            collection_id, auth_info, parent_sha, commit_msg=commit_msg
+
+    def del_collection_fn(doc, doc_id, auth_info, parent_sha, merged_sha, commit_msg):
+        return doc_id, docstore.delete_collection(
+            doc_id,
+            auth_info,
+            parent_sha,
+            commit_msg=commit_msg,
         )
-        if x.get("error") == 0:
-            api_utils.deferred_push_to_gh_call(
-                request, None, doc_type="collection", auth_token=auth_info["auth_token"]
-            )
-        return x
-    except GitWorkflowError as err:
-        raise HTTPBadRequest(err.msg)
-    except:
-        raise400("Unknown error in collection deletion")
+
+    blob = commit_doc_and_trigger_push(
+        request,
+        commit_fn=del_collection_fn,
+        doc=None,
+        doc_id=collection_id,
+        doc_type_name="collection",
+        auth_info=r_auth_info,
+        parent_sha=r_parent_sha,
+        merged_sha=None,
+        commit_msg=r_commit_msg,
+    )
+    return blob
diff --git a/phylesystem_api/phylesystem_api/views/study.py b/phylesystem_api/phylesystem_api/views/study.py
index a229e3e3..b5eec3fb 100644
--- a/phylesystem_api/phylesystem_api/views/study.py
+++ b/phylesystem_api/phylesystem_api/views/study.py
@@ -582,50 +582,32 @@ def update_study_fn(doc, doc_id, auth_info, parent_sha, merged_sha, commit_msg):
 
 @view_config(route_name="delete_study", renderer="json")
 def delete_study(request):
-    # _LOG = api_utils.get_logger(request, 'delete_study')
-    _LOG.debug("delete_study STARTING...")
     study_id = request.matchdict["study_id"]
-    _LOG.debug("study_id={}".format(study_id))
-
-    # this method requires authentication
-    auth_info = api_utils.auth_and_not_read_only(request)
-
-    # gather any user-provided git-commit message
-    try:
-        commit_msg = find_in_request(request, "commit_msg", "")
-        if commit_msg.strip() == "":
-            # git rejects empty commit messages
-            commit_msg = None
-    except:
-        commit_msg = None
-    _LOG.debug("commit_msg={}".format(commit_msg))
-
-    parent_sha = find_in_request(request, "starting_commit_SHA", None)
-    _LOG.debug("parent_sha={}".format(parent_sha))
+    _LOG.debug("delete study_id={}".format(study_id))
+    r_auth_info = api_utils.auth_and_not_read_only(request)
+    r_commit_msg = get_commit_message(request)
+    r_parent_sha = get_parent_sha(request)
+    _LOG.debug("parent_sha={}".format(r_parent_sha))
 
     phylesystem = api_utils.get_phylesystem(request)
 
-    _LOG.debug("trying to delete now... via:")
-    _LOG.debug(phylesystem.delete_study)
-    try:
-        x = phylesystem.delete_study(
-            study_id, auth_info, parent_sha, commit_msg=commit_msg
+    def del_study_fn(doc, doc_id, auth_info, parent_sha, merged_sha, commit_msg):
+        return doc_id, phylesystem.delete_study(
+            doc_id, auth_info, parent_sha, commit_msg=commit_msg
         )
-        _LOG.debug("initial return x:")
-        _LOG.debug(x)
-        if x.get("error") == 0:
-            _LOG.debug("calling deferred push...")
-            api_utils.deferred_push_to_gh_call(
-                request, None, doc_type="nexson", auth_token=auth_info["auth_token"]
-            )
-            _LOG.debug("back from deferred push")
-        return x
-    except GitWorkflowError as err:
-        _LOG.exception("got a GitWorkflowError:")
-        raise HTTPBadRequest(err.msg)
-    except Exception:
-        _LOG.exception("Exception getting nexson content in phylesystem.delete_study")
-        raise400("Unknown error in study deletion")
+
+    blob = commit_doc_and_trigger_push(
+        request,
+        commit_fn=del_study_fn,
+        doc=None,
+        doc_id=study_id,
+        doc_type_name="nexson",
+        auth_info=r_auth_info,
+        parent_sha=r_parent_sha,
+        merged_sha=None,
+        commit_msg=r_commit_msg,
+    )
+    return blob
 
 
 @view_config(route_name="get_study_file_list", renderer="json")

From c437287a519c1d09667e14e3340fef4257b3fc50 Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Mon, 31 Jul 2023 16:34:41 -0500
Subject: [PATCH 19/33] more refactoring. tests passing locally

---
 phylesystem_api/phylesystem_api/__init__.py   |   4 +-
 phylesystem_api/phylesystem_api/api_utils.py  | 395 +++++++++---------
 phylesystem_api/phylesystem_api/routes.py     |  16 +-
 .../phylesystem_api/views/amendment.py        |   4 -
 .../phylesystem_api/views/amendments.py       |  62 +--
 .../phylesystem_api/views/collection.py       |  43 +-
 .../phylesystem_api/views/collections.py      |  68 +--
 .../phylesystem_api/views/default.py          | 327 ++++++---------
 .../phylesystem_api/views/study.py            |  22 +-
 9 files changed, 385 insertions(+), 556 deletions(-)

diff --git a/phylesystem_api/phylesystem_api/__init__.py b/phylesystem_api/phylesystem_api/__init__.py
index 9cd383ea..efc48486 100644
--- a/phylesystem_api/phylesystem_api/__init__.py
+++ b/phylesystem_api/phylesystem_api/__init__.py
@@ -1,5 +1,5 @@
 from pyramid.config import Configurator
-from .api_utils import get_conf_object, get_phylesystem
+from .api_utils import get_conf_object, get_docstore_from_type
 
 
 def main(global_config, **settings):
@@ -14,5 +14,5 @@ def main(global_config, **settings):
     #     print("{k}: {v}".format(k=k, v=repr(v)))
     localconfig_filename = settings["config_file_path"]
     conf_obj = get_conf_object(localconfig_filename=localconfig_filename)
-    _ps = get_phylesystem(request=None, conf_obj=conf_obj)
+    _ps = get_docstore_from_type("study", request=None, conf_obj=conf_obj)
     return config.make_wsgi_app()
diff --git a/phylesystem_api/phylesystem_api/api_utils.py b/phylesystem_api/phylesystem_api/api_utils.py
index 95b10c25..abf9608f 100644
--- a/phylesystem_api/phylesystem_api/api_utils.py
+++ b/phylesystem_api/phylesystem_api/api_utils.py
@@ -92,6 +92,71 @@ def get_parent_sha(request):
     return parent_sha
 
 
+def get_last_modified_dict(docstore=None, doc_id=None, last_commit=None):
+    if last_commit is None:
+        commit_list = docstore.get_version_history_for_doc_id(doc_id)
+        if not commit_list:
+            return None
+        latest_commit = commit_list[0]
+    return {
+        "author_name": latest_commit.get("author_name"),
+        "relative_date": latest_commit.get("relative_date"),
+        "display_date": latest_commit.get("date"),
+        "ISO_date": latest_commit.get("date_ISO_8601"),
+        "sha": latest_commit.get("id"),  # this is the commit hash
+    }
+
+
+_doc_type2store = None
+
+
+def get_docstore_from_type(doc_type_name, request, conf_obj=None):
+    global _doc_type2store
+    if _doc_type2store is None:
+        p_s = get_phylesystem(request=request, conf_obj=conf_obj)
+        c_s = get_tree_collection_store(request=request, conf_obj=conf_obj)
+        a_s = get_taxonomic_amendment_store(request=request, conf_obj=conf_obj)
+        d = {
+            "nexson": p_s,
+            "study": p_s,
+            "studies": p_s,
+            "collection": c_s,
+            "collections": c_s,
+            "amendment": a_s,
+            "amendments": a_s,
+        }
+        _doc_type2store = d
+    try:
+        return _doc_type2store[doc_type_name]
+    except:
+        raise400("Unrecognized document type {}".format(doc_type_name))
+
+
+def get_config_impl(request, doc_type_name):
+    raise_on_CORS_preflight(request)
+    docstore = get_docstore_from_type(doc_type_name, request)
+    return docstore.get_configuration_dict()
+
+
+def get_update_coll_fn(docstore, owner_id):
+    """Returns a function to be a commit_fn for commit_doc_and_trigger_push."""
+
+    def update_collection_fn(
+        doc, doc_id, auth_info, parent_sha, merged_sha, commit_msg
+    ):
+        return doc_id, docstore.update_existing_collection(
+            owner_id,
+            doc_id,
+            doc,
+            auth_info,
+            parent_sha,
+            merged_sha,
+            commit_msg=commit_msg,
+        )
+
+    return update_collection_fn
+
+
 def commit_doc_and_trigger_push(
     request,
     commit_fn,
@@ -187,8 +252,10 @@ def fetch_doc(
 
 
 def get_private_dir(request):
-    _LOG.debug("WHY PRIVATE DIR")
-    return "~/private/"
+    pd = "scratch"
+    if not os.path.exists(pd):
+        os.makedirs(pd)
+    return pd
 
 
 def atomic_write_json_if_not_found(obj, dest, request):
@@ -223,11 +290,8 @@ def compose_push_to_github_url(request, resource_id, doc_type):
 def get_phylesystem(request, conf_obj=None):
     global READ_ONLY_MODE
     global _PHYLESYSTEM
-    # _LOG.debug('@@@ checking for _PHYLESYSTEM singleton...READ_ONLY_MODE? {}'.format(READ_ONLY_MODE))
     if _PHYLESYSTEM is not None:
-        _LOG.debug("@@@ FOUND it, returning now")
         return _PHYLESYSTEM
-    # _LOG.debug('@@@ NOT FOUND, creating now')
     from phylesystem_api.gitdata import GitData
 
     if conf_obj is None:
@@ -240,14 +304,6 @@ def get_phylesystem(request, conf_obj=None):
     peyotl_config.set(
         "phylesystem", "max_file_size", pc.max_filesize
     )  # overrides peyotl config with max phylesytem-api filesize
-    push_mirror = os.path.join(pc.repo_parent, "mirror")
-    pmi = {
-        "parent_dir": push_mirror,
-        "remote_map": {
-            "GitHubRemote": pc.git_hub_remote,
-        },
-    }
-    mirror_info = {"push": pmi}
     a = {}
     try:
         new_study_prefix = conf_obj.get("apis", "new_study_prefix")
@@ -259,7 +315,7 @@ def get_phylesystem(request, conf_obj=None):
         git_ssh=pc.git_ssh,
         pkey=pc.pkey,
         git_action_class=GitData,
-        mirror_info=mirror_info,
+        mirror_info=pc.mirror_info,
         **a
     )
     # _LOG.debug('[[[[[[ repo_nexml2json = {}'.format(_PHYLESYSTEM.repo_nexml2json))
@@ -273,108 +329,80 @@ def get_phylesystem(request, conf_obj=None):
 _TREE_COLLECTION_STORE = None
 
 
-def get_tree_collection_store(request):
+def get_tree_collection_store(request, conf_obj=None):
     global _TREE_COLLECTION_STORE
     if _TREE_COLLECTION_STORE is not None:
         return _TREE_COLLECTION_STORE
-    #    _LOG = get_logger(request, 'ot_api')
     from phylesystem_api.gitdata import GitData
 
-    (
-        repo_parent,
-        repo_remote,
-        git_ssh,
-        pkey,
-        git_hub_remote,
-        max_filesize,
-    ) = read_collections_config(request)
-    push_mirror = os.path.join(repo_parent, "mirror")
-    pmi = {
-        "parent_dir": push_mirror,
-        "remote_map": {
-            "GitHubRemote": git_hub_remote,
-        },
-    }
-    mirror_info = {"push": pmi}
-    a = {}
-    try:
-        # any keyword args to pass along from config?
-        # new_study_prefix = conf.get('apis', 'new_study_prefix')
-        # a['new_study_prefix'] = new_study_prefix
-        pass
-    except:
-        pass
+    if conf_obj is None:
+        conf_obj = get_conf_object(request)
+    cc = read_collections_config(request, conf_obj)
     _TREE_COLLECTION_STORE = TreeCollectionStore(
-        repos_par=repo_parent,
-        git_ssh=git_ssh,
-        pkey=pkey,
+        repos_par=cc.repo_parent,
+        git_ssh=cc.git_ssh,
+        pkey=cc.pkey,
         git_action_class=GitData,  # TODO?
-        mirror_info=mirror_info,
-        **a
+        mirror_info=cc.mirror_info,
     )
-    # _LOG.debug('assumed_doc_version = {}'.format(_TREE_COLLECTION_STORE.assumed_doc_version))
     return _TREE_COLLECTION_STORE
 
 
 _TAXONOMIC_AMENDMENT_STORE = None
 
 
-def get_taxonomic_amendment_store(request):
+def get_taxonomic_amendment_store(request, conf_obj=None):
     global _TAXONOMIC_AMENDMENT_STORE
     if _TAXONOMIC_AMENDMENT_STORE is not None:
         return _TAXONOMIC_AMENDMENT_STORE
-    #    _LOG = get_logger(request, 'ot_api')
     from phylesystem_api.gitdata import GitData
 
-    (
-        repo_parent,
-        repo_remote,
-        git_ssh,
-        pkey,
-        git_hub_remote,
-        max_filesize,
-    ) = read_amendments_config(request)
-    push_mirror = os.path.join(repo_parent, "mirror")
-    pmi = {
-        "parent_dir": push_mirror,
-        "remote_map": {
-            "GitHubRemote": git_hub_remote,
-        },
-    }
-    mirror_info = {"push": pmi}
-    a = {}
-    try:
-        # any keyword args to pass along from config?
-        # new_study_prefix = conf.get('apis', 'new_study_prefix')
-        # a['new_study_prefix'] = new_study_prefix
-        pass
-    except:
-        pass
+    if conf_obj is None:
+        conf_obj = get_conf_object(request)
+    cc = read_amendments_config(request, conf_obj)
     _TAXONOMIC_AMENDMENT_STORE = TaxonomicAmendmentStore(
-        repos_par=repo_parent,
-        git_ssh=git_ssh,
-        pkey=pkey,
-        git_action_class=GitData,  # TODO?
-        mirror_info=mirror_info,
-        **a
-    )
-    _LOG.debug(
-        "assumed_doc_version = {}".format(
-            _TAXONOMIC_AMENDMENT_STORE.assumed_doc_version
-        )
+        repos_par=cc.repo_parent,
+        git_ssh=cc.git_ssh,
+        pkey=cc.pkey,
+        git_action_class=GitData,
+        mirror_info=cc.mirror_info,
     )
+    tas = _TAXONOMIC_AMENDMENT_STORE
+    msg = "assumed_doc_version = {}".format(tas.assumed_doc_version)
+    _LOG.debug(msg)
     return _TAXONOMIC_AMENDMENT_STORE
 
 
-def get_failed_push_filepath(request, doc_type=None):
-    filenames_by_content_type = {
-        "nexson": "PUSH_FAILURE_nexson.json",
-        "collection": "PUSH_FAILURE_collection.json",
-        "amendment": "PUSH_FAILURE_amendment.json",
-        "favorites": "PUSH_FAILURE_favorites.json",
-    }
-    content_type = doc_type or request.matchdict.get("doc_type", "nexson")
-    failure_filename = filenames_by_content_type[content_type]
+filenames_by_content_type = {
+    "nexson": "PUSH_FAILURE_nexson.json",
+    "collection": "PUSH_FAILURE_collection.json",
+    "amendment": "PUSH_FAILURE_amendment.json",
+    "favorites": "PUSH_FAILURE_favorites.json",
+}
+filenames_by_content_type["collections"] = filenames_by_content_type["collection"]
+filenames_by_content_type["amendments"] = filenames_by_content_type["amendment"]
+filenames_by_content_type["study"] = filenames_by_content_type["nexson"]
+filenames_by_content_type["studies"] = filenames_by_content_type["nexson"]
+normalized_doc_type_names = {
+    "nexson": "nexson",
+    "studies": "nexson",
+    "study": "nexson",
+    "collections": "collection",
+    "collection": "collection",
+    "amendment": "amendment",
+    "amendments": "amendment",
+}
+
+
+def normalize_synon_doc_types(doc_type):
+    return normalized_doc_type_names[doc_type]
+
+
+def get_failed_push_filepath(request, doc_type):
+    try:
+        failure_filename = filenames_by_content_type[doc_type]
+    except:
+        raise400("Unrecognized document type {}".format(doc_type))
     return os.path.join(get_private_dir(request), failure_filename)
 
 
@@ -398,26 +426,21 @@ def get_conf_object(request=None, localconfig_filename=None):
     return conf
 
 
-_phylesystem_config = None
-_phylesystem_config_lock = threading.Lock()
-
-
-def read_phylesystem_config(request, conf_obj=None):
-    """Load settings for managing the main Nexson docstore"""
-    global _phylesystem_config
-    if _phylesystem_config is None:
-        if conf_obj is None:
-            conf_obj = get_conf_object(request)
-        with _phylesystem_config_lock:
-            if _phylesystem_config is None:
-                _phylesystem_config = PhylesystemConfig(conf_obj)
-    return _phylesystem_config
-
-
-class PhylesystemConfig(object):
-    def __init__(self, conf):
-        self._repo_parent = conf.get("apis", "repo_parent")
-        self._repo_remote = conf.get("apis", "repo_remote")
+class DocstoreConfig(object):
+    def __init__(self, conf, doc_type="nexson"):
+        if doc_type == "nexson":
+            prefix = ""
+            mfs = "peyotl_max_file_size"
+        elif doc_type == "collection":
+            prefix = "collections_"
+            mfs = "collections_max_file_size"
+        elif doc_type == "amendment":
+            prefix = "amendments_"
+            mfs = "amendments_max_file_size"
+        else:
+            raise ValueError("Unrecognized doc_type {}".format(doc_type))
+        self._repo_parent = conf.get("apis", prefix + "repo_parent")
+        self._repo_remote = conf.get("apis", prefix + "repo_remote")
         try:
             self._git_ssh = conf.get("apis", "git_ssh")
         except:
@@ -431,17 +454,20 @@ def __init__(self, conf):
         except:
             self._git_hub_remote = "git@github.com:OpenTreeOfLife"
         try:
-            self._max_filesize = conf.get("filesize", "peyotl_max_file_size")
+            self._max_filesize = conf.get("filesize", mfs)
         except:
             self._max_filesize = "20000000"
-        try:
-            self._max_num_trees = conf.get("filesize", "validation_max_num_trees")
-        except:
-            self._max_num_trees = 65
-        try:
-            self._max_num_trees = int(self._max_num_trees)
-        except ValueError:
-            raise400("max number of trees per study in config is not an integer")
+        if doc_type == "nexson":
+            try:
+                self._max_num_trees = conf.get("filesize", "validation_max_num_trees")
+            except:
+                self._max_num_trees = 65
+            try:
+                self._max_num_trees = int(self._max_num_trees)
+            except ValueError:
+                raise400("max number of trees per study in config is not an integer")
+        else:
+            self._max_num_trees = None
         try:
             self._read_only = conf.get("apis", "read_only") == "true"
         except:
@@ -479,67 +505,58 @@ def max_num_trees(self):
     def read_only(self):
         return self._read_only
 
+    @property
+    def mirror_info(self):
+        push_mirror = os.path.join(self._repo_parent, "mirror")
+        pmi = {
+            "parent_dir": push_mirror,
+            "remote_map": {
+                "GitHubRemote": self._git_hub_remote,
+            },
+        }
+        return {"push": pmi}
+
 
-def read_collections_config(request):
+_phylesystem_config = None
+_collections_config = None
+_amendments_config = None
+_docstore_config_lock = threading.Lock()
+
+
+def read_phylesystem_config(request, conf_obj=None):
+    """Load settings for managing the main Nexson docstore"""
+    global _phylesystem_config
+    if _phylesystem_config is None:
+        if conf_obj is None:
+            conf_obj = get_conf_object(request)
+        with _docstore_config_lock:
+            if _phylesystem_config is None:
+                _phylesystem_config = DocstoreConfig(conf_obj)
+    return _phylesystem_config
+
+
+def read_collections_config(request, conf_obj=None):
     """Load settings for a minor repo with shared tree collections"""
-    conf = get_conf_object(request)
-    collections_repo_parent = conf.get("apis", "collections_repo_parent")
-    collections_repo_remote = conf.get("apis", "collections_repo_remote")
-    try:
-        git_ssh = conf.get("apis", "git_ssh")
-    except:
-        git_ssh = "ssh"
-    try:
-        pkey = conf.get("apis", "pkey")
-    except:
-        pkey = None
-    try:
-        git_hub_remote = conf.get("apis", "git_hub_remote")
-    except:
-        git_hub_remote = "git@github.com:OpenTreeOfLife"
-    try:
-        max_filesize = conf.get("filesize", "collections_max_file_size")
-    except:
-        max_filesize = "20000000"
-    return (
-        collections_repo_parent,
-        collections_repo_remote,
-        git_ssh,
-        pkey,
-        git_hub_remote,
-        max_filesize,
-    )
+    global _collections_config
+    if _collections_config is None:
+        if conf_obj is None:
+            conf_obj = get_conf_object(request)
+        with _docstore_config_lock:
+            if _collections_config is None:
+                _collections_config = DocstoreConfig(conf_obj, "collection")
+    return _collections_config
 
 
-def read_amendments_config(request):
+def read_amendments_config(request, conf_obj=None):
     """Load settings for a minor repo with shared taxonomic amendments"""
-    conf = get_conf_object(request)
-    amendments_repo_parent = conf.get("apis", "amendments_repo_parent")
-    amendments_repo_remote = conf.get("apis", "amendments_repo_remote")
-    try:
-        git_ssh = conf.get("apis", "git_ssh")
-    except:
-        git_ssh = "ssh"
-    try:
-        pkey = conf.get("apis", "pkey")
-    except:
-        pkey = None
-    try:
-        git_hub_remote = conf.get("apis", "git_hub_remote")
-    except:
-        git_hub_remote = "git@github.com:OpenTreeOfLife"
-    try:
-        max_filesize = conf.get("filesize", "amendments_max_file_size")
-    except:
-        max_filesize = "20000000"
-    return (
-        amendments_repo_parent,
-        amendments_repo_remote,
-        git_ssh,
-        pkey,
-        git_hub_remote,
-        max_filesize,
-    )
+    global _amendments_config
+    if _amendments_config is None:
+        if conf_obj is None:
+            conf_obj = get_conf_object(request)
+        with _docstore_config_lock:
+            if _amendments_config is None:
+                _amendments_config = DocstoreConfig(conf_obj, "amendment")
+    return _amendments_config
 
 
 def read_favorites_config(request):
@@ -563,28 +580,14 @@ def read_favorites_config(request):
 
 
 def _raise_missing_auth_token():
-    raise HTTPBadRequest(
-        json.dumps(
-            {
-                "error": 1,
-                "description": "You must provide an auth_token to authenticate to the OpenTree API",
-            }
-        )
-    )
+    raise400("You must provide an auth_token to authenticate to the OpenTree API")
 
 
 def authenticate(request):
-    """Verify that we received a valid Github authentication token
-
-    This method takes a dict of keyword arguments and optionally
-    over-rides the author_name and author_email associated with the
-    given token, if they are present.
-
-    Returns a PyGithub object, author name and author email.
-
-    This method will return HTTP 400 if the auth token is not present
-    or if it is not valid, i.e. if PyGithub throws a BadCredentialsException.
+    """Verify that we received a valid GitHub in the request's auth_token
 
+    This method will raise HTTP 400 if the auth token is not missing or invalid.
+    Returns dict with "login", "auth_token", and (possibly) "name" and "email" keys
     """
     # this is the GitHub API auth-token for a logged-in curator
     auth_token = find_in_request(request, "auth_token", "")
@@ -598,7 +601,7 @@ def authenticate(request):
         auth_info["login"] = gh_user.login
     except BadCredentialsException:
         raise400("You have provided an invalid or expired authentication token")
-    # use the Github Oauth token to get a name/email if not specified
+    # use the GitHub Oauth token to get a name/email if not specified
     # we don't provide these as default values above because they would
     # generate API calls regardless of author_name/author_email being specifed
     auth_info["name"] = find_in_request(request, "author_name", gh_user.name)
diff --git a/phylesystem_api/phylesystem_api/routes.py b/phylesystem_api/phylesystem_api/routes.py
index 48ddcd35..61899fb2 100644
--- a/phylesystem_api/phylesystem_api/routes.py
+++ b/phylesystem_api/phylesystem_api/routes.py
@@ -94,16 +94,20 @@ def includeme(config):
     config.add_route("get_study_meta", "/{api_version}/study/{study_id}/meta")
     config.add_route("get_study_meta_slash", "/{api_version}/study/{study_id}/meta/")
     #
-    # TREE COLLECTION ROUTES
+    # GENERIC routes
     #
     config.add_route(
-        "list_all_collection_ids", "/{api_version}/collections/collection_list"
+        "get_docstore_config", "/{api_version}/{doc_type_name}/store_config"
     )
     config.add_route(
-        "get_collections_config", "/{api_version}/collections/store_config"
+        "docstore_push_failure", "/{api_version}/{doc_type_name}/push_failure"
     )
+
+    #
+    # TREE COLLECTION ROUTES
+    #
     config.add_route(
-        "collections_push_failure", "/{api_version}/collections/push_failure"
+        "list_all_collection_ids", "/{api_version}/collections/collection_list"
     )
     config.add_route("collection_properties", "/{api_version}/collections/properties")
     config.add_route(
@@ -138,10 +142,6 @@ def includeme(config):
         "list_all_amendment_ids", "/{api_version}/amendments/amendment_list"
     )
     config.add_route("list_all_amendments", "/{api_version}/amendments/list_all")
-    config.add_route("get_amendments_config", "/{api_version}/amendments/store_config")
-    config.add_route(
-        "amendments_push_failure", "/{api_version}/amendments/push_failure"
-    )
     config.add_route("create_amendment", "/{api_version}/amendment/")
     config.add_route(
         "amendment_CORS_preflight",
diff --git a/phylesystem_api/phylesystem_api/views/amendment.py b/phylesystem_api/phylesystem_api/views/amendment.py
index 147a7167..6a005be7 100644
--- a/phylesystem_api/phylesystem_api/views/amendment.py
+++ b/phylesystem_api/phylesystem_api/views/amendment.py
@@ -1,10 +1,7 @@
-import json
 import logging
-import traceback
 
 from peyotl.amendments import AMENDMENT_ID_PATTERN
 from peyotl.amendments.validation import validate_amendment
-from peyotl.phylesystem.git_workflows import GitWorkflowError
 
 # see exception subclasses at https://docs.pylonsproject.org/projects/pyramid/en/latest/api/httpexceptions.html
 from pyramid.httpexceptions import (
@@ -15,7 +12,6 @@
 
 import phylesystem_api.api_utils as api_utils
 from phylesystem_api.api_utils import (
-    find_in_request,
     extract_json_from_http_call,
     raise400,
     fetch_doc,
diff --git a/phylesystem_api/phylesystem_api/views/amendments.py b/phylesystem_api/phylesystem_api/views/amendments.py
index 115e0ff1..c9aba051 100644
--- a/phylesystem_api/phylesystem_api/views/amendments.py
+++ b/phylesystem_api/phylesystem_api/views/amendments.py
@@ -1,7 +1,4 @@
 import logging
-import os
-
-from peyotl.nexson_syntax import read_as_json
 
 # see exception subclasses at https://docs.pylonsproject.org/projects/pyramid/en/latest/api/httpexceptions.html
 from pyramid.httpexceptions import (
@@ -9,71 +6,38 @@
 )
 from pyramid.view import view_config
 
-import phylesystem_api.api_utils as api_utils
+from phylesystem_api.api_utils import (
+    get_taxonomic_amendment_store,
+    raise_on_CORS_preflight,
+    raise_int_server_err,
+    get_last_modified_dict,
+)
 
 _LOG = logging.getLogger("phylesystem_api")
 
 
 @view_config(route_name="list_all_amendment_ids", renderer="json")
 def list_all_amendment_ids(request):
-    docstore = api_utils.get_taxonomic_amendment_store(request)
+    docstore = get_taxonomic_amendment_store(request)
     return docstore.get_amendment_ids()
 
 
 @view_config(route_name="list_all_amendments", renderer="json")
 def list_all(request):
-    api_utils.raise_on_CORS_preflight(request)
+    raise_on_CORS_preflight(request)
     # TODO: proxy to oti for a filtered list?
     # For now, let's just return all collections (complete JSON)
     amendment_list = []
     try:
-        docstore = api_utils.get_taxonomic_amendment_store(request)
+        docstore = get_taxonomic_amendment_store(request)
         # Convert these to more closely resemble the output of find_all_studies
-        for id, props in docstore.iter_doc_objs():
-            # reckon and add 'lastModified' property, based on commit history?
-            latest_commit = docstore.get_version_history_for_doc_id(id)[0]
-            props.update(
-                {
-                    "id": id,
-                    "lastModified": {
-                        "author_name": latest_commit.get("author_name"),
-                        "relative_date": latest_commit.get("relative_date"),
-                        "display_date": latest_commit.get("date"),
-                        "ISO_date": latest_commit.get("date_ISO_8601"),
-                        "sha": latest_commit.get("id"),  # this is the commit hash
-                    },
-                }
-            )
+        for a_id, props in docstore.iter_doc_objs():
+            props["id"] = a_id
+            props["lastModified"] = get_last_modified_dict(docstore, a_id)
             amendment_list.append(props)
     except HTTPException:
         raise
     except Exception as x:
         msg = "Unexpected error calling oti: {}".format(",".join(x.args))
-        api_utils.raise_int_server_err(msg)
+        raise_int_server_err(msg)
     return amendment_list
-
-
-@view_config(route_name="get_amendments_config", renderer="json")
-def get_amendments_config(request):
-    _LOG.debug("get_amendments_config")
-    api_utils.raise_on_CORS_preflight(request)
-    docstore = api_utils.get_taxonomic_amendment_store(request)
-    return docstore.get_configuration_dict()
-
-
-@view_config(route_name="amendments_push_failure", renderer="json")
-def amendments_push_failure(request):
-    api_utils.raise_on_CORS_preflight(request)
-    # this should find a type-specific push_failure file
-    request.matchdict["doc_type"] = "amendment"
-    fail_file = api_utils.get_failed_push_filepath(request)
-    if os.path.exists(fail_file):
-        try:
-            blob = read_as_json(fail_file)
-        except:
-            blob = {"message": "could not read push fail file"}
-        blob["pushes_succeeding"] = False
-    else:
-        blob = {"pushes_succeeding": True}
-    blob["doc_type"] = request.matchdict.get("doc_type", "nexson")
-    return blob
diff --git a/phylesystem_api/phylesystem_api/views/collection.py b/phylesystem_api/phylesystem_api/views/collection.py
index 552a0650..8a317388 100644
--- a/phylesystem_api/phylesystem_api/views/collection.py
+++ b/phylesystem_api/phylesystem_api/views/collection.py
@@ -1,29 +1,27 @@
-import json
 import logging
-import traceback
 
 from peyotl.collections_store import COLLECTION_ID_PATTERN
 from peyotl.collections_store.validation import validate_collection
-from peyotl.phylesystem.git_workflows import GitWorkflowError
 
 # see exception subclasses at https://docs.pylonsproject.org/projects/pyramid/en/latest/api/httpexceptions.html
 from pyramid.httpexceptions import (
     HTTPException,
-    HTTPBadRequest,
 )
 from pyramid.view import view_config
 
 import phylesystem_api.api_utils as api_utils
 from phylesystem_api.api_utils import (
-    find_in_request,
+    commit_doc_and_trigger_push,
     extract_json_from_http_call,
-    raise400,
-    raise404,
     fetch_doc,
-    commit_doc_and_trigger_push,
-    get_parent_sha,
+    find_in_request,
     get_commit_message,
+    get_last_modified_dict,
     get_owner_id,
+    get_parent_sha,
+    get_update_coll_fn,
+    raise400,
+    raise404,
 )
 
 _LOG = logging.getLogger("phylesystem_api")
@@ -99,15 +97,9 @@ def add_collection_specific_fields(request, collection_id, result):
     # Add the lastModified field to the result JSON
     version_history = result.get("versionHistory")
     if version_history:
-        latest_commit = version_history[0]
-        last_modified = {
-            "author_name": latest_commit.get("author_name"),
-            "relative_date": latest_commit.get("relative_date"),
-            "display_date": latest_commit.get("date"),
-            "ISO_date": latest_commit.get("date_ISO_8601"),
-            "sha": latest_commit.get("id"),  # this is the commit hash
-        }
-        result["lastModified"] = last_modified
+        result["lastModified"] = get_last_modified_dict(
+            latest_commit=version_history[0]
+        )
     return result
 
 
@@ -187,25 +179,12 @@ def update_collection(request):
     # submit new json for this id, and read the results
     docstore = api_utils.get_tree_collection_store(request)
 
-    def update_collection_fn(
-        doc, doc_id, auth_info, parent_sha, merged_sha, commit_msg
-    ):
-        return doc_id, docstore.update_existing_collection(
-            owner_id,
-            doc_id,
-            doc,
-            auth_info,
-            parent_sha,
-            merged_sha,
-            commit_msg=commit_msg,
-        )
-
     r_parent_sha = find_in_request(request, "starting_commit_SHA", None)
     r_merged_sha = None  # TODO: find_in_request(request, '???', None)
     r_commit_msg = get_commit_message(request)
     blob = commit_doc_and_trigger_push(
         request,
-        commit_fn=update_collection_fn,
+        commit_fn=get_update_coll_fn(docstore, owner_id),
         doc=collection_obj,
         doc_id=collection_id,
         doc_type_name="collection",
diff --git a/phylesystem_api/phylesystem_api/views/collections.py b/phylesystem_api/phylesystem_api/views/collections.py
index d79a0eb2..23f6f7a9 100644
--- a/phylesystem_api/phylesystem_api/views/collections.py
+++ b/phylesystem_api/phylesystem_api/views/collections.py
@@ -9,43 +9,23 @@
 )
 from pyramid.view import view_config
 
-import phylesystem_api.api_utils as api_utils
-
-
-@view_config(route_name="get_collections_config", renderer="json")
-def get_collections_config(request):
-    api_utils.raise_on_CORS_preflight(request)
-    docstore = api_utils.get_tree_collection_store(request)
-    return docstore.get_configuration_dict()
-
-
-@view_config(route_name="collections_push_failure", renderer="json")
-def collections_push_failure(request):
-    # this should find a type-specific PUSH_FAILURE file
-    api_utils.raise_on_CORS_preflight(request)
-    request.matchdict["doc_type"] = "collection"
-    fail_file = api_utils.get_failed_push_filepath(request)
-    if os.path.exists(fail_file):
-        try:
-            blob = read_as_json(fail_file)
-        except:
-            blob = {"message": "could not read push fail file"}
-        blob["pushes_succeeding"] = False
-    else:
-        blob = {"pushes_succeeding": True}
-    blob["doc_type"] = request.matchdict["doc_type"]
-    return blob
+from phylesystem_api.api_utils import (
+    get_last_modified_dict,
+    get_tree_collection_store,
+    raise_on_CORS_preflight,
+    raise_int_server_err,
+)
 
 
 @view_config(route_name="list_all_collection_ids", renderer="json")
 def list_all_collection_ids(request):
-    docstore = api_utils.get_tree_collection_store(request)
+    docstore = get_tree_collection_store(request)
     return docstore.get_collection_ids()
 
 
 @view_config(route_name="collection_properties", renderer="json")
 def collection_properties(request):
-    api_utils.raise_on_CORS_preflight(request)
+    raise_on_CORS_preflight(request)
     raise HTTPNotImplemented(
         "Now we'd list all searchable properties in tree collections!"
     )
@@ -53,7 +33,7 @@ def collection_properties(request):
 
 @view_config(route_name="find_trees_in_collections", renderer="json")
 def find_trees_in_collections(request):
-    api_utils.raise_on_CORS_preflight(request)
+    raise_on_CORS_preflight(request)
     raise HTTPNotImplemented(
         "Now we'd list all collections holding trees that match the criteria provided!"
     )
@@ -61,31 +41,19 @@ def find_trees_in_collections(request):
 
 @view_config(route_name="find_collections", renderer="json")
 def find_collections(request):
-    api_utils.raise_on_CORS_preflight(request)
+    raise_on_CORS_preflight(request)
     # TODO: proxy to oti for a filtered list?
     # For now, let's just return all collections (complete JSON)
+    docstore = get_tree_collection_store(request)
+    # Convert these to more closely resemble the output of find_all_studies
+    collection_list = []
     try:
-        docstore = api_utils.get_tree_collection_store(request)
-        # Convert these to more closely resemble the output of find_all_studies
-        collection_list = []
-        for id, props in docstore.iter_doc_objs():
-            # reckon and add 'lastModified' property, based on commit history?
-            latest_commit = docstore.get_version_history_for_doc_id(id)[0]
-            props.update(
-                {
-                    "id": id,
-                    "lastModified": {
-                        "author_name": latest_commit.get("author_name"),
-                        "relative_date": latest_commit.get("relative_date"),
-                        "display_date": latest_commit.get("date"),
-                        "ISO_date": latest_commit.get("date_ISO_8601"),
-                        "sha": latest_commit.get("id"),  # this is the commit hash
-                    },
-                }
-            )
+        for c_id, props in docstore.iter_doc_objs():
+            props["id"] = c_id
+            props["lastModified"] = get_last_modified_dict(docstore, c_id)
             collection_list.append(props)
     except HTTPException:
         raise
-    except Exception as x:
-        api_utils.raise_int_server_err("Unexpected error gathering collections")
+    except:
+        raise_int_server_err("Unexpected error gathering collections")
     return collection_list
diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py
index fbddc0dd..ac0ef8c8 100644
--- a/phylesystem_api/phylesystem_api/views/default.py
+++ b/phylesystem_api/phylesystem_api/views/default.py
@@ -4,12 +4,14 @@
 import os
 import sys
 import traceback
+import threading
 from configparser import ConfigParser
 from io import StringIO
 
 import phylesystem_api.api_utils as api_utils
 import requests
 from peyotl import concatenate_collections, tree_is_in_collection
+from peyotl.nexson_syntax import read_as_json
 from peyotl.phylesystem.git_workflows import (
     GitWorkflowError,
     merge_from_master,
@@ -24,7 +26,15 @@
 )
 from pyramid.response import Response
 from pyramid.view import view_config
-from phylesystem_api.api_utils import raise_int_server_err, find_in_request
+from phylesystem_api.api_utils import (
+    raise_int_server_err,
+    find_in_request,
+    get_config_impl,
+    get_docstore_from_type,
+    get_update_coll_fn,
+    commit_doc_and_trigger_push,
+    normalize_synon_doc_types,
+)
 
 import json
 
@@ -235,13 +245,6 @@ def render_markdown(request):
     return Response(body=html, content_type="text/html")
 
 
-@view_config(route_name="phylesystem_config", renderer="json")
-def phylesystem_config(request):
-    # general information about the hosted phylesystem
-    phylesystem = api_utils.get_phylesystem(request)
-    return phylesystem.get_configuration_dict()
-
-
 @view_config(route_name="raw_study_list", renderer="json")
 def study_list(request):
     phylesystem = api_utils.get_phylesystem(request)
@@ -249,6 +252,34 @@ def study_list(request):
     return studies
 
 
+@view_config(route_name="get_docstore_config", renderer="json")
+def get_docstore_config(request):
+    return get_config_impl(request, request.matchdict["doc_type_name"])
+
+
+@view_config(route_name="docstore_push_failure", renderer="json")
+def docstore_push_failure(request):
+    # this should find a type-specific PUSH_FAILURE file
+    api_utils.raise_on_CORS_preflight(request)
+    doc_type = request.matchdict["doc_type_name"]
+    fail_file = api_utils.get_failed_push_filepath(request, doc_type)
+    if os.path.exists(fail_file):
+        try:
+            blob = read_as_json(fail_file)
+        except:
+            blob = {"message": "could not read push fail file"}
+        blob["pushes_succeeding"] = False
+    else:
+        blob = {"pushes_succeeding": True}
+    blob["doc_type"] = normalize_synon_doc_types(doc_type)
+    return blob
+
+
+@view_config(route_name="phylesystem_config", renderer="json")
+def phylesystem_config(request):
+    return get_docstore_config(request, "studies")
+
+
 @view_config(route_name="trees_in_synth", renderer="json")
 def trees_in_synth(request):
     """Return an "artificial" collection that contains all trees (and
@@ -336,37 +367,18 @@ def include_tree_in_synth(request):
     )
     # update (or add) the decision list for this collection
     coll["decisions"] = decision_list
-    # update the default collection (forces re-indexing)
-    try:
-        parent_sha = request.params.get("starting_commit_SHA", None)
-        merged_sha = None  # TODO: request.params.get('???', None)
-    except:
-        raise404("include_tree_in_synth(): fetch of starting_commit_SHA failed")
-    try:
-        r = cds.update_existing_collection(
-            owner_id,
-            default_collection_id,
-            coll,
-            auth_info,
-            parent_sha,
-            merged_sha,
-            commit_msg="Updated via API (include_tree_in_synth)",
-        )
-        commit_return = r
-    except GitWorkflowError as err:
-        raise HTTPBadRequest(body=err.msg)
-    except:
-        raise HTTPBadRequest(body=traceback.format_exc())
-
-    # check for 'merge needed'?
-    mn = commit_return.get("merge_needed")
-    if (mn is not None) and (not mn):
-        api_utils.deferred_push_to_gh_call(
-            request,
-            default_collection_id,
-            doc_type="collection",
-            auth_token=auth_info["auth_token"],
-        )
+    commit_fn = get_update_coll_fn(cds, owner_id)
+    commit_doc_and_trigger_push(
+        request,
+        commit_fn=commit_fn,
+        doc=coll,
+        doc_id=default_collection_id,
+        doc_type_name="collection",
+        auth_info=auth_info,
+        parent_sha=request.params.get("starting_commit_SHA", None),
+        merged_sha=None,  # TODO: request.params.get('???', None)
+        commit_msg="Updated via API (include_tree_from_synth)",
+    )
     # fetch and return the updated list of synth-input trees
     return trees_in_synth(request)
 
@@ -383,6 +395,7 @@ def exclude_tree_from_synth(request):
     cds = api_utils.get_tree_collection_store(request)
     auth_info = api_utils.auth_and_not_read_only(request)
     owner_id = auth_info.get("login", None)
+    commit_fn = get_update_coll_fn(cds, owner_id)
     for coll_id in coll_id_list:
         try:
             coll = cds.return_doc(coll_id, commit_sha=None, return_WIP_map=False)[0]
@@ -400,31 +413,17 @@ def exclude_tree_from_synth(request):
             # update the collection (forces re-indexing)
             parent_sha = request.params.get("starting_commit_SHA", None)
             merged_sha = None  # TODO: request.params.get('???', None)
-            try:
-                r = cds.update_existing_collection(
-                    owner_id,
-                    coll_id,
-                    coll,
-                    auth_info,
-                    parent_sha,
-                    merged_sha,
-                    commit_msg="Updated via API (include_tree_in_synth)",
-                )
-                commit_return = r
-            except GitWorkflowError as err:
-                raise HTTPBadRequest(body=err.msg)
-            except:
-                raise HTTPBadRequest(body=traceback.format_exc())
-
-            # check for 'merge needed'?
-            mn = commit_return.get("merge_needed")
-            if (mn is not None) and (not mn):
-                api_utils.deferred_push_to_gh_call(
-                    request,
-                    coll_id,
-                    doc_type="collection",
-                    auth_token=auth_info["auth_token"],
-                )
+            commit_doc_and_trigger_push(
+                request,
+                commit_fn=commit_fn,
+                doc=coll,
+                doc_id=coll_id,
+                doc_type_name="collection",
+                auth_info=auth_info,
+                parent_sha=parent_sha,
+                merged_sha=merged_sha,
+                commit_msg="Updated via API (exclude_tree_from_synth)",
+            )
     # fetch and return the updated list of synth-input trees
     return trees_in_synth(request)
 
@@ -499,11 +498,7 @@ def merge_docstore_changes(request):
         raise400(err.msg)
     except:
         m = traceback.format_exc()
-        raise HTTPConflict(
-            detail=json.dumps(
-                {"error": 1, "description": "Could not merge! Details: {}".format(m)}
-            )
-        )
+        raise_conflict("Could not merge! Details: {}".format(m))
 
 
 @view_config(route_name="push_docstore_changes", renderer="json")
@@ -514,155 +509,79 @@ def push_docstore_changes(request):
     curl -X POST https://devapi.opentreeoflife.org/v3/push_docstore_changes/nexson/ot_999
     """
     _LOG.debug("push_docstore_changes")
-    doc_type = request.matchdict["doc_type"]
+    doc_type = request.matchdict["doc_type"].lower()
     resource_id = request.matchdict.get("doc_id")
     if resource_id is None:
         resource_id = find_in_request(request, "doc_id", None)
     api_utils.auth_and_not_read_only(request)
     fail_file = api_utils.get_failed_push_filepath(request, doc_type=doc_type)
     # TODO
-    if doc_type.lower() == "nexson":
-        phylesystem = api_utils.get_phylesystem(request)
-        try:
-            phylesystem.push_study_to_remote("GitHubRemote", resource_id)
-        except:
-            m = traceback.format_exc()
-            _LOG.warning(
-                "Push of study {s} failed. Details: {m}".format(s=resource_id, m=m)
-            )
-            if os.path.exists(fail_file):
-                _LOG.warning(
-                    'push failure file "{f}" already exists. This event not logged there'.format(
-                        f=fail_file
-                    )
-                )
-            else:
-                timestamp = datetime.datetime.utcnow().isoformat()
-                try:
-                    ga = phylesystem.create_git_action(resource_id)
-                except:
-                    m = (
-                        'Could not create an adaptor for git actions on study ID "{}". '
-                        "If you are confident that this is a valid study ID, please report this as a bug."
-                    )
-                    raise400(m.format(resource_id))
-                master_sha = ga.get_master_sha()
-                obj = {
-                    "date": timestamp,
-                    "study": resource_id,
-                    "commit": master_sha,
-                    "stacktrace": m,
-                }
-                api_utils.atomic_write_json_if_not_found(obj, fail_file, request)
-                _LOG.warning('push failure file "{f}" created.'.format(f=fail_file))
-            raise HTTPConflict(
-                json.dumps(
-                    {
-                        "error": 1,
-                        "description": "Could not push! Details: {m}".format(m=m),
-                    }
-                )
-            )
+    docstore = get_docstore_from_type(doc_type, request)
+    doc_key_dict = {
+        "nexson": "study",
+        "study": "study",
+        "studies": "study",
+        "collection": "collection",
+        "collections": "collection",
+        "amendment": "amendment",
+        "amendments": "amendment",
+    }
+    doc_key = doc_key_dict[doc_type]
+    try:
+        docstore.push_doc_to_remote("GitHubRemote", resource_id)
+    except:
+        m = traceback.format_exc()
+        msg = "Push of {k} {s} failed. Details: {m}".format(
+            k=doc_key, s=resource_id, m=m
+        )
+        _LOG.warning(msg)
+        if os.path.exists(fail_file):
+            msg = '"{f}" already exists. This event not logged'.format(f=fail_file)
+            _LOG.warning(msg)
+        else:
+            timestamp = datetime.datetime.utcnow().isoformat()
+            try:
+                ga = docstore.create_git_action(resource_id)
+            except:
+                msg = 'Could not create an adaptor for git actions on ID "{}".  If you are confident that this is a valid ID, please report this as a bug.'
+                raise400(msg.format(resource_id))
+            master_sha = ga.get_master_sha()
+            obj = {
+                "date": timestamp,
+                doc_key: resource_id,
+                "commit": master_sha,
+                "stacktrace": m,
+            }
+            api_utils.atomic_write_json_if_not_found(obj, fail_file, request)
+            _LOG.warning('push failure file "{f}" created.'.format(f=fail_file))
+        raise_conflict("Could not push! Details: {m}".format(m=m))
 
-    elif doc_type.lower() == "collection":
-        _LOG.debug("in collections")
-        docstore = api_utils.get_tree_collection_store(request)
-        try:
-            _LOG.debug("calling peyotl push")
-            docstore.push_doc_to_remote("GitHubRemote", resource_id)
-        except:
-            _LOG.debug("in collections except")
-            m = traceback.format_exc()
-            _LOG.warning(
-                "Push of collection {s} failed. Details: {m}".format(s=resource_id, m=m)
-            )
-            if os.path.exists(fail_file):
-                _LOG.warning(
-                    'push failure file "{f}" already exists. This event not logged there'.format(
-                        f=fail_file
-                    )
-                )
-            else:
-                timestamp = datetime.datetime.utcnow().isoformat()
-                try:
-                    ga = docstore.create_git_action(resource_id)
-                except:
-                    m = (
-                        'Could not create an adaptor for git actions on collection ID "{}". '
-                        "If you are confident that this is a valid collection ID, please report this as a bug."
-                    )
-                    raise400(m.format(resource_id))
-                master_sha = ga.get_master_sha()
-                obj = {
-                    "date": timestamp,
-                    "collection": resource_id,
-                    "commit": master_sha,
-                    "stacktrace": m,
-                }
-                api_utils.atomic_write_json_if_not_found(obj, fail_file, request)
-                _LOG.warning('push failure file "{f}" created.'.format(f=fail_file))
-            raise HTTPConflict(
-                body=json.dumps(
-                    {
-                        "error": 1,
-                        "description": "Could not push! Details: {m}".format(m=m),
-                    }
-                )
-            )
+    if os.path.exists(fail_file):
+        remove_fail_file(fail_file)
+    return {"error": 0, "description": "Push succeeded"}
 
-    elif doc_type.lower() == "amendment":
-        docstore = api_utils.get_taxonomic_amendment_store(request)
-        try:
-            docstore.push_doc_to_remote("GitHubRemote", resource_id)
-        except:
-            m = traceback.format_exc()
-            _LOG.warning(
-                "Push of amendment {s} failed. Details: {m}".format(s=resource_id, m=m)
-            )
-            if os.path.exists(fail_file):
-                _LOG.warning(
-                    'push failure file "{f}" already exists. This event not logged there'.format(
-                        f=fail_file
-                    )
-                )
-            else:
-                timestamp = datetime.datetime.utcnow().isoformat()
-                try:
-                    ga = docstore.create_git_action(resource_id)
-                except:
-                    m = (
-                        'Could not create an adaptor for git actions on amendment ID "{}". '
-                        "If you are confident that this is a valid amendment ID, please report this as a bug."
-                    )
-                    raise400(m.format(resource_id))
-                master_sha = ga.get_master_sha()
-                obj = {
-                    "date": timestamp,
-                    "amendment": resource_id,
-                    "commit": master_sha,
-                    "stacktrace": m,
-                }
-                api_utils.atomic_write_json_if_not_found(obj, fail_file, request)
-                _LOG.warning('push failure file "{f}" created.'.format(f=fail_file))
-            raise HTTPConflict(
-                body=json.dumps(
-                    {
-                        "error": 1,
-                        "description": "Could not push! Details: {m}".format(m=m),
-                    }
-                )
-            )
 
-    else:
-        raise400("Can't push unknown doc_type '{}'".format(doc_type))
+def raise_conflict(msg):
+    bd = {
+        "error": 1,
+        "description": msg,
+    }
+    raise HTTPConflict(body=json.dumps(bd))
+
+
+fail_file_lock = threading.Lock()
+
+
+def remove_fail_file(fail_file):
+    with fail_file_lock:
 
-    if os.path.exists(fail_file):
         # log any old fail_file, and remove it because the pushes are working
+        if not os.path.exists(fail_file):
+            return
+
         with codecs.open(fail_file, "rU", encoding="utf-8") as inpf:
             prev_fail = json.load(inpf)
         os.unlink(fail_file)
         fail_log_file = codecs.open(fail_file + ".log", mode="a", encoding="utf-8")
         json.dump(prev_fail, fail_log_file, indent=2, encoding="utf-8")
         fail_log_file.close()
-
-    return {"error": 0, "description": "Push succeeded"}
diff --git a/phylesystem_api/phylesystem_api/views/study.py b/phylesystem_api/phylesystem_api/views/study.py
index b5eec3fb..50475d3d 100644
--- a/phylesystem_api/phylesystem_api/views/study.py
+++ b/phylesystem_api/phylesystem_api/views/study.py
@@ -1,9 +1,7 @@
 import json
 import logging
 import sys
-import traceback
 
-import phylesystem_api.api_utils as api_utils
 import requests
 from peyotl.external import import_nexson_from_treebase
 from peyotl.nexson_syntax import (
@@ -16,15 +14,6 @@
     GitWorkflowError,
     validate_and_convert_nexson,
 )
-from phylesystem_api.api_utils import (
-    find_in_request,
-    raise400,
-    raise404,
-    fetch_doc,
-    commit_doc_and_trigger_push,
-    get_parent_sha,
-    get_commit_message,
-)
 from pyramid.encode import quote_plus, urlencode
 
 # see exception subclasses at https://docs.pylonsproject.org/projects/pyramid/en/latest/api/httpexceptions.html
@@ -35,6 +24,17 @@
 from pyramid.renderers import render_to_response
 from pyramid.view import view_config
 
+import phylesystem_api.api_utils as api_utils
+from phylesystem_api.api_utils import (
+    find_in_request,
+    raise400,
+    raise404,
+    fetch_doc,
+    commit_doc_and_trigger_push,
+    get_parent_sha,
+    get_commit_message,
+)
+
 _LOG = logging.getLogger("phylesystem_api")
 
 

From a601e671568f8dafb360b17e92760c4bfb1ee00f Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Tue, 1 Aug 2023 10:46:37 -0500
Subject: [PATCH 20/33] adding tests of all routes

---
 phylesystem_api/phylesystem_api/routes.py     |   3 +-
 .../phylesystem_api/views/default.py          |   2 +
 ws-tests/opentreetesting.py                   |   6 +-
 ws-tests/test_api_root.py                     |  21 ++-
 ws-tests/test_render_markdown.py              | 122 ++++++++++++++++++
 5 files changed, 148 insertions(+), 6 deletions(-)
 create mode 100644 ws-tests/test_render_markdown.py

diff --git a/phylesystem_api/phylesystem_api/routes.py b/phylesystem_api/phylesystem_api/routes.py
index 61899fb2..2f0347df 100644
--- a/phylesystem_api/phylesystem_api/routes.py
+++ b/phylesystem_api/phylesystem_api/routes.py
@@ -4,7 +4,8 @@ def includeme(config):
     config.add_route("index", "/")
     # some (unused) URLs just show a tiny API description, with links to code + docs
     config.add_route("api_root", "/")
-    config.add_route("api_version_root", "/{api_version}/")
+    config.add_route("api_version_root", "/{api_version}")
+    config.add_route("api_version_root_slash", "/{api_version}/")
     config.add_route("studies_root", "/{api_version}/studies")
     config.add_route("studies_root_slash", "/{api_version}/studies/")
     config.add_route("collections_root", "/{api_version}/collections")
diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py
index ac0ef8c8..3b8b288a 100644
--- a/phylesystem_api/phylesystem_api/views/default.py
+++ b/phylesystem_api/phylesystem_api/views/default.py
@@ -59,6 +59,7 @@ def home_view(request):
 
 @view_config(route_name="api_root", renderer="json", request_method="POST")
 @view_config(route_name="api_version_root", renderer="json")
+@view_config(route_name="api_version_root_slash", renderer="json")
 @view_config(route_name="studies_root", renderer="json")
 @view_config(route_name="studies_root_slash", renderer="json")
 @view_config(route_name="amendments_root", renderer="json")
@@ -241,6 +242,7 @@ def render_markdown(request):
         src = request.params.get("src", "")
     else:
         src = request.body
+    _LOG.debug("render_markdown src:\n{}".format(src))
     html = api_utils.markdown_to_html(src, open_links_in_new_window=True)
     return Response(body=html, content_type="text/html")
 
diff --git a/ws-tests/opentreetesting.py b/ws-tests/opentreetesting.py
index d8eab88f..f2ca5988 100644
--- a/ws-tests/opentreetesting.py
+++ b/ws-tests/opentreetesting.py
@@ -192,7 +192,11 @@ def test_http_json_method(
         )
 
     exps = resp.status_code == expected_status
-    if (not exps) and (isinstance(expected_status, list) or isinstance(expected_status, tuple) or isinstance(expected_status, set)):
+    if (not exps) and (
+        isinstance(expected_status, list)
+        or isinstance(expected_status, tuple)
+        or isinstance(expected_status, set)
+    ):
         exps = resp.status_code in expected_status
     if not exps:
         debug("Sent {v} to {s}".format(v=verb, s=resp.url))
diff --git a/ws-tests/test_api_root.py b/ws-tests/test_api_root.py
index b873cbd2..4f1749fc 100644
--- a/ws-tests/test_api_root.py
+++ b/ws-tests/test_api_root.py
@@ -3,7 +3,20 @@
 from opentreetesting import test_http_json_method, config
 
 DOMAIN = config("host", "apihost")
-SUBMIT_URI = "{d}/v3/studies".format(d=DOMAIN)
-if test_http_json_method(SUBMIT_URI, "GET", expected_status=200):
-    sys.exit(0)
-sys.exit(1)
+
+frag_list = [
+    "{d}/",
+    "{d}/v3",
+    "{d}/v3/",
+    "{d}/v3/studies",
+    "{d}/v3/studies/",
+    "{d}/v3/collections",
+    "{d}/v3/collections/",
+    "{d}/v3/amendments",
+    "{d}/v3/amendments/",
+]
+for frag in frag_list:
+    SUBMIT_URI = frag.format(d=DOMAIN)
+    print(SUBMIT_URI)
+    if not test_http_json_method(SUBMIT_URI, "GET", expected_status=200):
+        sys.exit(1)
diff --git a/ws-tests/test_render_markdown.py b/ws-tests/test_render_markdown.py
new file mode 100644
index 00000000..fcee3aac
--- /dev/null
+++ b/ws-tests/test_render_markdown.py
@@ -0,0 +1,122 @@
+#!/usr/bin/env python
+import sys, os
+from opentreetesting import test_http_json_method, config
+
+DOMAIN = config("host", "apihost")
+
+src = """[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+# The Open Tree Of Life API
+
+[![Build Status](https://secure.travis-ci.org/OpenTreeOfLife/api.opentreeoflife.org.png)](http://travis-ci.org/OpenTreeOfLife/api.opentreeoflife.org)
+
+This repository holds the code that runs The Open Tree Of Life API, which talks
+to the backend datastore [phylesystem](https://github.com/OpenTreeOfLife/phylesystem).
+
+# Introduction
+
+This repo uses `peyotl` to interact with a copy of the phylesystem data on a server.
+The code here provides a web services API to that data store. 
+The best description of the phylesystem is in [the published paper in Bioinformatics](http://bioinformatics.oxfordjournals.org/content/31/17/2794).
+
+See [docs/](https://github.com/OpenTreeOfLife/api.opentreeoflife.org/blob/master/docs/) for examples of how to use the API with ```curl```.
+
+
+# Installation
+
+There are a dependencies installable from pypi using pip, and the open
+tree of life client-side python library is also used on the server side
+for handling some aspects of NexSON.
+
+    $ pip install -r requirements.txt
+    $ cd ..
+
+The first time you run, you'll need to:
+
+    $ git clone https://github.com/OpenTreeOfLife/peyotl.git
+    $ cd peyotl
+    $ pip install -r requirements.txt
+    $ python setup.py develop
+
+Subsequently changing to the <code>peyotl</code> directory and running
+
+    $ git pull origin master
+
+should be sufficient to get the latest changes.
+
+# Configuration
+
+    $ cp private/config.exampl private/config
+
+then open private/config in a text editor and tweak it. 
+
+  * `repo_parent` should be a file path which holds 1 or more phyleystem-# repositories
+with the data.
+
+  * `git_ssh` and `pkey`
+
+
+### Logging configuration
+
+The behavior of the log for functions run from with a request is determined by the config
+file. Specifically, the 
+
+    [logging]
+    level = debug
+    filepath = /tmp/my-api.log
+    formatter = rich
+
+section of that file.
+
+If you are developer of the phylesystem-api, and you want to see logging for functions
+that are not called in the context of a request, you can use the environmental variables:
+
+  * `OT_API_LOG_FILE_PATH` filepath of log file (StreamHandler if omitted)
+  * `OT_API_LOGGING_LEVEL` (NotSet, debug, info, warning, error, or critical)
+  * `OT_API_LOGGING_FORMAT` "rich", "simple" or "None" (None is default)
+
+# Deploying
+
+This git repository is meant to be a "web2py application", so you need to
+create a symlink in ```$WEB2PY_ROOT/applications``` to the API repo directory:
+
+   cd $WEB2PY_ROOT/application
+   # this will make the app available under /api
+   ln -sf /dir/with/api.opentreeoflife.org api
+
+# Using the API from the command-line
+
+See [docs/](https://github.com/OpenTreeOfLife/api.opentreeoflife.org/blob/master/docs/) for examples of how to use the API with ```curl```.
+
+# Using the API from Python
+
+See [peyotl](https://github.com/OpenTreeOfLife/peyotl) has wrappers for accessing phylesystem web services.
+See the [peyotl wiki](https://github.com/OpenTreeOfLife/peyotl/wiki) for details.
+
+# Authors
+
+See the CREDITS file
+
+# Code formatting
+Recommended `.git/hooks/pre-commit`:
+
+    #!/bin/sh
+    if ! black --check phylesystem_api/phylesystem_api phylesystem_api/setup.py phylesystem_api/tests ; then
+        black phylesystem_api/phylesystem_api phylesystem_api/setup.py phylesystem_api/tests
+        echo "code reformatted. commit again!"
+        exit 1
+    fi
+
+"""
+SUBMIT_URI = "{d}/v3/render_markdown".format(d=DOMAIN)
+data = {"src": src}
+r = test_http_json_method(
+    SUBMIT_URI,
+    "GET",
+    data=data,
+    expected_status=200,
+    return_bool_data=True,
+    is_json=False,
+)
+if not r[0]:
+    sys.exit(1)
+print(r[1])

From 810af9f10987037feecea9b60d10d0622e90d476 Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Tue, 1 Aug 2023 10:54:05 -0500
Subject: [PATCH 21/33] removed debugging text

---
 phylesystem_api/phylesystem_api/views/default.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py
index 3b8b288a..f0ebdae1 100644
--- a/phylesystem_api/phylesystem_api/views/default.py
+++ b/phylesystem_api/phylesystem_api/views/default.py
@@ -242,7 +242,6 @@ def render_markdown(request):
         src = request.params.get("src", "")
     else:
         src = request.body
-    _LOG.debug("render_markdown src:\n{}".format(src))
     html = api_utils.markdown_to_html(src, open_links_in_new_window=True)
     return Response(body=html, content_type="text/html")
 

From a094490b1c623469f4d9d11c9e716c817fbbddd4 Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Tue, 1 Aug 2023 12:41:40 -0500
Subject: [PATCH 22/33] store_config tests

---
 phylesystem_api/phylesystem_api/api_utils.py |  2 +-
 ws-tests/test_amendment_config.py            | 24 +++++++++++++++-----
 2 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/phylesystem_api/phylesystem_api/api_utils.py b/phylesystem_api/phylesystem_api/api_utils.py
index abf9608f..fe24ad20 100644
--- a/phylesystem_api/phylesystem_api/api_utils.py
+++ b/phylesystem_api/phylesystem_api/api_utils.py
@@ -129,7 +129,7 @@ def get_docstore_from_type(doc_type_name, request, conf_obj=None):
     try:
         return _doc_type2store[doc_type_name]
     except:
-        raise400("Unrecognized document type {}".format(doc_type_name))
+        raise404("Unrecognized document type {}".format(doc_type_name))
 
 
 def get_config_impl(request, doc_type_name):
diff --git a/ws-tests/test_amendment_config.py b/ws-tests/test_amendment_config.py
index 479c6cec..7c56aa51 100644
--- a/ws-tests/test_amendment_config.py
+++ b/ws-tests/test_amendment_config.py
@@ -4,10 +4,22 @@
 
 DOMAIN = config("host", "apihost")
 SUBMIT_URI = DOMAIN + "/v3/amendments/store_config"
-# NB - This poorly-named methd returns a list of just their IDs
-r = test_http_json_method(SUBMIT_URI, "GET", expected_status=200, return_bool_data=True)
-if not r[0]:
+frag_list = [
+    "{d}/phylesystem_config",
+    "{d}/v3/studies/store_config",
+    "{d}/v3/amendments/store_config",
+    "{d}/v3/collections/store_config",
+]
+for frag in frag_list:
+    SUBMIT_URI = frag.format(d=DOMAIN)
+    r = test_http_json_method(
+        SUBMIT_URI, "GET", expected_status=200, return_bool_data=True
+    )
+    if not r[0]:
+        sys.exit(1)
+    assert len(r) == 2
+    assert isinstance(r[1], dict)
+
+SUBMIT_URI = "{d}/v3/bogus/store_config".format(d=DOMAIN)
+if not test_http_json_method(SUBMIT_URI, "GET", expected_status=404):
     sys.exit(1)
-assert len(r) == 2
-assert isinstance(r[1], dict)
-# print r[1]

From 2dea3f2ad9e0265a71ab73a9980164d1c448df79 Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Tue, 1 Aug 2023 12:42:03 -0500
Subject: [PATCH 23/33] better name

---
 ws-tests/{test_amendment_config.py => test_docstore_config.py} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename ws-tests/{test_amendment_config.py => test_docstore_config.py} (100%)

diff --git a/ws-tests/test_amendment_config.py b/ws-tests/test_docstore_config.py
similarity index 100%
rename from ws-tests/test_amendment_config.py
rename to ws-tests/test_docstore_config.py

From a6bd8bbe578288d7fc2d162ad29416ca73b6ac76 Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Tue, 1 Aug 2023 12:57:10 -0500
Subject: [PATCH 24/33] bug_fix in which we were capturing any one-word url to
 api_root

---
 phylesystem_api/phylesystem_api/routes.py     | 16 +++++------
 .../phylesystem_api/views/default.py          | 28 ++++++++++++++-----
 ws-tests/test_amendment_list.py               | 24 ++++++++++------
 ws-tests/test_docstore_config.py              |  1 -
 4 files changed, 45 insertions(+), 24 deletions(-)

diff --git a/phylesystem_api/phylesystem_api/routes.py b/phylesystem_api/phylesystem_api/routes.py
index 2f0347df..b66ce596 100644
--- a/phylesystem_api/phylesystem_api/routes.py
+++ b/phylesystem_api/phylesystem_api/routes.py
@@ -4,14 +4,14 @@ def includeme(config):
     config.add_route("index", "/")
     # some (unused) URLs just show a tiny API description, with links to code + docs
     config.add_route("api_root", "/")
-    config.add_route("api_version_root", "/{api_version}")
-    config.add_route("api_version_root_slash", "/{api_version}/")
-    config.add_route("studies_root", "/{api_version}/studies")
-    config.add_route("studies_root_slash", "/{api_version}/studies/")
-    config.add_route("collections_root", "/{api_version}/collections")
-    config.add_route("collections_root_slash", "/{api_version}/collections/")
-    config.add_route("amendments_root", "/{api_version}/amendments")
-    config.add_route("amendments_root_slash", "/{api_version}/amendments/")
+    config.add_route("api_version_root", "/v{api_version}")
+    config.add_route("api_version_root_slash", "/v{api_version}/")
+    config.add_route("studies_root", "/v{api_version}/studies")
+    config.add_route("studies_root_slash", "/v{api_version}/studies/")
+    config.add_route("collections_root", "/v{api_version}/collections")
+    config.add_route("collections_root_slash", "/v{api_version}/collections/")
+    config.add_route("amendments_root", "/v{api_version}/amendments")
+    config.add_route("amendments_root_slash", "/v{api_version}/amendments/")
     # utilities (generally for internal use, not documented)
     config.add_route("render_markdown", "/{api_version}/render_markdown")
     config.add_route("phylesystem_config", "/{api_version}/phylesystem_config")
diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py
index f0ebdae1..627847b1 100644
--- a/phylesystem_api/phylesystem_api/views/default.py
+++ b/phylesystem_api/phylesystem_api/views/default.py
@@ -57,7 +57,26 @@ def home_view(request):
     return {"title": "phylesystem API"}
 
 
+_DOC_DICT = {
+    "description": "The Open Tree API",
+    "documentation_url": "https://github.com/OpenTreeOfLife/phylesystem-api/tree/master/docs",
+    "source_url": "https://github.com/OpenTreeOfLife/phylesystem-api",
+}
+
+
 @view_config(route_name="api_root", renderer="json", request_method="POST")
+def base_API_view(request):
+    return _DOC_DICT
+
+
+def validate_api_version_number(arg):
+    try:
+        iarg = int(arg)
+        assert 0 < iarg < 4
+    except:
+        raise404("Unrecognized API version v{}".format(arg))
+
+
 @view_config(route_name="api_version_root", renderer="json")
 @view_config(route_name="api_version_root_slash", renderer="json")
 @view_config(route_name="studies_root", renderer="json")
@@ -68,13 +87,8 @@ def home_view(request):
 @view_config(route_name="collections_root_slash", renderer="json")
 def base_API_view(request):
     # a tiny JSON description of the API and where to find documentation
-    api_version = request.matchdict["api_version"]
-    # TODO: Modify URLs if they differ across API versions
-    return {
-        "description": "The Open Tree API {}".format(api_version),
-        "documentation_url": "https://github.com/OpenTreeOfLife/phylesystem-api/tree/master/docs",
-        "source_url": "https://github.com/OpenTreeOfLife/phylesystem-api",
-    }
+    validate_api_version_number(request.matchdict["api_version"])
+    return _DOC_DICT
 
 
 # Create a unique cache key with the URL and any vars (GET *and* POST) to its "query string"
diff --git a/ws-tests/test_amendment_list.py b/ws-tests/test_amendment_list.py
index 1c93f819..c7b7270c 100644
--- a/ws-tests/test_amendment_list.py
+++ b/ws-tests/test_amendment_list.py
@@ -3,11 +3,19 @@
 from opentreetesting import test_http_json_method, config
 
 DOMAIN = config("host", "apihost")
-SUBMIT_URI = DOMAIN + "/v3/amendments/list_all"
-# NB - This vaguely-named methd returns all amendments (complete JSON)
-r = test_http_json_method(SUBMIT_URI, "GET", expected_status=200, return_bool_data=True)
-if not r[0]:
-    sys.exit(1)
-assert len(r) == 2
-assert isinstance(r[1], list)
-# print r[1]
+frag_list = [
+    "{d}/v3/study_list",
+    "{d}/v3/amendments/amendment_list",
+    "{d}/v3/collections/collection_list",
+]
+for frag in frag_list:
+    SUBMIT_URI = frag.format(d=DOMAIN)
+    print(SUBMIT_URI)
+    r = test_http_json_method(
+        SUBMIT_URI, "GET", expected_status=200, return_bool_data=True
+    )
+    if not r[0]:
+        sys.exit(1)
+    print(repr(r[1]))
+    assert len(r) == 2
+    assert isinstance(r[1], list)
diff --git a/ws-tests/test_docstore_config.py b/ws-tests/test_docstore_config.py
index 7c56aa51..62d17b2c 100644
--- a/ws-tests/test_docstore_config.py
+++ b/ws-tests/test_docstore_config.py
@@ -3,7 +3,6 @@
 from opentreetesting import test_http_json_method, config
 
 DOMAIN = config("host", "apihost")
-SUBMIT_URI = DOMAIN + "/v3/amendments/store_config"
 frag_list = [
     "{d}/phylesystem_config",
     "{d}/v3/studies/store_config",

From e4a8f42ad0980a9fdb76995ab7f594f9b75a3a61 Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Tue, 1 Aug 2023 13:10:39 -0500
Subject: [PATCH 25/33] generic doc listing and v{api_version} in routes

---
 phylesystem_api/phylesystem_api/routes.py     | 106 +++++++++---------
 .../phylesystem_api/views/default.py          |  15 ++-
 ws-tests/test_amendment_list.py               |   4 +
 3 files changed, 70 insertions(+), 55 deletions(-)

diff --git a/phylesystem_api/phylesystem_api/routes.py b/phylesystem_api/phylesystem_api/routes.py
index b66ce596..8a08847e 100644
--- a/phylesystem_api/phylesystem_api/routes.py
+++ b/phylesystem_api/phylesystem_api/routes.py
@@ -13,154 +13,156 @@ def includeme(config):
     config.add_route("amendments_root", "/v{api_version}/amendments")
     config.add_route("amendments_root_slash", "/v{api_version}/amendments/")
     # utilities (generally for internal use, not documented)
-    config.add_route("render_markdown", "/{api_version}/render_markdown")
-    config.add_route("phylesystem_config", "/{api_version}/phylesystem_config")
-    config.add_route("raw_study_list", "/{api_version}/study_list")
+    config.add_route("render_markdown", "/v{api_version}/render_markdown")
+    config.add_route("phylesystem_config", "/v{api_version}/phylesystem_config")
+    config.add_route("raw_study_list", "/v{api_version}/study_list")
     config.add_route("pull_through_cache", "/cached/{target_url:.*}")
     config.add_route("clear_cache_keys", "/clear_cache_keys/{key_pattern:.*}")
     config.add_route(
         "merge_docstore_changes",
-        "/{api_version}/merge_docstore_changes/{doc_id}/{starting_commit_SHA}",
+        "/v{api_version}/merge_docstore_changes/{doc_id}/{starting_commit_SHA}",
     )
     config.add_route(
         "push_docstore_changes",
-        "/{api_version}/push_docstore_changes/{doc_type}/{doc_id}",
+        "/v{api_version}/push_docstore_changes/{doc_type}/{doc_id}",
     )
     config.add_route(
         "push_docstore_changes_bare",
-        "/{api_version}/push_docstore_changes/{doc_type}",
+        "/v{api_version}/push_docstore_changes/{doc_type}",
         request_method="PUT",
     )
     #
     # STUDY/TREE ROUTES
     #
-    config.add_route("study_properties", "/{api_version}/studies/properties")
-    config.add_route("find_studies", "/{api_version}/studies/find_studies")
-    config.add_route("find_trees", "/{api_version}/studies/find_trees")
-    config.add_route("trees_in_synth", "/{api_version}/trees_in_synth")
-    config.add_route("include_tree_in_synth", "/{api_version}/include_tree_in_synth")
+    config.add_route("study_properties", "/v{api_version}/studies/properties")
+    config.add_route("conventional_study_list", "/v{api_version}/studies/study_list")
+
+    config.add_route("find_studies", "/v{api_version}/studies/find_studies")
+    config.add_route("find_trees", "/v{api_version}/studies/find_trees")
+    config.add_route("trees_in_synth", "/v{api_version}/trees_in_synth")
+    config.add_route("include_tree_in_synth", "/v{api_version}/include_tree_in_synth")
     config.add_route(
-        "exclude_tree_from_synth", "/{api_version}/exclude_tree_from_synth"
+        "exclude_tree_from_synth", "/v{api_version}/exclude_tree_from_synth"
     )
-    config.add_route("create_study", "/{api_version}/study/")
+    config.add_route("create_study", "/v{api_version}/study/")
     config.add_route(
         "study_CORS_preflight",
-        "/{api_version}/study/{study_id}",
+        "/v{api_version}/study/{study_id}",
         request_method="OPTIONS",
     )
     config.add_route(
-        "fetch_study", "/{api_version}/study/{study_id}", request_method="GET"
+        "fetch_study", "/v{api_version}/study/{study_id}", request_method="GET"
     )
     config.add_route(
-        "fetch_study_label", "/{api_version}/study/{study_id}/", request_method="GET"
+        "fetch_study_label", "/v{api_version}/study/{study_id}/", request_method="GET"
     )
     config.add_route(
-        "update_study", "/{api_version}/study/{study_id}", request_method="PUT"
+        "update_study", "/v{api_version}/study/{study_id}", request_method="PUT"
     )
     config.add_route(
-        "delete_study", "/{api_version}/study/{study_id}", request_method="DELETE"
+        "delete_study", "/v{api_version}/study/{study_id}", request_method="DELETE"
     )
-    config.add_route("get_study_file_list", "/{api_version}/study/{study_id}/file")
+    config.add_route("get_study_file_list", "/v{api_version}/study/{study_id}/file")
     config.add_route(
-        "get_study_single_file", "/{api_version}/study/{study_id}/file/{file_id}"
+        "get_study_single_file", "/v{api_version}/study/{study_id}/file/{file_id}"
     )
     config.add_route(
-        "get_study_external_url", "/{api_version}/study/external_url/{study_id}"
+        "get_study_external_url", "/v{api_version}/study/external_url/{study_id}"
     )
     config.add_route(
         "get_study_tree",
-        "/{api_version}/study/{study_id}/tree/{tree_id_with_extension}",
+        "/v{api_version}/study/{study_id}/tree/{tree_id_with_extension}",
     )
     config.add_route(
         "get_study_tree_label",
-        "/{api_version}/study/{study_id}/tree/{tree_id_with_extension}/",
+        "/v{api_version}/study/{study_id}/tree/{tree_id_with_extension}/",
     )
     #
     # STUDY/OTHER FINE-GRAINED ACCESS ROUTES
     #
     config.add_route(
-        "get_study_otu_by_id", "/{api_version}/study/{study_id}/otu/{otu_id}"
+        "get_study_otu_by_id", "/v{api_version}/study/{study_id}/otu/{otu_id}"
     )
     config.add_route(
-        "get_study_otu_by_id_slash", "/{api_version}/study/{study_id}/otu/{otu_id}/"
+        "get_study_otu_by_id_slash", "/v{api_version}/study/{study_id}/otu/{otu_id}/"
     )
-    config.add_route("get_study_otu", "/{api_version}/study/{study_id}/otu")
-    config.add_route("get_study_otu_slash", "/{api_version}/study/{study_id}/otu/")
-    config.add_route("get_study_otus", "/{api_version}/study/{study_id}/otus")
-    config.add_route("get_study_otus_slash", "/{api_version}/study/{study_id}/otus/")
-    config.add_route("get_study_otumap", "/{api_version}/study/{study_id}/otumap")
+    config.add_route("get_study_otu", "/v{api_version}/study/{study_id}/otu")
+    config.add_route("get_study_otu_slash", "/v{api_version}/study/{study_id}/otu/")
+    config.add_route("get_study_otus", "/v{api_version}/study/{study_id}/otus")
+    config.add_route("get_study_otus_slash", "/v{api_version}/study/{study_id}/otus/")
+    config.add_route("get_study_otumap", "/v{api_version}/study/{study_id}/otumap")
     config.add_route(
-        "get_study_otumap_slash", "/{api_version}/study/{study_id}/otumap/"
+        "get_study_otumap_slash", "/v{api_version}/study/{study_id}/otumap/"
     )
-    config.add_route("get_study_meta", "/{api_version}/study/{study_id}/meta")
-    config.add_route("get_study_meta_slash", "/{api_version}/study/{study_id}/meta/")
+    config.add_route("get_study_meta", "/v{api_version}/study/{study_id}/meta")
+    config.add_route("get_study_meta_slash", "/v{api_version}/study/{study_id}/meta/")
     #
     # GENERIC routes
     #
     config.add_route(
-        "get_docstore_config", "/{api_version}/{doc_type_name}/store_config"
+        "get_docstore_config", "/v{api_version}/{doc_type_name}/store_config"
     )
     config.add_route(
-        "docstore_push_failure", "/{api_version}/{doc_type_name}/push_failure"
+        "docstore_push_failure", "/v{api_version}/{doc_type_name}/push_failure"
     )
-
+    config.add_route("doc_list", "/v{api_version}/{doc_type_name}/doc_list")
     #
     # TREE COLLECTION ROUTES
     #
     config.add_route(
-        "list_all_collection_ids", "/{api_version}/collections/collection_list"
+        "list_all_collection_ids", "/v{api_version}/collections/collection_list"
     )
-    config.add_route("collection_properties", "/{api_version}/collections/properties")
+    config.add_route("collection_properties", "/v{api_version}/collections/properties")
     config.add_route(
-        "find_trees_in_collections", "/{api_version}/collections/find_trees"
+        "find_trees_in_collections", "/v{api_version}/collections/find_trees"
     )
-    config.add_route("find_collections", "/{api_version}/collections/find_collections")
-    config.add_route("create_collection", "/{api_version}/collection/")
+    config.add_route("find_collections", "/v{api_version}/collections/find_collections")
+    config.add_route("create_collection", "/v{api_version}/collection/")
     config.add_route(
         "collection_CORS_preflight",
-        "/{api_version}/collection/{collection_id:.*}",
+        "/v{api_version}/collection/{collection_id:.*}",
         request_method="OPTIONS",
     )
     config.add_route(
         "fetch_collection",
-        "/{api_version}/collection/{collection_id:.*}",
+        "/v{api_version}/collection/{collection_id:.*}",
         request_method="GET",
     )
     config.add_route(
         "update_collection",
-        "/{api_version}/collection/{collection_id:.*}",
+        "/v{api_version}/collection/{collection_id:.*}",
         request_method="PUT",
     )
     config.add_route(
         "delete_collection",
-        "/{api_version}/collection/{collection_id:.*}",
+        "/v{api_version}/collection/{collection_id:.*}",
         request_method="DELETE",
     )
     #
     # TAXONOMIC AMENDMENT ROUTES
     #
     config.add_route(
-        "list_all_amendment_ids", "/{api_version}/amendments/amendment_list"
+        "list_all_amendment_ids", "/v{api_version}/amendments/amendment_list"
     )
-    config.add_route("list_all_amendments", "/{api_version}/amendments/list_all")
-    config.add_route("create_amendment", "/{api_version}/amendment/")
+    config.add_route("list_all_amendments", "/v{api_version}/amendments/list_all")
+    config.add_route("create_amendment", "/v{api_version}/amendment/")
     config.add_route(
         "amendment_CORS_preflight",
-        "/{api_version}/amendment/{amendment_id}",
+        "/v{api_version}/amendment/{amendment_id}",
         request_method="OPTIONS",
     )
     config.add_route(
         "fetch_amendment",
-        "/{api_version}/amendment/{amendment_id}",
+        "/v{api_version}/amendment/{amendment_id}",
         request_method="GET",
     )
     config.add_route(
         "update_amendment",
-        "/{api_version}/amendment/{amendment_id}",
+        "/v{api_version}/amendment/{amendment_id}",
         request_method="PUT",
     )
     config.add_route(
         "delete_amendment",
-        "/{api_version}/amendment/{amendment_id}",
+        "/v{api_version}/amendment/{amendment_id}",
         request_method="DELETE",
     )
diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py
index 627847b1..f09c9bfc 100644
--- a/phylesystem_api/phylesystem_api/views/default.py
+++ b/phylesystem_api/phylesystem_api/views/default.py
@@ -267,9 +267,17 @@ def study_list(request):
     return studies
 
 
-@view_config(route_name="get_docstore_config", renderer="json")
-def get_docstore_config(request):
-    return get_config_impl(request, request.matchdict["doc_type_name"])
+@view_config(route_name="conventional_study_list", renderer="json")
+def conventional_study_list(request):
+    validate_api_version_number(request.matchdict["api_version"])
+    return study_list(request)
+
+
+@view_config(route_name="doc_list", renderer="json")
+def doc_list(request):
+    validate_api_version_number(request.matchdict["api_version"])
+    docstore = get_docstore_from_type(request.matchdict["doc_type_name"], request)
+    return docstore.get_doc_ids()
 
 
 @view_config(route_name="docstore_push_failure", renderer="json")
@@ -292,6 +300,7 @@ def docstore_push_failure(request):
 
 @view_config(route_name="phylesystem_config", renderer="json")
 def phylesystem_config(request):
+    validate_api_version_number(request.matchdict["api_version"])
     return get_docstore_config(request, "studies")
 
 
diff --git a/ws-tests/test_amendment_list.py b/ws-tests/test_amendment_list.py
index c7b7270c..8c26aba5 100644
--- a/ws-tests/test_amendment_list.py
+++ b/ws-tests/test_amendment_list.py
@@ -7,6 +7,10 @@
     "{d}/v3/study_list",
     "{d}/v3/amendments/amendment_list",
     "{d}/v3/collections/collection_list",
+    "{d}/v3/studies/study_list",
+    "{d}/v3/amendments/doc_list",
+    "{d}/v3/collections/doc_list",
+    "{d}/v3/studies/doc_list",
 ]
 for frag in frag_list:
     SUBMIT_URI = frag.format(d=DOMAIN)

From 565dfdd1447598b3b1ef172d019ad50a7d69564c Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Tue, 1 Aug 2023 16:24:27 -0500
Subject: [PATCH 26/33] narrowing pattern matching on routes

---
 phylesystem_api/phylesystem_api/routes.py | 145 +++++++++++++---------
 1 file changed, 84 insertions(+), 61 deletions(-)

diff --git a/phylesystem_api/phylesystem_api/routes.py b/phylesystem_api/phylesystem_api/routes.py
index 8a08847e..2fff6fff 100644
--- a/phylesystem_api/phylesystem_api/routes.py
+++ b/phylesystem_api/phylesystem_api/routes.py
@@ -4,165 +4,188 @@ def includeme(config):
     config.add_route("index", "/")
     # some (unused) URLs just show a tiny API description, with links to code + docs
     config.add_route("api_root", "/")
-    config.add_route("api_version_root", "/v{api_version}")
-    config.add_route("api_version_root_slash", "/v{api_version}/")
-    config.add_route("studies_root", "/v{api_version}/studies")
-    config.add_route("studies_root_slash", "/v{api_version}/studies/")
-    config.add_route("collections_root", "/v{api_version}/collections")
-    config.add_route("collections_root_slash", "/v{api_version}/collections/")
-    config.add_route("amendments_root", "/v{api_version}/amendments")
-    config.add_route("amendments_root_slash", "/v{api_version}/amendments/")
+    config.add_route("api_version_root", "/v{api_version:[123]}")
+    config.add_route("api_version_root_slash", "/v{api_version:[123]}/")
+    config.add_route("studies_root", "/v{api_version:[123]}/studies")
+    config.add_route("studies_root_slash", "/v{api_version:[123]}/studies/")
+    config.add_route("collections_root", "/v{api_version:[123]}/collections")
+    config.add_route("collections_root_slash", "/v{api_version:[123]}/collections/")
+    config.add_route("amendments_root", "/v{api_version:[123]}/amendments")
+    config.add_route("amendments_root_slash", "/v{api_version:[123]}/amendments/")
     # utilities (generally for internal use, not documented)
-    config.add_route("render_markdown", "/v{api_version}/render_markdown")
-    config.add_route("phylesystem_config", "/v{api_version}/phylesystem_config")
-    config.add_route("raw_study_list", "/v{api_version}/study_list")
+    config.add_route("render_markdown", "/v{api_version:[123]}/render_markdown")
+    config.add_route("phylesystem_config", "/v{api_version:[123]}/phylesystem_config")
+    config.add_route("raw_study_list", "/v{api_version:[123]}/study_list")
     config.add_route("pull_through_cache", "/cached/{target_url:.*}")
     config.add_route("clear_cache_keys", "/clear_cache_keys/{key_pattern:.*}")
     config.add_route(
         "merge_docstore_changes",
-        "/v{api_version}/merge_docstore_changes/{doc_id}/{starting_commit_SHA}",
+        "/v{api_version:[123]}/merge_docstore_changes/{doc_id}/{starting_commit_SHA}",
     )
     config.add_route(
         "push_docstore_changes",
-        "/v{api_version}/push_docstore_changes/{doc_type}/{doc_id}",
+        "/v{api_version:[123]}/push_docstore_changes/{doc_type}/{doc_id}",
     )
     config.add_route(
         "push_docstore_changes_bare",
-        "/v{api_version}/push_docstore_changes/{doc_type}",
+        "/v{api_version:[123]}/push_docstore_changes/{doc_type}",
         request_method="PUT",
     )
     #
     # STUDY/TREE ROUTES
     #
-    config.add_route("study_properties", "/v{api_version}/studies/properties")
-    config.add_route("conventional_study_list", "/v{api_version}/studies/study_list")
+    config.add_route("study_properties", "/v{api_version:[123]}/studies/properties")
+    config.add_route(
+        "conventional_study_list", "/v{api_version:[123]}/studies/study_list"
+    )
 
-    config.add_route("find_studies", "/v{api_version}/studies/find_studies")
-    config.add_route("find_trees", "/v{api_version}/studies/find_trees")
-    config.add_route("trees_in_synth", "/v{api_version}/trees_in_synth")
-    config.add_route("include_tree_in_synth", "/v{api_version}/include_tree_in_synth")
+    config.add_route("find_studies", "/v{api_version:[123]}/studies/find_studies")
+    config.add_route("find_trees", "/v{api_version:[123]}/studies/find_trees")
+    config.add_route("trees_in_synth", "/v{api_version:[123]}/trees_in_synth")
     config.add_route(
-        "exclude_tree_from_synth", "/v{api_version}/exclude_tree_from_synth"
+        "include_tree_in_synth", "/v{api_version:[123]}/include_tree_in_synth"
     )
-    config.add_route("create_study", "/v{api_version}/study/")
+    config.add_route(
+        "exclude_tree_from_synth", "/v{api_version:[123]}/exclude_tree_from_synth"
+    )
+    config.add_route("create_study", "/v{api_version:[123]}/study/")
     config.add_route(
         "study_CORS_preflight",
-        "/v{api_version}/study/{study_id}",
+        "/v{api_version:[123]}/study/{study_id}",
         request_method="OPTIONS",
     )
     config.add_route(
-        "fetch_study", "/v{api_version}/study/{study_id}", request_method="GET"
+        "fetch_study", "/v{api_version:[123]}/study/{study_id}", request_method="GET"
+    )
+    config.add_route(
+        "fetch_study_label",
+        "/v{api_version:[123]}/study/{study_id}/",
+        request_method="GET",
     )
     config.add_route(
-        "fetch_study_label", "/v{api_version}/study/{study_id}/", request_method="GET"
+        "update_study", "/v{api_version:[123]}/study/{study_id}", request_method="PUT"
     )
     config.add_route(
-        "update_study", "/v{api_version}/study/{study_id}", request_method="PUT"
+        "delete_study",
+        "/v{api_version:[123]}/study/{study_id}",
+        request_method="DELETE",
     )
     config.add_route(
-        "delete_study", "/v{api_version}/study/{study_id}", request_method="DELETE"
+        "get_study_file_list", "/v{api_version:[123]}/study/{study_id}/file"
     )
-    config.add_route("get_study_file_list", "/v{api_version}/study/{study_id}/file")
     config.add_route(
-        "get_study_single_file", "/v{api_version}/study/{study_id}/file/{file_id}"
+        "get_study_single_file", "/v{api_version:[123]}/study/{study_id}/file/{file_id}"
     )
     config.add_route(
-        "get_study_external_url", "/v{api_version}/study/external_url/{study_id}"
+        "get_study_external_url", "/v{api_version:[123]}/study/external_url/{study_id}"
     )
     config.add_route(
         "get_study_tree",
-        "/v{api_version}/study/{study_id}/tree/{tree_id_with_extension}",
+        "/v{api_version:[123]}/study/{study_id}/tree/{tree_id_with_extension}",
     )
     config.add_route(
         "get_study_tree_label",
-        "/v{api_version}/study/{study_id}/tree/{tree_id_with_extension}/",
+        "/v{api_version:[123]}/study/{study_id}/tree/{tree_id_with_extension}/",
     )
     #
     # STUDY/OTHER FINE-GRAINED ACCESS ROUTES
     #
     config.add_route(
-        "get_study_otu_by_id", "/v{api_version}/study/{study_id}/otu/{otu_id}"
+        "get_study_otu_by_id", "/v{api_version:[123]}/study/{study_id}/otu/{otu_id}"
+    )
+    config.add_route(
+        "get_study_otu_by_id_slash",
+        "/v{api_version:[123]}/study/{study_id}/otu/{otu_id}/",
     )
+    config.add_route("get_study_otu", "/v{api_version:[123]}/study/{study_id}/otu")
     config.add_route(
-        "get_study_otu_by_id_slash", "/v{api_version}/study/{study_id}/otu/{otu_id}/"
+        "get_study_otu_slash", "/v{api_version:[123]}/study/{study_id}/otu/"
     )
-    config.add_route("get_study_otu", "/v{api_version}/study/{study_id}/otu")
-    config.add_route("get_study_otu_slash", "/v{api_version}/study/{study_id}/otu/")
-    config.add_route("get_study_otus", "/v{api_version}/study/{study_id}/otus")
-    config.add_route("get_study_otus_slash", "/v{api_version}/study/{study_id}/otus/")
-    config.add_route("get_study_otumap", "/v{api_version}/study/{study_id}/otumap")
+    config.add_route("get_study_otus", "/v{api_version:[123]}/study/{study_id}/otus")
     config.add_route(
-        "get_study_otumap_slash", "/v{api_version}/study/{study_id}/otumap/"
+        "get_study_otus_slash", "/v{api_version:[123]}/study/{study_id}/otus/"
+    )
+    config.add_route(
+        "get_study_otumap", "/v{api_version:[123]}/study/{study_id}/otumap"
+    )
+    config.add_route(
+        "get_study_otumap_slash", "/v{api_version:[123]}/study/{study_id}/otumap/"
+    )
+    config.add_route("get_study_meta", "/v{api_version:[123]}/study/{study_id}/meta")
+    config.add_route(
+        "get_study_meta_slash", "/v{api_version:[123]}/study/{study_id}/meta/"
     )
-    config.add_route("get_study_meta", "/v{api_version}/study/{study_id}/meta")
-    config.add_route("get_study_meta_slash", "/v{api_version}/study/{study_id}/meta/")
     #
     # GENERIC routes
     #
     config.add_route(
-        "get_docstore_config", "/v{api_version}/{doc_type_name}/store_config"
+        "get_docstore_config", "/v{api_version:[123]}/{doc_type_name}/store_config"
     )
     config.add_route(
-        "docstore_push_failure", "/v{api_version}/{doc_type_name}/push_failure"
+        "docstore_push_failure", "/v{api_version:[123]}/{doc_type_name}/push_failure"
     )
-    config.add_route("doc_list", "/v{api_version}/{doc_type_name}/doc_list")
+    config.add_route("doc_list", "/v{api_version:[123]}/{doc_type_name}/doc_list")
     #
     # TREE COLLECTION ROUTES
     #
     config.add_route(
-        "list_all_collection_ids", "/v{api_version}/collections/collection_list"
+        "list_all_collection_ids", "/v{api_version:[123]}/collections/collection_list"
+    )
+    config.add_route(
+        "collection_properties", "/v{api_version:[123]}/collections/properties"
+    )
+    config.add_route(
+        "find_trees_in_collections", "/v{api_version:[123]}/collections/find_trees"
     )
-    config.add_route("collection_properties", "/v{api_version}/collections/properties")
     config.add_route(
-        "find_trees_in_collections", "/v{api_version}/collections/find_trees"
+        "find_collections", "/v{api_version:[123]}/collections/find_collections"
     )
-    config.add_route("find_collections", "/v{api_version}/collections/find_collections")
-    config.add_route("create_collection", "/v{api_version}/collection/")
+    config.add_route("create_collection", "/v{api_version:[123]}/collection/")
     config.add_route(
         "collection_CORS_preflight",
-        "/v{api_version}/collection/{collection_id:.*}",
+        "/v{api_version:[123]}/collection/{collection_id:.*}",
         request_method="OPTIONS",
     )
     config.add_route(
         "fetch_collection",
-        "/v{api_version}/collection/{collection_id:.*}",
+        "/v{api_version:[123]}/collection/{collection_id:.*}",
         request_method="GET",
     )
     config.add_route(
         "update_collection",
-        "/v{api_version}/collection/{collection_id:.*}",
+        "/v{api_version:[123]}/collection/{collection_id:.*}",
         request_method="PUT",
     )
     config.add_route(
         "delete_collection",
-        "/v{api_version}/collection/{collection_id:.*}",
+        "/v{api_version:[123]}/collection/{collection_id:.*}",
         request_method="DELETE",
     )
     #
     # TAXONOMIC AMENDMENT ROUTES
     #
     config.add_route(
-        "list_all_amendment_ids", "/v{api_version}/amendments/amendment_list"
+        "list_all_amendment_ids", "/v{api_version:[123]}/amendments/amendment_list"
     )
-    config.add_route("list_all_amendments", "/v{api_version}/amendments/list_all")
-    config.add_route("create_amendment", "/v{api_version}/amendment/")
+    config.add_route("list_all_amendments", "/v{api_version:[123]}/amendments/list_all")
+    config.add_route("create_amendment", "/v{api_version:[123]}/amendment/")
     config.add_route(
         "amendment_CORS_preflight",
-        "/v{api_version}/amendment/{amendment_id}",
+        "/v{api_version:[123]}/amendment/{amendment_id}",
         request_method="OPTIONS",
     )
     config.add_route(
         "fetch_amendment",
-        "/v{api_version}/amendment/{amendment_id}",
+        "/v{api_version:[123]}/amendment/{amendment_id}",
         request_method="GET",
     )
     config.add_route(
         "update_amendment",
-        "/v{api_version}/amendment/{amendment_id}",
+        "/v{api_version:[123]}/amendment/{amendment_id}",
         request_method="PUT",
     )
     config.add_route(
         "delete_amendment",
-        "/v{api_version}/amendment/{amendment_id}",
+        "/v{api_version:[123]}/amendment/{amendment_id}",
         request_method="DELETE",
     )

From fc5ac4b68e00231ca33622bde557360d8cc3b3e3 Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Tue, 1 Aug 2023 16:38:48 -0500
Subject: [PATCH 27/33] minor

---
 phylesystem_api/phylesystem_api/routes.py     |  3 +-
 .../phylesystem_api/views/default.py          | 35 +++++++------------
 2 files changed, 14 insertions(+), 24 deletions(-)

diff --git a/phylesystem_api/phylesystem_api/routes.py b/phylesystem_api/phylesystem_api/routes.py
index 2fff6fff..2e6b0016 100644
--- a/phylesystem_api/phylesystem_api/routes.py
+++ b/phylesystem_api/phylesystem_api/routes.py
@@ -119,7 +119,8 @@ def includeme(config):
     # GENERIC routes
     #
     config.add_route(
-        "get_docstore_config", "/v{api_version:[123]}/{doc_type_name}/store_config"
+        "get_docstore_config",
+        "/v{api_version:[123]}/{doc_type_name}/store_config",
     )
     config.add_route(
         "docstore_push_failure", "/v{api_version:[123]}/{doc_type_name}/push_failure"
diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py
index f09c9bfc..6e0df14b 100644
--- a/phylesystem_api/phylesystem_api/views/default.py
+++ b/phylesystem_api/phylesystem_api/views/default.py
@@ -139,7 +139,6 @@ def fetch_and_cache(url):
         fetch_url = base_url + root_relative_url
         _LOG.warning("NOT CACHED, FETCHING THIS URL: {}".format(fetch_url))
         _LOG.warning("  request.method = {}".format(request.method))
-
         # modify or discard "hop-by-hop" headers
         for bad_header in hop_by_hop_headers:
             request.headers.pop(bad_header, None)
@@ -165,37 +164,27 @@ def fetch_and_cache(url):
                 _LOG.warning("  treating as GET")
                 fetched = requests.get(fetch_url)
             # TODO: For more flexibility, we might examine and mimic the original request (headers, etc)
-            _LOG.warning(
-                "... and now we're back with fetched, which is a {}".format(
-                    type(fetched)
-                )
-            )
+            msg = "... and now we're back with fetched, which is a {}"
+            msg = msg.format(type(fetched))
+            _LOG.warning(msg)
             fetched.raise_for_status()
             fetched.encoding = "utf-8"  # Optional: requests infers this internally
 
             # modify or discard "hop-by-hop" headers
             for bad_header in hop_by_hop_headers:
                 fetched.headers.pop(bad_header, None)
-            # _LOG.warning("  MODIFIED fetched.headers:")
-            # _LOG.warning( dict(fetched.headers) )
-
             try:
                 fetched.json()  # missing JSON payload will raise an error
-                return Response(
-                    headers=fetched.headers,
-                    body=fetched.text,  # missing JSON payload will raise an error
-                    status="200 OK",
-                    charset="UTF-8",
-                    content_type="application/json",
-                )
+                ct = "application/json"
             except requests.exceptions.JSONDecodeError:
-                return Response(
-                    headers=fetched.headers,
-                    body=fetched.text,
-                    status="200 OK",
-                    charset="UTF-8",
-                    content_type="text/plain",
-                )
+                ct = "text/plain"
+            return Response(
+                headers=fetched.headers,
+                body=fetched.text,  # missing JSON payload will raise an error
+                status="200 OK",
+                charset="UTF-8",
+                content_type=ct,
+            )
         except requests.RequestException as e:
             # throw an exception (hopefully copying its status code and message) so we don't poison the cache!
             # NB - We don't want to cache this response, but we DO want to return its payload

From 3bc6019681f674029e6bd1270784b0963eaf6652 Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Tue, 1 Aug 2023 16:41:36 -0500
Subject: [PATCH 28/33] route fixed given change in api_version

---
 phylesystem_api/phylesystem_api/api_utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/phylesystem_api/phylesystem_api/api_utils.py b/phylesystem_api/phylesystem_api/api_utils.py
index fe24ad20..bae10a02 100644
--- a/phylesystem_api/phylesystem_api/api_utils.py
+++ b/phylesystem_api/phylesystem_api/api_utils.py
@@ -276,10 +276,10 @@ def atomic_write_json_if_not_found(obj, dest, request):
 def compose_push_to_github_url(request, resource_id, doc_type):
     if resource_id is None:
         return request.route_url(
-            "push_docstore_changes_bare", api_version="v3", doc_type=doc_type
+            "push_docstore_changes_bare", api_version="3", doc_type=doc_type
         )
     return request.route_url(
-        "push_docstore_changes", api_version="v3", doc_type=doc_type, doc_id=resource_id
+        "push_docstore_changes", api_version="3", doc_type=doc_type, doc_id=resource_id
     )
 
 

From 868eedac6f80382c80a794f64da2c32592928208 Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Tue, 1 Aug 2023 16:50:43 -0500
Subject: [PATCH 29/33] bug fixes

---
 phylesystem_api/phylesystem_api/routes.py        |  3 +++
 phylesystem_api/phylesystem_api/views/default.py | 13 ++++++++++++-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/phylesystem_api/phylesystem_api/routes.py b/phylesystem_api/phylesystem_api/routes.py
index 2e6b0016..3274741f 100644
--- a/phylesystem_api/phylesystem_api/routes.py
+++ b/phylesystem_api/phylesystem_api/routes.py
@@ -15,6 +15,9 @@ def includeme(config):
     # utilities (generally for internal use, not documented)
     config.add_route("render_markdown", "/v{api_version:[123]}/render_markdown")
     config.add_route("phylesystem_config", "/v{api_version:[123]}/phylesystem_config")
+    config.add_route("phylesystem_config_no_vers", "/phylesystem_config")
+    config.add_route("phylesystem_config_no_vers_slash", "/phylesystem_config/")
+
     config.add_route("raw_study_list", "/v{api_version:[123]}/study_list")
     config.add_route("pull_through_cache", "/cached/{target_url:.*}")
     config.add_route("clear_cache_keys", "/clear_cache_keys/{key_pattern:.*}")
diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py
index 6e0df14b..2ca01d63 100644
--- a/phylesystem_api/phylesystem_api/views/default.py
+++ b/phylesystem_api/phylesystem_api/views/default.py
@@ -287,10 +287,21 @@ def docstore_push_failure(request):
     return blob
 
 
+@view_config(route_name="get_docstore_config", renderer="json")
+def get_docstore_config(request):
+    return get_config_impl(request, request.matchdict["doc_type_name"])
+
+
+@view_config(route_name="phylesystem_config_no_vers", renderer="json")
+@view_config(route_name="phylesystem_config_no_vers_slash", renderer="json")
+def phylesystem_config_no_vers(request):
+    return get_config_impl(request, "studies")
+
+
 @view_config(route_name="phylesystem_config", renderer="json")
 def phylesystem_config(request):
     validate_api_version_number(request.matchdict["api_version"])
-    return get_docstore_config(request, "studies")
+    return get_config_impl(request, "studies")
 
 
 @view_config(route_name="trees_in_synth", renderer="json")

From 85ce5e155ab605f5fbc395a2f3c13fe0429352f2 Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Tue, 1 Aug 2023 16:51:47 -0500
Subject: [PATCH 30/33] better name

---
 phylesystem_api/phylesystem_api/routes.py                  | 1 -
 ws-tests/{test_amendment_list.py => test_docstore_list.py} | 0
 2 files changed, 1 deletion(-)
 rename ws-tests/{test_amendment_list.py => test_docstore_list.py} (100%)

diff --git a/phylesystem_api/phylesystem_api/routes.py b/phylesystem_api/phylesystem_api/routes.py
index 3274741f..ae743bc1 100644
--- a/phylesystem_api/phylesystem_api/routes.py
+++ b/phylesystem_api/phylesystem_api/routes.py
@@ -17,7 +17,6 @@ def includeme(config):
     config.add_route("phylesystem_config", "/v{api_version:[123]}/phylesystem_config")
     config.add_route("phylesystem_config_no_vers", "/phylesystem_config")
     config.add_route("phylesystem_config_no_vers_slash", "/phylesystem_config/")
-
     config.add_route("raw_study_list", "/v{api_version:[123]}/study_list")
     config.add_route("pull_through_cache", "/cached/{target_url:.*}")
     config.add_route("clear_cache_keys", "/clear_cache_keys/{key_pattern:.*}")
diff --git a/ws-tests/test_amendment_list.py b/ws-tests/test_docstore_list.py
similarity index 100%
rename from ws-tests/test_amendment_list.py
rename to ws-tests/test_docstore_list.py

From b8175921398e48795626990cd04188e1856677d6 Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Wed, 2 Aug 2023 16:06:06 -0500
Subject: [PATCH 31/33] push_docstore_changes tests

---
 phylesystem_api/phylesystem_api/routes.py     |  6 +++++
 .../phylesystem_api/views/default.py          |  1 +
 ws-tests/test_push_docstore_changes.py        | 22 +++++++++++++++++++
 3 files changed, 29 insertions(+)
 create mode 100644 ws-tests/test_push_docstore_changes.py

diff --git a/phylesystem_api/phylesystem_api/routes.py b/phylesystem_api/phylesystem_api/routes.py
index ae743bc1..592dc5d5 100644
--- a/phylesystem_api/phylesystem_api/routes.py
+++ b/phylesystem_api/phylesystem_api/routes.py
@@ -33,6 +33,12 @@ def includeme(config):
         "/v{api_version:[123]}/push_docstore_changes/{doc_type}",
         request_method="PUT",
     )
+    config.add_route(
+        "push_docstore_changes_slash",
+        "/v{api_version:[123]}/push_docstore_changes/{doc_type}/",
+        request_method="PUT",
+    )
+
     #
     # STUDY/TREE ROUTES
     #
diff --git a/phylesystem_api/phylesystem_api/views/default.py b/phylesystem_api/phylesystem_api/views/default.py
index 2ca01d63..1835bba5 100644
--- a/phylesystem_api/phylesystem_api/views/default.py
+++ b/phylesystem_api/phylesystem_api/views/default.py
@@ -527,6 +527,7 @@ def merge_docstore_changes(request):
 
 @view_config(route_name="push_docstore_changes", renderer="json")
 @view_config(route_name="push_docstore_changes_bare", renderer="json")
+@view_config(route_name="push_docstore_changes_slash", renderer="json")
 def push_docstore_changes(request):
     """OpenTree API method to update branch on master
 
diff --git a/ws-tests/test_push_docstore_changes.py b/ws-tests/test_push_docstore_changes.py
new file mode 100644
index 00000000..3a7ae2be
--- /dev/null
+++ b/ws-tests/test_push_docstore_changes.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+from opentreetesting import test_http_json_method, writable_api_host_and_oauth_or_exit
+
+
+DOMAIN, auth_token = writable_api_host_and_oauth_or_exit(__file__)
+data = {"auth_token": auth_token}
+frag_list = [
+    "{d}/v3/push_docstore_changes/studies",
+    "{d}/v3/push_docstore_changes/amendments",
+    "{d}/v3/push_docstore_changes/collections",
+]
+for frag in frag_list:
+    SUBMIT_URI = frag.format(d=DOMAIN)
+    r = test_http_json_method(
+        SUBMIT_URI, "PUT", data=data, expected_status=200, return_bool_data=True
+    )
+    if not r[0]:
+        sys.exit(1)
+
+SUBMIT_URI = "{d}/v3/push_docstore_changes/studies".format(d=DOMAIN)
+if not test_http_json_method(SUBMIT_URI, "PUT", data={}, expected_status=400):
+    sys.exit(1)

From 31adaa49db4c31fdbc66e5713e3617feff74b189 Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Wed, 2 Aug 2023 17:02:27 -0500
Subject: [PATCH 32/33] import for failures

---
 ws-tests/test_push_docstore_changes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ws-tests/test_push_docstore_changes.py b/ws-tests/test_push_docstore_changes.py
index 3a7ae2be..72ccb9f9 100644
--- a/ws-tests/test_push_docstore_changes.py
+++ b/ws-tests/test_push_docstore_changes.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 from opentreetesting import test_http_json_method, writable_api_host_and_oauth_or_exit
-
+import sys
 
 DOMAIN, auth_token = writable_api_host_and_oauth_or_exit(__file__)
 data = {"auth_token": auth_token}

From 9dec603cd1ba56b122c0befd6f663fee6a1231dd Mon Sep 17 00:00:00 2001
From: "Mark T. Holder" <mtholder@gmail.com>
Date: Mon, 7 Aug 2023 14:39:55 -0500
Subject: [PATCH 33/33] need to accept all arguments used by
 commit_doc_and_trigger_push

---
 phylesystem_api/phylesystem_api/views/amendment.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/phylesystem_api/phylesystem_api/views/amendment.py b/phylesystem_api/phylesystem_api/views/amendment.py
index 6a005be7..4942ff74 100644
--- a/phylesystem_api/phylesystem_api/views/amendment.py
+++ b/phylesystem_api/phylesystem_api/views/amendment.py
@@ -98,7 +98,9 @@ def create_amendment(request, **kwargs):
     # caller. It will assign the new amendment id accordingly!
     docstore = api_utils.get_taxonomic_amendment_store(request)
 
-    def amend_commit_fn(doc, doc_id, auth_info, commit_msg):
+    def amend_commit_fn(
+        doc, doc_id, auth_info, parent_sha=None, merged_sha=None, commit_msg=None
+    ):
         return docstore.add_new_amendment(
             doc, auth_info=auth_info, commit_msg=commit_msg
         )