feat(unit_tests): new unit tests for lit_processing

agr_literature_service/lit_processing/data_ingest/pubmed_ingest/xml_to_json.py

@@ -155,7 +155,7 @@ def get_medline_date_from_xml_date(pub_date):
         return medline_re_output.group(1)
 
 
-def get_alliance_category_from_pubmed_types(pubmed_types):     # noqa: C901
+def get_alliance_category_from_pubmed_types(pubmed_types: List[str]):     # noqa: C901
 
     # for functional tests work
     mapping_path = path.dirname(path.abspath(__file__)) + "/data_for_pubmed_processing/"

cleanup.sh

@@ -1,2 +1,3 @@
 rm -rf agr_literature_service/lit_processing/data_ingest/tmp
-rm -rf agr_literature_service/lit_processing/tests/tmp
+rm -rf agr_literature_service/lit_processing/tests/tmp
+rm -rf tests/lit_processing/tmp

pyproject.toml

@@ -52,3 +52,4 @@ multi_line_output = 3
 
 [tool.pytest.ini_options]
 addopts = "--cov --cov-fail-under=80 -vv --cov-report html --ignore non_pr_tests/"
+markers = ["webtest"]

run_tests.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
 export ENV_STATE=test
 # pytest -vv
-pytest
+pytest -m "not webtest"
 

tests/fixtures.py

@@ -9,6 +9,7 @@
 
 def delete_all_table_content(engine):
     if environ.get('TEST_DATABASE_DELETE') == "true":
+        print("***** Deleting tables *****")
         for table in reversed(Base.metadata.sorted_tables):
             if table != "users":
                 engine.execute(table.delete())
@@ -22,5 +23,5 @@ def db() -> Session:
     delete_all_table_content(engine)
     db = sessionmaker(bind=engine, autoflush=True)()
     yield db
-    print("***** Deleting DB connection *****")
     delete_all_table_content(engine)
+    print("***** Deleting DB connection *****")

tests/lit_processing/data_ingest/pubmed_ingest/test_process_single_pmid.py

@@ -0,0 +1,17 @@
+import os
+from os import environ
+
+import pytest
+
+from agr_literature_service.lit_processing.data_ingest.pubmed_ingest.process_single_pmid import process_pmid
+from tests.utils import cleanup_tmp_files
+
+
+class TestProcessSinglePMID:
+
+    @pytest.mark.webtest
+    def test_process_pmid(self):
+        base_path = environ.get('XML_PATH')
+        process_pmid("PMID:12345")
+        assert os.path.exists(os.path.join(base_path, "pubmed_xml", "12345.xml"))
+        cleanup_tmp_files()

tests/lit_processing/data_ingest/pubmed_ingest/test_xml_to_json.py

@@ -0,0 +1,11 @@
+from agr_literature_service.lit_processing.data_ingest.pubmed_ingest.xml_to_json import \
+    get_alliance_category_from_pubmed_types
+
+
+class TestXmlToJson:
+
+    def test_get_alliance_category_from_pubmed_types(self):
+        assert get_alliance_category_from_pubmed_types(["Journal Article", "Research Support, N.I.H., Extramural",
+                                                        "Research Support, Non-U.S. Gov't"]) == "Research_Article"
+        assert get_alliance_category_from_pubmed_types(["Journal Article"]) == "Research_Article"
+        # TODO: add more cases

tests/lit_processing/test_data_export.py

@@ -0,0 +1,20 @@
+import pytest
+
+from agr_literature_service.lit_processing.tests.mod_populate_load import post_mods
+from ..fixtures import db # noqa
+
+
+@pytest.fixture
+def test_data(db):
+    print("***** Loading data into the DB *****")
+    print("***** Adding mods *****")
+    post_mods()
+    print("***** Adding references *****")
+    yield None
+    print("***** Cleaning up data *****")
+
+
+class TestDataExport:
+
+    def test_dump_data(self, test_data):
+        assert True

tests/utils.py

@@ -0,0 +1,7 @@
+import shutil
+from os import environ
+
+
+def cleanup_tmp_files():
+    base_path = environ.get('XML_PATH')
+    shutil.rmtree(base_path)