refactor(lit_processing): created new project structure and moved files

.gitignore

@@ -1,21 +1,7 @@
 __pycache__/
 Pipfile.lock
-junk
 htmlcov
-agr_literature_service/lit_processing/pubmed_json
-agr_literature_service/lit_processing/pubmed_xml
-agr_literature_service/lit_processing/reference_primary_id_to_curie
-src/xml_processing/resource_primary_id_to_curie
-src/xml_processing/junk
-src/xml_processing/past_runs
-agr_literature_service/lit_processing/inputs
-src/xml_processing/J_Medline.txt
-src/xml_processing/auth0_token
-agr_literature_service/lit_processing/okta_token
-src/xml_processing/pmids_by_mods
-agr_literature_service/lit_processing/pmids_not_found
-
-src/xml_processing/code/venv/
+agr_literature_service/lit_processing/data_ingest/tmp/*
 *.coverage
 *.coverage.*
 .idea/*

Makefile

@@ -70,7 +70,7 @@ run-functest:
 	sleep 5
 
 	# load the mods
-	docker-compose --env-file .env.test run test_runner python3 agr_literature_service/lit_processing/mod_populate_load.py
+	docker-compose --env-file .env.test run test_runner python3 agr_literature_service/lit_processing/tests/mod_populate_load.py
 
 	# load the data
 	docker-compose --env-file .env.test run test_runner /bin/bash agr_literature_service/lit_processing/sample_reference_populate_load.sh

agr_literature_service/api/routers/reference_router.py

@@ -15,8 +15,8 @@
 
 import logging
 
-from agr_literature_service.lit_processing.process_single_pmid import process_pmid
-from agr_literature_service.lit_processing.dump_json_data import dump_data
+from agr_literature_service.lit_processing.data_ingest.pubmed_ingest.process_single_pmid import process_pmid
+from agr_literature_service.lit_processing.data_export.export_single_mod_references_to_json import dump_data
 
 
 logger = logging.getLogger(__name__)

agr_literature_service/api/s3/download.py

@@ -2,7 +2,7 @@
 from fastapi import HTTPException
 from fastapi.encoders import jsonable_encoder
 from os import environ, getcwd, path
-from agr_literature_service.lit_processing.helper_s3 import download_file_from_s3
+from agr_literature_service.lit_processing.utils.s3_utils import download_file_from_s3
 from fastapi.responses import FileResponse
 from agr_literature_service.api.config import config
 

agr_literature_service/lit_processing/dump_all_json_data.py → agr_literature_service/lit_processing/data_export/export_all_mod_references_to_json.py

@@ -1,7 +1,7 @@
 import logging
 import time
 
-from agr_literature_service.lit_processing.dump_json_data import dump_data
+from agr_literature_service.lit_processing.data_export.export_single_mod_references_to_json import dump_data
 
 logging.basicConfig(format='%(message)s')
 log = logging.getLogger()

agr_literature_service/lit_processing/dump_json_data.py → agr_literature_service/lit_processing/data_export/export_single_mod_references_to_json.py

@@ -7,9 +7,9 @@
 import gzip
 import shutil
 
-from agr_literature_service.lit_processing.helper_sqlalchemy import create_postgres_engine
-from agr_literature_service.lit_processing.helper_s3 import upload_file_to_s3
-from agr_literature_service.lit_processing.helper_email import send_email
+from agr_literature_service.lit_processing.utils.sqlalchemy_utils import create_postgres_engine
+from agr_literature_service.lit_processing.utils.s3_utils import upload_file_to_s3
+from agr_literature_service.lit_processing.utils.email_utils import send_email
 
 logging.basicConfig(format='%(message)s')
 log = logging.getLogger()

agr_literature_service/lit_processing/parse_dqm_json_reference.py → agr_literature_service/lit_processing/data_ingest/dqm_ingest/parse_dqm_json_reference.py

@@ -1,6 +1,5 @@
 import argparse
 import json
-import logging
 import logging.config
 import re
 import sys
@@ -11,8 +10,8 @@
 import bs4
 from dotenv import load_dotenv
 
-from agr_literature_service.lit_processing.helper_file_processing import (clean_up_keywords, split_identifier,
-                                                                          write_json)
+from agr_literature_service.lit_processing.utils.file_processing_utils import (clean_up_keywords, split_identifier,
+                                                                               write_json)
 
 warnings.filterwarnings("ignore", category=UserWarning, module='bs4')
 

agr_literature_service/lit_processing/parse_dqm_json_resource.py → agr_literature_service/lit_processing/data_ingest/dqm_ingest/parse_dqm_json_resource.py

@@ -1,19 +1,18 @@
 
 import json
-import logging
 import logging.config
 import re
 from os import environ, makedirs, path
 
 from dotenv import load_dotenv
 
-from agr_literature_service.lit_processing.helper_file_processing import (load_pubmed_resource_basic,
-                                                                          save_resource_file, split_identifier,
-                                                                          write_json)
+from agr_literature_service.lit_processing.utils.file_processing_utils import (load_pubmed_resource_basic,
+                                                                               save_resource_file, split_identifier,
+                                                                               write_json)
 
 load_dotenv()
 
-log_file_path = path.join(path.dirname(path.abspath(__file__)), '../../logging.conf')
+log_file_path = path.join(path.dirname(path.abspath(__file__)), '../../../../logging.conf')
 logging.config.fileConfig(log_file_path)
 logger = logging.getLogger('literature logger')
 

agr_literature_service/lit_processing/sort_dqm_json_reference_updates.py → agr_literature_service/lit_processing/data_ingest/dqm_ingest/sort_dqm_json_reference_updates.py

@@ -1,29 +1,28 @@
 import argparse
 import json
 import sys
-import logging
 import logging.config
 import warnings
 from os import environ, makedirs, path
 from dotenv import load_dotenv
 from fastapi.encoders import jsonable_encoder
 
-from agr_literature_service.lit_processing.filter_dqm_md5sum import load_s3_md5data,\
+from agr_literature_service.lit_processing.data_ingest.dqm_ingest.utils.md5sum_utils import load_s3_md5data,\
     generate_new_md5, save_s3_md5data
 from agr_literature_service.api.models import ReferenceModel, ModReferenceTypeModel,\
     ModCorpusAssociationModel, AuthorModel, CrossReferenceModel, ModModel
-from agr_literature_service.lit_processing.helper_file_processing import compare_authors_or_editors,\
+from agr_literature_service.lit_processing.utils.file_processing_utils import compare_authors_or_editors,\
     split_identifier, write_json
-from agr_literature_service.lit_processing.helper_sqlalchemy import create_postgres_session,\
+from agr_literature_service.lit_processing.utils.sqlalchemy_utils import create_postgres_session,\
     create_postgres_engine, sqlalchemy_load_ref_xref
-from agr_literature_service.lit_processing.helper_email import send_email
-from agr_literature_service.lit_processing.parse_dqm_json_reference import generate_pmid_data,\
+from agr_literature_service.lit_processing.utils.email_utils import send_email
+from agr_literature_service.lit_processing.data_ingest.dqm_ingest.parse_dqm_json_reference import generate_pmid_data,\
     aggregate_dqm_with_pubmed
-from agr_literature_service.lit_processing.get_pubmed_xml import download_pubmed_xml
-from agr_literature_service.lit_processing.xml_to_json import generate_json
-from agr_literature_service.lit_processing.post_reference_to_db import post_references
-from agr_literature_service.lit_processing.update_resource_pubmed_nlm import update_resource_pubmed_nlm
-from agr_literature_service.lit_processing.get_dqm_data import download_dqm_json
+from agr_literature_service.lit_processing.data_ingest.pubmed_ingest.xml.get_pubmed_xml import download_pubmed_xml
+from agr_literature_service.lit_processing.data_ingest.pubmed_ingest.xml.xml_to_json import generate_json
+from agr_literature_service.lit_processing.data_ingest.post_reference_to_db import post_references
+from agr_literature_service.lit_processing.data_ingest.pubmed_ingest.pubmed_update_resources_nlm import update_resource_pubmed_nlm
+from agr_literature_service.lit_processing.data_ingest.dqm_ingest.get_dqm_data import download_dqm_json
 from agr_literature_service.api.user import set_global_user_id
 
 # For WB needing 57578 references checked for updating,

agr_literature_service/lit_processing/sort_dqm_json_resource_updates.py → agr_literature_service/lit_processing/data_ingest/dqm_ingest/sort_dqm_json_resource_updates.py

@@ -1,5 +1,4 @@
 import json
-import logging
 import logging.config
 import warnings
 from os import environ, makedirs, path
@@ -8,10 +7,10 @@
 from fastapi.encoders import jsonable_encoder
 
 from agr_literature_service.api.models import ResourceModel, CrossReferenceModel
-from agr_literature_service.lit_processing.helper_sqlalchemy import create_postgres_session,\
+from agr_literature_service.lit_processing.utils.sqlalchemy_utils import create_postgres_session,\
     sqlalchemy_load_ref_xref
-from agr_literature_service.lit_processing.helper_file_processing import (compare_authors_or_editors,
-                                                                          save_resource_file, split_identifier)
+from agr_literature_service.lit_processing.utils.file_processing_utils import (compare_authors_or_editors,
+                                                                               save_resource_file, split_identifier)
 from agr_literature_service.api.user import set_global_user_id
 
 warnings.filterwarnings("ignore", category=UserWarning, module='bs4')
@@ -32,7 +31,7 @@
 # keep working off of lit-4003, comparing data from 20211025 files (loaded at lit-4005)
 
 
-log_file_path = path.join(path.dirname(path.abspath(__file__)), '../../logging.conf')
+log_file_path = path.join(path.dirname(path.abspath(__file__)), '../../../../logging.conf')
 logging.config.fileConfig(log_file_path)
 logger = logging.getLogger('literature logger')
 

agr_literature_service/lit_processing/filter_dqm_md5sum.py → agr_literature_service/lit_processing/data_ingest/dqm_ingest/utils/md5sum_utils.py

@@ -4,11 +4,10 @@
 import argparse
 import sys
 from os import environ, path, makedirs, listdir
-import logging
 import logging.config
 
-from agr_literature_service.lit_processing.helper_file_processing import split_identifier, write_json
-from agr_literature_service.lit_processing.helper_s3 import upload_file_to_s3, download_file_from_s3
+from agr_literature_service.lit_processing.utils.file_processing_utils import split_identifier, write_json
+from agr_literature_service.lit_processing.utils.s3_utils import upload_file_to_s3, download_file_from_s3
 
 from dotenv import load_dotenv
 

agr_literature_service/lit_processing/post_reference_to_db.py → agr_literature_service/lit_processing/data_ingest/post_reference_to_db.py

@@ -6,10 +6,8 @@
 from agr_literature_service.api.models import CrossReferenceModel, ReferenceModel,\
     AuthorModel, ModCorpusAssociationModel, ModReferenceTypeModel, ModModel,\
     ReferenceCommentAndCorrectionModel, MeshDetailModel
-from agr_literature_service.lit_processing.helper_sqlalchemy import create_postgres_session,\
-    create_postgres_engine
-from agr_literature_service.api.crud.reference_crud import get_citation_from_args,\
-    get_next_curie
+from agr_literature_service.lit_processing.utils.sqlalchemy_utils import create_postgres_session, create_postgres_engine
+from agr_literature_service.api.crud.reference_crud import get_citation_from_args, get_next_curie
 
 logging.basicConfig(format='%(message)s')
 log = logging.getLogger()

agr_literature_service/lit_processing/post_resource_to_api.py → agr_literature_service/lit_processing/data_ingest/post_resource_to_db.py

@@ -1,18 +1,17 @@
 import argparse
 import json
-import logging
 import logging.config
 import sqlalchemy
 import sys
 from os import environ
 
 from dotenv import load_dotenv
 
-from agr_literature_service.lit_processing.helper_sqlalchemy import create_postgres_session
+from agr_literature_service.lit_processing.utils.sqlalchemy_utils import create_postgres_session, \
+    sqlalchemy_load_ref_xref
 from agr_literature_service.api.models import ResourceModel, CrossReferenceModel, EditorModel
 from agr_literature_service.api.crud.resource_crud import create_next_curie
-from agr_literature_service.lit_processing.helper_sqlalchemy import sqlalchemy_load_ref_xref
-from agr_literature_service.lit_processing.helper_file_processing import split_identifier
+from agr_literature_service.lit_processing.utils.file_processing_utils import split_identifier
 
 load_dotenv()