disruptive-papers.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Disruptive Papers in Fetal Surgery\n",
    "\n",
    "This notebook:\n",
    "- Reads search terms from text files\n",
    "- Builds search hedges for each term list and writes them to TXT\n",
    "- Builds full search queries from each set of hedges and writes them to TXT\n",
    "- Pulls search results from PubMed and writes them to CSV\n",
    "- Tests search results against lists of PMIDs identified as papers to include or exclude\n",
    "- Maps PMIDs to MAG IDs via OpenAlex API\n",
    "- Pulls citation counts, development, and disruption scores from the AggregatedMAG.txt dataset\n",
    "- exports scoring data to CSV and XLSX\n",
    "- optionally creates datestamped snapshot of input/output files\n",
    "\n",
    "TODO:\n",
    "- more logging\n",
    "- function documentation\n",
    "- finish test_results function\n",
    "- MeSH analysis of search results/test articles"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "import logging\n",
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "from pymed import PubMed\n",
    "import requests\n",
    "import shutil\n",
    "from styleframe import StyleFrame, Styler, utils\n",
    "import sys\n",
    "import time\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Constants and Create Directories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# API parameters\n",
    "email = \"whimar@oshu.edu\"\n",
    "pubmed = PubMed(tool=\"FetalSurgerySearch\", email=email)\n",
    "openalex = \"https://api.openalex.org/works?per-page=100&filter=pmid:\"\n",
    "headers = {\"user-agent\": \"mailto:\" + email}\n",
    "max_results = 100000\n",
    "\n",
    "datestamp = time.strftime(\"%Y%m%d\")\n",
    "TOP_PAPERS = 100\n",
    "MAG_SCORES = \"AggregatedMAG.txt\"\n",
    "\n",
    "LOG_DIR = \"log/\"\n",
    "TEST_DIR = \"tests/*\"\n",
    "\n",
    "# data diectory paths\n",
    "DATA_DIR = \"data/\"\n",
    "JOURNAL_DIR = DATA_DIR + \"journals/*\"\n",
    "KEYWORD_DIR = DATA_DIR + \"keywords/*\"\n",
    "MESH_DIR = DATA_DIR + \"mesh-terms/*\"\n",
    "\n",
    "# output directory paths\n",
    "TARGET_DIR = \"target/\"\n",
    "CSV_DIR = TARGET_DIR + \"csv/\"\n",
    "HEDGE_DIR = TARGET_DIR + \"hedges/\"\n",
    "QUERY_DIR = TARGET_DIR + \"queries/\"\n",
    "XLSX_DIR = TARGET_DIR + \"xlsx/\"\n",
    "\n",
    "# archive directory\n",
    "ARCHIVE_DIR = \"archive/\"\n",
    "\n",
    "# create output directories if they don't exist\n",
    "Path(TARGET_DIR).mkdir(parents=True, exist_ok=True)\n",
    "Path(CSV_DIR).mkdir(parents=True, exist_ok=True)\n",
    "Path(HEDGE_DIR).mkdir(parents=True, exist_ok=True)\n",
    "Path(LOG_DIR).mkdir(parents=True, exist_ok=True)\n",
    "Path(QUERY_DIR).mkdir(parents=True, exist_ok=True)\n",
    "Path(XLSX_DIR).mkdir(parents=True, exist_ok=True)\n",
    "Path(ARCHIVE_DIR).mkdir(parents=True, exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Configure Logging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "logging.basicConfig(\n",
    "    filename=f\"{LOG_DIR}{datestamp}-hedgebill.log\",\n",
    "    filemode=\"w\",\n",
    "    force=True,\n",
    "    format=\"%(asctime)s.%(msecs)03d : %(levelname)s : %(message)s\",\n",
    "    level=logging.INFO,\n",
    "    datefmt=\"%Y-%m-%d %H:%M:%S\",\n",
    ")\n",
    "logger = logging.getLogger()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Function Definitions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### create_snapshot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_snapshot():\n",
    "    today_folder = f\"{ARCHIVE_DIR}{datestamp}/\"\n",
    "    data_folder = f\"{ARCHIVE_DIR}{datestamp}/data/\"\n",
    "\n",
    "    Path(today_folder).mkdir(parents=True, exist_ok=True)\n",
    "    Path(data_folder).mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "    shutil.copytree(DATA_DIR, data_folder, dirs_exist_ok=True)\n",
    "\n",
    "    if Path(TARGET_DIR).isdir():\n",
    "        shutil.move(TARGET_DIR, today_folder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### pmid_to_magid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pmid_to_magid(articles_df, querycount, querytype):\n",
    "    magsfile = f\"{CSV_DIR}{datestamp}-{querycount}-{querytype}-MAGID.csv\"\n",
    "    magsfilepath = Path(magsfile)\n",
    "\n",
    "    # Have we already pulled these MAG IDs?\n",
    "    if magsfilepath.is_file():\n",
    "        # If so, read the results we saved to CSV\n",
    "        logger.info(\"Reading MAGID data from file\")\n",
    "        mags_df = pd.read_csv(magsfile, index_col=\"long_pmid\")\n",
    "    else:\n",
    "        logger.info(\"Pulling new MAGIDs from OpenAlex\")\n",
    "        # If not, pull MAGIDs from OpenAlex\n",
    "        pmids = articles_df[\"pmid\"].tolist()\n",
    "        pmid_groups = [pmids[i : i + 50] for i in range(0, len(pmids), 50)]\n",
    "\n",
    "        pmid_strings = []\n",
    "        mags = {}\n",
    "        for pmid_group in pmid_groups:\n",
    "            pmid_strings.append(\"|\".join(map(str, pmid_group)))\n",
    "\n",
    "        for pmid_string in tqdm(pmid_strings, desc=\"openalex\"):\n",
    "            response = requests.get(openalex + pmid_string, headers=headers)\n",
    "            if response.status_code == 200:\n",
    "                works = response.json()\n",
    "                for work in works[\"results\"]:\n",
    "                    if \"mag\" in work[\"ids\"]:\n",
    "                        mags[work[\"ids\"][\"pmid\"]] = work[\"ids\"][\"mag\"]\n",
    "            else:\n",
    "                logger.error(\"Error making openalex request\")\n",
    "\n",
    "        mags_df = pd.DataFrame.from_dict(mags, orient=\"index\", columns=[\"magid\"])\n",
    "        mags_df.index.name = \"long_pmid\"\n",
    "        mags_df.sort_index(inplace=True)\n",
    "        mags_df.to_csv(magsfile)\n",
    "        mags_df = pd.read_csv(magsfile, index_col=\"long_pmid\")\n",
    "\n",
    "    return mags_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### run_query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_query(query, querycount, querytype):\n",
    "    resultsfile = f\"{CSV_DIR}{datestamp}-{querycount}-{querytype}-results.csv\"\n",
    "    samplefile = f\"{CSV_DIR}{datestamp}-{querycount}-{querytype}-SAMPLE.csv\"\n",
    "    resultsfilepath = Path(resultsfile)\n",
    "\n",
    "    # Have we already run this query?\n",
    "    if resultsfilepath.is_file():\n",
    "        # If so, read the results we saved to CSV\n",
    "        logger.info(\"Reading query results from file\")\n",
    "        articles_df = pd.read_csv(resultsfile, index_col=\"long_pmid\")\n",
    "    else:\n",
    "        # If not, run the query\n",
    "        logger.info(\"Running new PubMed query\")\n",
    "        queryresults = pubmed.query(query, max_results=max_results)\n",
    "        # drop results that are not journal articles\n",
    "        articles = [\n",
    "            [\n",
    "                article.pubmed_id.splitlines()[0],\n",
    "                article.title,\n",
    "                article.journal,\n",
    "                article.publication_date,\n",
    "                article.doi,\n",
    "                article.abstract,\n",
    "            ]\n",
    "            for article in tqdm(queryresults, total=querycount, desc=querytype)\n",
    "            if hasattr(article, \"journal\")\n",
    "        ]\n",
    "        articlecount = len(articles)\n",
    "        articles_df = pd.DataFrame(\n",
    "            articles,\n",
    "            columns=[\"pmid\", \"title\", \"journal\", \"pubdate\", \"doi\", \"abstract\"],\n",
    "        )\n",
    "\n",
    "        # add columns of helpful URLs\n",
    "        # hyperlinked URLs makes things more accessible in the XLSX output\n",
    "        articles_df[\"doi\"] = [\n",
    "            f'=HYPERLINK(\"https://doi.org/{doi}\")' if doi is not None else \"\"\n",
    "            for doi in articles_df[\"doi\"]\n",
    "        ]\n",
    "        articles_df[\"pubmed\"] = [\n",
    "            f'=HYPERLINK(\"https://pubmed.ncbi.nlm.nih.gov/{pmid}\", {pmid})'\n",
    "            for pmid in articles_df[\"pmid\"]\n",
    "        ]\n",
    "        articles_df[\"ohsu_library\"] = [\n",
    "            f'=HYPERLINK(\"https://librarysearch.ohsu.edu/openurl/OHSU/OHSU?sid=Entrez:PubMed&id=pmid:{pmid}\", \"Find @ OHSU\")'\n",
    "            for pmid in articles_df[\"pmid\"]\n",
    "        ]\n",
    "        articles_df[\"rush_library\"] = [\n",
    "            f'=HYPERLINK(\"https://i-share-rsh.primo.exlibrisgroup.com/openurl/01CARLI_RSH/01CARLI_RSH:CARLI_RSH?sid=Entrez:PubMed&id=pmid:{pmid}\", \"Find @ Rush\")'\n",
    "            for pmid in articles_df[\"pmid\"]\n",
    "        ]\n",
    "        articles_df[\"long_pmid\"] = [\n",
    "            f\"https://pubmed.ncbi.nlm.nih.gov/{pmid}\" for pmid in articles_df[\"pmid\"]\n",
    "        ]\n",
    "\n",
    "        articles_df.set_index(\"long_pmid\", inplace=True)\n",
    "        articles_df.sort_index(inplace=True)\n",
    "\n",
    "        articles_df.to_csv(resultsfile)\n",
    "        articles_df.sample(n=25).to_csv(samplefile)\n",
    "        articles_df = pd.read_csv(resultsfile, index_col=\"long_pmid\")\n",
    "\n",
    "    return articles_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### score_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def score_results(articles_df, querycount, querytype):\n",
    "    citefile = f\"{CSV_DIR}{datestamp}-{querycount}-{querytype}-topcited.csv\"\n",
    "    developfile = f\"{CSV_DIR}{datestamp}-{querycount}-{querytype}-topdevelopmental.csv\"\n",
    "    disruptfile = f\"{CSV_DIR}{datestamp}-{querycount}-{querytype}-topdisruptive.csv\"\n",
    "\n",
    "    mag_df = pd.read_csv(\n",
    "        f\"AggregatedMAG.txt\",\n",
    "        sep=\"\\t\",\n",
    "        usecols=[0, 5, 6],\n",
    "        names=[\"magid\", \"num_citations\", \"disruption_score\"],\n",
    "        index_col=\"magid\",\n",
    "        dtype={\"citation\": \"int64\", \"disruption\": \"float64\"},\n",
    "    )\n",
    "    mag_df.sort_index(inplace=True)\n",
    "\n",
    "    scored_df = articles_df.join(mag_df)\n",
    "    scored_df.reset_index(inplace=True)\n",
    "\n",
    "    cited_df = scored_df[\n",
    "        [\n",
    "            \"num_citations\",\n",
    "            \"title\",\n",
    "            \"journal\",\n",
    "            \"abstract\",\n",
    "            \"pubdate\",\n",
    "            \"magid\",\n",
    "            \"pmid\",\n",
    "            \"doi\",\n",
    "            \"pubmed\",\n",
    "            \"ohsu_library\",\n",
    "            \"rush_library\",\n",
    "        ]\n",
    "    ].nlargest(TOP_PAPERS, columns=\"num_citations\")\n",
    "    development_df = scored_df[\n",
    "        [\n",
    "            \"disruption_score\",\n",
    "            \"title\",\n",
    "            \"journal\",\n",
    "            \"abstract\",\n",
    "            \"pubdate\",\n",
    "            \"magid\",\n",
    "            \"pmid\",\n",
    "            \"doi\",\n",
    "            \"pubmed\",\n",
    "            \"ohsu_library\",\n",
    "            \"rush_library\",\n",
    "        ]\n",
    "    ].nsmallest(TOP_PAPERS, columns=\"disruption_score\")\n",
    "    disrupt_sf = scored_df[\n",
    "        [\n",
    "            \"disruption_score\",\n",
    "            \"title\",\n",
    "            \"journal\",\n",
    "            \"abstract\",\n",
    "            \"pubdate\",\n",
    "            \"magid\",\n",
    "            \"pmid\",\n",
    "            \"doi\",\n",
    "            \"pubmed\",\n",
    "            \"ohsu_library\",\n",
    "            \"rush_library\",\n",
    "        ]\n",
    "    ].nlargest(TOP_PAPERS, columns=\"disruption_score\")\n",
    "\n",
    "    cited_df.to_csv(\n",
    "        citefile,\n",
    "        index=False,\n",
    "        columns=[\n",
    "            \"num_citations\",\n",
    "            \"magid\",\n",
    "            \"pmid\",\n",
    "            \"title\",\n",
    "            \"journal\",\n",
    "            \"pubdate\",\n",
    "            \"doi\",\n",
    "            \"abstract\",\n",
    "        ],\n",
    "    )\n",
    "    development_df.to_csv(\n",
    "        developfile,\n",
    "        index=False,\n",
    "        columns=[\n",
    "            \"disruption_score\",\n",
    "            \"magid\",\n",
    "            \"pmid\",\n",
    "            \"title\",\n",
    "            \"journal\",\n",
    "            \"pubdate\",\n",
    "            \"doi\",\n",
    "            \"abstract\",\n",
    "        ],\n",
    "    )\n",
    "    disrupt_sf.to_csv(\n",
    "        disruptfile,\n",
    "        index=False,\n",
    "        columns=[\n",
    "            \"disruption_score\",\n",
    "            \"magid\",\n",
    "            \"pmid\",\n",
    "            \"title\",\n",
    "            \"journal\",\n",
    "            \"pubdate\",\n",
    "            \"doi\",\n",
    "            \"abstract\",\n",
    "        ],\n",
    "    )\n",
    "\n",
    "    return cited_df, development_df, disrupt_sf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### style_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def style_output(cited_df, develop_df, disrupt_sf, query, querycount, querytype):\n",
    "    scorefile = f\"{XLSX_DIR}{datestamp}-{querycount}-{querytype}-scores.xlsx\"\n",
    "\n",
    "    cited_sf = StyleFrame(cited_df.drop(\"pmid\", axis=1))\n",
    "    develop_sf = StyleFrame(develop_df.drop(\"pmid\", axis=1))\n",
    "    disrupt_sf = StyleFrame(disrupt_sf.drop(\"pmid\", axis=1))\n",
    "    query_s = pd.Series({f\"{querytype} query\": query})\n",
    "\n",
    "    default_style = Styler(\n",
    "        font=utils.fonts.calibri,\n",
    "        font_size=11,\n",
    "        border_type=utils.borders.default_grid,\n",
    "        horizontal_alignment=utils.horizontal_alignments.left,\n",
    "        wrap_text=False,\n",
    "        shrink_to_fit=False,\n",
    "    )\n",
    "\n",
    "    header_style = Styler(\n",
    "        bg_color=utils.colors.black,\n",
    "        bold=True,\n",
    "        font=utils.fonts.calibri,\n",
    "        font_color=utils.colors.white,\n",
    "        font_size=14,\n",
    "        horizontal_alignment=utils.horizontal_alignments.left,\n",
    "        shrink_to_fit=False,\n",
    "        wrap_text=False,\n",
    "        vertical_alignment=utils.vertical_alignments.center,\n",
    "    )\n",
    "    hyperlink_style = Styler(\n",
    "        font_color=utils.colors.blue,\n",
    "        protection=True,\n",
    "        underline=utils.underline.single,\n",
    "    )\n",
    "    float_style = Styler(\n",
    "        number_format=\"0.000000000000\",\n",
    "        horizontal_alignment=utils.horizontal_alignments.right,\n",
    "    )\n",
    "\n",
    "    cited_sf.set_column_width_dict(\n",
    "        col_width_dict={\n",
    "            (\"pubdate\", \"magid\", \"pubmed\"): 12,\n",
    "            (\"num_citations\", \"ohsu_library\", \"rush_library\"): 15,\n",
    "            (\"title\", \"journal\", \"abstract\", \"doi\"): 50,\n",
    "        }\n",
    "    )\n",
    "    cited_sf.apply_headers_style(header_style)\n",
    "    cited_sf.apply_column_style(cited_sf.columns, styler_obj=default_style)\n",
    "    cited_sf.apply_column_style(\n",
    "        [\"doi\", \"pubmed\", \"ohsu_library\", \"rush_library\"],\n",
    "        styler_obj=Styler.combine(default_style, hyperlink_style),\n",
    "    )\n",
    "\n",
    "    develop_sf.set_column_width_dict(\n",
    "        col_width_dict={\n",
    "            (\"pubdate\", \"magid\", \"pubmed\"): 12,\n",
    "            (\"ohsu_library\", \"rush_library\"): 15,\n",
    "            (\"disruption_score\"): 20,\n",
    "            (\"title\", \"journal\", \"abstract\", \"doi\"): 50,\n",
    "        }\n",
    "    )\n",
    "    develop_sf.apply_headers_style(header_style)\n",
    "    develop_sf.apply_column_style(develop_sf.columns, styler_obj=default_style)\n",
    "    develop_sf.apply_column_style(\n",
    "        \"disruption_score\", styler_obj=Styler.combine(default_style, float_style)\n",
    "    )\n",
    "    develop_sf.apply_column_style(\n",
    "        [\"doi\", \"pubmed\", \"ohsu_library\", \"rush_library\"],\n",
    "        styler_obj=Styler.combine(default_style, hyperlink_style),\n",
    "    )\n",
    "\n",
    "    disrupt_sf.set_column_width_dict(\n",
    "        col_width_dict={\n",
    "            (\"pubdate\", \"magid\", \"pubmed\"): 12,\n",
    "            (\"ohsu_library\", \"rush_library\"): 15,\n",
    "            (\"disruption_score\"): 20,\n",
    "            (\"title\", \"journal\", \"abstract\", \"doi\"): 50,\n",
    "        }\n",
    "    )\n",
    "    disrupt_sf.apply_headers_style(header_style)\n",
    "    disrupt_sf.apply_column_style(disrupt_sf.columns, styler_obj=default_style)\n",
    "    disrupt_sf.apply_column_style(\n",
    "        \"disruption_score\", styler_obj=Styler.combine(default_style, float_style)\n",
    "    )\n",
    "    disrupt_sf.apply_column_style(\n",
    "        [\"doi\", \"pubmed\", \"ohsu_library\", \"rush_library\"],\n",
    "        styler_obj=Styler.combine(default_style, hyperlink_style),\n",
    "    )\n",
    "\n",
    "    with pd.ExcelWriter(scorefile) as sfile:\n",
    "        cited_sf.to_excel(\n",
    "            sfile,\n",
    "            index=False,\n",
    "            columns=[\n",
    "                \"num_citations\",\n",
    "                \"title\",\n",
    "                \"journal\",\n",
    "                \"abstract\",\n",
    "                \"pubdate\",\n",
    "                \"magid\",\n",
    "                \"doi\",\n",
    "                \"pubmed\",\n",
    "                \"ohsu_library\",\n",
    "                \"rush_library\",\n",
    "            ],\n",
    "            sheet_name=f\"top {TOP_PAPERS} cited\",\n",
    "        )\n",
    "        develop_sf.to_excel(\n",
    "            sfile,\n",
    "            index=False,\n",
    "            columns=[\n",
    "                \"disruption_score\",\n",
    "                \"title\",\n",
    "                \"journal\",\n",
    "                \"abstract\",\n",
    "                \"pubdate\",\n",
    "                \"magid\",\n",
    "                \"doi\",\n",
    "                \"pubmed\",\n",
    "                \"ohsu_library\",\n",
    "                \"rush_library\",\n",
    "            ],\n",
    "            sheet_name=f\"top {TOP_PAPERS} developmental\",\n",
    "        )\n",
    "        disrupt_sf.to_excel(\n",
    "            sfile,\n",
    "            index=False,\n",
    "            columns=[\n",
    "                \"disruption_score\",\n",
    "                \"title\",\n",
    "                \"journal\",\n",
    "                \"abstract\",\n",
    "                \"pubdate\",\n",
    "                \"magid\",\n",
    "                \"doi\",\n",
    "                \"pubmed\",\n",
    "                \"ohsu_library\",\n",
    "                \"rush_library\",\n",
    "            ],\n",
    "            sheet_name=f\"top {TOP_PAPERS} disruptive\",\n",
    "        )\n",
    "        query_s.to_excel(sfile, index=False, header=False, sheet_name=\"pubmed query\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### test_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_results(articles_df):\n",
    "    # test_results_file = f\"{XLS}{}\"\n",
    "    tests = {}\n",
    "    results = {}\n",
    "    coverage = {}\n",
    "    testfiles = sorted(glob.glob(TEST_DIR))\n",
    "\n",
    "    for testfile in testfiles:\n",
    "        logger.info(\"test file: %s\", testfile)\n",
    "        tfile = Path(testfile)\n",
    "        tests[tfile.stem] = pd.read_csv(testfile, names=[\"pmid\"])\n",
    "        results[tfile.stem] = tests[tfile.stem].merge(\n",
    "            articles_df.reset_index(drop=True), how=\"inner\", on=\"pmid\"\n",
    "        )\n",
    "        coverage[tfile.stem] = (\n",
    "            str(len(results[tfile.stem]) / len(tests[tfile.stem]) * 100) + \"%\"\n",
    "        )\n",
    "\n",
    "    # save these to a file? or return results and coverage to pass into the XSLX?\n",
    "    # tests[\"exclude\"].merge(mesh_articles_df.reset_index(drop=True), how=\"inner\", on=\"pmid\")\n",
    "    # tests[\"include\"].merge(mesh_articles_df.reset_index(drop=True), how=\"inner\", on=\"pmid\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### write_hedge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_hedge(hedge, building_block, hedgetype):\n",
    "    hedgecount = str(pubmed.getTotalResultsCount(hedge))\n",
    "    hedgefile = f\"{HEDGE_DIR}{datestamp}-{building_block}-{hedgecount}-{hedgetype}.txt\"\n",
    "    with open(hedgefile, \"w+\") as hfile:\n",
    "        hfile.write(f\"({hedge})\")\n",
    "    logger.info(\"building block: %s\", building_block)\n",
    "    logger.info(\"hedge type: %s\", hedgetype)\n",
    "    logger.info(\"hedge result count: %s\", hedgecount)\n",
    "    logger.info(\"hedge file: %s\", hedgefile)\n",
    "    logger.info(\"hedge: %s\", hedge)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### write_query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_query(query, querytype):\n",
    "    try:\n",
    "        querycount = pubmed.getTotalResultsCount(query)\n",
    "    except requests.exceptions.HTTPError as httperr:\n",
    "        logger.error(httperr)\n",
    "        queryfile = f\"{QUERY_DIR}{datestamp}-{querycount}-TOOLONG.txt\"\n",
    "        with open(queryfile, \"w+\") as qfile:\n",
    "            qfile.write(query)\n",
    "        sys.exit(1)\n",
    "    except Exception as e:\n",
    "        logger.error(\"e.message\")\n",
    "        sys.exit(1)\n",
    "    else:\n",
    "        queryfile = f\"{QUERY_DIR}{datestamp}-{str(querycount)}-{querytype}.txt\"\n",
    "        with open(queryfile, \"w+\") as qfile:\n",
    "            qfile.write(query)\n",
    "        logger.info(\"query type: %s\", querytype)\n",
    "        logger.info(\"query result count: %d\", querycount)\n",
    "        logger.info(\"query file: %s\", queryfile)\n",
    "        logger.info(\"query: %s\", query)\n",
    "        return querycount"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build Hedges, Run Queries, Pull Scores, Write Outputs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MeSH Query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "mesh_querytype = \"[mesh]\"\n",
    "mesh_hedges = []\n",
    "mesh_termfiles = sorted(glob.glob(MESH_DIR))\n",
    "\n",
    "for mesh_termfile in mesh_termfiles:\n",
    "    logger.info(\"term file: %s\", mesh_termfile)\n",
    "    building_block = Path(mesh_termfile).stem\n",
    "    with open(mesh_termfile, \"r\") as tfile:\n",
    "        mesh_hedge = \" OR \".join(sorted(tfile.read().strip().split(\"\\n\")))\n",
    "        write_hedge(mesh_hedge, building_block, mesh_querytype)\n",
    "        mesh_hedges.append(f\"({mesh_hedge})\")\n",
    "\n",
    "mesh_query = \" AND \".join(mesh_hedges)\n",
    "mesh_querycount = write_query(mesh_query, mesh_querytype)\n",
    "\n",
    "mesh_articles_df = run_query(mesh_query, mesh_querycount, mesh_querytype)\n",
    "# test_results(mesh_articles_df)\n",
    "\n",
    "mesh_mags_df = pmid_to_magid(mesh_articles_df, mesh_querycount, mesh_querytype)\n",
    "\n",
    "mesh_df = mesh_mags_df.join(mesh_articles_df).set_index(\"magid\").sort_index()\n",
    "mesh_df.index = mesh_df.index.astype(\"int64\")\n",
    "\n",
    "mesh_cited_df, mesh_develop_df, mesh_disrupt_df = score_results(\n",
    "    mesh_df, mesh_querycount, mesh_querytype\n",
    ")\n",
    "\n",
    "style_output(\n",
    "    mesh_cited_df,\n",
    "    mesh_develop_df,\n",
    "    mesh_disrupt_df,\n",
    "    mesh_query,\n",
    "    mesh_querycount,\n",
    "    mesh_querytype,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create_snapshot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.13 ('fsdp-venv': venv)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "0869d01855810540252d0453a326789001cd1f6afb1862e845959a8760081659"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}