3. Validation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1 Compiling notebook 2 outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import math\n",
    "from pathlib import Path\n",
    "import numpy as np\n",
    "import re\n",
    "import glob\n",
    "import configparser\n",
    "import json  \n",
    "\n",
    "from utils.misc.regex_block import MutationFinder, TmVar, CustomWBregex, normalize_mutations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data/model_output/extracted_snippets_0_10.csv\n",
      "data/model_output/extracted_snippets_10_20.csv\n",
      "data/model_output/extracted_snippets_20_50.csv\n",
      "data/model_output/extracted_snippets_50_60.csv\n",
      "data/model_output/extracted_snippets_60_70.csv\n",
      "data/model_output/extracted_snippets_70_100.csv\n"
     ]
    }
   ],
   "source": [
    "# incase you've ran the prev notebook on splits of papers\n",
    "data = []\n",
    "for file in glob.glob(\"data/model_output/*.csv\"):\n",
    "    print(file)\n",
    "    # 'WBPaper ID', 'Method', '* Genes', '* Gene-Variant combo', 'Mutation', 'Sentence'\n",
    "    text = pd.read_csv(file).to_numpy().tolist()\n",
    "    data = data + text \n",
    "data = np.array(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# above cell takes a while to complete, so saving the data temporarily\n",
    "data = pd.DataFrame(data[:], columns=['WBPaper ID', 'Method', '*Genes', '*Gene-Variant combo ', 'Mutations', 'Sentence'])\n",
    "data.to_csv(\"data/model_output/processed/snippets_1.csv\", index=False, encoding='utf-8')\n",
    "data = None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2 Normalizing mutations to  one-letter amino acid codes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "These code imports would be doing the same thing done in notebook 2, but on a much small subset of data.   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(\"data/model_output/processed/snippets_1.csv\")\n",
    "data = data.to_numpy() # 'WBPaper ID', 'Method', 'Genes', '*Gene-Variant combo ', 'Mutations', 'Sentence'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/c/Users/Rishab/Documents/GitHub/genomic-info-from-papers/genomic-info-from-papers/utils/misc/regex_block.py:102: FutureWarning: Possible nested set at position 15\n",
      "  self._regular_expressions.append(re.compile(reg))\n"
     ]
    }
   ],
   "source": [
    "db_config = configparser.ConfigParser()\n",
    "db_config.read('utils/all_config.cfg')\n",
    "\n",
    "custom_mut_extract = CustomWBregex(db_config, locus_only=True)\n",
    "mf_mut_extract = MutationFinder('data/regexs/mutationfinder_regex/seth_modified.txt')\n",
    "tmvar_mut_extract = TmVar('data/regexs/tmvar_regex/final_regex_path')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def point_mut_block(sentence, span_size=150):\n",
    "    mut_and_snippets = []\n",
    "    \n",
    "    # MutationFinder\n",
    "    mut_and_snippets = mut_and_snippets + mf_mut_extract(sentence, span_size=span_size)\n",
    "    # tmVar\n",
    "    mut_and_snippets = mut_and_snippets + tmvar_mut_extract(sentence, span_size=span_size)\n",
    "    # Custom patterns\n",
    "    mut_and_snippets = mut_and_snippets + custom_mut_extract(sentence, span_size=span_size)\n",
    "\n",
    "    if mut_and_snippets:\n",
    "        mut_and_snippets = np.array(mut_and_snippets)\n",
    "        mut_and_snippets = mut_and_snippets[:, 0].tolist()\n",
    "        mut_and_snippets = list(set(mut_and_snippets))\n",
    "    return mut_and_snippets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['G-to-A change at nucleotide 673', 'T-to-G change at nucleotide 811']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "point_mut_block('ad465 , nucleotide 2862 of the coding region to 400 bp downstream G-to-A change at nucleotide 673 resulting in a stop codon from the stop site was amplified using primers ATGGATGAAC at amino acid 107; ad692 , T-to-G change at nucleotide 811 TATACAA')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('T811G', 'F230L', 'C4539T', 'M856K', 'G118R', 'V1247L')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normalize_mutations('T-to-G change at nucleotide 811'), normalize_mutations('Phe230Leu'), \\\n",
    "normalize_mutations('C to T at nucleotide 4539'), normalize_mutations('methionine for lysine-856'), \\\n",
    "normalize_mutations('glycine-118 is replaced by an arginine'), normalize_mutations('1247 (valine to leucine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'F230AMBER'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normalize_mutations('Phe230amber')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Working with the protein mutations from regex block for now"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Following mutations could NOT be normalized. Either a) normalize them manually and add in the csv file b) Make edits in the normalize_mutations fn\n",
      "Lys-82 to Lys\n",
      "53 bp del\n",
      "99 bp del\n",
      "serine at position 377 to phenylalanine\n",
      "arginine at position 551 with a histidine\n",
      "glycine at position 573 with a serine\n",
      "aspartic acid to asparagine at codon 652\n",
      "glutamine at codon 13 to a stop\n",
      "glutamic acid 230 to lysine\n",
      "glycine at position 560 to an arginine\n",
      "glycine at position 558 to an arginine\n",
      "glycine at position 76 changed to glutamic acid\n",
      "glycine at position 76 changed to glutamic acid\n",
      "glutamic acid for glycine at codon 13\n"
     ]
    }
   ],
   "source": [
    "# old - 'WBPaper ID', 'Method', 'Genes', '*Gene-Variant combo ', 'Mutations', 'Sentence'\n",
    "# new - 'WBPaper ID', 'Method', 'Genes', '*Gene-Variant combo ', 'Mutations', 'Normalized Mutations', 'Sentence'\n",
    "temp = []\n",
    "total_count = len(data)\n",
    "ner_count = 0\n",
    "regex_count = 0\n",
    "\n",
    "paper_mut_count = {}\n",
    "print('Following mutations could NOT be normalized. Either a) normalize them manually and add in the csv file b) Make edits in the normalize_mutations fn')\n",
    "for i, row in enumerate(data):\n",
    "    if row[1] != 'Regex':\n",
    "        if row[1] == 'NER':\n",
    "            ner_count += 1\n",
    "        temp.append(np.insert(data[i], -1, '').tolist())\n",
    "    else:\n",
    "        paper_id = row[0]\n",
    "        if paper_id not in paper_mut_count.keys():\n",
    "            paper_mut_count[paper_id] = {}\n",
    "        regex_count += 1\n",
    "        norm_mutations = []\n",
    "        mutations = data[i, -2][1:-1].split(\"', '\")\n",
    "        for raw_mut in mutations: \n",
    "            mut = point_mut_block(raw_mut)\n",
    "            if mut:\n",
    "                # helps filtering obvious ones\n",
    "                for m in mut:\n",
    "                    m = m.replace(\",\", \"\")\n",
    "                    if m.find(')') != -1:\n",
    "                        if m.find('(') == -1:\n",
    "                            continue\n",
    "                    try:\n",
    "                        # takes care of filtering out bad mutations where\n",
    "                        # wild residues and mutants are same e.g G123G\n",
    "                        norm_mut = normalize_mutations(mut[0])\n",
    "                        if norm_mut: \n",
    "                            if norm_mut not in paper_mut_count[paper_id].keys():\n",
    "                                paper_mut_count[paper_id][norm_mut] = 0\n",
    "                            paper_mut_count[paper_id][norm_mut] += 1\n",
    "                            norm_mutations.append(norm_mut)\n",
    "                    except KeyError:\n",
    "                        print(m)\n",
    "        if norm_mutations:\n",
    "            norm_mutations = list(set(norm_mutations))\n",
    "            norm_mutations = \"'\" + \"', '\".join(norm_mutations) + \"'\"\n",
    "        else: \n",
    "            norm_mutations = ''\n",
    "        temp.append(np.insert(data[i], -1, norm_mutations).tolist())\n",
    "        \n",
    "data = temp\n",
    "temp = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data/model_output/processed/temp_paper_mut_count.json\", \"w\") as outfile: \n",
    "    json.dump(paper_mut_count, outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All 1694 NER data rows were ignored. Only 620 regex data rows were used.\n"
     ]
    }
   ],
   "source": [
    "print('All', ner_count, 'NER data rows were ignored. Only', regex_count, 'regex data rows were used.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# saving things\n",
    "data = pd.DataFrame(data[:], columns=['WBPaper ID', 'Method', 'Genes', '*Gene-Variant combo ', 'Mutations', 'Normalized Mutations', 'Sentence'])\n",
    "data.to_csv(\"data/model_output/processed/snippets_2.csv\", index=False, encoding='utf-8')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3 Normalizing common gene name to its WormBase ID\n",
    "And getting the gene and mutation frequency in a paper."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(\"data/model_output/processed/snippets_2.csv\")\n",
    "data = data.to_numpy() # 'WBPaper ID', 'Method', 'Genes', '*Gene-Variant combo ', 'Mutations', 'Normalized Mutations', 'Sentence'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Inefficient way to do this. Have to work on better search algo."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "306123"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wb_genes_1 = Path('data/gsoc/Gene_alias.1.txt').read_text().split('\\n')\n",
    "wb_genes_2 = Path('data/gsoc/Gene_alias.2.txt').read_text().split('\\n')\n",
    "wb_genes_3 = Path('data/gsoc/Gene_alias.3.txt').read_text().split('\\n')\n",
    "\n",
    "wb_genes_1 = [r.split('\\t') for r in wb_genes_1]\n",
    "wb_genes_2 = [r.split(' ') for r in wb_genes_2]\n",
    "wb_genes_3 = [r.split(' ') for r in wb_genes_3]\n",
    "\n",
    "all_wb_genes = dict()\n",
    "\n",
    "for row in wb_genes_1+wb_genes_2+wb_genes_3:\n",
    "    if row[0] not in all_wb_genes.keys():\n",
    "        all_wb_genes[row[0]] = []\n",
    "    for gene in row[1:]: \n",
    "        if len(gene) and gene.lower() not in all_wb_genes[row[0]]: \n",
    "            all_wb_genes[row[0]].append(gene.lower())\n",
    "len(all_wb_genes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total sentences: 18276, processed count:  100 200 300 400 500 600 700 800 900 1000 1100 1200 1300 1400 1500 1600 1700 1800 1900 2000 2100 2200 2300 2400 2500 2600 2700 2800 2900 3000 3100 3200 3300 3400 3500 3600 3700 3800 3900 4000 4100 4200 4300 4400 4500 4600 4700 4800 4900 5000 5100 5200 5300 5400 5500 5600 5700 5800 5900 6000 6100 6200 6300 6400 6500 6600 6700 6800 6900 7000 7100 7200 7300 7400 7500 7600 7700 7800 7900 8000 8100 8200 8300 8400 8500 8600 8700 8800 8900 9000 9100 9200 9300 9400 9500 9600 9700 9800 9900 10000 10100 10200 10300 10400 10500 10600 10700 10800 10900 11000 11100 11200 11300 11400 11500 11600 11700 11800 11900 12000 12100 12200 12300 12400 12500 12600 12700 12800 12900 13000 13100 13200 13300 13400 13500 13600 13700 13800 13900 14000 14100 14200 14300 14400 14500 14600 14700 14800 14900 15000 15100 15200 15300 15400 15500 15600 15700 15800 15900 16000 16100 16200 16300 16400 16500 16600 16700 16800 16900 17000 17100 17200 17300 17400 17500 17600 17700 17800 17900 18000 18100 18200 "
     ]
    }
   ],
   "source": [
    "print('Total sentences: {}, processed count: '.format(len(data)), end=' ')\n",
    "updated_data = []\n",
    "paper_wbgene_count = {}\n",
    "\n",
    "# 'WBPaper ID', 'Method', 'Genes', '*Gene-Variant combo ', 'Mutations', 'Normalized Mutations', 'Sentence'\n",
    "for i, row in enumerate(data):\n",
    "    if (i+1) % 100 == 0: print(f\"{i+1}\", end = \" \")\n",
    "    paper_id = row[0]\n",
    "    genes = row[2]\n",
    "    sentence = row[-1]\n",
    "    # checking if nan\n",
    "    if type(genes) == float:\n",
    "        col_genes = ''\n",
    "    else:\n",
    "        if paper_id not in paper_wbgene_count.keys():\n",
    "            paper_wbgene_count[paper_id] = {}\n",
    "        genes = genes[1:-1].split(\"', '\")\n",
    "        col_genes = []\n",
    "        \n",
    "        for gene in genes:\n",
    "            for key, value in all_wb_genes.items():\n",
    "                if gene.lower() in value:\n",
    "                    if key not in paper_wbgene_count[paper_id]:\n",
    "                        paper_wbgene_count[paper_id][key] = 0\n",
    "                    paper_wbgene_count[paper_id][key] += 1\n",
    "                    col_genes.append(key)\n",
    "                    break\n",
    "        if col_genes:\n",
    "            col_genes = list(set(col_genes))\n",
    "            col_genes = \"'\" + \"', '\".join(col_genes) + \"'\"\n",
    "        else: \n",
    "            col_genes = ''\n",
    "    updated_data.append([data[i,0], data[i,1], data[i,2], col_genes, data[i,3], data[i,4], data[i,5], data[i,6]])\n",
    "    \n",
    "data = updated_data # 'WBPaper ID', 'Method', 'Genes', 'WBGenes', '*Gene-Variant combo ', 'Mutations', 'Sentence'\n",
    "updated_data = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data/model_output/processed/temp_paper_wbgene_count.json\", \"w\") as outfile: \n",
    "    json.dump(paper_wbgene_count, outfile)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Checking if any detected gene was NOT in the WB gene dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([], shape=(0, 18276, 8), dtype='<U11771')"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = np.array(data)\n",
    "data[len(data[:,2]) != len(data[:,3])] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# above cell takes a while to complete, so saving the data temporarily\n",
    "data = pd.DataFrame(data[:], columns=['WBPaper ID', 'Method', 'Genes', 'WBGenes', '*Gene-Variant combo ', 'Mutations', 'Normalized Mutations', 'Sentence'])\n",
    "data.to_csv(\"data/model_output/processed/snippets_3.csv\", index=False, encoding='utf-8')\n",
    "data = None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5 Validation\n",
    "Finding the gene and mutation matches using the transcripts in c_elegans.PRJNA13758.WS281.protein.fa   \n",
    "Get the file here - ftp://ftp.ebi.ac.uk/pub/databases/wormbase/releases/WS281/species/c_elegans/PRJNA13758/c_elegans.PRJNA13758.WS281.protein.fa.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "306123"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wb_genes_1 = Path('data/gsoc/Gene_alias.1.txt').read_text().split('\\n')\n",
    "wb_genes_2 = Path('data/gsoc/Gene_alias.2.txt').read_text().split('\\n')\n",
    "wb_genes_3 = Path('data/gsoc/Gene_alias.3.txt').read_text().split('\\n')\n",
    "\n",
    "wb_genes_1 = [r.split('\\t') for r in wb_genes_1]\n",
    "wb_genes_2 = [r.split(' ') for r in wb_genes_2]\n",
    "wb_genes_3 = [r.split(' ') for r in wb_genes_3]\n",
    "\n",
    "all_wb_genes = dict()\n",
    "\n",
    "for row in wb_genes_1+wb_genes_2+wb_genes_3:\n",
    "    if row[0] not in all_wb_genes.keys():\n",
    "        all_wb_genes[row[0]] = []\n",
    "    for gene in row[1:]: \n",
    "        if len(gene) and gene.lower() not in all_wb_genes[row[0]]: \n",
    "            all_wb_genes[row[0]].append(gene.lower())\n",
    "len(all_wb_genes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(\"data/model_output/processed/snippets_3.csv\")\n",
    "data = data.to_numpy() # 'WBPaper ID', 'Method', 'Genes', 'WBGenes', '*Gene-Variant combo ', 'Mutations', 'Normalized Mutations', 'Sentence'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "proteinfa = Path('data/gsoc/proteinfa/c_elegans.PRJNA13758.WS281.protein.fa').read_text().split('>')[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19987"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wb_gene_and_prot = dict() # {wbgene: [transcript, protein]}\n",
    "\n",
    "for row in proteinfa:\n",
    "    wbgene = re.findall(\"WBGene[0-9]+\", row)[0]\n",
    "    protein = \"\".join(re.findall(\"\\n.*\", row)).replace('\\n','')\n",
    "    transcript = row.split(' ')[0]\n",
    "    if wbgene not in wb_gene_and_prot.keys():\n",
    "        wb_gene_and_prot[wbgene] = []\n",
    "    wb_gene_and_prot[wbgene].append([transcript, protein])\n",
    "    \n",
    "len(wb_gene_and_prot)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create a pair of gene and mutation only when BOTH are present in same sentence."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "paper_raw_info_compiled = []\n",
    "# 'WBPaper ID', 'Method', 'Genes', 'WBGenes', '*Gene-Variant combo ', 'Mutations', 'Normalized Mutations', 'Sentence'\n",
    "for row in data:\n",
    "    ppr_id = row[0]\n",
    "    norm_muts = row[-2]\n",
    "    wbgenes = row[3]\n",
    "    sentence = row[-1]\n",
    "    gene_var = row[4]\n",
    "        \n",
    "    # filtering out nan values\n",
    "    if type(norm_muts) != float and type(wbgenes) != float:    \n",
    "        norm_muts = norm_muts[1:-1].split(\"', '\")\n",
    "        wbgenes = wbgenes[1:-1].split(\"', '\")\n",
    "        for m in norm_muts:\n",
    "            for g in wbgenes:\n",
    "                if len(m) and len(g):\n",
    "                    paper_raw_info_compiled.append([ppr_id, g, m, sentence, gene_var])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "matches = [] \n",
    "final_sheet = [] # ppr_id, gene, transcript\n",
    "\n",
    "for info_from_ppr in paper_raw_info_compiled:\n",
    "    ppr_id = info_from_ppr[0]\n",
    "    gene = info_from_ppr[1]\n",
    "    mut = info_from_ppr[2]\n",
    "    sent = info_from_ppr[3]\n",
    "    gene_var = info_from_ppr[4]\n",
    "    if not len(mut):\n",
    "        continue\n",
    "    if gene not in wb_gene_and_prot.keys():\n",
    "        continue\n",
    "    for row in wb_gene_and_prot[gene]:\n",
    "        transcript, protein_string = row\n",
    "        wt_res = mut[0]\n",
    "        pos = int(''.join(n for n in mut if n.isdigit()))\n",
    "        mut_res = mut[-1]\n",
    "        try:\n",
    "            if protein_string[pos-1] == wt_res:\n",
    "                matches.append([ppr_id, gene, mut, gene_var, transcript, sent])\n",
    "        except IndexError:\n",
    "            pass\n",
    "    \n",
    "for r in matches:\n",
    "    p = r[0]\n",
    "    p, wbg, mut, gene_var, transcript, sent = r\n",
    "    # Adding gene common names column, again\n",
    "    # Current code doesn't keep any link between the WB gene name and the common name\n",
    "    g_common_name = all_wb_genes[wbg]\n",
    "    g_common_name = ', '.join(g_common_name)\n",
    "    final_sheet.append([p, wbg, g_common_name, mut, gene_var, transcript, sent])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "977"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(final_sheet)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Getting metadata on genes and mutations, and adding warnings column "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data/model_output/processed/temp_paper_wbgene_count.json\", \"r\") as f: \n",
    "    paper_wbgene_count = json.loads(f.read())\n",
    "with open(\"data/model_output/processed/temp_paper_mut_count.json\", \"r\") as f: \n",
    "    paper_mut_count = json.loads(f.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_sheet = np.array(final_sheet)\n",
    "updated_sheet = []\n",
    "for i, row in enumerate(final_sheet):\n",
    "    warnings = []\n",
    "    paper_id = row[0]\n",
    "    wbgene = row[1]\n",
    "    mut = row[3]\n",
    "    sentence = row[-1]\n",
    "    for ppr_mut, count in paper_mut_count[paper_id].items():\n",
    "        if mut == ppr_mut and count == 1:\n",
    "            warnings.append(f'{mut} mentioned only once in entire paper')\n",
    "            break\n",
    "            \n",
    "    rows_with_same_mut = final_sheet[np.logical_and(final_sheet[:, 0] == paper_id, final_sheet[:,3] == mut)]\n",
    "    same_mut_all_genes = list(set(rows_with_same_mut[:, 1]))\n",
    "    # If the same variant is found in two different genes in the same paper - WARN! \n",
    "    # It is more likely to belong to the gene it is most frequently encountered\n",
    "    if len(same_mut_all_genes) > 1:\n",
    "        temp_warn_store = f'{mut} was paired with other genes too:'\n",
    "        for ppr_gene, count in paper_wbgene_count[paper_id].items():\n",
    "            if ppr_gene in same_mut_all_genes:\n",
    "                temp_warn_store += (f' {ppr_gene} (seen {count} times),')\n",
    "        warnings.append(temp_warn_store)\n",
    "    \n",
    "    cut_mut = re.sub(\"([A-Z])([0-9]+)([A-Za-z]+)\", r'\\1\\2', mut)\n",
    "    remaining_mut = mut.replace(cut_mut, \"\")\n",
    "    same_cut_muts = [i for i,m in enumerate(final_sheet[:,3]) if (m[:len(cut_mut)] == cut_mut and m[len(cut_mut):] != remaining_mut)]\n",
    "    if same_cut_muts:\n",
    "        temp_warn_store = f'{mut} similar to:'\n",
    "        for temp_i in same_cut_muts:\n",
    "            temp_warn_store += (f' {final_sheet[:,3][temp_i]} (line {temp_i}),')\n",
    "        warnings.append(temp_warn_store)\n",
    "        \n",
    "    all_muts_in_sentence = data[np.logical_and(data[:, 0] == paper_id, data[:,-1] == sentence)][:,-2]\n",
    "    all_muts_in_sentence = all_muts_in_sentence[0][1:-1].split(\"', '\")\n",
    "    \n",
    "    all_matched_muts_in_sentence = final_sheet[np.logical_and(final_sheet[:, 0] == paper_id, final_sheet[:,-1] == sentence)][:,3]\n",
    "    all_matched_muts_in_sentence = list(set(all_matched_muts_in_sentence))\n",
    "    \n",
    "    unmatched_muts_in_sentence = [m for m in all_muts_in_sentence if m not in all_matched_muts_in_sentence]\n",
    "    if len(unmatched_muts_in_sentence) >= 2:\n",
    "        temp_warn_store = f'Sentence has multiple mutations which did not match:'\n",
    "        for m in unmatched_muts_in_sentence:\n",
    "            temp_warn_store += (f' {m},')\n",
    "        warnings.append(temp_warn_store)\n",
    "    \n",
    "    all_genes_with_this_mut = final_sheet[np.logical_and(final_sheet[:, 0] == paper_id, final_sheet[:, 3] == mut)][:, 1]\n",
    "    all_genes_with_this_mut = list(set(all_genes_with_this_mut))\n",
    "    if len(all_genes_with_this_mut) > 3:\n",
    "        temp_warn_store = f'{mut} was matched with {len(all_genes_with_this_mut)} genes:'\n",
    "        for g in all_genes_with_this_mut:\n",
    "            temp_warn_store += (f' {g},')\n",
    "        warnings.append(temp_warn_store)\n",
    "            \n",
    "    if warnings:\n",
    "        warnings = \" || \".join(warnings)\n",
    "    else:\n",
    "        warnings = \"\"\n",
    "    updated_sheet.append(np.insert(row, -1, warnings).tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# saving things\n",
    "updated_sheet = pd.DataFrame(updated_sheet[:], columns=['WBPaper ID', 'WBGene', 'Gene', 'Mutation', 'Gene-Var combo', 'Transcript', 'Warnings', 'Sentence'])\n",
    "updated_sheet.to_csv(\"data/model_output/processed/snippets_4.csv\", index=False, encoding='utf-8')\n",
    "updated_sheet = None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6 Additional details"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6.1 Strains"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(\"data/model_output/processed/snippets_4.csv\").to_numpy() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "strains = Path('data/gsoc/Strains.txt').read_text().split('\\n')\n",
    "strains = [r.split('\\t') for r in strains][:-1]\n",
    "all_wb_strains = dict()\n",
    "\n",
    "for row in strains:\n",
    "    if row[0] not in all_wb_strains.keys():\n",
    "        all_wb_strains[row[0]] = []\n",
    "    for strain in row[1:]: \n",
    "        if len(strain) and strain.lower() not in all_wb_strains[row[0]]: \n",
    "            all_wb_strains[row[0]].append(strain.lower())\n",
    "            \n",
    "strains = [s for row in strains for s in row[1:] if len(s) and not s.isdigit()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total sentences: 977, processed count:  100 200 300 400 500 600 700 800 900 "
     ]
    }
   ],
   "source": [
    "OPENING_CLOSING_REGEXES = [r'(?:^|[^0-9A-Za-z])(', r')(?:^|[^0-9A-Za-z])']\n",
    "all_strain = OPENING_CLOSING_REGEXES[0] + '|'.join(strains) + OPENING_CLOSING_REGEXES[1]\n",
    "all_strain = [re.compile(r,re.IGNORECASE) for r in [all_strain]]\n",
    "\n",
    "# 'WBPaper ID', 'WBGene', 'Gene', 'Mutation', 'Gene-Var combo', 'Transcript', 'Warnings', 'Sentence'\n",
    "updated_data = []\n",
    "total = len(data)\n",
    "print('Total sentences: {}, processed count: '.format(total), end=' ')\n",
    "for i, sent in enumerate(data[:, -1]):\n",
    "    if (i+1) % 100 == 0: print(f\"{i+1}\", end = \" \")\n",
    "    paper_strains = []\n",
    "    for regex in all_strain:      \n",
    "        for m in regex.finditer(sent):\n",
    "            span = (m.start(0), m.end(0))    \n",
    "            raw = (sent[span[0]:span[1]]).strip()\n",
    "            raw = raw[1:] if not raw[0].isalnum() else raw\n",
    "            raw = raw[:-1] if not raw[-1].isalnum() else raw\n",
    "            if len(raw.strip()) > 1 and not raw.strip().isdigit(): paper_strains.append(raw.strip())\n",
    "    if paper_strains:\n",
    "        paper_strains  = list(set(paper_strains))\n",
    "        col_wbid = []\n",
    "        for strain in paper_strains:\n",
    "            for key, value in all_wb_strains.items():\n",
    "                if strain.lower() in value:\n",
    "                    col_wbid.append(key)\n",
    "                    break\n",
    "        paper_strains = \"'\" + \"', '\".join(paper_strains) + \"'\"\n",
    "        if col_wbid:\n",
    "            col_wbid = list(set(col_wbid))\n",
    "            col_wbid = \", \".join(col_wbid)\n",
    "        else: \n",
    "            col_wbid = ''\n",
    "            # lazy way to deal with bad snippets due to special characters in the Strains.txt file\n",
    "            # which are caught in regex\n",
    "            paper_strains = '' \n",
    "    else:\n",
    "        paper_strains = ''\n",
    "        col_wbid = ''\n",
    "    updated_data.append([data[i,0], data[i,1], data[i,2], col_wbid, paper_strains, data[i,3], data[i,-4], data[i,-3], data[i,-2], data[i,-1]])\n",
    "data = np.array(updated_data) # 'WBPaper ID', 'WBGene', 'Gene', 'WBStrain', 'Strains', 'Mutation', 'Gene-Var combo', 'Transcript', 'Warnings', 'Sentence'\n",
    "updated_data = None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6.2 Variants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total sentences: 977, processed count:  100 200 300 400 500 600 700 800 900 "
     ]
    }
   ],
   "source": [
    "OPENING_CLOSING_REGEXES = [r'(?:^|[^0-9A-Za-z])(', r')(?:^|[^0-9A-Za-z])']\n",
    "\n",
    "# the allele regex and db idea was stolen from wbtools\n",
    "allele_designations = np.load('data/gsoc/wbtools/wb_allele_designations.npy').astype('U6')\n",
    "alleles_variations = np.load('data/gsoc/wbtools/wb_alleles_variations.npy').astype('U6')\n",
    "DB_VAR_REGEX = r'({designations}|m|p|ts|gf|lf|d|sd|am|cs)([0-9]+)'\n",
    "var_regex_1 = OPENING_CLOSING_REGEXES[0] + DB_VAR_REGEX.format(designations=\"|\".join(allele_designations)) + OPENING_CLOSING_REGEXES[1]\n",
    "all_var = OPENING_CLOSING_REGEXES[0] + '|'.join(alleles_variations) + '|' + var_regex_1 + OPENING_CLOSING_REGEXES[1]\n",
    "all_var = [re.compile(r,re.IGNORECASE) for r in [all_var]]\n",
    "\n",
    "# 'WBPaper ID', 'WBGene', 'Gene', 'WBStrain', 'Strains', 'Mutation', 'Transcript', 'Warnings', 'Sentence'\n",
    "updated_data = []\n",
    "total = len(data)\n",
    "print('Total sentences: {}, processed count: '.format(total), end=' ')\n",
    "for i, sent in enumerate(data[:, -1]):\n",
    "    if (i+1) % 100 == 0: print(f\"{i+1}\", end = \" \")\n",
    "    variants = []\n",
    "    for regex in all_var:      \n",
    "        for m in regex.finditer(sent):\n",
    "            span = (m.start(0), m.end(0))    \n",
    "            raw = (sent[span[0]:span[1]]).strip()\n",
    "            raw = raw[1:] if not raw[0].isalnum() else raw\n",
    "            raw = raw[:-1] if not raw[-1].isalnum() else raw\n",
    "            if len(raw.strip()) > 1: variants.append(raw.strip())\n",
    "    if variants:\n",
    "        variants  = list(set(variants))\n",
    "        variants = \"'\" + \"', '\".join(variants) + \"'\"\n",
    "    else:\n",
    "        variants = ''\n",
    "    updated_data.append([data[i,0], data[i,1], data[i,2], data[i,3], data[i,4], variants, data[i,-5], data[i,-4], data[i,-3], data[i,-2], data[i,-1]])\n",
    "    \n",
    "data = np.array(updated_data) # 'WBPaper ID', 'WBGene', 'Gene', 'WBStrain', 'Strains', 'Variants', 'Mutation', 'Gene-Var combo', 'Transcript', 'Warnings', 'Sentence'\n",
    "updated_data = None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6.3 Variation type\n",
    "Extraction rate would be very low as most snippets from notebook 2 are discarded due to limitation in the mutation normalization block above."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "Variation_type = pd.read_csv(\"data/gsoc/Variation_type.csv\").to_numpy() \n",
    "Variation_type = [t.replace(\"_\", \" \") for t in Variation_type[:,2] if type(t)!=float]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "updated_sheet = []\n",
    "# 'WBPaper ID', 'WBGene', 'Gene', 'WBStrain', 'Strains', 'Variants', 'Mutation', 'Gene-Var combo', 'Transcript', 'Warnings', 'Sentence'\n",
    "for i, row in enumerate(data):\n",
    "    sent = row[-1]\n",
    "    col_var_type = []\n",
    "    for sub in Variation_type:\n",
    "        if re.search(sub, sent, re.IGNORECASE):\n",
    "            col_var_type.append(sub)\n",
    "    if col_var_type:\n",
    "        col_var_type = list(set(col_var_type))\n",
    "        col_var_type = \", \".join(col_var_type)\n",
    "    else:\n",
    "        col_var_type = ''\n",
    "    updated_sheet.append(np.insert(row, -3, col_var_type).tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = np.array(updated_sheet)\n",
    "updated_sheet = None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6.3 Functional effect & Generation method \n",
    "These type of data were in a few subset of papers tested during dev - expect these columns to be mostly empty."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "functional_effect = ['function uncertain', 'transcript function', 'translational product function', \\\n",
    "                 'decreased transcript level', 'increased transcript level', 'decreased transcript stability', \\\n",
    "                 'gain of function', 'dominant negative', 'dominant negativ', 'antimorphic', \\\n",
    "                 'hypermorphic', 'neomorphic', 'conditional activity', 'hypomorphic', 'amorphic', \\\n",
    "                 'repressible', 'misexpressed']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "common_gen_methods = ['CRISPR', 'ENU', 'EMS']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "updated_sheet = []\n",
    "# 'WBPaper ID', 'WBGene', 'Gene', 'WBStrain', 'Strains', 'Variants', 'Mutation', 'Gene-Var combo', 'Variation type', 'Transcript', 'Warnings', 'Sentence'\n",
    "for i, row in enumerate(data):\n",
    "    sent = row[-1]\n",
    "    col_functional_effect = []\n",
    "    col_gen_method = [] \n",
    "    for sub in functional_effect:\n",
    "        if re.search(sub, sent, re.IGNORECASE):\n",
    "            col_functional_effect.append(sub)\n",
    "            \n",
    "    for sub in common_gen_methods:\n",
    "        if re.search(sub, sent):\n",
    "            col_gen_method.append(sub)\n",
    "            \n",
    "    if col_functional_effect:\n",
    "        col_functional_effect = list(set(col_functional_effect))\n",
    "        col_functional_effect = \", \".join(col_functional_effect)\n",
    "    else:\n",
    "        col_functional_effect = ''\n",
    "        \n",
    "    if col_gen_method:\n",
    "        col_gen_method = list(set(col_gen_method))\n",
    "        col_gen_method = \", \".join(col_gen_method)\n",
    "    else:\n",
    "        col_gen_method = ''\n",
    "    row = np.insert(row, -3, col_functional_effect)\n",
    "    row = np.insert(row, -3, col_gen_method)\n",
    "    updated_sheet.append(row.tolist())\n",
    "data = np.array(updated_sheet)\n",
    "updated_sheet = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# saving things\n",
    "updated_sheet = pd.DataFrame(data[:], columns=['WBPaper ID', 'WBGene', 'Gene', 'WBStrain', 'Strains', \\\n",
    "                                               'Variants', 'Mutation', 'Gene-Var combo', 'Variation type', 'Functional effect', \\\n",
    "                                               'Generation method', 'Transcript', 'Warnings', 'Sentence'])\n",
    "updated_sheet.to_csv(\"data/model_output/processed/final.csv\", index=False, encoding='utf-8')\n",
    "updated_sheet = None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 7 Verification\n",
    "Finding precision by cross-checking with the manually curated data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(\"data/model_output/processed/final.csv\")\n",
    "data = data.to_numpy() \n",
    "paper_ids_processed = np.unique(data[:,0])\n",
    "paper_ids_processed = np.sort(paper_ids_processed)\n",
    "\n",
    "temp = pd.read_csv(\"data/model_output/processed/snippets_1.csv\")\n",
    "temp = temp.to_numpy()\n",
    "total_paper_ids_processed = np.unique(temp[:,0])\n",
    "temp = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total count of papers processed: 100\n",
      "Count of papers: 53\n"
     ]
    }
   ],
   "source": [
    "print('Total count of papers processed:', len(total_paper_ids_processed))\n",
    "print('Count of papers:', len(paper_ids_processed))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 7.1 Original ground truth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "ground_truth = Path('data/gsoc/variantsDB.txt').read_text().split('\\n')\n",
    "ground_truth = [r.split('\\t') for r in ground_truth][:-1]\n",
    "ground_truth = np.array(ground_truth, dtype=object)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WBPaper00002627 WBPaper00028462 WBPaper00029073 WBPaper00029143 WBPaper00030864 WBPaper00031335 WBPaper00045258 "
     ]
    }
   ],
   "source": [
    "# Checking if any processed paper is not in the ground truth file\n",
    "for id in total_paper_ids_processed:\n",
    "    if id not in ground_truth[:,0]:\n",
    "        print(id, end = ' ')\n",
    "        if id in paper_ids_processed:\n",
    "            print(' false positive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "tp_col = []\n",
    "for row in data:\n",
    "    paper_id = row[0]\n",
    "    gene = row[1]\n",
    "    mutation = row[6]\n",
    "    mutation = mutation.upper()\n",
    "    transcript = row[-3]\n",
    "    bool_found = False\n",
    "    for label in ground_truth[ground_truth[:,0] == paper_id]:\n",
    "        label[-2] = label[-2].upper()\n",
    "        if transcript == label[-1] and mutation == label[-2]:\n",
    "            bool_found = True\n",
    "            # continue bc we're storing all the labels from a paper\n",
    "            continue\n",
    "    if bool_found:\n",
    "        tp_col.append('True Positive')\n",
    "    else:\n",
    "        tp_col.append('False Positive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(472, 505)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tp_col.count('True Positive'), tp_col.count('False Positive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision  48.31115660184238 %\n"
     ]
    }
   ],
   "source": [
    "print('Precision ',tp_col.count('True Positive')*100/(tp_col.count('True Positive') + tp_col.count('False Positive')), '%')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 7.2 Curated ground truth  \n",
    "Updated after manually looking at the \"false positives\" which aren't false positives and adding them in the ground truth file.   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "ground_truth = Path('data/gsoc/variantsDB_curated.txt').read_text().split('\\n')\n",
    "ground_truth = [r.split('\\t') for r in ground_truth][:-1]\n",
    "ground_truth = np.array(ground_truth, dtype=object)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WBPaper00002627 WBPaper00028462 WBPaper00029073 WBPaper00029143 WBPaper00030864 WBPaper00031335 WBPaper00045258 "
     ]
    }
   ],
   "source": [
    "# Checking if any processed paper is not in the ground truth file\n",
    "for id in total_paper_ids_processed:\n",
    "    if id not in ground_truth[:,0]:\n",
    "        print(id, end = ' ')\n",
    "        if id in paper_ids_processed:\n",
    "            print(' false positive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "tp_col = []\n",
    "for row in data:\n",
    "    paper_id = row[0]\n",
    "    gene = row[1]\n",
    "    mutation = row[6]\n",
    "    mutation = mutation.upper()\n",
    "    transcript = row[-3]\n",
    "    bool_found = False\n",
    "    for label in ground_truth[ground_truth[:,0] == paper_id]:\n",
    "        label[-2] = label[-2].upper()\n",
    "        if transcript == label[-1] and mutation == label[-2]:\n",
    "            bool_found = True\n",
    "            # continue bc we're storing all the labels from a paper\n",
    "            continue\n",
    "    if bool_found:\n",
    "        tp_col.append('True Positive')\n",
    "    else:\n",
    "        tp_col.append('False Positive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(807, 170)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tp_col.count('True Positive'), tp_col.count('False Positive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision  82.59979529170931 %\n"
     ]
    }
   ],
   "source": [
    "print('Precision ',tp_col.count('True Positive')*100/(tp_col.count('True Positive') + tp_col.count('False Positive')), '%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "tp_col = np.array(tp_col).T.reshape(-1, 1)\n",
    "final_sheet = np.hstack((data,tp_col))\n",
    "# saving things\n",
    "final_sheet = pd.DataFrame(final_sheet[:], columns=['WBPaper ID', 'WBGene', 'Gene', 'WBStrain', 'Strains', \\\n",
    "                                               'Variants', 'Mutation', 'Gene-Var combo', 'Variation type', 'Functional effect', \\\n",
    "                                               'Generation method', 'Transcript', 'Warnings', 'Sentence', 'Result'])\n",
    "final_sheet.to_csv(\"data/model_output/processed/final_verified.csv\", index=False, encoding='utf-8')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking how many matches are present in the ground truth for the processed papers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2433"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_from_truth = []\n",
    "for ppr in paper_ids_processed:\n",
    "    for label in ground_truth[ground_truth[:,0] == ppr]:\n",
    "        label[-2] = label[-2].upper()\n",
    "        all_from_truth.append(label)\n",
    "len(all_from_truth)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Results:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Out of 100 papers, we could find gene-mutation matches on 53 papers.   \n",
    "Those 53 papers had total of 2433 matches in ground truth (which were manually curated).  \n",
    "We were able to find 977 matches.  \n",
    "TP: 472, FP: 505  \n",
    "Precision: 48.3%  \n",
    "After manually checking the false positives and updating the ground truth file -  \n",
    "TP: 807, FP: 170  \n",
    "Precision: 82.59%  \n",
    "  \n",
    "-> Not all \"FP\" are FP. After manual verification of the final output, some were noticied to be true positive which were originally missed during the manual curation. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "90bac3f7a4bb879b9d06605bdeda624e0779c88b1a5b8631d7aaa6d430fa2aec"
  },
  "kernelspec": {
   "display_name": "wb_env",
   "language": "python",
   "name": "wb_env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}