Compress disease-gene-pairs-association.csv

Merges #43 Day 2 of Snorkel Week #39
greenelab · May 1, 2018 · 7b95a5d · 7b95a5d
1 parent a8003b0
commit 7b95a5d
Show file tree

Hide file tree

Showing 5 changed files with 11 additions and 5 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1 @@
+*.xz filter=lfs diff=lfs merge=lfs -text
diff --git a/All_Relationships/1a.stratify-candidates.ipynb b/All_Relationships/1a.stratify-candidates.ipynb
@@ -715,7 +715,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "dg_map_df.to_csv(\"disease-gene-pairs-association.csv\", index=False, float_format='%.5g')"
+    "dg_map_df.to_csv(\"data/disease-gene-pairs-association.csv.xz\", index=False, float_format='%.5g', compression='xz')"
    ]
   },
   {

diff --git a/All_Relationships/1a.stratify-candidates.py b/All_Relationships/1a.stratify-candidates.py
@@ -186,7 +186,7 @@ def get_split(partition_rank, training=0.7, dev=0.2, test=0.1):
 # In[16]:
 
 
-dg_map_df.to_csv("disease-gene-pairs-association.csv", index=False, float_format='%.5g')
+dg_map_df.to_csv("data/disease-gene-pairs-association.csv.xz", index=False, float_format='%.5g', compression='xz')
 
 
 # In[17]:

diff --git a/All_Relationships/data/disease-gene-pairs-association.csv.xz b/All_Relationships/data/disease-gene-pairs-association.csv.xz
diff --git a/All_Relationships/utils/disease_gene_lf.py b/All_Relationships/utils/disease_gene_lf.py
@@ -12,6 +12,7 @@
     rule_regex_search_before_B,
 )
 import re
+import pathlib
 import pandas as pd
 import nltk
 from nltk.stem import WordNetLemmatizer
@@ -61,7 +62,8 @@ def ltp(tokens):
 """
 DISTANT SUPERVISION
 """
-pair_df = pd.read_csv("disease-gene-pairs-association.csv")
+path = pathlib.Path(__file__).joinpath('../../data/disease-gene-pairs-association.csv.xz').resolve()
+pair_df = pd.read_csv(path, dtype={"sources": str})
 knowledge_base = set()
 for row in pair_df.itertuples():
     if not row.sources or pd.isnull(row.sources):
@@ -121,14 +123,14 @@ def LF_CHECK_DISEASE_TAG(c):
     c- the candidate object to be passed in.
     """
     sen = c[0].get_parent()
-    disease_name = re.sub("\)", "", c[0].get_span())
+    disease_name = re.sub("\) ?", "", c[0].get_span())
 
     # If abbreviation skip since no means of easy resolution
     if len(disease_name) <=5 and disease_name.isupper():
         return 0
 
     disease_name = [wordnet_lemmatizer.lemmatize(word) for word in disease_name.split(" ")]
-    disease_name = " ".join(list(map(lambda x: x[0], filter(lambda x: x[1] == 'NN', nltk.pos_tag(disease_name)))))
+    disease_name = " ".join(list(map(lambda x: x[0], filter(lambda x: 'NN' in x[1], nltk.pos_tag(disease_name)))))
 
     disease_id = sen.entity_cids[c[0].get_word_start()]
     disease_entry_df = disease_desc.query("doid_code == @disease_id")