build: update requirements (#241)

* build: update requirements.txt

- split requirements into two files, one for main django app, other for supplementary scripts

* build: update requirements.txt

* fix: remove init files

* build: further simplify requirements

* test(literature_review): update tests and code

* build: remove duplicated psycopg2

* test: update test to match new screening pipeline

README.md

@@ -42,12 +42,6 @@ Use pip to install requirements (you will need `g++` to install fasttext):
 (cruise-literature)$ pip install -r requirements.txt
 ```
 
-If you have a GPU-enabled device:
-
-```bash
-(cruise-literature)$ pip install -r requirements-gpu.txt
-```
-
 npm install bulma-calendar
 
 

requirements.txt

@@ -1,34 +1,21 @@
-# main app
-Django~=4.1.5
-wikipedia~=1.4.0
-fuzzywuzzy~=0.18.0
-requests~=2.25.1
-torch>=1.13.1
-transformers==4.14.1
-rdflib~=6.1.1
-faiss-cpu~=1.7.2
-numpy~=1.22.3
-pandas==1.4.2
-xmltodict~=0.13.0
-psycopg2~=2.9.3
-psycopg2-binary~=2.9.3
-bibtexparser~=1.3.0
-biopython~=1.80
-djangorestframework~=3.13.1
-django-environ~=0.10.0
-crispy-bulma~=0.9.0
+bibtexparser==1.4.1
+biopython==1.81
+crispy-bulma==0.10.0
 django-crispy-forms~=2.0
-
-# scripts
-pycld3~=0.22
-tqdm~=4.64.0
-beautifulsoup4~=4.11.1
-elasticsearch~=7.15.1
-keybert==0.5.1
-cso-classifier==3.0
-scholarly~=1.7.2
+django-environ==0.11.2
+django-filter==23.3
+Django==4.2.6
+djangorestframework==3.14.0
 fasttext~=0.9.2
-joblib~=1.1.0
-openai~=0.24.0
-rispy~=0.7.1
-scikit-learn~=1.1.0
+fuzzywuzzy==0.18.0
+grobid_tei_xml==0.1.3
+markdown==3.5
+numpy==1.26.1
+pandas==2.1.1
+psycopg2-binary~=2.9.3
+rispy==0.8.1
+rdflib==7.0.0
+requests==2.31.0
+scholarly~=1.7.2
+wikipedia~=1.4.0
+xmltodict~=0.13.0

scripts/requirements.txt

@@ -0,0 +1,14 @@
+# scripts
+pycld3~=0.22
+tqdm~=4.64.0
+beautifulsoup4~=4.11.1
+elasticsearch~=7.15.1
+keybert==0.5.1
+cso-classifier==3.0
+scholarly~=1.7.2
+fasttext~=0.9.2
+joblib~=1.1.0
+openai~=0.24.0
+rispy~=0.7.1
+scikit-learn~=1.1.0
+transformers==4.34.1

src/cruise_literature/citation_screening/tests.py

@@ -1,10 +1,12 @@
+import datetime
 import time
 
 from django.test import TestCase, Client
 from django.urls import reverse
 from django.utils.datastructures import MultiValueDictKeyError
 
 from literature_review.models import LiteratureReview, LiteratureReviewMember
+from .models import CitationScreening
 from users.models import User
 from .views import (
     screen_papers,
@@ -86,9 +88,21 @@ def setUpClass(cls):
                 ],
             },
             papers={_fake_paper["id"]: _fake_paper},
+            ready_for_screening=True,
         )
         cls.member = LiteratureReviewMember.objects.create(
-            member=cls.user, literature_review=cls.lit_rev
+            member=cls.user, literature_review=cls.lit_rev, added_by=cls.user
+        )
+        cls.cs_for_review = CitationScreening.objects.create(
+            literature_review=cls.lit_rev,
+            tasks={
+                "new": {cls.user.username: [_fake_paper["id"]]},
+                "in_progress": {},
+                "done": {},
+            },
+            screening_level=1,
+            tasks_updated_at=datetime.datetime.now(),
+            distributed_papers=[_fake_paper["id"]],
         )
 
     def setUp(self):

src/cruise_literature/citation_screening/views.py

@@ -158,7 +158,9 @@ def screening_home(request, review_id):
         ).first()
         if screening_task:
             new = screening_task.tasks["new"].get(request.user.username, [])
-            new = [paper for paper in _papers if str(paper["id"]) in new]  # todo: fix paper ID type
+            new = [
+                paper for paper in _papers if str(paper["id"]) in new
+            ]  # todo: fix paper ID type
             done = screening_task.tasks["done"].get(request.user.username, [])
             done = [paper for paper in _papers if str(paper["id"]) in done]
 
@@ -212,7 +214,6 @@ def screen_papers(request, review_id):
             {"review": review, "paper": paper, "start_time": time.time()},
         )
     elif request.method == "POST":
-
         paper_id = request.POST["paper_id"]
         review, request = create_screening_decisions(request, review, paper_id)
         review.save()
@@ -265,11 +266,15 @@ def create_screening_decisions(request, review, paper_id):
     }
 
     reason = request.POST["reason"]
-    topic_relevance : Optional[int] = request.POST.get("topic_relevance", None)
-    domain_relevance : Optional[int] = request.POST.get("domain_relevance", None)
+    topic_relevance: Optional[int] = request.POST.get("topic_relevance", None)
+    domain_relevance: Optional[int] = request.POST.get("domain_relevance", None)
     decision = request.POST["decision"]
-    paper_prior_knowledge : Optional[int] = request.POST.get("paper_prior_knowledge", None)
-    authors_prior_knowledge : Optional[int] = request.POST.get("authors_prior_knowledge", None)
+    paper_prior_knowledge: Optional[int] = request.POST.get(
+        "paper_prior_knowledge", None
+    )
+    authors_prior_knowledge: Optional[int] = request.POST.get(
+        "authors_prior_knowledge", None
+    )
 
     review.papers[paper_id]["decisions"] = [
         {
@@ -279,10 +284,18 @@ def create_screening_decisions(request, review, paper_id):
             "exclusion_decisions": exclusion_decisions,
             "inclusion_decisions": inclusion_decisions,
             "stage": "title_abstract",
-            "domain_relevance": int(domain_relevance) if domain_relevance else domain_relevance,
-            "topic_relevance": int(topic_relevance) if topic_relevance else topic_relevance,
-            "paper_prior_knowledge": int(paper_prior_knowledge) if paper_prior_knowledge else paper_prior_knowledge,
-            "authors_prior_knowledge": int(authors_prior_knowledge) if authors_prior_knowledge else authors_prior_knowledge,
+            "domain_relevance": int(domain_relevance)
+            if domain_relevance
+            else domain_relevance,
+            "topic_relevance": int(topic_relevance)
+            if topic_relevance
+            else topic_relevance,
+            "paper_prior_knowledge": int(paper_prior_knowledge)
+            if paper_prior_knowledge
+            else paper_prior_knowledge,
+            "authors_prior_knowledge": int(authors_prior_knowledge)
+            if authors_prior_knowledge
+            else authors_prior_knowledge,
             "screening_time": screening_time,
         }
     ]
@@ -333,7 +346,6 @@ def screen_paper(request, review_id, paper_id):
             },
         )
     if request.method == "POST":
-
         paper_id = request.POST["paper_id"]
         review, request = create_screening_decisions(request, review, paper_id)
         review.save()

src/cruise_literature/literature_review/tests.py

@@ -89,7 +89,7 @@ def setUpClass(cls):
             papers={_fake_paper["id"]: _fake_paper},
         )
         cls.member = LiteratureReviewMember.objects.create(
-            member=cls.user, literature_review=cls.lit_rev
+            member=cls.user, literature_review=cls.lit_rev, added_by=cls.user
         )
 
     def setUp(self):
@@ -117,6 +117,7 @@ def setUp(self):
             "top_k": 10,
             "search_engines": [SearchEngine.objects.filter(name="CRUISE").first().id],
             "annotations_per_paper": 1,
+            "review_type": "AN",
         }
 
     def test_literature_review_list_GET(self):
@@ -171,7 +172,7 @@ def test_create_new_review_GET_unauthenticated(self):
     def test_create_new_review_POST_success_test_redirect(self):
         response = self.client.post(self.create_new_review_url, self.new_review_params)
         self.assertEqual(response.status_code, 302)
-        self.assertRedirects(response, "/")
+        self.assertRedirects(response, "/manage_review/3/")
 
     def test_create_new_review_POST_success_test_db(self):
         self.client.post(self.create_new_review_url, self.new_review_params)

src/cruise_literature/literature_review/views.py

@@ -32,7 +32,9 @@ def create_new_review(request):
             form.save()
             title = form.cleaned_data.get("title")
             messages.success(request, f"New review created: {title}")
-            return redirect("literature_review:manage_review", review_id=form.instance.id)
+            return redirect(
+                "literature_review:manage_review", review_id=form.instance.id
+            )
         else:
             if "error_messages" in form:
                 for msg in form.error_messages:
@@ -406,8 +408,9 @@ def add_seed_studies(request, review_id):
                 review.papers[doc.pdf_md5] = _new_papers
                 review.save()
                 added_studies.append(seed_studies_url)
-            except HTTPError:
-                print("HTTPError")
+            except (HTTPError, ValueError) as e:
+                print(e)
+                messages.error(request, f"Error adding {seed_studies_url}")
 
         if added_studies:
             messages.success(