Better ClinVar tooltips and added exclude clinsig status for SNVs filter

CHANGELOG.md

@@ -117,6 +117,7 @@ About changelog [here](https://keepachangelog.com/en/1.0.0/)
 - Export variant type and callers-related info fields when exporting variants from variantS pages
 - Cases advanced search on the dashboard page
 - Possibility to use only signed off panels when building the PanelApp GREEN panel
+- Option to exclude ClinVar significance status in SNVs filters form
 ### Changed
 - On genes panel page and gene panel PDF export, it's more evident which genes were newly introduced into the panel
 - WTS outlier position copy button on WTS outliers page
@@ -135,6 +136,7 @@ About changelog [here](https://keepachangelog.com/en/1.0.0/)
 - Refactor the loading of PanelApp panels to use the maintained API - Customised PanelApp GREEN panels
 - Better layout for Consequence cell on cancer SNVs page
 - Merged `Qual` and `Callers` cell on cancer SNVs page
+- Improved tooltips for ClinVar filter in SNVs filter form
 ### Fixed
 - Empty custom_images dicts in case load config do not crash
 - Tracks missing alignment files are skipped on generating IGV views

scout/adapter/mongo/query.py

@@ -202,6 +202,7 @@ def build_query(
                 'region_annotations': list,
                 'functional_annotations': list,
                 'clinsig': list,
+                'clinsig_exclude': bool,
                 'clinsig_confident_always_returned': boolean,
                 'variant_type': str(('research', 'clinical')),
                 'chrom': str or list of str,
@@ -319,14 +320,16 @@ def build_query(
         for term in PRIMARY_CRITERIA:
             if query.get(term):
                 primary_terms = True
+                break
 
         # check if any of the secondary criteria was specified in the query:
         for term in SECONDARY_CRITERIA:
             if query.get(term):
                 secondary_terms = True
+                break
 
         if primary_terms is True:
-            clinsign_filter = self.clinsig_query(query, mongo_query)
+            clinsign_filter = self.clinsig_query(query)
 
         # Secondary, excluding filter criteria will hide variants in general,
         # but can be overridden by an including, major filter criteria
@@ -355,9 +358,15 @@ def build_query(
                     secondary_filter.append(clinsign_filter)
                     mongo_query["$and"] = secondary_filter
 
-        elif primary_terms is True:  # clisig is provided without secondary terms query
+        elif primary_terms is True:  # clnsig is provided without secondary terms query
             # use implicit and
-            mongo_query["clnsig"] = clinsign_filter["clnsig"]
+            if query.get("clinsig_exclude"):
+                mongo_query["$or"] = [
+                    {"clnsig": {"$exists": False}},
+                    {"clnsig": {"$not": clinsign_filter["clnsig"]}},
+                ]
+            else:
+                mongo_query["clnsig"] = clinsign_filter["clnsig"]
 
         # if chromosome coordinates exist in query, add them as first element of the mongo_query['$and']
         if coordinate_query:
@@ -366,6 +375,7 @@ def build_query(
             else:
                 mongo_query["$and"] = coordinate_query
 
+        LOG.warning(mongo_query)
         return mongo_query
 
     def affected_inds_query(self, mongo_query, case_id, gt_query):
@@ -401,60 +411,40 @@ def affected_inds_query(self, mongo_query, case_id, gt_query):
         ]:  # Consider situation where all individuals are unaffected
             mongo_query["samples"] = affected_query
 
-    def clinsig_query(self, query, mongo_query):
-        """Add clinsig filter values to the mongo query object
-
-        Args:
-            query(dict): a dictionary of query filters specified by the users
-            mongo_query(dict): the query that is going to be submitted to the database
-
-        Returns:
-            clinsig_query(dict): a dictionary with clinsig key-values
+    def clinsig_query(self, query: dict) -> dict:
+        """Add clinsig filter values to the mongo query object"""
 
-        """
-        LOG.debug("clinsig is a query parameter")
         trusted_revision_level = TRUSTED_REVSTAT_LEVEL
         rank = []
         str_rank = []
-        clnsig_query = {}
 
         for item in query["clinsig"]:
             rank.append(int(item))
             # search for human readable clinsig values in newer cases
             rank.append(CLINSIG_MAP[int(item)])
             str_rank.append(CLINSIG_MAP[int(item)])
 
-        if query.get("clinsig_confident_always_returned") is True:
-            LOG.debug("add CLINSIG filter with trusted_revision_level")
+        elem_match_value = {
+            "$or": [
+                {"value": {"$in": rank}},
+                {"value": re.compile("|".join(str_rank))},
+            ]
+        }
 
+        if query.get("clinsig_confident_always_returned") is True:
             clnsig_query = {
                 "clnsig": {
                     "$elemMatch": {
                         "$and": [
-                            {
-                                "$or": [
-                                    {"value": {"$in": rank}},
-                                    {"value": re.compile("|".join(str_rank))},
-                                ]
-                            },
+                            elem_match_value,
                             {"revstat": re.compile("|".join(trusted_revision_level))},
                         ]
                     }
                 }
             }
         else:
-            LOG.debug("add CLINSIG filter for rank: %s" % ", ".join(str(query["clinsig"])))
+            clnsig_query = {"clnsig": {"$elemMatch": elem_match_value}}
 
-            clnsig_query = {
-                "clnsig": {
-                    "$elemMatch": {
-                        "$or": [
-                            {"value": {"$in": rank}},
-                            {"value": re.compile("|".join(str_rank))},
-                        ]
-                    }
-                }
-            }
         return clnsig_query
 
     def coordinate_filter(self, query, mongo_query):

scout/server/blueprints/variants/forms.py

@@ -113,6 +113,7 @@ class VariantFiltersForm(FlaskForm):
     compound_follow_filter = BooleanField("Compounds follow filter")
     cadd_inclusive = BooleanField("CADD inclusive")
     clinsig = NonValidatingSelectMultipleField("ClinVar CLINSIG", choices=CLINSIG_OPTIONS)
+    clinsig_exclude = BooleanField("Exclude significance")
 
     gnomad_frequency = BetterDecimalField("gnomadAF", validators=[validators.Optional()])
     local_obs_old = IntegerField("Local obs. (archive)", validators=[validators.Optional()])

scout/server/blueprints/variants/templates/variants/utils.html

@@ -308,16 +308,17 @@
       {{ form.spidex_human(class="selectpicker", data_style="btn-secondary") }}
     </div>
     <div class="col-2">
-      {{ form.clinsig.label(class="control-label") }}
+      <span>{{ form.clinsig.label(class="control-label") }}</span>
+      <span style="float:right;">{{ form.clinsig_exclude.label(class="control-label", data_bs_toggle="tooltip", data_bs_placement="left", title="Exclude variants with clinical significance among the selected categories.") }} {{form.clinsig_exclude}}</span>
       {{ form.clinsig(class="selectpicker", data_style="btn-secondary") }}
     </div>
     <div class="col-1">
-      {{ form.clinsig_confident_always_returned.label(class="control-label", data_bs_toggle="tooltip", data_bs_placement="top", title="Always show selected CLINSIG entries with trusted revision status levels.") }}
-      <div>{{ form.clinsig_confident_always_returned() }}</div>
+      <span>{{ form.clinsig_confident_always_returned.label(class="control-label", data_bs_toggle="tooltip", data_bs_placement="top", title="Limit search to variants with trusted ClinVar revision status levels: mult, multiple_submitters, single, single_submitter, exp, reviewed_by_expert_panel, guideline, practice_guideline.") }}</span>
+      <span>{{ form.clinsig_confident_always_returned() }}</span>
     </div>
     <div class="col-1">
+      {{ form.clinvar_tag.label(class="control-label", data_bs_toggle="tooltip", data_bs_placement="top", title="Return only variants with annotated ClinVar significance.") }}
       {{ form.clinvar_tag() }}
-      {{ form.clinvar_tag.label(class="ms-2") }}
     </div>
   </div>
   <div class="row mb-2">

tests/adapter/mongo/test_query.py

@@ -275,6 +275,7 @@ def test_build_clinsig(adapter):
 
 
 def test_build_clinsig_filter(real_variant_database):
+    """Test building a variants query with ClinVar status."""
     adapter = real_variant_database
     case_id = "cust000"
     clinsig_items = [4, 5]
@@ -349,7 +350,7 @@ def test_build_clinsig_filter(real_variant_database):
         {"$set": {"clnsig.0.value": "Pathogenic, Likely pathogenic"}},
     )
 
-    # One variant has multiple clssig now:
+    # One variant has multiple clinsig now:
     res = adapter.variant_collection.find({"clnsig.value": "Pathogenic, Likely pathogenic"})
     assert sum(1 for _ in res) == 1
 
@@ -384,7 +385,8 @@ def test_build_clinsig_filter(real_variant_database):
     assert n_results_raw_query == n_filtered_variants
 
 
-def test_build_clinsig_always(real_variant_database):
+def test_build_clinsig_high_confidence_plus_region_and_gnomad(real_variant_database):
+    """Test building a variants query with ClinVar status and high confidence."""
     adapter = real_variant_database
     case_id = "cust000"
     clinsig_confident_always_returned = True
@@ -545,7 +547,8 @@ def test_build_has_cosmic_ids(
     assert {"cosmic_ids": {"$ne": None}} in mongo_query["$and"]
 
 
-def test_build_clinsig_always_only(adapter):
+def test_build_clinsig_high_confidence(adapter):
+    """Test building a variants query with high confidence of ClinVar status."""
     case_id = "cust000"
     clinsig_confident_always_returned = True
     trusted_revstat_lev = TRUSTED_REVSTAT_LEVEL
@@ -557,6 +560,7 @@ def test_build_clinsig_always_only(adapter):
         all_clinsig.append(CLINSIG_MAP[item])
         clinsig_mapped_items.append(CLINSIG_MAP[item])
 
+    # Testing with INCLUDE ClinVar terms criterion
     query = {
         "clinsig": clinsig_items,
         "clinsig_confident_always_returned": clinsig_confident_always_returned,