Merge pull request datajoint#39 from CBroz1/main

Add CaImAn QC metrics

.gitignore

@@ -104,3 +104,4 @@ temp/*
 /docs/site/
 /docs/src/tutorials/*ipynb
 /docs/mike-mkdocs*
+element_miniscope/qc.py

CHANGELOG.md

@@ -7,6 +7,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 
 - Update - Attribute names relative to issues #20, #22, #26
 - Add - Tutorial pages
+- Add - Quality Control plotting tool and report schema
 
 ## [0.1.4] - 2022-10-21
 

cspell.json

@@ -4,7 +4,8 @@
     "version": "0.2", // Version of the setting file.  Always 0.2
     "language": "en", // language - current active spelling language
     "enabledLanguageIds": [
-        "markdown", "yaml"
+        "markdown",
+        "yaml"
     ],
     // flagWords - list of words to be always considered incorrect
     // This is useful for offensive words and common spelling errors.
@@ -16,6 +17,7 @@
     ],
     "words": [
         "Adrien",
+        "aggr",
         "Ahanonu",
         "Aharoni",
         "Aharoni's",
@@ -27,7 +29,6 @@
         "Corre",
         "deconvolution",
         "Denk",
-        "Denk",
         "denoising",
         "Ephys",
         "Epub",
@@ -38,19 +39,25 @@
         "Guillaume",
         "IDAS",
         "IDPS",
+        "ifnull",
+        "Inscopix",
         "Jinghao",
         "Kamata",
         "Lavi",
         "Liron",
         "Manolis",
-        "Manolis",
-        "Miniscopes ",
         "miniscopes",
+        "Miniscopes ",
         "miniscoPy",
-        "neuroinformatics",
+        "nchannels",
         "Neurodata",
+        "neuroinformatics",
+        "neuropil",
+        "Neuropil",
         "Neurosci",
+        "nframes",
         "nmeth",
+        "npix",
         "nVoke",
         "Pengcheng",
         "Peyman",
@@ -59,6 +66,7 @@
         "PIMPN",
         "PMCID",
         "PMID",
+        "pname",
         "Reimer",
         "Rong",
         "Sano",
@@ -77,19 +85,12 @@
         "UCSF",
         "Veshkini",
         "Viejo",
+        "xpix",
         "Yaniv",
+        "ypix",
         "Zhang",
         "Zhou",
         "Ziv's",
-        "npix",
-        "xpix",
-        "ypix",
-        "nchannels",
-        "nframes",
-        "pname",
-        "neuropil",
-        "Neuropil",
-        "aggr",
-        "ifnull"
+        "Zong"
     ]
 }

docs/docker-compose.yaml

@@ -1,4 +1,4 @@
-# MODE="LIVE|QA|PUSH" PACKAGE=element_miniscope UPSTREAM_REPO=https://github.com/datajoint/element-miniscope.git HOST_UID=$(id -u) docker compose -f docs/docker-compose.yaml up --build
+# MODE="LIVE" PACKAGE=element_miniscope UPSTREAM_REPO=https://github.com/datajoint/element-miniscope.git HOST_UID=$(id -u) docker compose -f docs/docker-compose.yaml up --build
 #
 # navigate to http://localhost/
 version: "2.4"
@@ -36,6 +36,7 @@ services:
           mv /main/delete/notebooks/*ipynb /main/docs/src/tutorials/
           rm -fR /main/delete
         fi
+        ln -fs ./plotting/qc.py ./element_miniscope/qc.py # Workaround for subfolder error
         if echo "$${MODE}" | grep -i live &>/dev/null; then
             mkdocs serve --config-file ./docs/mkdocs.yaml -a 0.0.0.0:80
         elif echo "$${MODE}" | grep -iE "qa|push" &>/dev/null; then

docs/mkdocs.yaml

@@ -13,10 +13,11 @@ nav:
     - Configure: tutorials/01-Configure.ipynb
     - Workflow Structure: tutorials/02-WorkflowStructure_Optional.ipynb
     - Process: tutorials/03-Process.ipynb
-    # - Automate: tutorials/04-Automate_Optional.ipynb
+    - Automate: tutorials/04-Automate_Optional.ipynb
     - Explore: tutorials/05-Explore.ipynb
     - Drop: tutorials/06-Drop_Optional.ipynb
     - Downstream Analysis: tutorials/07-DownstreamAnalysis_Optional.ipynb
+    - Visualizations: tutorials/08-Visualizations.ipynb
   - Citation: citation.md
   - API: api/ # defer to gen-files + literate-nav
   - Changelog: changelog.md
@@ -101,6 +102,12 @@ plugins:
   #       "index.md": "getting_started.md"
   - mkdocstrings:
       default_handler: python
+      handlers:
+        python:
+          options:
+            members_order: source
+            group_by_category: false
+            line_length: 88
   - gen-files:
       scripts:
         - ./src/api/make_pages.py
@@ -111,7 +118,6 @@ plugins:
         - "*/navigation.md"
   - mkdocs-jupyter:
       ignore_h1_titles: True
-      ignore: ["*make_pages.py"]
 
 markdown_extensions:
   - attr_list
@@ -175,4 +181,4 @@ extra_css:
   - assets/stylesheets/extra.css
 
 extra_javascript:
-  - https://js-na1.hs-scripts.com/23133402.js # HubSpot chatbot
+  - https://js-na1.hs-scripts.com/23133402.js  # HubSpot chatbot

docs/src/api/make_pages.py

@@ -9,13 +9,17 @@
 import os
 
 package = os.getenv("PACKAGE")
-
 element = package.split("_", 1)[1]
+
 nav = mkdocs_gen_files.Nav()
+
 for path in sorted(Path(package).glob("**/*.py")) + sorted(
     Path(f"workflow_{element}").glob("**/*.py")
 ):
-    if path.stem == "__init__":
+    if (
+        path.stem == "__init__"
+        or "plotting" in path.parts  # Workaround for mkdocstring-python subfolder error
+    ):
         continue
     with mkdocs_gen_files.open(f"api/{path.with_suffix('')}.md", "w") as f:
         module_path = ".".join(

docs/src/concepts.md

@@ -1,3 +1,4 @@
+<!-- markdownlint-disable MD053 -->
 # Concepts
 
 ## Miniscopes in Neuroscience Research
@@ -153,6 +154,21 @@ Tables related to importing, analyzing, and exporting miniscope data.
 | ActivityExtractionMethod | A table with information about the activity extraction method (e.g. deconvolution) applied on the fluorescence trace. |
 | Activity | A table with neuronal activity traces from fluorescence trace (e.g. spikes). |
 
+### `miniscope_report` schema ([API docs](../api/element_miniscope/miniscope_report))
+
+Tables related to summary reports of miniscope data.
+
+| Table | Description |
+| --- | --- |
+| QualityMetrics | A table containing information about CaImAn estimates. |
+
+The above `QualityMetrics` table includes the following for each component in the CaImAn
+analysis:
+
+- `r_values`: Space correlation.
+- `snr`: Trace SNR.
+- `cnn_preds`: CNN predictions.
+
 ## Pipeline Development
 
 With assistance from Dr. Peyman Golshani's Lab (UCLA) we have added support for the UCLA

docs/src/index.md

@@ -12,4 +12,4 @@ combined with other Elements to assemble a fully functional pipeline.
 
 Visit the [Concepts page](./concepts.md) for more information about miniscope calcium
 imaging and Element Miniscope. To get started with building your data pipeline visit
-the [Tutorials page](./tutorials.md).
+the [Tutorials page](./tutorials/).

docs/src/tutorials/index.md

@@ -38,12 +38,16 @@ Jupyter environment with access to example data, visit
    tables.
 - [Process](./03-Process.ipynb) steps through adding data to these tables and launching
    key Miniscope features, like model training.
-<!-- - [Automate](./04-Automate_Optional.ipynb)
-   highlights the same steps as above, but utilizing all built-in automation tools. -->
-- [Visualization](./05-Explore.ipynb)
+- [Automate](./04-Automate_Optional.ipynb)
+   highlights the same steps as above, but utilizing all built-in automation tools.
+- [Explore](./05-Explore.ipynb)
    demonstrates how to fetch data from the Element.
 - [Drop schemas](./06-Drop_Optional.ipynb)
-   provides the steps for dropping all the tables to start fresh.
+   provides the steps for dropping all the tables to start fresh.]
+- [Downstream Analysis](./07-DownstreamAnalysis_Optional.ipynb) highlights how to link
+   this Element to Element Event for event-based analyses.
+- [Visualizations](./08-Visualizations.ipynb) highlights how to use a built-in module 
+   for visualizing Quality Metrics from CaImAn analyses.
 - `NWB-Export` (coming soon!) will describe how to export into NWB files. 
 
 <!-- ## Data Export to Neurodata Without Borders (NWB)

element_miniscope/miniscope.py

@@ -11,6 +11,8 @@
 import numpy as np
 from element_interface.utils import dict_to_uuid, find_full_path, find_root_directory
 
+from . import miniscope_report
+
 schema = dj.Schema()
 
 _linking_module = None
@@ -65,6 +67,7 @@ def activate(
         create_tables=create_tables,
         add_objects=_linking_module.__dict__,
     )
+    miniscope_report.activate(f"{miniscope_schema_name}_report", miniscope_schema_name)
 
 
 # Functions required by the element-miniscope  -----------------------------------------
@@ -1174,7 +1177,7 @@ def make(self, key):
 }
 
 
-def get_loader_result(key, table):
+def get_loader_result(key, table) -> tuple:
     """Retrieve the loaded processed imaging results from the loader (e.g. caiman, etc.)
 
     Args:
@@ -1183,7 +1186,7 @@ def get_loader_result(key, table):
             the loaded results from (e.g. ProcessingTask, Curation).
 
     Returns:
-        tuple: method string and loader object with results (e.g. caiman.CaImAn, etc.)
+        method, loaded_output (tuple): method string and loader object with results (e.g. caiman.CaImAn, etc.)
     """
 
     method, output_dir = (ProcessingParamSet * table & key).fetch1(

element_miniscope/miniscope_report.py

@@ -0,0 +1,63 @@
+import datajoint as dj
+
+schema = dj.schema()
+
+miniscope = None
+
+
+def activate(
+    schema_name, miniscope_schema_name, *, create_schema=True, create_tables=True
+):
+    """Activate this schema.
+
+    The "activation" of miniscope_report should be evoked by the miniscope module
+
+    Args:
+        schema_name (str): schema name on the database server to activate the
+            `miniscope_report` schema
+        miniscope_schema_name (str): schema name of the activated miniscope element for
+            which this miniscope_report schema will be downstream from
+        create_schema (bool): when True (default), create schema in the database if it
+            does not yet exist.
+        create_tables (str): when True (default), create schema takes in the database
+            if they do not yet exist.
+    """
+    global miniscope
+    miniscope = dj.create_virtual_module("miniscope", miniscope_schema_name)
+    schema.activate(
+        schema_name,
+        create_schema=create_schema,
+        create_tables=create_tables,
+        add_objects=miniscope.__dict__,
+    )
+
+
+@schema
+class QualityMetrics(dj.Imported):
+    definition = """
+    -> miniscope.Curation
+    ---
+    r_values=null  : longblob # space correlation for each component
+    snr=null       : longblob # trace SNR for each component
+    cnn_preds=null : longblob # CNN predictions for each component
+    """
+
+    def make(self, key):
+        from .miniscope import get_loader_result
+
+        method, loaded_result = get_loader_result(key, miniscope.Curation)
+        assert (
+            method == "caiman"
+        ), f"Quality figures for {method} not yet implemented. Try CaImAn."
+
+        key.update(
+            {
+                attrib_name: getattr(loaded_result.cnmf.estimates, attrib, None)
+                for attrib_name, attrib in zip(
+                    ["r_values", "snr", "cnn_preds"],
+                    ["r_values", "SNR_comp", "cnn_preds"],
+                )
+            }
+        )
+
+        self.insert1(key)