diff --git a/.gitignore b/.gitignore index 9eb6eb3..6f84772 100644 --- a/.gitignore +++ b/.gitignore @@ -104,3 +104,4 @@ temp/* /docs/site/ /docs/src/tutorials/*ipynb /docs/mike-mkdocs* +element_miniscope/qc.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 960fe4d..4f13304 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and - Update - Attribute names relative to issues #20, #22, #26 - Add - Tutorial pages +- Add - Quality Control plotting tool and report schema ## [0.1.4] - 2022-10-21 diff --git a/cspell.json b/cspell.json index 1d58f64..8af5def 100644 --- a/cspell.json +++ b/cspell.json @@ -4,7 +4,8 @@ "version": "0.2", // Version of the setting file. Always 0.2 "language": "en", // language - current active spelling language "enabledLanguageIds": [ - "markdown", "yaml" + "markdown", + "yaml" ], // flagWords - list of words to be always considered incorrect // This is useful for offensive words and common spelling errors. @@ -16,6 +17,7 @@ ], "words": [ "Adrien", + "aggr", "Ahanonu", "Aharoni", "Aharoni's", @@ -27,7 +29,6 @@ "Corre", "deconvolution", "Denk", - "Denk", "denoising", "Ephys", "Epub", @@ -38,19 +39,25 @@ "Guillaume", "IDAS", "IDPS", + "ifnull", + "Inscopix", "Jinghao", "Kamata", "Lavi", "Liron", "Manolis", - "Manolis", - "Miniscopes ", "miniscopes", + "Miniscopes ", "miniscoPy", - "neuroinformatics", + "nchannels", "Neurodata", + "neuroinformatics", + "neuropil", + "Neuropil", "Neurosci", + "nframes", "nmeth", + "npix", "nVoke", "Pengcheng", "Peyman", @@ -59,6 +66,7 @@ "PIMPN", "PMCID", "PMID", + "pname", "Reimer", "Rong", "Sano", @@ -77,19 +85,12 @@ "UCSF", "Veshkini", "Viejo", + "xpix", "Yaniv", + "ypix", "Zhang", "Zhou", "Ziv's", - "npix", - "xpix", - "ypix", - "nchannels", - "nframes", - "pname", - "neuropil", - "Neuropil", - "aggr", - "ifnull" + "Zong" ] } diff --git a/docs/docker-compose.yaml b/docs/docker-compose.yaml index 4ccf733..c53aa3c 100644 --- a/docs/docker-compose.yaml +++ b/docs/docker-compose.yaml @@ -1,4 +1,4 @@ -# MODE="LIVE|QA|PUSH" PACKAGE=element_miniscope UPSTREAM_REPO=https://github.com/datajoint/element-miniscope.git HOST_UID=$(id -u) docker compose -f docs/docker-compose.yaml up --build +# MODE="LIVE" PACKAGE=element_miniscope UPSTREAM_REPO=https://github.com/datajoint/element-miniscope.git HOST_UID=$(id -u) docker compose -f docs/docker-compose.yaml up --build # # navigate to http://localhost/ version: "2.4" @@ -36,6 +36,7 @@ services: mv /main/delete/notebooks/*ipynb /main/docs/src/tutorials/ rm -fR /main/delete fi + ln -fs ./plotting/qc.py ./element_miniscope/qc.py # Workaround for subfolder error if echo "$${MODE}" | grep -i live &>/dev/null; then mkdocs serve --config-file ./docs/mkdocs.yaml -a 0.0.0.0:80 elif echo "$${MODE}" | grep -iE "qa|push" &>/dev/null; then diff --git a/docs/mkdocs.yaml b/docs/mkdocs.yaml index ed0ba29..71cdc78 100644 --- a/docs/mkdocs.yaml +++ b/docs/mkdocs.yaml @@ -13,10 +13,11 @@ nav: - Configure: tutorials/01-Configure.ipynb - Workflow Structure: tutorials/02-WorkflowStructure_Optional.ipynb - Process: tutorials/03-Process.ipynb - # - Automate: tutorials/04-Automate_Optional.ipynb + - Automate: tutorials/04-Automate_Optional.ipynb - Explore: tutorials/05-Explore.ipynb - Drop: tutorials/06-Drop_Optional.ipynb - Downstream Analysis: tutorials/07-DownstreamAnalysis_Optional.ipynb + - Visualizations: tutorials/08-Visualizations.ipynb - Citation: citation.md - API: api/ # defer to gen-files + literate-nav - Changelog: changelog.md @@ -101,6 +102,12 @@ plugins: # "index.md": "getting_started.md" - mkdocstrings: default_handler: python + handlers: + python: + options: + members_order: source + group_by_category: false + line_length: 88 - gen-files: scripts: - ./src/api/make_pages.py @@ -111,7 +118,6 @@ plugins: - "*/navigation.md" - mkdocs-jupyter: ignore_h1_titles: True - ignore: ["*make_pages.py"] markdown_extensions: - attr_list @@ -175,4 +181,4 @@ extra_css: - assets/stylesheets/extra.css extra_javascript: - - https://js-na1.hs-scripts.com/23133402.js # HubSpot chatbot + - https://js-na1.hs-scripts.com/23133402.js # HubSpot chatbot diff --git a/docs/src/api/make_pages.py b/docs/src/api/make_pages.py index 7b96ced..b5341d1 100644 --- a/docs/src/api/make_pages.py +++ b/docs/src/api/make_pages.py @@ -9,13 +9,17 @@ import os package = os.getenv("PACKAGE") - element = package.split("_", 1)[1] + nav = mkdocs_gen_files.Nav() + for path in sorted(Path(package).glob("**/*.py")) + sorted( Path(f"workflow_{element}").glob("**/*.py") ): - if path.stem == "__init__": + if ( + path.stem == "__init__" + or "plotting" in path.parts # Workaround for mkdocstring-python subfolder error + ): continue with mkdocs_gen_files.open(f"api/{path.with_suffix('')}.md", "w") as f: module_path = ".".join( diff --git a/docs/src/concepts.md b/docs/src/concepts.md index 88f21c3..8fad6e4 100644 --- a/docs/src/concepts.md +++ b/docs/src/concepts.md @@ -1,3 +1,4 @@ + # Concepts ## Miniscopes in Neuroscience Research @@ -153,6 +154,21 @@ Tables related to importing, analyzing, and exporting miniscope data. | ActivityExtractionMethod | A table with information about the activity extraction method (e.g. deconvolution) applied on the fluorescence trace. | | Activity | A table with neuronal activity traces from fluorescence trace (e.g. spikes). | +### `miniscope_report` schema ([API docs](../api/element_miniscope/miniscope_report)) + +Tables related to summary reports of miniscope data. + +| Table | Description | +| --- | --- | +| QualityMetrics | A table containing information about CaImAn estimates. | + +The above `QualityMetrics` table includes the following for each component in the CaImAn +analysis: + +- `r_values`: Space correlation. +- `snr`: Trace SNR. +- `cnn_preds`: CNN predictions. + ## Pipeline Development With assistance from Dr. Peyman Golshani's Lab (UCLA) we have added support for the UCLA diff --git a/docs/src/index.md b/docs/src/index.md index 1f69f67..1ad0a37 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -12,4 +12,4 @@ combined with other Elements to assemble a fully functional pipeline. Visit the [Concepts page](./concepts.md) for more information about miniscope calcium imaging and Element Miniscope. To get started with building your data pipeline visit -the [Tutorials page](./tutorials.md). +the [Tutorials page](./tutorials/). diff --git a/docs/src/tutorials/index.md b/docs/src/tutorials/index.md index aae3c19..05f004c 100644 --- a/docs/src/tutorials/index.md +++ b/docs/src/tutorials/index.md @@ -38,12 +38,16 @@ Jupyter environment with access to example data, visit tables. - [Process](./03-Process.ipynb) steps through adding data to these tables and launching key Miniscope features, like model training. - -- [Visualization](./05-Explore.ipynb) +- [Automate](./04-Automate_Optional.ipynb) + highlights the same steps as above, but utilizing all built-in automation tools. +- [Explore](./05-Explore.ipynb) demonstrates how to fetch data from the Element. - [Drop schemas](./06-Drop_Optional.ipynb) - provides the steps for dropping all the tables to start fresh. + provides the steps for dropping all the tables to start fresh.] +- [Downstream Analysis](./07-DownstreamAnalysis_Optional.ipynb) highlights how to link + this Element to Element Event for event-based analyses. +- [Visualizations](./08-Visualizations.ipynb) highlights how to use a built-in module + for visualizing Quality Metrics from CaImAn analyses. - `NWB-Export` (coming soon!) will describe how to export into NWB files.