Merge branch 'main' of datajoint/element-miniscope

.gitignore

@@ -86,12 +86,22 @@ site
 # mypy
 .mypy_cache/
 
-# datajoint
-dj_local_con*.json
-temp*
-
 # emacs
 **/*~
 **/#*#
 **/.#*
 docker-compose.yml
+
+.vscode*
+
+# datajoint, notes, nwb export
+dj_local_c*.json
+temp*
+temp/*
+*nwb
+
+# docs
+/docs/site/
+/docs/src/tutorials/*ipynb
+/docs/mike-mkdocs*
+element_miniscope/qc.py

.markdownlint.yaml

@@ -0,0 +1,17 @@
+# Markdown Linter configuration for docs
+# https://github.com/DavidAnson/markdownlint
+# https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md
+MD009: false # permit trailing spaces
+MD007: false # List indenting - permit 4 spaces
+MD013:
+    line_length: "88" # Line length limits
+    tables: false # disable for tables
+    headings: false # disable for headings
+MD030: false # Number of spaces after a list
+MD033: # HTML elements allowed
+    allowed_elements:
+        - "br"
+        - "sup"
+MD034: false # Permit bare URLs
+MD031: false # Spacing w/code blocks. Conflicts with `??? Note` and code tab styling
+MD046: false # Spacing w/code blocks. Conflicts with `??? Note` and code tab styling

.pre-commit-config.yaml

@@ -0,0 +1,56 @@
+default_stages: [commit, push]
+exclude: (^.github/|^docs/|^images/)
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files # prevent giant files from being committed
+      - id: requirements-txt-fixer
+      - id: mixed-line-ending
+        args: ["--fix=lf"]
+        description: Forces to replace line ending by the UNIX 'lf' character.
+
+  # black
+  - repo: https://github.com/psf/black
+    rev: 22.12.0
+    hooks:
+      - id: black
+      - id: black-jupyter
+        args:
+          - --line-length=88
+
+  # isort
+  - repo: https://github.com/pycqa/isort
+    rev: 5.11.2
+    hooks:
+      - id: isort
+        args: ["--profile", "black"]
+        description: Sorts imports in an alphabetical order
+
+  # flake8
+  - repo: https://github.com/pycqa/flake8
+    rev: 4.0.1
+    hooks:
+      - id: flake8
+        args: # arguments to configure flake8
+          # making isort line length compatible with black
+          - "--max-line-length=88"
+          - "--max-complexity=18"
+          - "--select=B,C,E,F,W,T4,B9"
+
+          # these are errors that will be ignored by flake8
+          # https://www.flake8rules.com/rules/{code}.html
+          - "--ignore=E203,E501,W503,W605"
+          # E203 - Colons should not have any space before them.
+          #        Needed for list indexing
+          # E501 - Line lengths are recommended to be no greater than 79 characters.
+          #        Needed as we conform to 88
+          # W503 - Line breaks should occur after the binary operator.
+          #        Needed because not compatible with black
+          # W605 - a backslash-character pair that is not a valid escape sequence now
+          #        generates a DeprecationWarning. This will eventually become a SyntaxError.
+          #        Needed because we use \d as an escape sequence

CHANGELOG.md

@@ -1,26 +1,45 @@
 # Changelog
 
-Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention.
+Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
+[Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention.
+
+## [0.2.1] - 2023-05-11
+
++ Fix - `.ipynb` dark mode output for all notebooks.
++ Fix - Remove `GOOGLE_ANALYTICS_KEY` from `u24_element_release_call.yml`.
+
+## [0.2.0] - 2023-04-28
+
++ Update - Attribute names relative to issues #20, #22, #26
++ Add - Tutorial pages
++ Add - Quality Control plotting tool and report schema
++ Fix - `.ipynb` output in tutorials is not visible in dark mode.
 
 ## [0.1.4] - 2022-10-21
+
 + Add - mkdocs deployment with workflow API docs
 + Update - processing_method: char(16) -> varchar(16)
 
 ## [0.1.3] - 2022-10-11
+
 + Update - CICD workflows for PyPI release
 
 ## [0.1.2] - 2022-05-10
+
 + Add - Load data acquired with Miniscope-DAQ-V4
 + Add - Load data analyzed with CaImAn
 + Add - Trigger CaImAn analysis
 + Remove - Load data analyzed with MiniscopeAnalysis
 + Add - Adopted black formatting into code base
 
 ## [0.1.1] - 2021-04-01
+
 + Add - Load data acquired with Miniscope-DAQ-V3
 + Add - Load data analyzed with MiniscopeAnalysis
 
+[0.2.1]: https://github.com/datajoint/element-miniscope/releases/tag/0.2.1
 [0.2.0]: https://github.com/datajoint/element-miniscope/releases/tag/0.2.0
+[0.1.4]: https://github.com/datajoint/element-miniscope/releases/tag/0.1.4
 [0.1.3]: https://github.com/datajoint/element-miniscope/releases/tag/0.1.3
 [0.1.2]: https://github.com/datajoint/element-miniscope/releases/tag/0.1.2
 [0.1.1]: https://github.com/datajoint/element-miniscope/releases/tag/0.1.1

CODE_OF_CONDUCT.md

@@ -1,4 +1,3 @@
-
 # Contributor Covenant Code of Conduct
 
 ## Our Pledge

CONTRIBUTING.md

@@ -1,3 +1,5 @@
 # Contribution Guidelines
 
-This project follows the [DataJoint Contribution Guidelines](https://docs.datajoint.io/python/community/02-Contribute.html). Please reference the link for more full details.
+This project follows the
+[DataJoint Contribution Guidelines](https://datajoint.com/docs/community/contribute/).
+Please reference the link for more full details.

README.md

@@ -1,14 +1,15 @@
 # DataJoint Element - Miniscope Calcium Imaging
 
-DataJoint Element for functional calcium imaging data acquired with the [UCLA Miniscope]
-(https://github.com/Aharoni-Lab/Miniscope-v4) and [Miniscope DAQ]
-(https://github.com/Aharoni-Lab/Miniscope-DAQ-QT-Software) acquisition system, and
-analyzed with `CaImAn`. DataJoint Elements collectively standardize and automate data
-collection and analysis for neuroscience experiments.  Each Element is a modular
-pipeline for data storage and processing with corresponding database tables that can be
-combined with other Elements to assemble a fully functional pipeline.
+DataJoint Element for functional calcium imaging data acquired with the
+[UCLA Miniscope](https://github.com/Aharoni-Lab/Miniscope-v4) and
+[Miniscope DAQ](https://github.com/Aharoni-Lab/Miniscope-DAQ-QT-Software)
+acquisition system, and analyzed with `CaImAn`. DataJoint Elements collectively
+standardize and automate data collection and analysis for neuroscience experiments.
+Each Element is a modular pipeline for data storage and processing with corresponding
+database tables that can be combined with other Elements to assemble a fully functional
+pipeline.
 
 ![diagram](https://raw.githubusercontent.com/datajoint/element-miniscope/main/images/diagram_flowchart.svg)
 
-Installation and usage instructions can be found at the 
-[Element documentation](datajoint.com/docs/elements/element-miniscope).
+Installation and usage instructions can be found at the
+[Element documentation](https://datajoint.com/docs/elements/element-miniscope).

cspell.json

@@ -0,0 +1,99 @@
+// cSpell Settings
+//https://github.com/streetsidesoftware/vscode-spell-checker
+{
+    "version": "0.2", // Version of the setting file.  Always 0.2
+    "language": "en", // language - current active spelling language
+    "enabledLanguageIds": [
+        "markdown",
+        "yaml"
+    ],
+    // flagWords - list of words to be always considered incorrect
+    // This is useful for offensive words and common spelling errors.
+    // For example "hte" should be "the"
+    "flagWords": [],
+    "allowCompoundWords": true,
+    "ignorePaths": [
+        "src/archive"
+    ],
+    "words": [
+        "Adrien",
+        "aggr",
+        "Ahanonu",
+        "Aharoni",
+        "Aharoni's",
+        "Atah",
+        "Baumgaertel",
+        "Biane",
+        "Cheng",
+        "CNMF",
+        "Corre",
+        "deconvolution",
+        "Denk",
+        "denoising",
+        "Ephys",
+        "Epub",
+        "Etter",
+        "fncel",
+        "Froudaraki",
+        "Golshani",
+        "Guillaume",
+        "IDAS",
+        "IDPS",
+        "ifnull",
+        "Inscopix",
+        "Jinghao",
+        "Kamata",
+        "Lavi",
+        "Liron",
+        "Manolis",
+        "miniscopes",
+        "Miniscopes ",
+        "miniscoPy",
+        "nchannels",
+        "Neurodata",
+        "neuroinformatics",
+        "neuropil",
+        "Neuropil",
+        "Neurosci",
+        "nframes",
+        "nmeth",
+        "npix",
+        "nVoke",
+        "Pengcheng",
+        "Peyman",
+        "Peyrache",
+        "Peyrache's",
+        "PIMPN",
+        "PMCID",
+        "PMID",
+        "pname",
+        "Reimer",
+        "Rong",
+        "Sano",
+        "Sasaki",
+        "Schnitzer",
+        "Schnitzer's",
+        "Sheintuch",
+        "Shen",
+        "Shobe",
+        "Shuman",
+        "Sitonic",
+        "Sylvain",
+        "Sylvian",
+        "Tolias",
+        "Tuszynski",
+        "UCSF",
+        "Veshkini",
+        "Viejo",
+        "xpix",
+        "Yaniv",
+        "ypix",
+        "Zhang",
+        "Zhou",
+        "Ziv's",
+        "Zong",
+        "Yatsenko",
+        "RRID",
+        "Rxiv"
+    ]
+}

docs/.docker/pip_requirements.txt

@@ -9,4 +9,4 @@ mkdocs-literate-nav
 mkdocs-exclude-search
 mkdocs-markdownextradata-plugin
 mkdocs-jupyter
-mkdocs-section-index
+mkdocs-section-index

docs/docker-compose.yaml

@@ -1,4 +1,4 @@
-# MODE="LIVE|QA|PUSH" PACKAGE=element_miniscope UPSTREAM_REPO=https://github.com/datajoint/element-miniscope.git HOST_UID=$(id -u) docker compose -f docs/docker-compose.yaml up --build
+# MODE="LIVE" PACKAGE=element_miniscope UPSTREAM_REPO=https://github.com/datajoint/element-miniscope.git HOST_UID=$(id -u) docker compose -f docs/docker-compose.yaml up --build
 #
 # navigate to http://localhost/
 version: "2.4"
@@ -27,7 +27,15 @@ services:
       - |
         git config --global --add safe.directory /main
         set -e
+        export ELEMENT_NAME=$$(echo $${PACKAGE} | sed 's/element_//g')
         export PATCH_VERSION=$$(cat /main/$${PACKAGE}/version.py | grep -oE '\d+\.\d+\.[a-z0-9]+')
+        git clone https://github.com/datajoint/workflow-$${ELEMENT_NAME}.git /main/delete || true
+        if [ -d /main/delete/ ]; then
+          mv /main/delete/workflow_$${ELEMENT_NAME} /main/
+          mv /main/delete/notebooks/*ipynb /main/docs/src/tutorials/
+          rm -fR /main/delete
+        fi
+        ln -fs ./plotting/qc.py ./element_miniscope/qc.py # Workaround for subfolder error
         if echo "$${MODE}" | grep -i live &>/dev/null; then
             mkdocs serve --config-file ./docs/mkdocs.yaml -a 0.0.0.0:80
         elif echo "$${MODE}" | grep -iE "qa|push" &>/dev/null; then

docs/mkdocs.yaml

@@ -7,19 +7,29 @@ repo_name: datajoint/element-miniscope
 nav:
   - Element Miniscope: index.md
   - Concepts: concepts.md
-  - Tutorials: tutorials.md
+  - Tutorials: 
+    - Overview: tutorials/index.md
+    - Data Download: tutorials/00-DataDownload_Optional.ipynb
+    - Configure: tutorials/01-Configure.ipynb
+    - Workflow Structure: tutorials/02-WorkflowStructure_Optional.ipynb
+    - Process: tutorials/03-Process.ipynb
+    - Automate: tutorials/04-Automate_Optional.ipynb
+    - Explore: tutorials/05-Explore.ipynb
+    - Drop: tutorials/06-Drop_Optional.ipynb
+    - Downstream Analysis: tutorials/07-DownstreamAnalysis_Optional.ipynb
+    - Visualizations: tutorials/08-Visualizations.ipynb
   - Citation: citation.md
   - API: api/ # defer to gen-files + literate-nav
   - Changelog: changelog.md
 
 # --------------------- NOTES TO CONTRIBUTORS -----------------------
 # Markdown in mkdocs
-# 01. Redering concatenates across single line breaks. This means...
+# 01. Rendering concatenates across single line breaks. This means...
 #     A. We have to be careful to add extra line breaks around paragraphs,
-#        including between the end of a pgf and the beginnign of bullets.
+#        including between the end of a pgf and the beginning of bullets.
 #     B. We can use hard wrapping to make github reviews easier to read.
 #        VSCode Rewrap extension offers a keyboard shortcut for hard wrap
-#        at the ruler, but don't add breaks in [multiword links](example.com)
+#        at the ruler, but don't add breaks in [multi-word links](example.com)
 # 02. Instead of designating codeblocks with bash, use console. For example..
 #     ```console
 #     cd ../my_dir
@@ -86,6 +96,12 @@ plugins:
   - search
   - mkdocstrings:
       default_handler: python
+      handlers:
+        python:
+          options:
+            members_order: source
+            group_by_category: false
+            line_length: 88
   - gen-files:
       scripts:
         - ./src/api/make_pages.py
@@ -94,6 +110,9 @@ plugins:
   - exclude-search:
       exclude:
         - "*/navigation.md"
+  - mkdocs-jupyter:
+      ignore_h1_titles: True
+
 markdown_extensions:
   - attr_list
   - toc:
@@ -117,6 +136,7 @@ markdown_extensions:
   - pymdownx.magiclink # Displays bare URLs as links
   - pymdownx.tasklist: # Renders check boxes in tasks lists
       custom_checkbox: true
+  - footnotes
 extra:
   PATCH_VERSION: !ENV PATCH_VERSION
   generator: false # Disable watermark
@@ -154,4 +174,4 @@ extra_css:
   - assets/stylesheets/extra.css
 
 extra_javascript:
-  - https://js-na1.hs-scripts.com/23133402.js # HubSpot chatbot
+  - https://js-na1.hs-scripts.com/23133402.js  # HubSpot chatbot

docs/src/.overrides/assets/stylesheets/extra.css

@@ -91,3 +91,8 @@ html a[title="YouTube"].md-social__link svg {
     /* previous/next text */
     /* --md-footer-fg-color: var(--dj-white); */
 }
+
+[data-md-color-scheme="slate"] td,
+th {
+    color: var(--dj-black)
+}