WIP: database management doc

docs/mkdocs.yml

@@ -51,12 +51,14 @@ nav:
       - Session Groups: misc/session_groups.md
       - Insert Data: misc/insert_data.md
       - Merge Tables: misc/merge_tables.md
+      - Database Management: misc/database_management.md
   - Tutorials:
       - Overview: notebooks/README.md
       - General:
           - Setup: notebooks/00_Setup.ipynb
           - Insert Data: notebooks/01_Insert_Data.ipynb
           - Data Sync: notebooks/02_Data_Sync.ipynb
+          - Merge Tables: notebooks/03_Merge_Tables.ipynb
       - Ephys:
           - Spike Sorting: notebooks/10_Spike_Sorting.ipynb
           - Curation: notebooks/11_Curation.ipynb
@@ -101,8 +103,6 @@ plugins:
             group_by_category: false
             line_length: 80
             docstring_style: numpy
-      # watch:
-      # - src/spyglass/
   - literate-nav:
       nav_file: navigation.md
   - exclude-search:
@@ -111,11 +111,11 @@ plugins:
   - gen-files:
       scripts:
         - ./src/api/make_pages.py
-  - mkdocs-jupyter: # Comment this block during dev to reduce build time
-      execute: False # Very slow, needs gh-action edit to work/link to db
-      include_source: False
-      ignore_h1_titles: True
-      ignore: ["*make_pages.py", "**checkpoints**", "*/py_scripts/*"]
+  # - mkdocs-jupyter: # Comment this block during dev to reduce build time
+  #     execute: False # Very slow, needs gh-action edit to work/link to db
+  #     include_source: False
+  #     ignore_h1_titles: True
+  #     ignore: ["*make_pages.py", "**checkpoints**", "*/py_scripts/*"]
 
 markdown_extensions:
   - attr_list

docs/src/api/make_pages.py

@@ -4,15 +4,30 @@
 from pathlib import Path
 
 import mkdocs_gen_files
+from mkdocs.utils import log
+
+ignored_stems = ["__init__", "_version"]
+
+added = 0
+add_limit = None
 
 nav = mkdocs_gen_files.Nav()
-for path in sorted(Path("src").glob("**/*.py")):
-    if path.stem == "__init__" or "cython" in path.stem:
+for path in sorted(Path("src/spyglass/").glob("**/*.py")):
+    if path.stem in ignored_stems or "cython" in path.stem:
         continue
-    with mkdocs_gen_files.open(f"api/{path.with_suffix('')}.md", "w") as f:
+    rel_path = path.relative_to("src/spyglass")
+    with mkdocs_gen_files.open(f"api/{rel_path.with_suffix('')}.md", "w") as f:
         module_path = ".".join([p for p in path.with_suffix("").parts])
         print(f"::: {module_path}", file=f)
-    nav[path.parts] = f"{path.with_suffix('')}.md"
+    nav[rel_path.parts] = f"{rel_path.with_suffix('')}.md"
+
+    if add_limit is not None:
+        if added < add_limit:
+            log.warning(f"Generated {rel_path.with_suffix('')}.md")
+            added += 1
+        else:
+            break
+
 
 with mkdocs_gen_files.open("api/navigation.md", "w") as nav_file:
     nav_file.writelines(nav.build_literate_nav())

docs/src/misc/database_management.md

@@ -0,0 +1,89 @@
+# Database Management
+
+While Spyglass can help you organize your data, there are a number of things
+you'll need to do to manage users, database backups, and file cleanup.
+
+Some these tasks should be set to run regularly. [Cron jobs](https://www.hostinger.com/tutorials/cron-job)
+can help with automation.
+
+## MySQL Version
+
+The Frank Lab's database is running MySQL 8.0 with a number of custom
+configurations set by our system admin to reflect UCSF's IT security
+requirements.
+
+DataJoint's default docker container for MySQL is version 5.7. As the Spyglass
+team has hit select compatibility issues, we've worked with the DataJoint team
+to update the open source package to support MySQL 8.0.
+
+While the Spyglass team won't be able to support earlier versions, if you run
+into any issues declaring Spyglass tables with an 8.0 instance, please let us
+know.
+
+## User Management
+
+The [DatabaseSettings](../api/utils/database_settings.md) class provides a
+number of methods to help you manage users. By default, it will write out a
+temporary `.sql` file and execute it on the database.
+
+### Privileges
+
+DataJoint schemas correspond to MySQL databases. Priveleges are managed by
+schema/database prefix.
+
+- `SELECT` privileges allow users to read, write, and delete data.
+- `ALL` privileges allow users to create, alter, or drop tables and schemas in
+    addition to operations above.
+
+In practice, DataJoint only permits alerations of secondary keys on existing
+tables, and more derstructive operations would require using DataJoint to
+execeute MySQL commands.
+
+Shared schema prefixes are those defined in the Spyglass package (e.g.,
+`common`, `lfp`, etc.). A 'user schema' is any schema with the username as
+prefix. User types differ in the privileges they are granted on  
+these prifixes.
+
+### Users types
+
+- `collab_user`: `ALL` on user schema, `SELECT` on all other schemas.
+- `dj_guest`: `SELECT` on all schemas.
+- `dj_user`: `ALL` on shared and user schema, `SELECT` on all other schemas.
+
+### Setting Passwords
+
+New users are generted with the password `temppass`. In order to change this,
+we recommend downloading DataJoint `0.14.2` (currently pre-release).
+
+```console
+git clone https://github.com/datajoint/datajoint-python/
+pip install ./datajoint-python
+```
+
+Then, you the user can reset within Python:
+
+```python
+import datajoint as dj 
+dj.set_password()
+```
+
+## Database Backups
+
+Coming soon...
+
+## File Cleanup
+
+Spyglass is designed to hold metadata for analyses that reference NWB files on
+disk. There are several tables that retain lists of files that have been generated
+during analyses. If someone deletes analysis entries, files will still be on disk.
+
+To remove orphaned files, we run the following commands in our cron jobs:
+
+```python
+from spyglass.common import AnalysisNwbfile
+from spyglass.spikesorting import SpikeSorting
+
+def main():
+    AnalysisNwbfile().nightly_cleanup()
+    SpikeSorting().nightly_cleanup()
+```

docs/src/misc/merge_tables.md

@@ -70,7 +70,7 @@ The Merge class in Spyglass's utils is a subclass of DataJoint's [Manual
 Table](https://datajoint.com/docs/core/design/tables/tiers/#data-entry-lookup-and-manual)
 and adds functions to make the awkwardness of part tables more manageable.
 These functions are described in the
-[API section](../../api/src/spyglass/utils/dj_merge_tables.md), under
+[API section](../api/utils/dj_merge_tables.md), under
 `utils.dj_merge_tables`.
 
 ### Restricting

notebooks/00_Setup.ipynb

@@ -219,12 +219,16 @@
     "- Add yourself to the\n",
     "  [`docker` group](https://docs.docker.com/engine/install/linux-postinstall/) so\n",
     "  that you don't have to be sudo to run docker.\n",
-    "- Download the docker image for datajoint/mysql\n",
+    "- Download the docker image for `datajoint/mysql:8.0`.\n",
     "\n",
     "  ```bash\n",
-    "  docker pull datajoint/mysql\n",
+    "  docker pull datajoint/mysql:8.0\n",
     "  ```\n",
     "\n",
+    "_Note_: For demonstation, MySQL version won't matter. Some\n",
+    "  [database management](https://lorenfranklab.github.io/spyglass/latest/misc/database_management/#mysql-version)\n",
+    "  features of Spyglass, however, expect MySQL >= 8.\n",
+    "\n",
     "- When run, this is referred to as a 'Docker container'\n",
     "- Next start the container with a couple additional pieces of info...\n",
     "\n",
@@ -233,7 +237,7 @@
     "  - Port mapping. Here, we map 3306 across the local machine and container.\n",
     "\n",
     "  ```bash\n",
-    "  docker run --name spyglass-db -p 3306:3306 -e MYSQL_ROOT_PASSWORD=tutorial datajoint/mysql\n",
+    "  docker run --name spyglass-db -p 3306:3306 -e MYSQL_ROOT_PASSWORD=tutorial datajoint/mysql:8.0\n",
     "  ```\n",
     "\n",
     "- For data to persist after terminating the container,\n",

notebooks/py_scripts/00_Setup.py

@@ -5,7 +5,7 @@
 #       extension: .py
 #       format_name: light
 #       format_version: '1.5'
-#       jupytext_version: 1.14.5
+#       jupytext_version: 1.15.2
 #   kernelspec:
 #     display_name: Python 3 (ipykernel)
 #     language: python
@@ -185,12 +185,16 @@
 # - Add yourself to the
 #   [`docker` group](https://docs.docker.com/engine/install/linux-postinstall/) so
 #   that you don't have to be sudo to run docker.
-# - Download the docker image for datajoint/mysql
+# - Download the docker image for `datajoint/mysql:8.0`.
 #
 #   ```bash
-#   docker pull datajoint/mysql
+#   docker pull datajoint/mysql:8.0
 #   ```
 #
+# _Note_: For demonstation, MySQL version won't matter. Some
+#   [database management](https://lorenfranklab.github.io/spyglass/latest/misc/database_management/#mysql-version)
+#   features of Spyglass, however, expect MySQL >= 8.
+#
 # - When run, this is referred to as a 'Docker container'
 # - Next start the container with a couple additional pieces of info...
 #
@@ -199,7 +203,7 @@
 #   - Port mapping. Here, we map 3306 across the local machine and container.
 #
 #   ```bash
-#   docker run --name spyglass-db -p 3306:3306 -e MYSQL_ROOT_PASSWORD=tutorial datajoint/mysql
+#   docker run --name spyglass-db -p 3306:3306 -e MYSQL_ROOT_PASSWORD=tutorial datajoint/mysql:8.0
 #   ```
 #
 # - For data to persist after terminating the container,

src/spyglass/common/common_filter.py

@@ -225,7 +225,7 @@ def filter_data_nwb(
         electrode_ids: list,
         decimation: int,
         description: str = "filtered data",
-        type: Union[None, str] = None,
+        data_type: Union[None, str] = None,
     ):
         """
         Filter data from an NWB electrical series using the ghostipy package,
@@ -256,6 +256,10 @@ def filter_data_nwb(
             The NWB object ID of the filtered data and a list containing the
             first and last timestamps.
         """
+        # Note: type -> data_type to avoid conflict with builtin type
+        # All existing refs to this func use positional args, so no need to
+        # adjust elsewhere, but low probability of issues with custom scripts
+
         MEM_USE_LIMIT = 0.9  # % of RAM use permitted
 
         gsp = _import_ghostipy()
@@ -323,7 +327,7 @@ def filter_data_nwb(
                 timestamps=np.empty(output_shape_list[time_axis]),
                 description=description,
             )
-            if type == "LFP":
+            if data_type == "LFP":
                 ecephys_module = nwbf.create_processing_module(
                     name="ecephys", description=description
                 )

src/spyglass/settings.py

@@ -18,17 +18,14 @@ class SpyglassConfig:
     facilitate testing.
     """
 
-    def __init__(self, base_dir=None, **kwargs):
+    def __init__(self, base_dir: str = None, **kwargs):
         """
         Initializes a new instance of the class.
 
         Parameters
         ----------
-            base_dir (str): The base directory.
-
-        Returns
-        -------
-            None
+        base_dir (str)
+            The base directory.
         """
         self.supplied_base_dir = base_dir
         self._config = dict()

src/spyglass/utils/database_settings.py

@@ -65,9 +65,10 @@ def add_collab_user(self):
 
     @property
     def _add_dj_guest_sql(self):
+        # Note: changing to temppass for uniformity
         return [
             # Create the user (if not already created) and set the password
-            f"{CREATE_USR}'{self.user}'@'%' IDENTIFIED BY 'Data_$haring';\n",
+            f"{CREATE_USR}'{self.user}'@'%'{TEMP_PASS}\n",
             # Grant privileges
             f"{GRANT_SEL}`%`.* TO '{self.user}'@'%';\n",
         ]