diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml deleted file mode 100644 index 0a8b8e925..000000000 --- a/.github/workflows/black.yml +++ /dev/null @@ -1,10 +0,0 @@ -name: Lint - -on: [push, pull_request] - -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: psf/black@stable \ No newline at end of file diff --git a/.github/workflows/codespell.yml b/.github/workflows/lint.yml similarity index 61% rename from .github/workflows/codespell.yml rename to .github/workflows/lint.yml index 44c1e44a5..a75f17768 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/lint.yml @@ -1,13 +1,14 @@ ---- -name: Codespell +name: Lint -on: - push: - branches: [master] - pull_request: - branches: [master] +on: [push, pull_request] jobs: + black: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: psf/black@stable + codespell: name: Check for spelling errors runs-on: ubuntu-latest diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index a7a31758e..8cabef999 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -40,7 +40,9 @@ jobs: - name: Deploy run: | - echo "OWNER: ${REPO_OWNER}" + FULL_VERSION=${{ github.ref }} + export MAJOR_VERSION=${FULL_VERSION:0:3} + echo "OWNER: ${REPO_OWNER}. BUILD: ${MAJOR_VERSION}" bash ./docs/build-docs.sh push $REPO_OWNER env: USERNAME: github-actions[bot] diff --git a/.github/workflows/test-conda.yml b/.github/workflows/test-conda.yml index 594a7b2b8..56347a38e 100644 --- a/.github/workflows/test-conda.yml +++ b/.github/workflows/test-conda.yml @@ -2,6 +2,8 @@ name: Test conda env and run tests on: push: + branches: + - '!test_branch' schedule: # once a day at midnight UTC - cron: '0 0 * * *' diff --git a/.github/workflows/test-package-build.yml b/.github/workflows/test-package-build.yml index 3d87548fc..8dab627fd 100644 --- a/.github/workflows/test-package-build.yml +++ b/.github/workflows/test-package-build.yml @@ -5,6 +5,7 @@ on: branches: - master - maint/* + - '!test_branch' tags: - "*" pull_request: diff --git a/CHANGELOG.md b/CHANGELOG.md index dc991e655..eb05dd9ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,30 +1,47 @@ # Change Log -## [0.4.4] (Unreleased) +## Unreleased ### Infrastructure -- Additional documentation. #690 -- Clean up following pre-commit checks. #688 -- Add Mixin class to centralize `fetch_nwb` functionality. #692, #734 -- Refactor restriction use in `delete_downstream_merge` #703 -- Add `cautious_delete` to Mixin class - - Initial implementation. #711, #762 - - More robust caching of join to downstream tables. #806 - - Overwrite datajoint `delete` method to use `cautious_delete`. #806 - - Reverse join order for session summary. #821 - - Add temporary logging of use to `common_usage`. #811, #821 -- Add `deprecation_factory` to facilitate table migration. #717 -- Add Spyglass logger. #730 -- IntervalList: Add secondary key `pipeline` #742 -- Increase pytest coverage for `common`, `lfp`, and `utils`. #743 -- Update docs to reflect new notebooks. #776 -- Add overview of Spyglass to docs. #779 -- Update linting for Black 24. #808 -- Steamline dependency management. #822 +- Add user roles to `database_settings.py`. #832 + +## [0.5.0] (February 9, 2024) + +### Infrastructure + +- Docs: + - Additional documentation. #690 + - Add overview of Spyglass to docs. #779 + - Update docs to reflect new notebooks. #776 +- Mixin: + - Add Mixin class to centralize `fetch_nwb` functionality. #692, #734 + - Refactor restriction use in `delete_downstream_merge` #703 + - Add `cautious_delete` to Mixin class + - Initial implementation. #711, #762 + - More robust caching of join to downstream tables. #806 + - Overwrite datajoint `delete` method to use `cautious_delete`. #806 + - Reverse join order for session summary. #821 + - Add temporary logging of use to `common_usage`. #811, #821 +- Merge Tables: + - UUIDs: Revise Merge table uuid generation to include source. #824 + - UUIDs: Remove mutual exclusivity logic due to new UUID generation. #824 + - Add method for `merge_populate`. #824 +- Linting: + - Clean up following pre-commit checks. #688 + - Update linting for Black 24. #808 +- Misc: + - Add `deprecation_factory` to facilitate table migration. #717 + - Add Spyglass logger. #730 + - Increase pytest coverage for `common`, `lfp`, and `utils`. #743 + - Steamline dependency management. #822 ### Pipelines +- Common: + - `IntervalList`: Add secondary key `pipeline` #742 + - Add `common_usage` table. #811, #821, #824 + - Add catch errors during `populate_all_common`. #824 - Spike sorting: - Add SpikeSorting V1 pipeline. #651 - Move modules into spikesorting.v0 #807 @@ -159,4 +176,4 @@ [0.4.1]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.4.1 [0.4.2]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.4.2 [0.4.3]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.4.3 -[0.4.4]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.4.4 +[0.5.0]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.5.0 diff --git a/CITATION.cff b/CITATION.cff index 4dce7514b..2460e4494 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -146,5 +146,5 @@ keywords: - spike sorting - kachery license: MIT -version: 0.4.3 -date-released: '2023-11-07' +version: 0.5.0 +date-released: '2024-02-09' diff --git a/README.md b/README.md index 2241868e1..a1625f8d0 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,8 @@ ![Spyglass Figure](docs/src/images/fig1.png) +[Demo](https://spyglass.hhmi.2i2c.cloud/hub/user-redirect/git-pull?repo=https%3A%2F%2Fgithub.com%2FLorenFrankLab%2Fspyglass-demo&urlpath=lab%2Ftree%2Fspyglass-demo%2Fnotebooks%2F01_Insert_Data.ipynb&branch=main) | [Installation](https://lorenfranklab.github.io/spyglass/latest/installation/) | [Docs](https://lorenfranklab.github.io/spyglass/) | [Tutorials](https://github.com/LorenFrankLab/spyglass/tree/master/notebooks) | [Citation](#citation) + `spyglass` is a data analysis framework that facilitates the storage, analysis, visualization, and sharing of neuroscience data to support reproducible research. It is designed to be interoperable with the NWB format and integrates @@ -81,11 +83,7 @@ License and Copyright notice can be found at ## Citation -> Lee, K.H.\*, Denovellis, E.L.\*, Ly, R., Magland, J., Soules, J., Gramling, D.P., - Guidera, J.A., Nevers, R., Adenekan, P., Bray, S., et al. (2024). Spyglass: -a data analysis framework for reproducible and shareable neuroscience research. - bioRxiv. - [10.1101/2024.01.25.577295](https://doi.org/10.1101/2024.01.25.577295 ). +> Lee, K.H.\*, Denovellis, E.L.\*, Ly, R., Magland, J., Soules, J., Comrie, A.E., Gramling, D.P., Guidera, J.A., Nevers, R., Adenekan, P., Brozdowski, C., Bray, S., Monroe, E., Bak, J.H., Coulter, M.E., Sun, X., Tritt, A., Rübel, O., Nguyen, T., Yatsenko, D., Chu, J., Kemere, C., Garcia, S., Buccino, A., Frank, L.M., 2024. Spyglass: a data analysis framework for reproducible and shareable neuroscience research. bioRxiv. [10.1101/2024.01.25.577295](https://doi.org/10.1101/2024.01.25.577295 ). *\* Equal contribution* diff --git a/config/add_dj_collaborator.py b/config/add_dj_collaborator.py index cf6c0c48e..13dbf889d 100755 --- a/config/add_dj_collaborator.py +++ b/config/add_dj_collaborator.py @@ -9,4 +9,4 @@ "This script is deprecated. " + "Use spyglass.utils.database_settings.DatabaseSettings instead." ) - DatabaseSettings(user_name=sys.argv[1]).add_collab_user() + DatabaseSettings(user_name=sys.argv[1]).add_collab() diff --git a/config/add_dj_guest.py b/config/add_dj_guest.py index 1a087a3a8..80f55a141 100755 --- a/config/add_dj_guest.py +++ b/config/add_dj_guest.py @@ -9,4 +9,4 @@ "This script is deprecated. " + "Use spyglass.utils.database_settings.DatabaseSettings instead." ) - DatabaseSettings(user_name=sys.argv[1]).add_dj_guest() + DatabaseSettings(user_name=sys.argv[1]).add_guest() diff --git a/docs/build-docs.sh b/docs/build-docs.sh index b36b0533d..74174e9ca 100755 --- a/docs/build-docs.sh +++ b/docs/build-docs.sh @@ -1,8 +1,7 @@ #!/bin/bash # Run this script from repo root to serve site: > bash ./docs/build-docs.sh serve # Then, navigate to localhost:8000/ to inspect site, then ctrl+c to exit -# For auto-reload during dev, use `mkdocs serve -f ./docs/mkdosc.yaml` - +# For auto-reload during dev, use `mkdocs serve -f ./docs/mkdocs.yaml` # Copy top-level repo files for docs display cp ./CHANGELOG.md ./docs/src/ @@ -14,26 +13,29 @@ mv ./docs/src/notebooks/README.md ./docs/src/notebooks/index.md cp -r ./notebook-images ./docs/src/notebooks/ cp -r ./notebook-images ./docs/src/ -# Get major version -version_line=$(grep "__version__ =" ./src/spyglass/_version.py) -version_string=$(echo "$version_line" | awk -F"[\"']" '{print $2}') -export MAJOR_VERSION="${version_string:0:3}" -echo "$MAJOR_VERSION" +if [ -z "$MAJOR_VERSION" ]; then # Get version from file + version_line=$(grep "__version__ =" ./src/spyglass/_version.py) + version_string=$(echo "$version_line" | awk -F"[\"']" '{print $2}') + export MAJOR_VERSION="${version_string:0:3}" +fi +echo "$MAJOR_VERSION" # May be available as env var # Get ahead of errors export JUPYTER_PLATFORM_DIRS=1 -# jupyter notebook --generate-config +jupyter notebook --generate-config -y &> /dev/null +jupyter trust ./docs/src/notebooks/*.ipynb &> /dev/null # Generate site docs -mike deploy "$MAJOR_VERSION" --config ./docs/mkdocs.yml -b documentation +mike deploy "$MAJOR_VERSION" --config ./docs/mkdocs.yml -b documentation \ + 2>&1 | grep -v 'kernel_spec' # Suppress kernel_spec errors # Label this version as latest, set as default -mike alias "$MAJOR_VERSION" latest --config ./docs/mkdocs.yml -b documentation -mike set-default latest --config ./docs/mkdocs.yml -b documentation +mike alias "$MAJOR_VERSION" latest -u --config ./docs/mkdocs.yml -b documentation +# mike set-default latest --config ./docs/mkdocs.yml -b documentation # # Serve site to localhost if [ "$1" == "serve" ]; then # If first arg is serve, serve docs - mike serve --config ./docs/mkdocs.yml -b documentation + mike serve --config ./docs/mkdocs.yml -b documentation | grep -v 'kernel_spec' elif [ "$1" == "push" ]; then # if first arg is push if [ -z "$2" ]; then # When no second arg, use local git user git_user=$(git config user.name) diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 3767f8dd4..5eb02d0b6 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -43,13 +43,6 @@ theme: nav: - Home: index.md - Installation: installation.md - - Miscellaneous: - - Overview: misc/index.md - - FigURL: misc/figurl_views.md - - Session Groups: misc/session_groups.md - - Insert Data: misc/insert_data.md - - Merge Tables: misc/merge_tables.md - - Database Management: misc/database_management.md - Tutorials: - Overview: notebooks/index.md - Intro: @@ -61,7 +54,7 @@ nav: - Spikes: - Spike Sorting V0: notebooks/10_Spike_SortingV0.ipynb - Spike Sorting V1: notebooks/10_Spike_SortingV1.ipynb - - Curation: notebooks/11_Curation.ipynb + - Curation: notebooks/11_CurationV0.ipynb - Position: - Position Trodes: notebooks/20_Position_Trodes.ipynb - DLC Models: notebooks/21_DLC.ipynb @@ -72,9 +65,17 @@ nav: - Theta: notebooks/31_Theta.ipynb - Ripple Detection: notebooks/32_Ripple_Detection.ipynb - Decoding: - - Extract Clusterless: notebooks/41_Extracting_Clusterless_Waveform_Features.ipynb - - Decoding Clusterless: notebooks/42_Decoding_Clusterless.ipynb - - Decoding Sorted Spikes: notebooks/43_Decoding_SortedSpikes.ipynb + - Extracting Waveforms: notebooks/40_Extracting_Clusterless_Waveform_Features.ipynb + - Decoding Clusterless: notebooks/41_Decoding_Clusterless.ipynb + - Decoding Sorted Spikes: notebooks/42_Decoding_SortedSpikes.ipynb + - MUA Detection: notebooks/50_MUA_Detection.ipynb + - Miscellaneous: + - Overview: misc/index.md + - FigURL: misc/figurl_views.md + - Session Groups: misc/session_groups.md + - Insert Data: misc/insert_data.md + - Merge Tables: misc/merge_tables.md + - Database Management: misc/database_management.md - API Reference: api/ # defer to gen-files + literate-nav - How to Contribute: contribute.md - Change Log: CHANGELOG.md diff --git a/docs/src/index.md b/docs/src/index.md index 1a0233192..3f70c1c9c 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -68,12 +68,10 @@ contains license information. ## Citing Spyglass -Kyu Hyun Lee, Eric Denovellis, Ryan Ly, Jeremy Magland, Jeff Soules, Alison -Comrie, Jennifer Guidera, Rhino Nevers, Daniel Gramling, Philip Adenekan, Ji -Hyun Bak, Emily Monroe, Andrew Tritt, Oliver Rübel, Thinh Nguyen, Dimitri -Yatsenko, Joshua Chu, Caleb Kemere, Samuel Garcia, Alessio Buccino, Emily Aery -Jones, Lisa Giocomo, and Loren Frank. 'Spyglass: A Data Analysis Framework for -Reproducible and Shareable Neuroscience Research.' (2022) Society for -Neuroscience, San Diego, CA. +> Lee, K.H.\*, Denovellis, E.L.\*, Ly, R., Magland, J., Soules, J., Comrie, A.E., Gramling, D.P., Guidera, J.A., Nevers, R., Adenekan, P., Brozdowski, C., Bray, S., Monroe, E., Bak, J.H., Coulter, M.E., Sun, X., Tritt, A., Rübel, O., Nguyen, T., Yatsenko, D., Chu, J., Kemere, C., Garcia, S., Buccino, A., Frank, L.M., 2024. Spyglass: a data analysis framework for reproducible and shareable neuroscience research. bioRxiv. [10.1101/2024.01.25.577295](https://doi.org/10.1101/2024.01.25.577295 ). + +*\* Equal contribution* + +See paper related code [here](https://github.com/LorenFrankLab/spyglass-paper). diff --git a/docs/src/misc/database_management.md b/docs/src/misc/database_management.md index 42f3125e0..5c00d2688 100644 --- a/docs/src/misc/database_management.md +++ b/docs/src/misc/database_management.md @@ -36,19 +36,28 @@ schema/database prefix. - `ALL` privileges allow users to create, alter, or drop tables and schemas in addition to operations above. -In practice, DataJoint only permits alerations of secondary keys on existing +In practice, DataJoint only permits alterations of secondary keys on existing tables, and more derstructive operations would require using DataJoint to execeute MySQL commands. Shared schema prefixes are those defined in the Spyglass package (e.g., `common`, `lfp`, etc.). A 'user schema' is any schema with the username as prefix. User types differ in the privileges they are granted on these prifixes. +Declaring a table with the SpyglassMixin on a schema other than a shared module +or the user's own prefix will raise a warning. -### Users types +### Users roles + +When a database is first initialized, the team should run `add_roles` to create +the following roles: -- `collab_user`: `ALL` on user schema, `SELECT` on all other schemas. - `dj_guest`: `SELECT` on all schemas. +- `dj_collab`: `ALL` on user schema, `SELECT` on all other schemas. - `dj_user`: `ALL` on shared and user schema, `SELECT` on all other schemas. +- `dj_admin`: `ALL` on all schemas. + +If new shared modules are introduced, the `add_module` method should be used to +expand the privileges of the `dj_user` role. ### Setting Passwords @@ -224,9 +233,19 @@ To remove orphaned files, we run the following commands in our cron jobs: ```python from spyglass.common import AnalysisNwbfile from spyglass.spikesorting import SpikeSorting +from spyglass.common.common_nwbfile import schema as nwbfile_schema +from spyglass.decoding.v1.sorted_spikes import schema as spikes_schema +from spyglass.decoding.v1.clusterless import schema as clusterless_schema def main(): AnalysisNwbfile().nightly_cleanup() SpikeSorting().nightly_cleanup() + nwbfile_schema.external['analysis'].delete(delete_external_files=True)) + nwbfile_schema.external['raw'].delete(delete_external_files=True)) + spikes_schema.external['analysis'].delete(delete_external_files=True)) + clusterless_schema.external['analysis'].delete(delete_external_files=True)) ``` + +The `delete` calls above use DataJoint's `ExternalTable.delete` method, which +will remove files from disk that are no longer referenced in the database. diff --git a/environment_dlc.yml b/environment_dlc.yml index e175e7408..45fd107c8 100644 --- a/environment_dlc.yml +++ b/environment_dlc.yml @@ -22,7 +22,6 @@ dependencies: - libgcc # dlc-only - matplotlib - non_local_detector - - numpy<1.24 - pip>=20.2.* - position_tools - pybind11 # req by mountainsort4 -> isosplit5 @@ -47,4 +46,5 @@ dependencies: - pynwb>=2.2.0,<3 - sortingview>=0.11 - spikeinterface>=0.98.2,<0.99 + - tensorflow<=2.12 # dlc-only - .[dlc] diff --git a/notebooks/04_PopulateConfigFile.ipynb b/notebooks/04_PopulateConfigFile.ipynb index 23dce0f84..4ead237fb 100644 --- a/notebooks/04_PopulateConfigFile.ipynb +++ b/notebooks/04_PopulateConfigFile.ipynb @@ -2,6 +2,7 @@ "cells": [ { "cell_type": "markdown", + "id": "68303e8a", "metadata": {}, "source": [ "# Customizing Data Insertion into Spyglass\n", @@ -44,6 +45,7 @@ }, { "cell_type": "markdown", + "id": "bcc87f67", "metadata": {}, "source": [ "## Define a Configuration YAML File\n", @@ -85,6 +87,7 @@ }, { "cell_type": "markdown", + "id": "fc6d0986", "metadata": {}, "source": [ "## Create Entries to Reference in the Configuration YAML\n", @@ -145,6 +148,7 @@ }, { "cell_type": "markdown", + "id": "9d641e00", "metadata": {}, "source": [ "## Example Ingestion with Real Data\n", @@ -171,6 +175,7 @@ { "cell_type": "code", "execution_count": null, + "id": "37aa5182", "metadata": {}, "outputs": [], "source": [ @@ -180,6 +185,7 @@ { "cell_type": "code", "execution_count": null, + "id": "aab5b47a", "metadata": {}, "outputs": [], "source": [ @@ -195,6 +201,7 @@ }, { "cell_type": "markdown", + "id": "d132e797", "metadata": {}, "source": [ "Then call `insert_sessions` as usual." @@ -203,6 +210,7 @@ { "cell_type": "code", "execution_count": null, + "id": "bed5c6e1", "metadata": {}, "outputs": [], "source": [ @@ -213,6 +221,7 @@ }, { "cell_type": "markdown", + "id": "d875b158", "metadata": {}, "source": [ "Confirm the session was inserted with the correct `DataAcquisitionDevice`" @@ -221,6 +230,7 @@ { "cell_type": "code", "execution_count": null, + "id": "8411cb43", "metadata": {}, "outputs": [], "source": [ @@ -234,6 +244,7 @@ }, { "cell_type": "markdown", + "id": "d85b1416", "metadata": {}, "source": [] } diff --git a/notebooks/41_Extracting_Clusterless_Waveform_Features.ipynb b/notebooks/40_Extracting_Clusterless_Waveform_Features.ipynb similarity index 100% rename from notebooks/41_Extracting_Clusterless_Waveform_Features.ipynb rename to notebooks/40_Extracting_Clusterless_Waveform_Features.ipynb diff --git a/notebooks/42_Decoding_Clusterless.ipynb b/notebooks/41_Decoding_Clusterless.ipynb similarity index 100% rename from notebooks/42_Decoding_Clusterless.ipynb rename to notebooks/41_Decoding_Clusterless.ipynb diff --git a/notebooks/43_Decoding_SortedSpikes.ipynb b/notebooks/42_Decoding_SortedSpikes.ipynb similarity index 100% rename from notebooks/43_Decoding_SortedSpikes.ipynb rename to notebooks/42_Decoding_SortedSpikes.ipynb diff --git a/notebooks/51_MUA_Detection.ipynb b/notebooks/50_MUA_Detection.ipynb similarity index 100% rename from notebooks/51_MUA_Detection.ipynb rename to notebooks/50_MUA_Detection.ipynb diff --git a/src/spyglass/common/common_behav.py b/src/spyglass/common/common_behav.py index 530cdac36..a2fa0125f 100644 --- a/src/spyglass/common/common_behav.py +++ b/src/spyglass/common/common_behav.py @@ -43,6 +43,25 @@ class SpatialSeries(SpyglassMixin, dj.Part): name=null: varchar(32) # name of spatial series """ + def populate(self, keys=None): + """Insert position source data from NWB file. + + WARNING: populate method on Manual table is not protected by transaction + protections like other DataJoint tables. + """ + if not isinstance(keys, list): + keys = [keys] + if isinstance(keys[0], dj.Table): + keys = [k for tbl in keys for k in tbl.fetch("KEY", as_dict=True)] + for key in keys: + nwb_file_name = key.get("nwb_file_name") + if not nwb_file_name: + raise ValueError( + "PositionSource.populate is an alias for a non-computed table " + + "and must be passed a key with nwb_file_name" + ) + self.insert_from_nwbfile(nwb_file_name) + @classmethod def insert_from_nwbfile(cls, nwb_file_name): """Add intervals to ItervalList and PositionSource. @@ -487,6 +506,7 @@ def _no_transaction_make(self, key): # Skip populating if no pos interval list names if len(pos_intervals) == 0: + # TODO: Now that populate_all accept errors, raise here? logger.error(f"NO POS INTERVALS FOR {key}; {no_pop_msg}") return @@ -524,6 +544,7 @@ def _no_transaction_make(self, key): # Check that each pos interval was matched to only one epoch if len(matching_pos_intervals) != 1: + # TODO: Now that populate_all accept errors, raise here? logger.error( f"Found {len(matching_pos_intervals)} pos intervals for {key}; " + f"{no_pop_msg}\n{matching_pos_intervals}" diff --git a/src/spyglass/common/common_usage.py b/src/spyglass/common/common_usage.py index ccd58091d..fdf7ae99d 100644 --- a/src/spyglass/common/common_usage.py +++ b/src/spyglass/common/common_usage.py @@ -1,6 +1,7 @@ """A schema to store the usage of advanced Spyglass features. -Records show usage of features such as table chains, which will be used to +Records show usage of features such as cautious delete and fault-permitting +insert, which will be used to determine which features are used, how often, and by whom. This will help plan future development of Spyglass. """ @@ -21,3 +22,18 @@ class CautiousDelete(dj.Manual): restriction: varchar(255) merge_deletes = null: blob """ + + +@schema +class InsertError(dj.Manual): + definition = """ + id: int auto_increment + --- + dj_user: varchar(64) + connection_id: int # MySQL CONNECTION_ID() + nwb_file_name: varchar(64) + table: varchar(64) + error_type: varchar(64) + error_message: varchar(255) + error_raw = null: blob + """ diff --git a/src/spyglass/common/populate_all_common.py b/src/spyglass/common/populate_all_common.py index b2fa7d760..2972ed145 100644 --- a/src/spyglass/common/populate_all_common.py +++ b/src/spyglass/common/populate_all_common.py @@ -1,3 +1,5 @@ +import datajoint as dj + from spyglass.common.common_behav import ( PositionSource, RawPosition, @@ -14,51 +16,58 @@ from spyglass.common.common_nwbfile import Nwbfile from spyglass.common.common_session import Session from spyglass.common.common_task import TaskEpoch +from spyglass.common.common_usage import InsertError from spyglass.utils import logger def populate_all_common(nwb_file_name): - # Insert session one by one - fp = [(Nwbfile & {"nwb_file_name": nwb_file_name}).proj()] - logger.info("Populate Session...") - Session.populate(fp) - - # If we use Kachery for data sharing we can uncomment the following two lines. TBD - # logger.info('Populate NwbfileKachery...') - # NwbfileKachery.populate() - - logger.info("Populate ElectrodeGroup...") - ElectrodeGroup.populate(fp) - - logger.info("Populate Electrode...") - Electrode.populate(fp) - - logger.info("Populate Raw...") - Raw.populate(fp) - - logger.info("Populate SampleCount...") - SampleCount.populate(fp) - - logger.info("Populate DIOEvents...") - DIOEvents.populate(fp) - - # sensor data (from analog ProcessingModule) is temporarily removed from NWBFile - # to reduce file size while it is not being used. add it back in by commenting out - # the removal code in spyglass/data_import/insert_sessions.py when ready - # logger.info('Populate SensorData') - # SensorData.populate(fp) - - logger.info("Populate TaskEpochs") - TaskEpoch.populate(fp) - logger.info("Populate StateScriptFile") - StateScriptFile.populate(fp) - logger.info("Populate VideoFile") - VideoFile.populate(fp) - logger.info("RawPosition...") - PositionSource.insert_from_nwbfile(nwb_file_name) - RawPosition.populate(fp) - - logger.info("Populate ImportedSpikeSorting...") + """Insert all common tables for a given NWB file.""" from spyglass.spikesorting.imported import ImportedSpikeSorting - ImportedSpikeSorting.populate(fp) + key = [(Nwbfile & f"nwb_file_name LIKE '{nwb_file_name}'").proj()] + tables = [ + Session, + # NwbfileKachery, # Not used by default + ElectrodeGroup, + Electrode, + Raw, + SampleCount, + DIOEvents, + # SensorData, # Not used by default. Generates large files + RawPosition, + TaskEpoch, + StateScriptFile, + VideoFile, + PositionSource, + RawPosition, + ImportedSpikeSorting, + ] + error_constants = dict( + dj_user=dj.config["database.user"], + connection_id=dj.conn().connection_id, + nwb_file_name=nwb_file_name, + ) + + for table in tables: + logger.info(f"Populating {table.__name__}...") + try: + table.populate(key) + except Exception as e: + InsertError.insert1( + dict( + **error_constants, + table=table.__name__, + error_type=type(e).__name__, + error_message=str(e), + error_raw=str(e), + ) + ) + query = InsertError & error_constants + if query: + err_tables = query.fetch("table") + logger.error( + f"Errors occurred during population for {nwb_file_name}:\n\t" + + f"Failed tables {err_tables}\n\t" + + "See common_usage.InsertError for more details" + ) + return query.fetch("KEY") diff --git a/src/spyglass/decoding/v1/sorted_spikes.py b/src/spyglass/decoding/v1/sorted_spikes.py index 55d174e37..43a71d91e 100644 --- a/src/spyglass/decoding/v1/sorted_spikes.py +++ b/src/spyglass/decoding/v1/sorted_spikes.py @@ -47,6 +47,7 @@ class SortedSpikesDecodingSelection(SpyglassMixin, dj.Manual): -> IntervalList.proj(decoding_interval='interval_list_name') estimate_decoding_params = 1 : bool # whether to estimate the decoding parameters """ + # NOTE: Excessive key length fixed by reducing UnitSelectionParams.unit_filter_params_name @schema diff --git a/src/spyglass/linearization/__init__.py b/src/spyglass/linearization/__init__.py index 681df507c..e6a9504ea 100644 --- a/src/spyglass/linearization/__init__.py +++ b/src/spyglass/linearization/__init__.py @@ -1 +1,3 @@ -from spyglass.linearization.merge import LinearizedPositionOutput +# CB: Circular import if only importing PositionOutput + +# from spyglass.linearization.merge import LinearizedPositionOutput diff --git a/src/spyglass/position/v1/dlc_reader.py b/src/spyglass/position/v1/dlc_reader.py index aed66ac4a..8d6c18c23 100644 --- a/src/spyglass/position/v1/dlc_reader.py +++ b/src/spyglass/position/v1/dlc_reader.py @@ -114,7 +114,8 @@ def pkl(self): def yml(self): if self._yml is None: with open(self.yml_path, "rb") as f: - self._yml = yaml.safe_load(f) + safe_yaml = yaml.YAML(typ="safe", pure=True) + self._yml = safe_yaml.load(f) return self._yml @property diff --git a/src/spyglass/position/v1/dlc_utils.py b/src/spyglass/position/v1/dlc_utils.py index 8ef76074e..4d55bd844 100644 --- a/src/spyglass/position/v1/dlc_utils.py +++ b/src/spyglass/position/v1/dlc_utils.py @@ -6,6 +6,7 @@ import pathlib import pwd import subprocess +import sys from collections import abc from contextlib import redirect_stdout from itertools import groupby @@ -419,8 +420,10 @@ def get_video_path(key): """ import pynwb - vf_key = {"nwb_file_name": key["nwb_file_name"], "epoch": key["epoch"]} - VideoFile()._no_transaction_make(vf_key, verbose=False) + valid_fields = VideoFile.fetch().dtype.fields.keys() + vf_key = {k: val for k, val in key.items() if k in valid_fields} + if not VideoFile & vf_key: + VideoFile()._no_transaction_make(vf_key, verbose=False) video_query = VideoFile & vf_key if len(video_query) != 1: @@ -537,17 +540,23 @@ def _convert_mp4( "copy", f"{dest_path.as_posix()}", ] - try: - convert_process = subprocess.Popen( - convert_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT - ) - except subprocess.CalledProcessError as err: - raise RuntimeError( - f"command {err.cmd} return with error (code {err.returncode}): {err.output}" - ) from err - out, _ = convert_process.communicate() - print(out.decode("utf-8")) - print(f"finished converting {filename}") + if dest_path.exists(): + print(f"{dest_path} already exists, skipping conversion") + else: + try: + sys.stdout.flush() + convert_process = subprocess.Popen( + convert_command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + except subprocess.CalledProcessError as err: + raise RuntimeError( + f"command {err.cmd} return with error (code {err.returncode}): {err.output}" + ) from err + out, _ = convert_process.communicate() + print(out.decode("utf-8")) + print(f"finished converting {filename}") print( f"Checking that number of packets match between {orig_filename} and {dest_filename}" ) diff --git a/src/spyglass/position/v1/position_dlc_model.py b/src/spyglass/position/v1/position_dlc_model.py index 30bd3b1e8..5b1cf265b 100644 --- a/src/spyglass/position/v1/position_dlc_model.py +++ b/src/spyglass/position/v1/position_dlc_model.py @@ -196,7 +196,8 @@ def make(self, key): raise OSError(f"config_path {config_path} does not exist.") if config_path.suffix in (".yml", ".yaml"): with open(config_path, "rb") as f: - dlc_config = yaml.safe_load(f) + safe_yaml = yaml.YAML(typ="safe", pure=True) + dlc_config = safe_yaml.load(f) if isinstance(params["params"], dict): dlc_config.update(params["params"]) del params["params"] diff --git a/src/spyglass/spikesorting/analysis/v1/group.py b/src/spyglass/spikesorting/analysis/v1/group.py index 573697f1a..3403ad0b5 100644 --- a/src/spyglass/spikesorting/analysis/v1/group.py +++ b/src/spyglass/spikesorting/analysis/v1/group.py @@ -14,11 +14,12 @@ @schema class UnitSelectionParams(SpyglassMixin, dj.Manual): definition = """ - unit_filter_params_name: varchar(128) + unit_filter_params_name: varchar(32) --- include_labels = Null: longblob exclude_labels = Null: longblob """ + # NOTE: pk reduced from 128 to 32 to avoid long primary key error contents = [ [ "all_units", @@ -96,9 +97,10 @@ def filter_units( for ind, unit_labels in enumerate(labels): if isinstance(unit_labels, str): unit_labels = [unit_labels] - if np.all(~np.isin(unit_labels, include_labels)) or np.any( - np.isin(unit_labels, exclude_labels) - ): + if ( + include_labels.size > 0 + and np.all(~np.isin(unit_labels, include_labels)) + ) or np.any(np.isin(unit_labels, exclude_labels)): # if the unit does not have any of the include labels # or has any of the exclude labels, skip continue diff --git a/src/spyglass/utils/database_settings.py b/src/spyglass/utils/database_settings.py index 7fd1b44ff..8dacb3be1 100755 --- a/src/spyglass/utils/database_settings.py +++ b/src/spyglass/utils/database_settings.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -import grp import os import sys import tempfile @@ -9,11 +8,6 @@ from spyglass.utils.logging import logger -GRANT_ALL = "GRANT ALL PRIVILEGES ON " -GRANT_SEL = "GRANT SELECT ON " -CREATE_USR = "CREATE USER IF NOT EXISTS " -TEMP_PASS = " IDENTIFIED BY 'temppass';" -ESC = r"\_%" SHARED_MODULES = [ "common", "spikesorting", @@ -25,6 +19,12 @@ "waveform", "mua", ] +GRANT_ALL = "GRANT ALL PRIVILEGES ON " +GRANT_SEL = "GRANT SELECT ON " +CREATE_USR = "CREATE USER IF NOT EXISTS " +CREATE_ROLE = "CREATE ROLE IF NOT EXISTS " +TEMP_PASS = " IDENTIFIED BY 'temppass';" +ESC = r"\_%" class DatabaseSettings: @@ -32,20 +32,29 @@ def __init__( self, user_name=None, host_name=None, - target_group=None, debug=False, target_database=None, ): """Class to manage common database settings + Roles: + - dj_guest: select for all prefix + - dj_collab: select for all prefix, all for user prefix + - dj_user: select for all prefix, all for user prefix, all for shared + - dj_admin: all for all prefix + + Note: To add dj_user role to all those with common access, run: + query = "SELECT user, host FROM mysql.db WHERE Db LIKE 'common%';" + users = dj.conn().query(query).fetchall() + for user in users: + dj.conn().query(f"GRANT dj_user TO '{user[0][0]}'@'%';") + Parameters ---------- user_name : str, optional The name of the user to add to the database. Default from dj.config host_name : str, optional The name of the host to add to the database. Default from dj.config - target_group : str, optional - Group to which user belongs. Default is kachery-users debug : bool, optional Default False. If True, pprint sql instead of running target_database : str, optional @@ -56,91 +65,78 @@ def __init__( self.host = ( host_name or dj.config["database.host"] or "lmf-db.cin.ucsf.edu" ) - self.target_group = target_group or "kachery-users" self.debug = debug self.target_database = target_database or "mysql" @property - def _add_collab_usr_sql(self): - return [ - # Create the user (if not already created) and set the password - f"{CREATE_USR}'{self.user}'@'%'{TEMP_PASS}\n", - # Grant privileges to databases matching the user_name pattern - f"{GRANT_ALL}`{self.user}{ESC}`.* TO '{self.user}'@'%';\n", - # Grant SELECT privileges on all databases - f"{GRANT_SEL}`%`.* TO '{self.user}'@'%';\n", + def _create_roles_sql(self): + guest_role = [ + f"{CREATE_ROLE}'dj_guest';\n", + f"{GRANT_SEL}`%`.* TO 'dj_guest';\n", + ] + collab_role = [ + f"{CREATE_ROLE}'dj_collab';\n", + f"{GRANT_SEL}`%`.* TO 'dj_collab';\n", + ] # also gets own prefix below + user_role = [ + f"{CREATE_ROLE}'dj_user';\n", + f"{GRANT_SEL}`%`.* TO 'dj_user';\n", + ] + [ + f"{GRANT_ALL}`{module}`.* TO 'dj_user';\n" + for module in self.shared_modules + ] # also gets own prefix below + admin_role = [ + f"{CREATE_ROLE}'dj_admin';\n", + f"{GRANT_ALL}`%`.* TO 'dj_admin';\n", ] - def add_collab_user(self): - """Add collaborator user with full permissions to shared modules""" - file = self.write_temp_file(self._add_collab_usr_sql) - self.exec(file) + return guest_role + collab_role + user_role + admin_role + + def _create_user_sql(self, role): + """Create user and grant role""" + return [ + f"{CREATE_USR}'{self.user}'@'%'{TEMP_PASS}\n", # create user + f"GRANT {role} TO '{self.user}'@'%';\n", # grant role + ] @property - def _add_dj_guest_sql(self): - # Note: changing to temppass for uniformity + def _user_prefix_sql(self): + """Grant user all permissions for user prefix""" return [ - # Create the user (if not already created) and set the password - f"{CREATE_USR}'{self.user}'@'%'{TEMP_PASS}\n", - # Grant privileges - f"{GRANT_SEL}`%`.* TO '{self.user}'@'%';\n", + f"{GRANT_ALL}`{self.user}{ESC}`.* TO '{self.user}'@'%';\n", ] - def add_dj_guest(self, method="file"): - """Add guest user with select permissions to shared modules""" - file = self.write_temp_file(self._add_dj_guest_sql) - self.exec(file) + @property + def _add_guest_sql(self): + return self._create_user_sql("dj_guest") - def _find_group(self): - # find the kachery-users group - groups = grp.getgrall() - group_found = False # initialize the flag as False - for group in groups: - if group.gr_name == self.target_group: - group_found = ( - True # set the flag to True when the group is found - ) - break + @property + def _add_collab_sql(self): + return self._create_user_sql("dj_collab") + self._user_prefix_sql - # Check if the group was found - if not group_found: - if self.debug: - logger.info(f"All groups: {[g.gr_name for g in groups]}") - sys.exit( - f"Error: The target group {self.target_group} was not found." - ) + @property + def _add_user_sql(self): + return self._create_user_sql("dj_user") + self._user_prefix_sql - return group + def _add_module_sql(self, module_name): + return [f"{GRANT_ALL}`{module_name}{ESC}`.* TO dj_user;\n"] - def _add_module_sql(self, module_name, group): - return [ - f"{GRANT_ALL}`{module_name}{ESC}`.* TO `{user}`@'%';\n" - # get a list of usernames - for user in group.gr_mem - ] + def add_collab(self): + """Add collaborator user with full permissions to shared modules""" + file = self.write_temp_file(self._add_collab_sql) + self.exec(file) + + def add_guest(self): + """Add guest user with select permissions to shared modules""" + file = self.write_temp_file(self._add_guest_sql) + self.exec(file) def add_module(self, module_name): """Add module to database. Grant permissions to all users in group""" logger.info(f"Granting everyone permissions to module {module_name}") - group = self._find_group() - file = self.write_temp_file(self._add_module_sql(module_name, group)) + file = self.write_temp_file(self._add_module_sql(module_name)) self.exec(file) - @property - def _add_dj_user_sql(self): - return ( - [ - f"{CREATE_USR}'{self.user}'@'%' " - + "IDENTIFIED BY 'temppass';\n", - f"{GRANT_ALL}`{self.user}{ESC}`.* TO '{self.user}'@'%';" + "\n", - ] - + [ - f"{GRANT_ALL}`{module}`.* TO '{self.user}'@'%';\n" - for module in self.shared_modules - ] - + [f"{GRANT_SEL}`%`.* TO '{self.user}'@'%';\n"] - ) - def add_dj_user(self, check_exists=True): """Add user to database with permissions to shared modules""" if check_exists: @@ -149,10 +145,15 @@ def add_dj_user(self, check_exists=True): logger.info("Creating database user ", self.user) else: sys.exit( - f"Error: could not find {self.user} in home dir: {user_home}" + f"Error: couldn't find {self.user} in home dir: {user_home}" ) - file = self.write_temp_file(self._add_dj_user_sql) + file = self.write_temp_file(self._add_user_sql) + self.exec(file) + + def add_roles(self): + """Add roles to database""" + file = self.write_temp_file(self._create_roles_sql) self.exec(file) def write_temp_file(self, content: list) -> tempfile.NamedTemporaryFile: @@ -176,11 +177,10 @@ def exec(self, file): if self.debug: return - if self.target_database == "mysql": - cmd = f"mysql -p -h {self.host} < {file.name}" - else: - cmd = ( - f"docker exec -i {self.target_database} mysql -u {self.user} " - + f"--password=tutorial < {file.name}" - ) + cmd = ( + f"mysql -p -h {self.host} < {file.name}" + if self.target_database == "mysql" + else f"docker exec -i {self.target_database} mysql -u {self.user} " + + f"--password=tutorial < {file.name}" + ) os.system(cmd) diff --git a/src/spyglass/utils/dj_chains.py b/src/spyglass/utils/dj_chains.py index 457e3e613..f281bfb37 100644 --- a/src/spyglass/utils/dj_chains.py +++ b/src/spyglass/utils/dj_chains.py @@ -221,7 +221,7 @@ def objects(self) -> List[dj.FreeTable]: ) def join( - self, restricton: str = None, reverse_order: bool = False + self, restriction: str = None, reverse_order: bool = False ) -> dj.expression.QueryExpression: """Return join of tables in chain with restriction applied to parent. @@ -237,7 +237,7 @@ def join( return None objects = self.objects[::-1] if reverse_order else self.objects - restriction = restricton or self.parent.restriction or True + restriction = restriction or self.parent.restriction or True join = objects[0] & restriction for table in objects[1:]: try: diff --git a/src/spyglass/utils/dj_merge_tables.py b/src/spyglass/utils/dj_merge_tables.py index f15183dfe..14c8c20db 100644 --- a/src/spyglass/utils/dj_merge_tables.py +++ b/src/spyglass/utils/dj_merge_tables.py @@ -3,13 +3,14 @@ from inspect import getmodule from itertools import chain as iter_chain from pprint import pprint +from time import time from typing import Union import datajoint as dj from datajoint.condition import make_condition from datajoint.errors import DataJointError from datajoint.preview import repr_html -from datajoint.utils import from_camel_case, get_master, to_camel_case +from datajoint.utils import from_camel_case, to_camel_case from IPython.core.display import HTML from spyglass.utils.logging import logger @@ -248,6 +249,9 @@ def _merge_repr(cls, restriction: str = True) -> dj.expression.Union: return_empties=False, # motivated by SpikeSortingOutput.Import ) ] + if not parts: + logger.warning("No parts found. Try adjusting restriction.") + return attr_dict = { # NULL for non-numeric, 0 for numeric attr.name: "0" if attr.numeric else "NULL" @@ -274,10 +278,8 @@ def _proj_part(part): return query @classmethod - def _merge_insert( - cls, rows: list, part_name: str = None, mutual_exclusvity=True, **kwargs - ) -> None: - """Insert rows into merge, ensuring db integrity and mutual exclusivity + def _merge_insert(cls, rows: list, part_name: str = None, **kwargs) -> None: + """Insert rows into merge, ensuring data exists in part parent(s). Parameters --------- @@ -291,18 +293,17 @@ def _merge_insert( TypeError If rows is not a list of dicts ValueError - If entry already exists, mutual exclusivity errors If data doesn't exist in part parents, integrity error """ cls._ensure_dependencies_loaded() + type_err_msg = "Input `rows` must be a list of dictionaries" try: for r in iter(rows): - assert isinstance( - r, dict - ), 'Input "rows" must be a list of dictionaries' + if not isinstance(r, dict): + raise TypeError(type_err_msg) except TypeError: - raise TypeError('Input "rows" must be a list of dictionaries') + raise TypeError(type_err_msg) parts = cls._merge_restrict_parts(as_objects=True) if part_name: @@ -315,30 +316,24 @@ def _merge_insert( master_entries = [] parts_entries = {p: [] for p in parts} for row in rows: - keys = [] # empty to-be-inserted key + keys = [] # empty to-be-inserted keys for part in parts: # check each part - part_parent = part.parents(as_objects=True)[-1] part_name = cls._part_name(part) + part_parent = part.parents(as_objects=True)[-1] if part_parent & row: # if row is in part parent - if keys and mutual_exclusvity: # if key from other part - raise ValueError( - "Mutual Exclusivity Error! Entry exists in more " - + f"than one table - Entry: {row}" - ) - keys = (part_parent & row).fetch("KEY") # get pk if len(keys) > 1: raise ValueError( "Ambiguous entry. Data has mult rows in " + f"{part_name}:\n\tData:{row}\n\t{keys}" ) - master_pk = { # make uuid - cls()._reserved_pk: dj.hash.key_hash(keys[0]), - } - parts_entries[part].append({**master_pk, **keys[0]}) - master_entries.append( - {**master_pk, cls()._reserved_sk: part_name} - ) + key = keys[0] + master_sk = {cls()._reserved_sk: part_name} + uuid = dj.hash.key_hash(key | master_sk) + master_pk = {cls()._reserved_pk: uuid} + + master_entries.append({**master_pk, **master_sk}) + parts_entries[part].append({**master_pk, **key}) if not keys: raise ValueError( @@ -369,27 +364,22 @@ def _ensure_dependencies_loaded(cls) -> None: if not dj.conn.connection.dependencies._loaded: dj.conn.connection.dependencies.load() - def insert(self, rows: list, mutual_exclusvity=True, **kwargs): - """Merges table specific insert - - Ensuring db integrity and mutual exclusivity + def insert(self, rows: list, **kwargs): + """Merges table specific insert, ensuring data exists in part parents. Parameters --------- rows: List[dict] An iterable where an element is a dictionary. - mutual_exclusvity: bool - Check for mutual exclusivity before insert. Default True. Raises ------ TypeError If rows is not a list of dicts ValueError - If entry already exists, mutual exclusivity errors If data doesn't exist in part parents, integrity error """ - self._merge_insert(rows, mutual_exclusvity=mutual_exclusvity, **kwargs) + self._merge_insert(rows, **kwargs) @classmethod def merge_view(cls, restriction: str = True): @@ -586,6 +576,8 @@ def merge_get_part( + "Try adding a restriction before invoking `get_part`.\n\t" + "Or permitting multiple sources with `multi_source=True`." ) + if len(sources) == 0: + return None parts = [ ( @@ -770,12 +762,33 @@ def merge_fetch(self, restriction: str = True, *attrs, **kwargs) -> list: ) return results[0] if len(results) == 1 else results - @classmethod - def merge_populate(source: str, key=None): - raise NotImplementedError( - "CBroz: In the future, this command will support executing " - + "part_parent `make` and then inserting all entries into Merge" - ) + def merge_populate(self, source: str, keys=None): + """Populate the merge table with entries from the source table.""" + logger.warning("CBroz: Not fully tested. Use with caution.") + parent_class = self.merge_get_parent_class(source) + if not keys: + keys = parent_class.key_source + parent_class.populate(keys) + successes = (parent_class & keys).fetch("KEY", as_dict=True) + self.insert(successes) + + def delete(self, force_permission=False, *args, **kwargs): + """Alias for cautious_delete, overwrites datajoint.table.Table.delete""" + for part in self.merge_get_part( + restriction=self.restriction, + multi_source=True, + return_empties=False, + ): + part.delete(force_permission=force_permission, *args, **kwargs) + + def super_delete(self, *args, **kwargs): + """Alias for datajoint.table.Table.delete. + + Added to support MRO of SpyglassMixin""" + logger.warning("!! Using super_delete. Bypassing cautious_delete !!") + + self._log_use(start=time(), super_delete=True) + super().delete(*args, **kwargs) _Merge = Merge diff --git a/src/spyglass/utils/dj_mixin.py b/src/spyglass/utils/dj_mixin.py index 0e18e3a5c..1b4b24ff6 100644 --- a/src/spyglass/utils/dj_mixin.py +++ b/src/spyglass/utils/dj_mixin.py @@ -10,6 +10,7 @@ from datajoint.utils import get_master, user_choice from pymysql.err import DataError +from spyglass.utils.database_settings import SHARED_MODULES from spyglass.utils.dj_chains import TableChain, TableChains from spyglass.utils.dj_helper_fn import fetch_nwb from spyglass.utils.dj_merge_tables import RESERVED_PRIMARY_KEY as MERGE_PK @@ -30,7 +31,7 @@ class SpyglassMixin: a NWBFile table (including AnalysisNwbfile) or a _nwb_table attribute to determine which table to use. delte_downstream_merge(restriction=None, dry_run=True, reload_cache=False) - Delete downstream merge table entries associated with restricton. + Delete downstream merge table entries associated with restriction. Requires caching of merge tables and links, which is slow on first call. `restriction` can be set to a string to restrict the delete. `dry_run` can be set to False to commit the delete. `reload_cache` can be set to @@ -55,6 +56,26 @@ class SpyglassMixin: _session_pk = None # Session primary key. Mixin is ambivalent to Session pk _member_pk = None # LabMember primary key. Mixin ambivalent table structure + def __init__(self, *args, **kwargs): + """Initialize SpyglassMixin. + + Checks that schema prefix is in SHARED_MODULES. + """ + if ( + self.database # Connected to a database + and not self.is_declared # New table + and self.database.split("_")[0] # Prefix + not in [ + *SHARED_MODULES, # Shared modules + dj.config["database.user"], # User schema + "temp", + "test", + ] + ): + logger.error( + f"Schema prefix not in SHARED_MODULES: {self.database}" + ) + # ------------------------------- fetch_nwb ------------------------------- @cached_property @@ -178,7 +199,7 @@ def delete_downstream_merge( return_parts: bool = True, **kwargs, ) -> Union[List[QueryExpression], Dict[str, List[QueryExpression]]]: - """Delete downstream merge table entries associated with restricton. + """Delete downstream merge table entries associated with restriction. Requires caching of merge tables and links, which is slow on first call. @@ -215,8 +236,8 @@ def delete_downstream_merge( if not merge_join_dict and not disable_warning: logger.warning( f"No merge deletes found w/ {self.table_name} & " - + f"{restriction}.\n\tIf this is unexpected, try running with " - + "`reload_cache`." + + f"{restriction}.\n\tIf this is unexpected, try importing " + + " Merge table(s) and running with `reload_cache`." ) if dry_run: @@ -365,7 +386,7 @@ def _usage_table(self): return CautiousDelete() - def _log_use(self, start, merge_deletes=None): + def _log_use(self, start, merge_deletes=None, super_delete=False): """Log use of cautious_delete.""" if isinstance(merge_deletes, QueryExpression): merge_deletes = merge_deletes.fetch(as_dict=True) @@ -374,15 +395,13 @@ def _log_use(self, start, merge_deletes=None): dj_user=dj.config["database.user"], origin=self.full_table_name, ) + restr_str = "Super delete: " if super_delete else "" + restr_str += "".join(self.restriction) if self.restriction else "None" try: self._usage_table.insert1( dict( **safe_insert, - restriction=( - "".join(self.restriction)[255:] # handle list - if self.restriction - else "None" - ), + restriction=restr_str[:255], merge_deletes=merge_deletes, ) ) @@ -455,4 +474,5 @@ def delete(self, force_permission=False, *args, **kwargs): def super_delete(self, *args, **kwargs): """Alias for datajoint.table.Table.delete.""" logger.warning("!! Using super_delete. Bypassing cautious_delete !!") + self._log_use(start=time(), super_delete=True) super().delete(*args, **kwargs) diff --git a/tests/common/test_position.py b/tests/common/test_position.py index 8a7261c74..b10c0654b 100644 --- a/tests/common/test_position.py +++ b/tests/common/test_position.py @@ -118,11 +118,11 @@ def test_fetch1_dataframe(interval_position_info, interval_pos_key): df_sums = {c: df[c].iloc[:5].sum() for c in df.columns} df_sums_exp = { "head_orientation": 4.4300073600180125, - "head_position_x": 111.25, - "head_position_y": 141.75, - "head_speed": 0.6084872579024899, - "head_velocity_x": -0.4329520555149495, - "head_velocity_y": 0.42756198762527325, + "head_position_x": 222.5, + "head_position_y": 283.5, + "head_speed": 1.2245733375331014, + "head_velocity_x": -0.865904111029899, + "head_velocity_y": 0.8551239752505465, } for k in df_sums: assert k in df_sums_exp, err_msg