From 2e057c4815ab02d0f8b35162c6b363293625d4a8 Mon Sep 17 00:00:00 2001 From: James Munroe Date: Tue, 12 Mar 2024 21:25:52 +0000 Subject: [PATCH] add video and mysql command --- ...populated.ipynb => 00_HubQuickStart.ipynb} | 376 ++++++++---------- 1 file changed, 167 insertions(+), 209 deletions(-) rename notebooks/{00_All_from_Prepopulated.ipynb => 00_HubQuickStart.ipynb} (97%) diff --git a/notebooks/00_All_from_Prepopulated.ipynb b/notebooks/00_HubQuickStart.ipynb similarity index 97% rename from notebooks/00_All_from_Prepopulated.ipynb rename to notebooks/00_HubQuickStart.ipynb index 1490560..c8ea6cf 100644 --- a/notebooks/00_All_from_Prepopulated.ipynb +++ b/notebooks/00_HubQuickStart.ipynb @@ -1,10 +1,67 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "# Spyglass: Quick Start Demo\n" + ] + }, { "cell_type": "markdown", "metadata": {}, "source": [ - "# Spyglass: Quick Demo\n" + "Welcome to [Spyglass](https://lorenfranklab.github.io/spyglass/0.4/),\n", + "a [DataJoint](https://github.com/datajoint/datajoint-python/)\n", + "pipeline maintained by the [Frank Lab](https://franklab.ucsf.edu/) at UCSF." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This is a Jupyter Notebook. To run the selected *cell* and advance in this document, use `SHIFT+ENTER`\n", + "\n", + "\n", + "To learn more about Spyglass from Dr. Loren Frank by running the code block below and then click the play button to start the video." + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 50, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from IPython.display import IFrame\n", + "IFrame(src=\"https://player.vimeo.com/video/922578532?title=0&portrait=0\",\n", + " width='100%', height='500')" ] }, { @@ -16,6 +73,54 @@ "To learn how this data is loaded, please review the other notebooks in this series.\n" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Quick Start Setup" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This `00_HubQuickStart.ipynb` notebook is designed to be run on the JupyterHub set up at spyglass.hhmi.2i2c.cloud. If you are exploring Spyglass from your own machine, start with the notebook `00_Setup.ipynb` instead." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will start by pre-loading some data into a the MySQL database used by Spyglass." + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "mysql: [Warning] Using a password on the command line interface can be insecure.\n" + ] + } + ], + "source": [ + "%%bash\n", + "mysql -h 127.0.0.1 -u root --password=tutorial < /home/jovyan/spyglass-demo/data/prepopulate.sql" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This step should take less than 20 seconds and the warning `mysql: [Warning] Using a password on the command line interface can be insecure.` is expected for this quick start demo." + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -32,8 +137,10 @@ }, { "cell_type": "code", - "execution_count": 14, - "metadata": {}, + "execution_count": 58, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "import datajoint as dj\n", @@ -65,9 +172,20 @@ }, { "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [], + "execution_count": 59, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-03-12 21:18:20,771][INFO]: Connecting root@localhost:3306\n", + "[2024-03-12 21:18:20,783][INFO]: Connected root@localhost:3306\n" + ] + } + ], "source": [ "import spyglass.common as sgc\n", "\n", @@ -105,8 +223,10 @@ }, { "cell_type": "code", - "execution_count": 16, - "metadata": {}, + "execution_count": 60, + "metadata": { + "tags": [] + }, "outputs": [ { "data": { @@ -187,7 +307,7 @@ " (Total: 1)" ] }, - "execution_count": 16, + "execution_count": 60, "metadata": {}, "output_type": "execute_result" } @@ -207,8 +327,10 @@ }, { "cell_type": "code", - "execution_count": 17, - "metadata": {}, + "execution_count": 61, + "metadata": { + "tags": [] + }, "outputs": [ { "data": { @@ -297,7 +419,7 @@ " (Total: 1)" ] }, - "execution_count": 17, + "execution_count": 61, "metadata": {}, "output_type": "execute_result" } @@ -315,8 +437,10 @@ }, { "cell_type": "code", - "execution_count": 18, - "metadata": {}, + "execution_count": 62, + "metadata": { + "tags": [] + }, "outputs": [ { "data": { @@ -332,7 +456,7 @@ " 'upsampling_interpolation_method': 'linear'}" ] }, - "execution_count": 18, + "execution_count": 62, "metadata": {}, "output_type": "execute_result" } @@ -366,200 +490,34 @@ }, { "cell_type": "code", - "execution_count": 19, - "metadata": {}, + "execution_count": 63, + "metadata": { + "tags": [] + }, "outputs": [ { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
video_frame_indposition_xposition_yorientationvelocity_xvelocity_yspeed
time
1.625936e+090NaNNaNNaNNaNNaNNaN
1.625936e+091NaNNaNNaNNaNNaNNaN
1.625936e+092NaNNaNNaNNaNNaNNaN
1.625936e+093NaNNaNNaNNaNNaNNaN
1.625936e+094NaNNaNNaNNaNNaNNaN
........................
1.625937e+094418638.483603113.574868-1.4021991.008364-0.8651171.328618
1.625937e+094418738.483603113.521270-1.3986060.603148-0.5349380.806192
1.625937e+094418838.430005113.574868-1.4164780.256839-0.2198710.338096
1.625937e+094418938.376407113.574868-1.4321570.0177720.0253490.030958
1.625937e+094419038.376407113.628467-1.435269-0.1072310.1747920.205063
\n", - "

44191 rows × 7 columns

\n", - "
" - ], - "text/plain": [ - " video_frame_ind position_x position_y orientation \\\n", - "time \n", - "1.625936e+09 0 NaN NaN NaN \n", - "1.625936e+09 1 NaN NaN NaN \n", - "1.625936e+09 2 NaN NaN NaN \n", - "1.625936e+09 3 NaN NaN NaN \n", - "1.625936e+09 4 NaN NaN NaN \n", - "... ... ... ... ... \n", - "1.625937e+09 44186 38.483603 113.574868 -1.402199 \n", - "1.625937e+09 44187 38.483603 113.521270 -1.398606 \n", - "1.625937e+09 44188 38.430005 113.574868 -1.416478 \n", - "1.625937e+09 44189 38.376407 113.574868 -1.432157 \n", - "1.625937e+09 44190 38.376407 113.628467 -1.435269 \n", - "\n", - " velocity_x velocity_y speed \n", - "time \n", - "1.625936e+09 NaN NaN NaN \n", - "1.625936e+09 NaN NaN NaN \n", - "1.625936e+09 NaN NaN NaN \n", - "1.625936e+09 NaN NaN NaN \n", - "1.625936e+09 NaN NaN NaN \n", - "... ... ... ... \n", - "1.625937e+09 1.008364 -0.865117 1.328618 \n", - "1.625937e+09 0.603148 -0.534938 0.806192 \n", - "1.625937e+09 0.256839 -0.219871 0.338096 \n", - "1.625937e+09 0.017772 0.025349 0.030958 \n", - "1.625937e+09 -0.107231 0.174792 0.205063 \n", - "\n", - "[44191 rows x 7 columns]" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" + "name": "stderr", + "output_type": "stream", + "text": [ + "[21:18:32][INFO] Spyglass: NWB file not found locally; checking kachery for /home/jovyan/spyglass-demo/data/nwb/analysis/mediumnwb20230802/mediumnwb20230802_H9HJCJSOW9.nwb\n" + ] + }, + { + "ename": "Exception", + "evalue": "mediumnwb20230802_H9HJCJSOW9.nwb cannot be downloaded", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mException\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[63], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mnumpy\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mnp\u001b[39;00m\n\u001b[0;32m----> 3\u001b[0m position_info \u001b[38;5;241m=\u001b[39m \u001b[43mPositionOutput\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfetch1_dataframe\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 4\u001b[0m position_info\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.9/site-packages/spyglass/position/position_merge.py:89\u001b[0m, in \u001b[0;36mPositionOutput.fetch1_dataframe\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 82\u001b[0m key \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmerge_restrict(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mproj())\u001b[38;5;241m.\u001b[39mproj()\n\u001b[1;32m 83\u001b[0m query \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m 84\u001b[0m source_class_dict[\n\u001b[1;32m 85\u001b[0m to_camel_case(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmerge_get_parent(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mproj())\u001b[38;5;241m.\u001b[39mtable_name)\n\u001b[1;32m 86\u001b[0m ]\n\u001b[1;32m 87\u001b[0m \u001b[38;5;241m&\u001b[39m key\n\u001b[1;32m 88\u001b[0m )\n\u001b[0;32m---> 89\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mquery\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfetch1_dataframe\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.9/site-packages/spyglass/position/v1/position_trodes_position.py:228\u001b[0m, in \u001b[0;36mTrodesPosV1.fetch1_dataframe\u001b[0;34m(self, add_frame_ind)\u001b[0m\n\u001b[1;32m 223\u001b[0m logger\u001b[38;5;241m.\u001b[39mwarn(\n\u001b[1;32m 224\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUpsampled position data, frame indices are invalid. Setting add_frame_ind=False\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 225\u001b[0m )\n\u001b[1;32m 226\u001b[0m add_frame_ind \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 227\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m IntervalPositionInfo\u001b[38;5;241m.\u001b[39m_data_to_df(\n\u001b[0;32m--> 228\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfetch_nwb\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m[\u001b[38;5;241m0\u001b[39m], prefix\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m, add_frame_ind\u001b[38;5;241m=\u001b[39madd_frame_ind\n\u001b[1;32m 229\u001b[0m )\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.9/site-packages/spyglass/utils/dj_mixin.py:101\u001b[0m, in \u001b[0;36mSpyglassMixin.fetch_nwb\u001b[0;34m(self, *attrs, **kwargs)\u001b[0m\n\u001b[1;32m 93\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfetch_nwb\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39mattrs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 94\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Fetch NWBFile object from relevant table.\u001b[39;00m\n\u001b[1;32m 95\u001b[0m \n\u001b[1;32m 96\u001b[0m \u001b[38;5;124;03m Implementing class must have a foreign key reference to Nwbfile or\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 99\u001b[0m \u001b[38;5;124;03m precedence.\u001b[39;00m\n\u001b[1;32m 100\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 101\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfetch_nwb\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_nwb_table_tuple\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mattrs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.9/site-packages/spyglass/utils/dj_helper_fn.py:157\u001b[0m, in \u001b[0;36mfetch_nwb\u001b[0;34m(query_expression, nwb_master, *attrs, **kwargs)\u001b[0m\n\u001b[1;32m 154\u001b[0m file_path \u001b[38;5;241m=\u001b[39m file_path_fn(file_name)\n\u001b[1;32m 155\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mexists(file_path):\n\u001b[1;32m 156\u001b[0m \u001b[38;5;66;03m# retrieve the file from kachery. This also opens the file and stores the file object\u001b[39;00m\n\u001b[0;32m--> 157\u001b[0m \u001b[43mget_nwb_file\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfile_path\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 159\u001b[0m rec_dicts \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m 160\u001b[0m query_expression \u001b[38;5;241m*\u001b[39m tbl\u001b[38;5;241m.\u001b[39mproj(nwb2load_filepath\u001b[38;5;241m=\u001b[39mattr_name)\n\u001b[1;32m 161\u001b[0m )\u001b[38;5;241m.\u001b[39mfetch(\u001b[38;5;241m*\u001b[39mattrs, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnwb2load_filepath\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 163\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m rec_dicts \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m np\u001b[38;5;241m.\u001b[39many(\n\u001b[1;32m 164\u001b[0m [\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mobject_id\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m key \u001b[38;5;28;01mfor\u001b[39;00m key \u001b[38;5;129;01min\u001b[39;00m rec_dicts[\u001b[38;5;241m0\u001b[39m]]\n\u001b[1;32m 165\u001b[0m ):\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.9/site-packages/spyglass/utils/nwb_helper_fn.py:61\u001b[0m, in \u001b[0;36mget_nwb_file\u001b[0;34m(nwb_file_path)\u001b[0m\n\u001b[1;32m 57\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01msharing\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01msharing_kachery\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m AnalysisNwbfileKachery\n\u001b[1;32m 59\u001b[0m \u001b[38;5;66;03m# the download functions assume just the filename, so we need to\u001b[39;00m\n\u001b[1;32m 60\u001b[0m \u001b[38;5;66;03m# get that from the path\u001b[39;00m\n\u001b[0;32m---> 61\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[43mAnalysisNwbfileKachery\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdownload_file\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 62\u001b[0m \u001b[43m \u001b[49m\u001b[43mos\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpath\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbasename\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnwb_file_path\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 63\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m:\n\u001b[1;32m 64\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 65\u001b[0m \u001b[38;5;66;03m# now open the file\u001b[39;00m\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.9/site-packages/spyglass/sharing/sharing_kachery.py:215\u001b[0m, in \u001b[0;36mAnalysisNwbfileKachery.download_file\u001b[0;34m(analysis_file_name)\u001b[0m\n\u001b[1;32m 211\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m(\n\u001b[1;32m 212\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mLinked file \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mlinked_file_path\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m cannot be downloaded\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 213\u001b[0m )\n\u001b[1;32m 214\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m downloaded:\n\u001b[0;32m--> 215\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00manalysis_file_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m cannot be downloaded\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 217\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m\n", + "\u001b[0;31mException\u001b[0m: mediumnwb20230802_H9HJCJSOW9.nwb cannot be downloaded" + ] } ], "source": [ @@ -1194,7 +1152,7 @@ ], "metadata": { "kernelspec": { - "display_name": "spyglass", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -1212,5 +1170,5 @@ } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 }