diff --git a/docs/src/index.md b/docs/src/index.md index bbb6d4807..c5eaf0338 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -22,7 +22,7 @@ interested in in hosting a Spyglass instance for their own data should read the We have a series of additional docs under the [misc](./misc/index.md) folder that may be helpful. Our [changelog](./CHANGELOG.md) highlights the changes that -have been made to Spyglass over time and the [copyright](./copyright.md) page +have been made to Spyglass over time and the [copyright](./LICENSE.md) page contains license information. ## Citing Spyglass diff --git a/docs/src/misc/database_management.md b/docs/src/misc/database_management.md index 332d124cf..0d4471510 100644 --- a/docs/src/misc/database_management.md +++ b/docs/src/misc/database_management.md @@ -70,7 +70,142 @@ dj.set_password() ## Database Backups -Coming soon... +The following codeblockes are a series of files used to back up our database and +migrate the contents to another server. Some conventions to note: + +- `.host`: files used in the host's context +- `.container`: files used inside the database Docker container +- `.env`: files used to set environment variables used by the scripts for + database name, backup name, and backup credentials + +### mysql.env.host + +
+MySQL host environment variables + +```bash +ROOT_PATH=/usr/local/containers/mysql # path to this container's working area + +# variables for building image +SRC=ubuntu +VER=20.04 +DOCKERFILE=Dockerfile.base + +# variables for referencing image +IMAGE=mysql8 +TAG=u20 +# variables for running the container +CNAME=mysql-datajoint +MACADDR=4e:b0:3d:42:e0:70 +RPORT=3306 + +# variables for initializing/relaunching the container +# - where the mysql data and backups will live - these values +# are examples +DB_PATH=/data/db +DB_DATA=mysql +DB_BACKUP=/data/mysql-backups + +# backup info +BACK_USER=mysql-backup +BACK_PW={password} +BACK_DBNAME={database} +# mysql root password - make sure to remove this AFTER the container +# is initialized - and this file will be replicated inside the container +# on initialization, so remove it from there: /opt/bin/mysql.env +``` + +
+ +### backup-database.sh.host + +This script runs the mysql-backup container script (exec inside the container) +that dumps the database contents for each database as well as the entire +database. Use cron to set this to run on your desired schedule. + +
+MySQL host docker exec + +```bash +#!/bin/bash + +PRIOR_DIR=$(pwd) +cd /usr/local/containers/mysql || exit +. mysql.env +cd "$(dirname ${ROOT_PATH})" +# +docker exec ${CNAME} /opt/bin/mysql-backup.csh +# +cd "$(dirname ${DB_BACKUP})" +# +cd ${PRIOR_DIR} +``` + +
+ +### mysql-backup-xfer.csh.host + +This script transfers the backup to another server 'X' and is specific for us - +it uses passwordless ssh keys to a local unprivileged user on X that has the +mysql backup area on X as that user's home + +
+MySQL host transfer script + +```bash +#!/bin/csh +set td=`date +"%Y%m%d"` +cd /data/mysql-backups +scp -P {port} -i ~/mysql-backup -r ${database}-${td} mysql-backup@${X}:~/ +/bin/rm -r lmf-db-${td} +``` + +
+ +### myenv.csh.container + +
+Docker container environment variables + +```bash +set db_backup=mysql-backups +set back_user=mysql-backup +set back_pw={password} +set back_dbname={database} +``` + +
+ +### mysql-backup.csh.container + +
+Generate backups from within container + +```bash +#!/bin/csh +source /opt/bin/myenv.csh +set td=`date +"%Y%m%d"` +cd /${db_backup} +mkdir ${back_dbname}-${td} + +set list=`echo "show databases;" | mysql --user=${back_user} --password=${back_pw}` +set cnt=0 + +foreach db ($list) + if ($cnt == 0) then + echo "dumping mysql databases on $td" + else + echo "dumping MySQL database : $db" + # Per-schema backups + mysqldump $db --max_allowed_packet=512M --user=${back_user} --password=${back_pw} > /${db_backup}/${back_dbname}-${td}/mysql.${db}.sql + endif +@ cnt = $cnt + 1 +end +# Full database backup +mysqldump --all-databases --max_allowed_packet=512M --user=${back_user} --password=${back_pw} > /${db_backup}/${back_dbname}-${td}/mysql-all.sql +``` + +
## File Cleanup diff --git a/src/spyglass/position/v1/dlc_utils.py b/src/spyglass/position/v1/dlc_utils.py index 5e8dde55f..ba8b6d85f 100644 --- a/src/spyglass/position/v1/dlc_utils.py +++ b/src/spyglass/position/v1/dlc_utils.py @@ -51,11 +51,8 @@ def validate_option( ValueError If option is not in options. """ - if option is None: - if permit_none: - return - else: - raise ValueError(f"{name} cannot be None") + if option is None and not permit_none: + raise ValueError(f"{name} cannot be None") if options and option not in options: raise KeyError( diff --git a/src/spyglass/position/v1/position_dlc_centroid.py b/src/spyglass/position/v1/position_dlc_centroid.py index 380ef291a..d1e7e6dba 100644 --- a/src/spyglass/position/v1/position_dlc_centroid.py +++ b/src/spyglass/position/v1/position_dlc_centroid.py @@ -366,8 +366,7 @@ def fetch1_dataframe(self): def four_led_centroid(pos_df: pd.DataFrame, **params): - """ - Determines the centroid of 4 LEDS on an implant LED ring. + """Determines the centroid of 4 LEDS on an implant LED ring. Assumed to be the Green LED, and 3 red LEDs called: redLED_C, redLED_L, redLED_R By default, uses (greenled + redLED_C) / 2 to calculate centroid If Green LED is NaN, but red center LED is not,