From ba890ea58138607f505cb2087c033fd6b6235150 Mon Sep 17 00:00:00 2001 From: R Date: Tue, 18 Sep 2018 12:26:55 +0100 Subject: [PATCH 01/13] Wait for 2 minutes while trying to obtain lock on a sync, for cases with several clients --- bin/bitpocket | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/bin/bitpocket b/bin/bitpocket index aee472a..6170921 100755 --- a/bin/bitpocket +++ b/bin/bitpocket @@ -502,13 +502,19 @@ function release_lock { } function acquire_remote_lock { + tries=0 $REMOTE_RUNNER "mkdir -p \"$REMOTE_TMP_DIR\"; cd \"$REMOTE_PATH\" && mkdir \"$LOCK_DIR\" 2>/dev/null" - - if [[ $? != 0 ]]; then - echo "Couldn't acquire remote lock. Another client is syncing with $REMOTE or lock file couldn't be created. Exiting." - release_lock - exit 3 - fi + while [[ $? != 0 ]]; do + if [ $tries -gt 20 ]; then + echo "Couldn't acquire remote lock. Another client is syncing with $REMOTE or lock file couldn't be created. Exiting." + release_lock + exit 3 + else + ((tries++)) + sleep 10 + $REMOTE_RUNNER "mkdir -p \"$REMOTE_TMP_DIR\"; cd \"$REMOTE_PATH\" && mkdir \"$LOCK_DIR\" 2>/dev/null" + fi + done } function release_remote_lock { @@ -571,7 +577,7 @@ function list { } function usage { - cat <] | sync | help | pack | log | cron | list } From a417a6d88cc015ef47adce6fdb907bb9f019e95f Mon Sep 17 00:00:00 2001 From: R Date: Thu, 20 Sep 2018 09:26:52 +0100 Subject: [PATCH 02/13] Add an option for a lock acquire timeout on master --- bin/bitpocket | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/bin/bitpocket b/bin/bitpocket index 6170921..a9122bc 100755 --- a/bin/bitpocket +++ b/bin/bitpocket @@ -21,6 +21,7 @@ RSYNC_RSH="ssh" REMOTE_BACKUPS=false BACKUPS=true REMOTE_MOUNTPOINT=false +LOCK_ACQUIRE_TIMEOUT=60 # Default command-line options and such COMMANDS=() @@ -128,6 +129,9 @@ REMOTE_BACKUPS=false ## SSH command with options for connecting to \$REMOTE # RSYNC_RSH="ssh -p 22 -i $DOT_DIR/id_rsa" +## By default, the program will wait for 60 seconds to acquire a lock on the remote host (in case other clients are currently syncing). Change this value to the desired timeout value in seconds 0 to disable the wait time. +# LOCK_ACQUIRE_TIMEOUT=60 + ## Uncomment following line to follow symlinks (transform it into referent file/dir) # RSYNC_OPTS="-L" @@ -502,16 +506,16 @@ function release_lock { } function acquire_remote_lock { - tries=0 + start=$SECONDS $REMOTE_RUNNER "mkdir -p \"$REMOTE_TMP_DIR\"; cd \"$REMOTE_PATH\" && mkdir \"$LOCK_DIR\" 2>/dev/null" while [[ $? != 0 ]]; do - if [ $tries -gt 20 ]; then + duration=$(( $SECONDS - $start )) + if [ $duration -gt $LOCK_ACQUIRE_TIMEOUT ] || [ $LOCK_ACQUIRE_TIMEOUT -eq 0 ]; then echo "Couldn't acquire remote lock. Another client is syncing with $REMOTE or lock file couldn't be created. Exiting." release_lock exit 3 else - ((tries++)) - sleep 10 + sleep 5 $REMOTE_RUNNER "mkdir -p \"$REMOTE_TMP_DIR\"; cd \"$REMOTE_PATH\" && mkdir \"$LOCK_DIR\" 2>/dev/null" fi done @@ -595,8 +599,11 @@ Available commands: help Show this message. Options: - -p, --pretend Don't really perform the sync or update the current - state. Instead, show what would be synchronized. + -p, --pretend Don't really perform the sync or update the current + state. Instead, show what would be synchronized. + -w=WAIT_TIME, --wait=WAIT_TIME Number of seconds to wait when trying to acquire a + lock on a remote host. This is useful if several + are syncing at the same time. Note: All commands (apart from help), must be run in the root of a new or existing bitpocket directory structure. @@ -608,6 +615,7 @@ function parseargs() { case $1 in # Switches and configuration -p|--pretend) OPTIONS+=('pretend');; + -w=*|--wait=*) LOCK_ACQUIRE_TIMEOUT="${1#*=}";; -h|--help|-*) COMMANDS+=('help');; # Arguments (commands) init) if [[ $# < 2 ]]; then From 747f50d0af360679c05f3d5c4fb2aeff65bd5e33 Mon Sep 17 00:00:00 2001 From: R Date: Wed, 17 Oct 2018 10:59:51 +0100 Subject: [PATCH 03/13] Bugfix - Exit if rsync fails with error --- bin/bitpocket | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/bin/bitpocket b/bin/bitpocket index a9122bc..fdccf39 100755 --- a/bin/bitpocket +++ b/bin/bitpocket @@ -13,6 +13,7 @@ CFG_FILE="$DOT_DIR/config" TMP_DIR="$DOT_DIR/tmp" STATE_DIR="$DOT_DIR/state" LOCK_DIR="$TMP_DIR/lock" # Use a lock directory for atomic locks. See the Bash FAQ http://mywiki.wooledge.org/BashFAQ/045 +PARTIAL_DIR="$DOT_DIR/partials" # Default settings SLOW_SYNC_TIME=10 @@ -195,7 +196,7 @@ function pull() { # # Order of includes/excludes/filters is EXTREMELY important prefix "R " < "$TMP_DIR/remote-del" \ - | rsync -auzxi --delete --exclude "/$DOT_DIR" \ + | rsync -auzxi --delete --partial --partial-dir="$PARTIAL_DIR" --exclude "/$DOT_DIR" \ --exclude-from="$TMP_DIR/local-add" \ --exclude-from="$TMP_DIR/local-del" \ --filter=". -" \ @@ -203,7 +204,11 @@ function pull() { $DO_BACKUP \ $RSYNC_OPTS $USER_RULES $REMOTE/ . \ | detect_changes \ - | prefix " | " || die "PULL" + | prefix " | " + if [[ ${PIPESTATUS[1]} != 0 ]] + then + die "PULL" + fi # Some versions of rsync will create the backup dir, even if it doesn't get # populated with any backups @@ -261,17 +266,21 @@ function push() { # Do not push back remotely deleted files prefix "R " < "$TMP_DIR/local-del" \ - | rsync -auzxi --delete $RSYNC_OPTS --exclude "/$DOT_DIR" \ + | rsync -auzxi --delete --partial --partial-dir="$PARTIAL_DIR" $RSYNC_OPTS --exclude "/$DOT_DIR" \ --exclude-from="$TMP_DIR/remote-del" \ --filter=". -" \ --filter="P **" \ $DO_BACKUP \ $USER_RULES . $REMOTE/ \ - | prefix " | " || die "PUSH" + | prefix " | " + if [[ ${PIPESTATUS[1]} != 0 ]] + then + die "PUSH" + fi # Some versions of rsync will create the backup dir, even if it doesn't get # populated with any backups - if [[ $REMOTE_BACKUPS == true ]] + if [[ $REMOTE_BACKUPS == true && success == 0 ]] then $REMOTE_RUNNER " cd '$REMOTE_PATH' From 75fefe2599888191c0704b4ea4b4927683f80a9a Mon Sep 17 00:00:00 2001 From: R Date: Wed, 17 Oct 2018 11:16:08 +0100 Subject: [PATCH 04/13] mc --- bin/bitpocket | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/bitpocket b/bin/bitpocket index fdccf39..e2be042 100755 --- a/bin/bitpocket +++ b/bin/bitpocket @@ -196,7 +196,7 @@ function pull() { # # Order of includes/excludes/filters is EXTREMELY important prefix "R " < "$TMP_DIR/remote-del" \ - | rsync -auzxi --delete --partial --partial-dir="$PARTIAL_DIR" --exclude "/$DOT_DIR" \ + | rsync -auzxi --delete --partial-dir="$PARTIAL_DIR" --exclude "/$DOT_DIR" \ --exclude-from="$TMP_DIR/local-add" \ --exclude-from="$TMP_DIR/local-del" \ --filter=". -" \ @@ -266,7 +266,7 @@ function push() { # Do not push back remotely deleted files prefix "R " < "$TMP_DIR/local-del" \ - | rsync -auzxi --delete --partial --partial-dir="$PARTIAL_DIR" $RSYNC_OPTS --exclude "/$DOT_DIR" \ + | rsync -auzxi --delete --partial-dir="$PARTIAL_DIR" $RSYNC_OPTS --exclude "/$DOT_DIR" \ --exclude-from="$TMP_DIR/remote-del" \ --filter=". -" \ --filter="P **" \ From cfd05d048f94f1b5228d7d7fd8f21dc34c834bc3 Mon Sep 17 00:00:00 2001 From: R Date: Wed, 17 Oct 2018 12:22:34 +0100 Subject: [PATCH 05/13] mc --- bin/bitpocket | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/bitpocket b/bin/bitpocket index e2be042..c94a995 100755 --- a/bin/bitpocket +++ b/bin/bitpocket @@ -13,7 +13,7 @@ CFG_FILE="$DOT_DIR/config" TMP_DIR="$DOT_DIR/tmp" STATE_DIR="$DOT_DIR/state" LOCK_DIR="$TMP_DIR/lock" # Use a lock directory for atomic locks. See the Bash FAQ http://mywiki.wooledge.org/BashFAQ/045 -PARTIAL_DIR="$DOT_DIR/partials" +PARTIAL_DIR="`pwd`/$DOT_DIR/partials" # Default settings SLOW_SYNC_TIME=10 From efba2debe595188adf6b5c5985b4748dd60e92cf Mon Sep 17 00:00:00 2001 From: R Date: Wed, 17 Oct 2018 12:40:11 +0100 Subject: [PATCH 06/13] mc --- bin/bitpocket | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/bitpocket b/bin/bitpocket index c94a995..d368b62 100755 --- a/bin/bitpocket +++ b/bin/bitpocket @@ -331,8 +331,11 @@ function analyse { # Collect the current snapshot of the local tree rsync --list-only --recursive --exclude "/$DOT_DIR" $USER_RULES . \ | scrub_rsync_list \ - | sort > "$STATE_DIR/tree-current" \ - || die "SNAPSHOT" + | sort > "$STATE_DIR/tree-current" + if [[ ${PIPESTATUS[0]} != 0 ]] + then + die "SNAPSHOT" + fi # Prevent bringing back locally deleted files if [[ -s "$STATE_DIR/tree-prev" ]]; then From 79c150f535093ceffc413eb7c4039f36a8d0b64f Mon Sep 17 00:00:00 2001 From: R Date: Wed, 17 Oct 2018 13:24:16 +0100 Subject: [PATCH 07/13] mc --- bin/bitpocket | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/bitpocket b/bin/bitpocket index d368b62..5aa5069 100755 --- a/bin/bitpocket +++ b/bin/bitpocket @@ -280,7 +280,7 @@ function push() { # Some versions of rsync will create the backup dir, even if it doesn't get # populated with any backups - if [[ $REMOTE_BACKUPS == true && success == 0 ]] + if [[ $REMOTE_BACKUPS == true ]] then $REMOTE_RUNNER " cd '$REMOTE_PATH' From c12f0972f49e520ccb4a5f7f668820f16ab25c1b Mon Sep 17 00:00:00 2001 From: R Date: Wed, 17 Oct 2018 13:30:05 +0100 Subject: [PATCH 08/13] mc --- bin/bitpocket | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/bitpocket b/bin/bitpocket index 5aa5069..abeb967 100755 --- a/bin/bitpocket +++ b/bin/bitpocket @@ -8,12 +8,12 @@ fi export LC_ALL=$LANG # for stable "sort" output # Paths -DOT_DIR=.bitpocket +DOT_DIR=`pwd`/.bitpocket CFG_FILE="$DOT_DIR/config" TMP_DIR="$DOT_DIR/tmp" STATE_DIR="$DOT_DIR/state" LOCK_DIR="$TMP_DIR/lock" # Use a lock directory for atomic locks. See the Bash FAQ http://mywiki.wooledge.org/BashFAQ/045 -PARTIAL_DIR="`pwd`/$DOT_DIR/partials" +PARTIAL_DIR="$DOT_DIR/partials" # Default settings SLOW_SYNC_TIME=10 From 10347f321a1f9696e1ad2f488d20172943cff12d Mon Sep 17 00:00:00 2001 From: R Date: Wed, 17 Oct 2018 17:12:12 +0100 Subject: [PATCH 09/13] mc --- bin/bitpocket | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/bin/bitpocket b/bin/bitpocket index abeb967..ee7877f 100755 --- a/bin/bitpocket +++ b/bin/bitpocket @@ -205,8 +205,7 @@ function pull() { $RSYNC_OPTS $USER_RULES $REMOTE/ . \ | detect_changes \ | prefix " | " - if [[ ${PIPESTATUS[1]} != 0 ]] - then + if [[ ${PIPESTATUS[1]} != 0 ]]; then die "PULL" fi @@ -273,8 +272,7 @@ function push() { $DO_BACKUP \ $USER_RULES . $REMOTE/ \ | prefix " | " - if [[ ${PIPESTATUS[1]} != 0 ]] - then + if [[ ${PIPESTATUS[1]} != 0 ]]; then die "PUSH" fi @@ -332,8 +330,7 @@ function analyse { rsync --list-only --recursive --exclude "/$DOT_DIR" $USER_RULES . \ | scrub_rsync_list \ | sort > "$STATE_DIR/tree-current" - if [[ ${PIPESTATUS[0]} != 0 ]] - then + if [[ ${PIPESTATUS[0]} != 0 ]]; then die "SNAPSHOT" fi From 3d723854e4fd043004e86736802c16a61e060d40 Mon Sep 17 00:00:00 2001 From: R Date: Sat, 20 Oct 2018 09:34:09 +0100 Subject: [PATCH 10/13] Add delay-updates option to pull and ensure correct permissions for local partial dir --- bin/bitpocket | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/bin/bitpocket b/bin/bitpocket index ee7877f..67f50b3 100755 --- a/bin/bitpocket +++ b/bin/bitpocket @@ -148,7 +148,7 @@ REMOTE_BACKUPS=false ## mountpoint of the local and/or remote target. # REMOTE_MOUNTPOINT=/ EOF - + mkdir -m 700 "$PARTIAL_DIR" #Force create the partials dir with correct permissions echo "Initialized bitpocket directory at `pwd`" echo "Please have a look at the config file ($DOT_DIR/config)" } @@ -187,6 +187,9 @@ function pull() { # for logging of deleted files, which need to be sorted exec 3> >(sort > "$TMP_DIR/pull-delete") + # Ensure the partial directory has the correct permissions + ensure_correct_partial_dir + # Determine what will be fetched from server and make backup copies of any # local files to be deleted or overwritten. # @@ -201,6 +204,7 @@ function pull() { --exclude-from="$TMP_DIR/local-del" \ --filter=". -" \ --filter="P **" \ + --delay-updates \ $DO_BACKUP \ $RSYNC_OPTS $USER_RULES $REMOTE/ . \ | detect_changes \ @@ -247,6 +251,23 @@ function detect_changes() { done } +function ensure_correct_partial_dir() { + if [ ! -d "$PARTIAL_DIR" ]; then + mkdir -m 700 "$PARTIAL_DIR" + fi + if [ $(stat -c '%U' "$PARTIAL_DIR") != $(whoami) ]; then + echo "File does not belong to current user, cannot continue!" + die "PARTIAL DIR CHECK" + fi + if [ $(stat -c "%a" "$PARTIAL_DIR") != "700" ]; then + echo "IMPORTANT: the --partial-dir should not be writable by other users or it is a security risk." + chmod 700 "$PARTIAL_DIR" + fi + if [[ $? != 0 ]]; then + die "PARTIAL DIR CHECK" + fi +} + function push() { # Actual push @@ -265,7 +286,7 @@ function push() { # Do not push back remotely deleted files prefix "R " < "$TMP_DIR/local-del" \ - | rsync -auzxi --delete --partial-dir="$PARTIAL_DIR" $RSYNC_OPTS --exclude "/$DOT_DIR" \ + | rsync -auzxi --delete $RSYNC_OPTS --exclude "/$DOT_DIR" \ --exclude-from="$TMP_DIR/remote-del" \ --filter=". -" \ --filter="P **" \ @@ -501,7 +522,7 @@ function acquire_lock { echo -e "${RED}bitpocket error:${CLEAR} Bitpocket found a stale lock directory:" echo " | Root dir: $(pwd)" echo " | Lock dir: $LOCK_DIR" - echo " | Command: LOCK_PATH=$(pwd)/$LOCK_DIR && rm \$LOCK_PATH/pid && rmdir \$LOCK_PATH" + echo " | Command: LOCK_PATH=$LOCK_DIR && rm \$LOCK_PATH/pid && rmdir \$LOCK_PATH" echo "Please remove the lock directory and try again." exit 2 fi From 60fca8f0d8bd92e8239ef00d2143947fbaa7e3dc Mon Sep 17 00:00:00 2001 From: R Date: Sat, 20 Oct 2018 10:18:20 +0100 Subject: [PATCH 11/13] mc --- bin/bitpocket | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/bitpocket b/bin/bitpocket index f08a3ca..404709e 100755 --- a/bin/bitpocket +++ b/bin/bitpocket @@ -8,12 +8,12 @@ fi export LC_ALL=$LANG # for stable "sort" output # Paths -DOT_DIR=`pwd`/.bitpocket +DOT_DIR=.bitpocket CFG_FILE="$DOT_DIR/config" TMP_DIR="$DOT_DIR/tmp" STATE_DIR="$DOT_DIR/state" LOCK_DIR="$TMP_DIR/lock" # Use a lock directory for atomic locks. See the Bash FAQ http://mywiki.wooledge.org/BashFAQ/045 -PARTIAL_DIR="$DOT_DIR/partials" +PARTIAL_DIR="$(pwd)/$DOT_DIR/partials" # Default settings SLOW_SYNC_TIME=10 From 4e38193796c07de99ef951b1b6fe4a766d291275 Mon Sep 17 00:00:00 2001 From: R Date: Mon, 22 Oct 2018 16:59:08 +0100 Subject: [PATCH 12/13] mc --- bin/bitpocket | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/bitpocket b/bin/bitpocket index 404709e..4e415d4 100755 --- a/bin/bitpocket +++ b/bin/bitpocket @@ -223,7 +223,6 @@ function pull() { --exclude-from="$TMP_DIR/local-add-change" \ --filter=". -" \ --filter="P **" \ - --delay-updates \ $DO_BACKUP \ --out-format=%i:%B:%n \ $RSYNC_OPTS $USER_RULES $REMOTE/ . \ From 25b0c09de23d60ce8ff2b41271e0320025e9a483 Mon Sep 17 00:00:00 2001 From: R Date: Mon, 22 Oct 2018 17:26:25 +0100 Subject: [PATCH 13/13] mc --- bin/bitpocket | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/bin/bitpocket b/bin/bitpocket index 4e415d4..89a6108 100755 --- a/bin/bitpocket +++ b/bin/bitpocket @@ -13,7 +13,6 @@ CFG_FILE="$DOT_DIR/config" TMP_DIR="$DOT_DIR/tmp" STATE_DIR="$DOT_DIR/state" LOCK_DIR="$TMP_DIR/lock" # Use a lock directory for atomic locks. See the Bash FAQ http://mywiki.wooledge.org/BashFAQ/045 -PARTIAL_DIR="$(pwd)/$DOT_DIR/partials" # Default settings SLOW_SYNC_TIME=10 @@ -168,7 +167,6 @@ REMOTE_BACKUPS=false ## mountpoint of the local and/or remote target. # REMOTE_MOUNTPOINT=/ EOF - mkdir -m 700 "$PARTIAL_DIR" #Force create the partials dir with correct permissions echo "Initialized bitpocket directory at $(pwd)" echo "Please have a look at the config file ($DOT_DIR/config)" } @@ -203,9 +201,6 @@ function pull() { cp "$STATE_DIR/tree-current" "$TMP_DIR/tree-after" - # Ensure the partial directory has the correct permissions - ensure_correct_partial_dir - # Determine what will be fetched from server and make backup copies of any # local files to be deleted or overwritten. # @@ -218,7 +213,7 @@ function pull() { # TODO: Consider adding %U and %G to the output format to capture owner and # group changes prefix "R " < "$TMP_DIR/remote-del" \ - | rsync -auzxi --delete --partial-dir="$PARTIAL_DIR" --exclude "/$DOT_DIR" \ + | rsync -auzxi --delete --exclude "/$DOT_DIR" \ --exclude-from="$TMP_DIR/local-del" \ --exclude-from="$TMP_DIR/local-add-change" \ --filter=". -" \ @@ -272,23 +267,6 @@ function detect_changes() { exec 3>&- } -function ensure_correct_partial_dir() { - if [ ! -d "$PARTIAL_DIR" ]; then - mkdir -m 700 "$PARTIAL_DIR" - fi - if [ $(stat -c '%U' "$PARTIAL_DIR") != $(whoami) ]; then - echo "File does not belong to current user, cannot continue!" - die "PARTIAL DIR CHECK" - fi - if [ $(stat -c "%a" "$PARTIAL_DIR") != "700" ]; then - echo "IMPORTANT: the --partial-dir should not be writable by other users or it is a security risk." - chmod 700 "$PARTIAL_DIR" - fi - if [[ $? != 0 ]]; then - die "PARTIAL DIR CHECK" - fi -} - function push() { # Actual push