Skip to content

Commit

Permalink
backups: Use rsync to do the backing up
Browse files Browse the repository at this point in the history 
rsync has a --backup option which will accomplish what the backup_inline
function is accomplishing. However, it will add the ability to do remote
backups without doing remote shell magic.

Coincidently, this removes the parallel rsync used to do the pull(), which
may reduce the CPU and disk overhead slightly.

Outstanding is to detect if remote backups were performed and emit a local
message accordingly.
  • Loading branch information
Jared Hancock committed Aug 30, 2017
1 parent 3cf7531 commit fd99bab
Showing 1 changed file with 35 additions and 156 deletions.
191 changes: 35 additions & 156 deletions bin/bitpocket
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,11 +40,8 @@ if [ -n "$REMOTE_HOST" ]; then
else
REMOTE_RUNNER="bash -c"
REMOTE="$REMOTE_PATH"
REMOTE_BACKUPS=false
fi

LOCAL="$(pwd -P)"

REMOTE_TMP_DIR="$REMOTE_PATH/$DOT_DIR/tmp"

# Don't sync user excluded files
Expand All @@ -67,6 +64,13 @@ USER_RULES="$user_filter $user_include $user_exclude"

TIMESTAMP=$(date "+%Y-%m-%d.%H%M%S")

GREEN=""
RED=""
if [[ -t 1 ]]; then
GREEN="\x1b\x5b1;32m"
RED="\x1b\x5b1;31m"
fi

export RSYNC_RSH

function init {
Expand All @@ -87,7 +91,8 @@ function init {
REMOTE_HOST=$1
REMOTE_PATH="$2"
# Make revisions of files on the REMOTE_HOST in the push phase.
## Backups -----------------------------------
## Make revisions of files on the REMOTE_HOST in the push phase.
REMOTE_BACKUPS=false
## SSH command with options for connecting to \$REMOTE
Expand Down Expand Up @@ -119,78 +124,22 @@ function pull() {
echo "# Pulling changes from server"
echo "# >> Saving current state and backing up files (if needed)"

cp "$STATE_DIR/tree-current" "$TMP_DIR/tree-after"
local BACKUP_TARGET="$DOT_DIR/backups/$TIMESTAMP"
local DO_BACKUP="--backup --backup-dir="$DOT_DIR"/backups/$TIMESTAMP"

# Create a duplicate of STDOUT for logging of backed-up files, and use fd#4
# for logging of deleted files, which need to be sorted
exec 3>&1
exec 4> >(sort > "$TMP_DIR/pull-delete")
cp "$STATE_DIR/tree-current" "$TMP_DIR/tree-after"

# Determine what will be fetched from server and make backup
# copies of any local files to be deleted or overwritten.
# Order of includes/excludes/filters is EXTREMELY important
rsync --dry-run -auzxi --delete --exclude "/$DOT_DIR" \
rsync -auzxi --delete --exclude "/$DOT_DIR" \
--exclude-from="$TMP_DIR/local-add-del" \
$DO_BACKUP \
$RSYNC_OPTS $USER_RULES $REMOTE/ . \
| backup_inline \
| rsync --files-from=- --from0 -auzxi $RSYNC_OPTS $USER_RULES \
$REMOTE/ . \
| sed "s/^/ | /" || die "PULL"

[[ -d "$DOT_DIR"/backups/$TIMESTAMP ]] \
&& echo " | Some files were backed up to $DOT_DIR/backups/$TIMESTAMP"

# Close extra file descriptors
exec 4>&-
exec 3>&-
}

function backup_inline() {
# Read file names from STDIN, and, if they exist locally, create a backup
# of each of the files. If they are requested to be deleted (indicated by
# the tag at the line beginning), then they are deleted here. Write the
# filenames to STDOUT which need to be synchronized by `rsync`
while read line
do
filename=$(sed "s:^\S*\s*::" <<< "$line" | sed 's:\d96:\\\`:g')
operation=${line%% *}
if [[ "$line" =~ ^[ch\<\>][fd]|^\*deleting ]]
then
if [[ -f "$filename" ]]
then
[[ -d "$DOT_DIR"/backups/$TIMESTAMP ]] \
|| mkdir -p "$DOT_DIR"/backups/$TIMESTAMP
cp --parents -p --reflink=auto "$filename" \
"$DOT_DIR"/backups/$TIMESTAMP || die "BACKUP"
echo " B $filename" >&3
fi
# rsync does not support DELETE via --files-from, so delete inline here
# after the backup was made
if [[ "$operation" == "*deleting" ]]
then
# Only delete locally if deleted remotely
if grep -q -x "/$filename" "$TMP_DIR/remote-del"
then
# Delete here rather than syncing
echo " | *deleting $filename" >&3
echo "/$filename" >&4
[[ -d "$filename" ]] && rmdir "$filename" || rm "$filename"
continue
fi
elif [[ "$operation" =~ \+\+\+\+$ ]]
then
# Mark as added locally
echo "/$filename" >> "$TMP_DIR/tree-after"
fi
fi
# Don't sync changes to folders -- only the files in them
if [[ ${operation:0:5} == '.d..t' ]]
then
continue
fi
# Sync the file. Use a NULL byte delimiter
printf '%s\0' "$filename"
done
[[ -d "$BACKUP_TARGET" ]] \
&& echo " | Some files were backed up to $BACKUP_TARGET"
}

function push() {
Expand All @@ -201,93 +150,23 @@ function push() {
echo
echo "# Pushing changes to server"

local DO_BACKUP=""
if [[ -n "$REMOTE_BACKUPS" ]]
then
echo "# >> Saving current state and backing up files (if needed)"
DO_BACKUP="--backup --backup-dir="$DOT_DIR"/backups/$TIMESTAMP"
fi

# Do not push back remotely deleted files
cat "$TMP_DIR/remote-del" \
| rsync -auzxi --delete $RSYNC_OPTS --exclude "/$DOT_DIR" \
--exclude-from - $USER_RULES . $REMOTE/ \
rsync -auzxi --delete $RSYNC_OPTS --exclude "/$DOT_DIR" \
--exclude-from="$TMP_DIR/remote-del" \
$DO_BACKUP \
$USER_RULES . $REMOTE/ \
| sed "s/^/ | /" || die "PUSH"
}

function remote_pull() {
# Similar to the pull(), except the rsync output it sent to an rsync
# session on the remote server. Bitpocket is used on the remote side
# so that backups can be captured before the files are overwritten
# by the push sync.
#
# Here, the files to sync are sent to STDIN, piped through
# backup_inline, and piped through an rsync process to fetch the files

exec 3>&1
exec 4>/dev/null

local REMOTE_PATH="$2"
local REMOTE="$1:$2"

echo "# >> Saving current state and backing up files (if needed)"

# Send new and updated, remotely remove files deleted locally
# Order of includes/excludes/filters is EXTREMELY important
backup_inline \
| rsync --files-from=- --from0 -auzxi $RSYNC_OPTS $USER_RULES \
$REMOTE/ . \
| sed "s/^/ | /" \
|| die "REMOTE-PULL"

[[ -d "$DOT_DIR"/backups/$TIMESTAMP ]] \
&& echo " | Some files were backed up to $DOT_DIR/backups/$TIMESTAMP"

# Close extra file descriptors
exec 4>&-
exec 3>&-
}

# Thanks, https://stackoverflow.com/a/246128
function bitpocket_source() {
SOURCE="${BASH_SOURCE[0]}"
# resolve $SOURCE until the file is no longer a symlink
while [ -h "$SOURCE" ]; do
TARGET="$(readlink "$SOURCE")"
if [[ $TARGET == /* ]]; then
SOURCE="$TARGET"
else
DIR="$( dirname "$SOURCE" )"
# if $SOURCE was a relative symlink, we need to resolve it relative
# to the path where the symlink file was located
SOURCE="$DIR/$TARGET"
fi
done
echo "$SOURCE"
}

function remote_bitpocket() {
# Transfer this bitpocket script source and the local-add-del as the
# remotely deleted files. Then run the `bitpocket` script with the
# args passed to this function.
# This assumes `bash` is in the PATH of the remote environment
$REMOTE_RUNNER "
cd \"$REMOTE_PATH\";
cat << '=EOF:' > .bitpocket/bin
$(cat "$(bitpocket_source)" | sed "s/=EOF:/=EOF2:/")
=EOF:
cat << '=EOF:' > .bitpocket/tmp/remote-del
$(cat "$TMP_DIR/local-add-del")
=EOF:
bash .bitpocket/bin $*"
}

function push_remote() {
# Works with remote_pull() to send the files which would be sent by the
# push() method, but allows for a backup on the remote side.

echo
echo "# Pushing changes to server"

rsync --dry-run -auzxi --delete --exclude "/$DOT_DIR" \
--exclude-from="$TMP_DIR/remote-del" \
$USER_RULES . $REMOTE/ \
| remote_bitpocket remote-pull `hostname` $LOCAL \
| sed "s/^/(remote) /" \
|| die "REMOTE-PUSH"
if $REMOTE_RUNNER "[ -d '$BACKUP_TARGET' ]"; then
echo " | Some files were backed up to $BACKUP_TARGET"
fi
}

# Do the actual synchronization
Expand All @@ -297,7 +176,7 @@ function sync {
acquire_remote_lock

echo
echo -e "\x1b\x5b1;32mbitpocket started\x1b\x5b0m at `date`."
echo -e "${GREEN}bitpocket started\x1b\x5b0m at `date`."
echo

# Fire off slow sync start notifier in background
Expand Down Expand Up @@ -348,7 +227,7 @@ function sync {
fi

pull
[[ $REMOTE_BACKUPS == true ]] && push_remote || push
push

# Save after-sync state

Expand All @@ -366,7 +245,7 @@ function sync {

cleanup
echo
echo -e "\x1b\x5b1;32mbitpocket finished\x1b\x5b0m at `date`."
echo -e "${GREEN}bitpocket finished\x1b\x5b0m at `date`."
echo

}
Expand Down Expand Up @@ -464,7 +343,7 @@ function acquire_lock {
echo "There's already an instance of BitPocket syncing this directory. Exiting."
exit 1
else
echo -e "\x1b\x5b1;31mbitpocket error:\x1b\x5b0m Bitpocket found a stale lock directory:"
echo -e "${RED}bitpocket error:\x1b\x5b0m Bitpocket found a stale lock directory:"
echo " | Root dir: $(pwd)"
echo " | Lock dir: $LOCK_DIR"
echo " | Command: LOCK_PATH=$(pwd)/$LOCK_DIR && rm \$LOCK_PATH/pid && rmdir \$LOCK_PATH"
Expand Down Expand Up @@ -525,7 +404,7 @@ function die {

# List all files in the sync set
function list {
echo -e "\x1b\x5b1;32mbitpocket\x1b\x5b0m will sync the following files:"
echo -e "${GREEN}bitpocket\x1b\x5b0m will sync the following files:"
rsync -av --list-only --exclude "/$DOT_DIR" $USER_RULES . | grep "^-\|^d" \
| sed "s:^\S*\s*\S*\s*\S*\s*\S*\s*:/:" | sed "s:^/\.::" | sort
}
Expand Down

0 comments on commit fd99bab

Please sign in to comment.