From eb32ed541cb18173425bb4d59591e6192852bab0 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Sun, 20 Jan 2019 10:36:37 +0100
Subject: [PATCH 01/19] initial version without ssh, fixed rsync ssh options

---
 bin/bitpocket | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/bin/bitpocket b/bin/bitpocket
index 2e97ddb..44740e0 100755
--- a/bin/bitpocket
+++ b/bin/bitpocket
@@ -1,4 +1,5 @@
 #!/bin/bash
+#set -x
 
 LANG=$(locale | grep LANG= | sed 's:LANG=::')
 if [[ -z "$LANG" ]]; then
@@ -122,10 +123,10 @@ function init {
 
   REMOTE_PATH="$1"
   setup_remote
-  if $REMOTE_RUNNER "[ ! -d '$1' ]"; then
-    echo "fatal: '$REMOTE': Remote path is not accessible"
-    exit 128
-  fi
+  #if $REMOTE_RUNNER "[ ! -d '$1' ]"; then
+  #  echo "fatal: '$REMOTE': Remote path is not accessible"
+  #  exit 128
+  #fi
 
   mkdir "$DOT_DIR"
 
@@ -210,7 +211,7 @@ function pull() {
   # TODO: Consider adding %U and %G to the output format to capture owner and
   # group changes
   prefix "R " < "$TMP_DIR/remote-del" \
-  | rsync -auzx --delete --exclude "/$DOT_DIR" \
+  | rsync -e "ssh -o stricthostkeychecking=no" -auzx --delete --exclude "/$DOT_DIR" \
         --exclude-from="$TMP_DIR/local-del" \
         --exclude-from="$TMP_DIR/local-add-change" \
         --filter=". -" \
@@ -279,7 +280,7 @@ function push() {
 
   # Do not push back remotely deleted files
   prefix "R " < "$TMP_DIR/local-del" \
-  | rsync -auzxi --delete $RSYNC_OPTS --exclude "/$DOT_DIR" \
+  | rsync -e "ssh -o stricthostkeychecking=no" -auzxi --delete $RSYNC_OPTS --exclude "/$DOT_DIR" \
         --exclude-from="$TMP_DIR/remote-del" \
         --filter=". -" \
         --filter="P **" \
@@ -333,7 +334,7 @@ function analyse {
   # snapshot is available locally
   if [[ -s "$STATE_DIR/tree-prev" ]]; then
     echo "  | Root dir: $REMOTE"
-    rsync --list-only --recursive --exclude "/$DOT_DIR" $USER_RULES $REMOTE/ \
+    rsync -e "ssh -o stricthostkeychecking=no" $RSYNC_OPTS --list-only --recursive --exclude "/$DOT_DIR" $USER_RULES $REMOTE/ \
       | scrub_rsync_list \
       | sort -k 1.12 \
       > "$STATE_DIR/remote-tree-current" &
@@ -426,7 +427,7 @@ function sync {
   assert_dotdir
   assert_mountpoints
   acquire_lock
-  acquire_remote_lock
+  #acquire_remote_lock
   check_state_version
   detect_fatfs
 
@@ -701,7 +702,7 @@ function assert_mountpoints {
 
 function cleanup {
   release_lock
-  release_remote_lock
+  #release_remote_lock
 }
 
 ##

From 4cab4d89f5cf63cefc4f01c169b34c9d6fc94cea Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Sun, 20 Jan 2019 11:14:44 +0100
Subject: [PATCH 02/19] add -b basedir option

---
 bin/bitpocket | 126 +++++++++++++++++++++++++++-----------------------
 1 file changed, 69 insertions(+), 57 deletions(-)

diff --git a/bin/bitpocket b/bin/bitpocket
index 44740e0..c538f6f 100755
--- a/bin/bitpocket
+++ b/bin/bitpocket
@@ -15,6 +15,7 @@ TMP_DIR="$DOT_DIR/tmp"
 STATE_DIR="$DOT_DIR/state"
 LOCK_DIR="$TMP_DIR/lock"  # Use a lock directory for atomic locks. See the Bash FAQ http://mywiki.wooledge.org/BashFAQ/045
 
+
 # Default settings
 SLOW_SYNC_TIME=10
 SLOW_SYNC_FILE="$TMP_DIR/slow"
@@ -28,6 +29,74 @@ REMOTE_MOUNTPOINT=false
 COMMANDS=()
 ARGS=()
 OPTIONS=()
+function usage {
+  cat <<EOF  
+usage:  bitpocket { init [<REMOTE_HOST>] <REMOTE_PATH>
+                  | sync | help | pack | log | cron | list }
+
+Available commands:
+   sync    Run the sync process. If no command is specified, sync is run by
+           default.
+   init    Initialize a new bitpocket folder. Requires path and optional
+           remote host params. Remote path must already exist.
+   pack    Pack any existing (automatic) backups into a git repository.
+   cron    Run sync optimized for cron, logging output to file instead of
+           stdout.
+   log     Display the log generated by the cron command
+   list    List all files in the sync set (honoring include/exclude/filter
+           config).
+   help    Show this message.
+
+Options:
+   -f, --force      Clean up stale lock files automatically
+   -p, --pretend    Don't really perform the sync or update the current
+                    state. Instead, show what would be synchronized.
+   -b, --base       base dir
+
+Note: All commands (apart from help), must be run in the root of a
+      new or existing bitpocket directory structure.
+EOF
+}
+
+function parseargs() {
+    while [[ -n $1 ]]; do
+        case $1 in
+            # Switches and configuration
+            -p|--pretend)   OPTIONS+=('pretend');;
+            -f|--force)     OPTIONS+=('force');;
+	    -b|--basedir)   if [[ $# -lt 2 ]] ; then
+		    		usage
+				exit 128
+			    fi
+		            BASEDIR=$2
+		            shift
+		            cd $BASEDIR || exit 1
+		            ;;	    
+            -h|--help|-*)   COMMANDS+=('help');;
+            # Arguments (commands)
+            init)           if [[ $# -lt 2 ]]; then
+                                echo "usage: bitpocket init [<REMOTE_HOST>] <REMOTE_PATH>"
+                                exit 128
+                            fi
+                            COMMANDS+=("$1")
+                            ARGS+=("$2")
+                            if [[ $# -gt 2 ]]; then
+                                ARGS+=("$3")
+                                shift;
+                            fi
+                            shift;;
+            sync|pack|cron|log|list|help)
+                            COMMANDS+=("$1");;
+            # Anything else
+            *)              echo "!!! Invalid command: $1"
+	    		    exit 1;;
+        esac
+        shift
+    done
+}
+
+parseargs "$@"
+
 
 # Load config file
 [[ -f "$CFG_FILE" ]] && . "$CFG_FILE"
@@ -751,63 +820,6 @@ function list {
       | sort
 }
 
-function usage {
-  cat <<EOF  
-usage:  bitpocket { init [<REMOTE_HOST>] <REMOTE_PATH>
-                  | sync | help | pack | log | cron | list }
-
-Available commands:
-   sync    Run the sync process. If no command is specified, sync is run by
-           default.
-   init    Initialize a new bitpocket folder. Requires path and optional
-           remote host params. Remote path must already exist.
-   pack    Pack any existing (automatic) backups into a git repository.
-   cron    Run sync optimized for cron, logging output to file instead of
-           stdout.
-   log     Display the log generated by the cron command
-   list    List all files in the sync set (honoring include/exclude/filter
-           config).
-   help    Show this message.
-
-Options:
-   -f, --force      Clean up stale lock files automatically
-   -p, --pretend    Don't really perform the sync or update the current
-                    state. Instead, show what would be synchronized.
-
-Note: All commands (apart from help), must be run in the root of a
-      new or existing bitpocket directory structure.
-EOF
-}
-
-function parseargs() {
-    while [[ -n $1 ]]; do
-        case $1 in
-            # Switches and configuration
-            -p|--pretend)   OPTIONS+=('pretend');;
-            -f|--force)     OPTIONS+=('force');;
-            -h|--help|-*)   COMMANDS+=('help');;
-            # Arguments (commands)
-            init)           if [[ $# -lt 2 ]]; then
-                                echo "usage: bitpocket init [<REMOTE_HOST>] <REMOTE_PATH>"
-                                exit 128
-                            fi
-                            COMMANDS+=("$1")
-                            ARGS+=("$2")
-                            if [[ $# -gt 2 ]]; then
-                                ARGS+=("$3")
-                                shift;
-                            fi
-                            shift;;
-            sync|pack|cron|log|list|help)
-                            COMMANDS+=("$1");;
-            # Anything else
-            *)              echo "!!! Invalid command: $1";;
-        esac
-        shift
-    done
-}
-
-parseargs "$@"
 
 # By default, run the sync process
 [[ ${#COMMANDS} == 0 ]] && COMMANDS+=('sync')

From b27cbebd043d1322ad3f759133c80ec8fd03cf23 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Sun, 20 Jan 2019 11:23:12 +0100
Subject: [PATCH 03/19] add RSYNC_SSH_OPTIONS

---
 bin/bitpocket | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/bin/bitpocket b/bin/bitpocket
index c538f6f..ce577a2 100755
--- a/bin/bitpocket
+++ b/bin/bitpocket
@@ -20,6 +20,7 @@ LOCK_DIR="$TMP_DIR/lock"  # Use a lock directory for atomic locks. See the Bash
 SLOW_SYNC_TIME=10
 SLOW_SYNC_FILE="$TMP_DIR/slow"
 RSYNC_RSH="ssh"
+[ "$RSYNC_SSH_OPTIONS" = "" ] && RSYNC_SSH_OPTIONS="ssh"
 REMOTE_BACKUPS=false
 BACKUPS=true
 LOCAL_MOUNTPOINT=false
@@ -203,6 +204,7 @@ function init {
 ## Host and path of central storage
 REMOTE_HOST=$REMOTE_HOST
 REMOTE_PATH="$REMOTE_PATH"
+RSYNC_SSH_OPTIONS="$RSYNC_SSH_OPTIONS"
 
 ## Backups -----------------------------------
 ## Enable file revisioning locally in the pull phase (>false< to disable)
@@ -280,7 +282,7 @@ function pull() {
   # TODO: Consider adding %U and %G to the output format to capture owner and
   # group changes
   prefix "R " < "$TMP_DIR/remote-del" \
-  | rsync -e "ssh -o stricthostkeychecking=no" -auzx --delete --exclude "/$DOT_DIR" \
+  | rsync -e "$RSYNC_SSH_OPTIONS" -auzx --delete --exclude "/$DOT_DIR" \
         --exclude-from="$TMP_DIR/local-del" \
         --exclude-from="$TMP_DIR/local-add-change" \
         --filter=". -" \
@@ -349,7 +351,7 @@ function push() {
 
   # Do not push back remotely deleted files
   prefix "R " < "$TMP_DIR/local-del" \
-  | rsync -e "ssh -o stricthostkeychecking=no" -auzxi --delete $RSYNC_OPTS --exclude "/$DOT_DIR" \
+  | rsync -e "$RSYNC_SSH_OPTIONS" -auzxi --delete $RSYNC_OPTS --exclude "/$DOT_DIR" \
         --exclude-from="$TMP_DIR/remote-del" \
         --filter=". -" \
         --filter="P **" \
@@ -403,7 +405,7 @@ function analyse {
   # snapshot is available locally
   if [[ -s "$STATE_DIR/tree-prev" ]]; then
     echo "  | Root dir: $REMOTE"
-    rsync -e "ssh -o stricthostkeychecking=no" $RSYNC_OPTS --list-only --recursive --exclude "/$DOT_DIR" $USER_RULES $REMOTE/ \
+    rsync -e "$RSYNC_SSH_OPTIONS" $RSYNC_OPTS --list-only --recursive --exclude "/$DOT_DIR" $USER_RULES $REMOTE/ \
       | scrub_rsync_list \
       | sort -k 1.12 \
       > "$STATE_DIR/remote-tree-current" &

From 3c6cfc674e8dda5b24ea769507c593d6b2772df5 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Sun, 20 Jan 2019 21:04:05 +0100
Subject: [PATCH 04/19] add remote lock handling with rsync only

---
 bin/bitpocket | 154 ++++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 137 insertions(+), 17 deletions(-)

diff --git a/bin/bitpocket b/bin/bitpocket
index ce577a2..4b2b646 100755
--- a/bin/bitpocket
+++ b/bin/bitpocket
@@ -8,11 +8,13 @@ fi
 
 export LC_ALL=$LANG # for stable "sort" output
 
+REMOTE_LOCK_NAME=remotelock
 # Paths
 DOT_DIR=.bitpocket
 CFG_FILE="$DOT_DIR/config"
 TMP_DIR="$DOT_DIR/tmp"
 STATE_DIR="$DOT_DIR/state"
+REMOTE_LOCK_DIR="$DOT_DIR/$REMOTE_LOCK_NAME"
 LOCK_DIR="$TMP_DIR/lock"  # Use a lock directory for atomic locks. See the Bash FAQ http://mywiki.wooledge.org/BashFAQ/045
 
 
@@ -21,10 +23,13 @@ SLOW_SYNC_TIME=10
 SLOW_SYNC_FILE="$TMP_DIR/slow"
 RSYNC_RSH="ssh"
 [ "$RSYNC_SSH_OPTIONS" = "" ] && RSYNC_SSH_OPTIONS="ssh"
+[ "$REMOTE_LOCK_WAIT" = "" ] && REMOTE_LOCK_WAIT=1
+[ "$REMOTE_LOCK_TIMEOUT" = "" ] && REMOTE_LOCK_TIMEOUT=1800
 REMOTE_BACKUPS=false
 BACKUPS=true
 LOCAL_MOUNTPOINT=false
 REMOTE_MOUNTPOINT=false
+NO_SSH=false
 
 # Default command-line options and such
 COMMANDS=()
@@ -53,6 +58,7 @@ Options:
    -p, --pretend    Don't really perform the sync or update the current
                     state. Instead, show what would be synchronized.
    -b, --base       base dir
+   -n, --nossh      omit ssh calls - only use rsync (no remote locking)
 
 Note: All commands (apart from help), must be run in the root of a
       new or existing bitpocket directory structure.
@@ -65,14 +71,15 @@ function parseargs() {
             # Switches and configuration
             -p|--pretend)   OPTIONS+=('pretend');;
             -f|--force)     OPTIONS+=('force');;
-	    -b|--basedir)   if [[ $# -lt 2 ]] ; then
-		    		usage
-				exit 128
-			    fi
-		            BASEDIR=$2
-		            shift
-		            cd $BASEDIR || exit 1
-		            ;;	    
+            -n|--nossh)     NO_SSH=true;;
+        -b|--basedir)   if [[ $# -lt 2 ]] ; then
+                    usage
+                exit 128
+                fi
+                    BASEDIR=$2
+                    shift
+                    cd $BASEDIR || exit 1
+                    ;;      
             -h|--help|-*)   COMMANDS+=('help');;
             # Arguments (commands)
             init)           if [[ $# -lt 2 ]]; then
@@ -90,7 +97,7 @@ function parseargs() {
                             COMMANDS+=("$1");;
             # Anything else
             *)              echo "!!! Invalid command: $1"
-	    		    exit 1;;
+                    exit 1;;
         esac
         shift
     done
@@ -171,6 +178,7 @@ fi
 USER_RULES="$user_filter $user_include $user_exclude"
 
 TIMESTAMP=$(date "+%Y-%m-%d.%H%M%S")
+LOCK_TIMESTAMP=$(date "+%s")
 
 export RSYNC_RSH
 
@@ -193,10 +201,18 @@ function init {
 
   REMOTE_PATH="$1"
   setup_remote
-  #if $REMOTE_RUNNER "[ ! -d '$1' ]"; then
-  #  echo "fatal: '$REMOTE': Remote path is not accessible"
-  #  exit 128
-  #fi
+  if [ "$NO_SSH" != true ] ; then
+    if $REMOTE_RUNNER "[ ! -d '$1' ]"; then
+      echo "fatal: '$REMOTE': Remote path is not accessible"
+      exit 128
+    fi
+  else
+    rsync -e "$RSYNC_SSH_OPTIONS" $RSYNC_OPTS --list-only --include="." --exclude="**" "$REMOTE/"
+    if [ $? != 0 ] ; then
+        echo "fatal: $REMOTE: remote not accessible"
+        exit 128
+    fi
+  fi
 
   mkdir "$DOT_DIR"
 
@@ -205,6 +221,7 @@ function init {
 REMOTE_HOST=$REMOTE_HOST
 REMOTE_PATH="$REMOTE_PATH"
 RSYNC_SSH_OPTIONS="$RSYNC_SSH_OPTIONS"
+NO_SSH="$NO_SSH"
 
 ## Backups -----------------------------------
 ## Enable file revisioning locally in the pull phase (>false< to disable)
@@ -498,7 +515,11 @@ function sync {
   assert_dotdir
   assert_mountpoints
   acquire_lock
-  #acquire_remote_lock
+  if [ "$NO_SSH"  = true ] ; then
+      acquire_remote_lock_rsync
+  else
+      acquire_remote_lock
+  fi
   check_state_version
   detect_fatfs
 
@@ -658,7 +679,95 @@ function acquire_lock {
 function release_lock {
   rm "$LOCK_DIR/pid" &>/dev/null && rmdir "$LOCK_DIR" &>/dev/null
 }
-
+function filter_outdated(){
+    while read -r line
+    do
+        IFS=":" read -ra info <<< "$line"
+        if [ "${info[2]}" -gt "$1" ] ; then
+            echo "$line"
+        fi    
+    done
+}
+#read the remote locks
+#return locks in the file REMOTE_STATE
+#return 0 if our own only, 1 if other, -1 if own not there
+#p1 our own lock file name
+#TODO: handle timeouts for locks
+function read_remote_locks() {
+    REMOTE_STATE="$STATE_DIR/remote-state"
+    local old_ts=$(date "+%s")
+    let "old_ts=$old_ts-$REMOTE_LOCK_TIMEOUT"
+    rm -f "$REMOTE_STATE"
+    rsync -e "$RSYNC_SSH_OPTIONS" $RSYNC_OPTS -r --list-only --filter="+ $DOT_DIR" --filter="+ $REMOTE_LOCK_DIR" --filter="+ $REMOTE_LOCK_DIR/*" --filter="- *" $REMOTE/ | sed -n "s?.*$REMOTE_LOCK_DIR/??p" | sed '/^ *$/d' > "$REMOTE_STATE"
+    [ $? != 0 ] && return -2
+    grep "$1" "$REMOTE_STATE" > /dev/null 2>&1
+    [ $? != 0 ] && return -1
+    local n=`cat "$REMOTE_STATE" | filter_outdated $old_ts | wc -l`
+    [ $n -gt 1 ] && return 1
+    return 0    
+}
+
+function acquire_remote_lock_rsync {
+    #write our own lock file to the remote
+    #and check if we are the only one
+    #consider the REMOTE_LOCK_TIMEOUT and REMOTE_LOCK_WAIT
+    local INFO="$HOSTNAME:$$:$LOCK_TIMESTAMP"
+    if [ -d $REMOTE_LOCK_DIR ] ; then
+        rm -rf $REMOTE_LOCK_DIR
+        mkdir -p $REMOTE_LOCK_DIR
+        touch $REMOTE_LOCK_DIR/$INFO
+    fi    
+    #write our lockfile to remote
+    echo "#creating remote lock $REMOTE_LOCK_DIR/$INFO"
+    rsync  -e "$RSYNC_SSH_OPTIONS" $RSYNC_OPTS -ra  --filter="+ $DOT_DIR" --filter="+ $REMOTE_LOCK_DIR" --filter="+ $REMOTE_LOCK_DIR/$INFO" --filter="- *" ./ $REMOTE/
+    
+    read_remote_locks "$INFO"
+    local rstate=$?
+    [ $rstate = 0 ] && return 0
+    if [ "$rstate" -ne 1 ] ; then
+        die "unable to acquire remote lock"
+    fi
+    local wait_time=0
+    while [ $wait_time -lt "$REMOTE_LOCK_WAIT" ] 
+    do
+        local other_hosts=`cat "$REMOTE_STATE" | grep -v "^$HOSTNAME:"`
+        if [ "$other_hosts" != "" ] ; then
+            echo "other hosts having locks: $other_hosts"
+        else    
+            local pids_same_host=`cat "$REMOTE_STATE" | grep -v "$INFO" | sed -n "s?^$HOSTNAME:??p" | sed 's/:.*//'`
+            if [ "$pids_same_host" != "" ] ; then
+                echo "found pids: $pids_same_host from our host, checking"
+                local all_stopped=1
+                for pid in $pids_same_host
+                do
+                    kill -0 $pid > /dev/null 2>&1
+                    if [ $? = 0 ] ; then
+                        all_stopped=0
+                        break
+                    fi
+                done
+                if [ $all_stopped = 1 ] ; then
+                    echo "locks from our host found without process, ignoring"
+                    return 0
+                fi
+            fi
+        fi    
+        sleep 1
+        let "wait_time=$wait_time+1"
+        echo "waiting for remote lock"
+        read_remote_locks "$INFO"
+        rstate=$?
+        [ $rstate = 0 ] && return 0
+        if [ "$rstate" -ne 1 ] ; then
+            die "unable to acquire remote lock"
+        fi
+    done    
+    local code=3
+    echo -e "${YELLOW}Another client is syncing with '$REMOTE'${CLEAR}"
+    cat $REMOTE_STATE | grep -v "$INFO"
+    cleanup
+    exit $code
+}
 function acquire_remote_lock {
     # TODO: Place the local hostname and this PID in a file, which will make
     # automatic lock file cleanup possible. It will also offer better output if
@@ -712,10 +821,17 @@ function acquire_remote_lock {
         fi
     fi
 
-    release_lock
+    cleanup
     exit $code
 }
 
+function release_remote_lock_rsync {
+    rm -rf $REMOTE_LOCK_DIR
+    mkdir -p $REMOTE_LOCK_DIR
+    local INFO="$HOSTNAME:$$:$LOCK_TIMESTAMP"
+    echo "#releasing remote lock $REMOTE_LOCK_DIR/$INFO"
+    rsync -e "$RSYNC_SSH_OPTIONS" $RSYNC_OPTS -ra --delete --filter="+ $DOT_DIR" --filter="+ $REMOTE_LOCK_DIR" --filter="+ $REMOTE_LOCK_DIR/$INFO" --filter="- *" ./ $REMOTE/
+}
 function release_remote_lock {
   $REMOTE_RUNNER "cd \"$REMOTE_PATH\" && grep -q '$HOSTNAME:$$' '$LOCK_DIR/remote' && rm '$LOCK_DIR/remote' && rmdir '$LOCK_DIR' &>/dev/null"
 }
@@ -773,7 +889,11 @@ function assert_mountpoints {
 
 function cleanup {
   release_lock
-  #release_remote_lock
+  if [ "$NO_SSH" = true ] ; then
+      release_remote_lock_rsync
+  else
+      release_remote_lock
+  fi    
 }
 
 ##

From 3962d03f840ddcbe9bdb384a85cb3beadee5e421 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Sun, 20 Jan 2019 21:45:45 +0100
Subject: [PATCH 05/19] correctly create lock dir on first run, improve logging

---
 bin/bitpocket | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/bin/bitpocket b/bin/bitpocket
index 4b2b646..705799e 100755
--- a/bin/bitpocket
+++ b/bin/bitpocket
@@ -221,6 +221,11 @@ function init {
 REMOTE_HOST=$REMOTE_HOST
 REMOTE_PATH="$REMOTE_PATH"
 RSYNC_SSH_OPTIONS="$RSYNC_SSH_OPTIONS"
+#set a timeout after which remote locks are ignored (only with NO_SSH=true)
+REMOTE_LOCK_TIMEOUT="$REMOTE_LOCK_TIMEOUT"
+#set a time to wait for acquiring a remote lock (only with NO_SSH=true)
+REMOTE_LOCK_WAIT="$REMOTE_LOCK_WAIT"
+#avoid any direct ssh commands, always use rsync
 NO_SSH="$NO_SSH"
 
 ## Backups -----------------------------------
@@ -714,9 +719,9 @@ function acquire_remote_lock_rsync {
     local INFO="$HOSTNAME:$$:$LOCK_TIMESTAMP"
     if [ -d $REMOTE_LOCK_DIR ] ; then
         rm -rf $REMOTE_LOCK_DIR
-        mkdir -p $REMOTE_LOCK_DIR
-        touch $REMOTE_LOCK_DIR/$INFO
     fi    
+    mkdir -p $REMOTE_LOCK_DIR    
+    touch $REMOTE_LOCK_DIR/$INFO
     #write our lockfile to remote
     echo "#creating remote lock $REMOTE_LOCK_DIR/$INFO"
     rsync  -e "$RSYNC_SSH_OPTIONS" $RSYNC_OPTS -ra  --filter="+ $DOT_DIR" --filter="+ $REMOTE_LOCK_DIR" --filter="+ $REMOTE_LOCK_DIR/$INFO" --filter="- *" ./ $REMOTE/
@@ -727,12 +732,12 @@ function acquire_remote_lock_rsync {
     if [ "$rstate" -ne 1 ] ; then
         die "unable to acquire remote lock"
     fi
-    local wait_time=0
-    while [ $wait_time -lt "$REMOTE_LOCK_WAIT" ] 
+    local wait_time=$REMOTE_LOCK_WAIT
+    while [ $wait_time -gt 0 ] 
     do
         local other_hosts=`cat "$REMOTE_STATE" | grep -v "^$HOSTNAME:"`
         if [ "$other_hosts" != "" ] ; then
-            echo "other hosts having locks: $other_hosts"
+            echo "#other hosts having locks: $other_hosts"
         else    
             local pids_same_host=`cat "$REMOTE_STATE" | grep -v "$INFO" | sed -n "s?^$HOSTNAME:??p" | sed 's/:.*//'`
             if [ "$pids_same_host" != "" ] ; then
@@ -753,8 +758,8 @@ function acquire_remote_lock_rsync {
             fi
         fi    
         sleep 1
-        let "wait_time=$wait_time+1"
-        echo "waiting for remote lock"
+        let "wait_time=$wait_time-1"
+        echo "#waiting for remote lock for $wait_time seconds"
         read_remote_locks "$INFO"
         rstate=$?
         [ $rstate = 0 ] && return 0

From f609a1de0b334e8849be3660a1513f07c3c58d97 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Sat, 26 Jan 2019 12:01:58 +0100
Subject: [PATCH 06/19] improve remote lock handling, use arrays and check if
 we have the oldest timestamp to continue

---
 bin/bitpocket | 96 ++++++++++++++++++++++++++++++++++-----------------
 1 file changed, 64 insertions(+), 32 deletions(-)

diff --git a/bin/bitpocket b/bin/bitpocket
index 705799e..29dd249 100755
--- a/bin/bitpocket
+++ b/bin/bitpocket
@@ -694,22 +694,49 @@ function filter_outdated(){
     done
 }
 #read the remote locks
-#return locks in the file REMOTE_STATE
-#return 0 if our own only, 1 if other, -1 if own not there
+#return locks in the variable REMOTE_STATE
+#outdated in REMOTE_OUTDATED, other in REMOTE_OTHER, other from same host in LOCAL_OTHER
+#return 0 if our own only or our is oldest, 1 if other, -1 if own not there or other error
 #p1 our own lock file name
 #TODO: handle timeouts for locks
 function read_remote_locks() {
-    REMOTE_STATE="$STATE_DIR/remote-state"
     local old_ts=$(date "+%s")
     let "old_ts=$old_ts-$REMOTE_LOCK_TIMEOUT"
     rm -f "$REMOTE_STATE"
-    rsync -e "$RSYNC_SSH_OPTIONS" $RSYNC_OPTS -r --list-only --filter="+ $DOT_DIR" --filter="+ $REMOTE_LOCK_DIR" --filter="+ $REMOTE_LOCK_DIR/*" --filter="- *" $REMOTE/ | sed -n "s?.*$REMOTE_LOCK_DIR/??p" | sed '/^ *$/d' > "$REMOTE_STATE"
-    [ $? != 0 ] && return -2
-    grep "$1" "$REMOTE_STATE" > /dev/null 2>&1
-    [ $? != 0 ] && return -1
-    local n=`cat "$REMOTE_STATE" | filter_outdated $old_ts | wc -l`
-    [ $n -gt 1 ] && return 1
-    return 0    
+    REMOTE_STATE=()
+    while IFS= read -r line
+    do
+        REMOTE_STATE+=("$line")
+    done < <( rsync -e "$RSYNC_SSH_OPTIONS" $RSYNC_OPTS -r --list-only --filter="+ $DOT_DIR" --filter="+ $REMOTE_LOCK_DIR" --filter="+ $REMOTE_LOCK_DIR/*" --filter="- *" $REMOTE/ | sed -n "s?.*$REMOTE_LOCK_DIR/??p" | sed '/^ *$/d' )
+    [ ${#REMOTE_STATE[@]} -lt 1 ] && return -2
+    REMOTE_OUTDATED=()
+    REMOTE_OTHER=()
+    LOCAL_OTHER=()
+    local hasOwn=0
+    local ownIsOldest=1
+    IFS=":" read -ra ownInfo <<< "$1"
+    for line in "${REMOTE_STATE[@]}"
+    do
+        IFS=":" read -ra info <<< "$line"
+        if [ "${info[2]}" -lt "$old_ts" ] ; then
+            REMOTE_OUTDATED+=("$line")
+        else
+            if [ "$line" = "$1" ] ; then
+                hasOwn=1
+            else
+                if [ "${info[0]}" = "${ownInfo[0]}" ] ; then
+                    LOCAL_OTHER+=("$line")
+                else
+                    REMOTE_OTHER+=("$line")
+                fi
+                [ "${info[2]}" -lt "${ownInfo[2]}" ] && ownIsOldest=0
+            fi 
+        fi
+    done
+    [ "$hasOwn" != "1" ] && return -1
+    [ "${#REMOTE_OTHER[@]}" = "0" -a "${#LOCAL_OTHER[@]}" = "0" ] && return 0
+    [ "$ownIsOldest" = "1" ] && return 0
+    return 1    
 }
 
 function acquire_remote_lock_rsync {
@@ -718,9 +745,10 @@ function acquire_remote_lock_rsync {
     #consider the REMOTE_LOCK_TIMEOUT and REMOTE_LOCK_WAIT
     local INFO="$HOSTNAME:$$:$LOCK_TIMESTAMP"
     if [ -d $REMOTE_LOCK_DIR ] ; then
-        rm -rf $REMOTE_LOCK_DIR
+        rm -f $REMOTE_LOCK_DIR/$INFO
+    else
+        mkdir -p $REMOTE_LOCK_DIR    
     fi    
-    mkdir -p $REMOTE_LOCK_DIR    
     touch $REMOTE_LOCK_DIR/$INFO
     #write our lockfile to remote
     echo "#creating remote lock $REMOTE_LOCK_DIR/$INFO"
@@ -735,26 +763,27 @@ function acquire_remote_lock_rsync {
     local wait_time=$REMOTE_LOCK_WAIT
     while [ $wait_time -gt 0 ] 
     do
-        local other_hosts=`cat "$REMOTE_STATE" | grep -v "^$HOSTNAME:"`
-        if [ "$other_hosts" != "" ] ; then
-            echo "#other hosts having locks: $other_hosts"
+        if [ "${#REMOTE_OTHER[@]}" != "0" ] ; then
+            echo "#other hosts having locks:"
+            printf "%s\n" "${REMOTE_OTHER[@]}"
         else    
-            local pids_same_host=`cat "$REMOTE_STATE" | grep -v "$INFO" | sed -n "s?^$HOSTNAME:??p" | sed 's/:.*//'`
-            if [ "$pids_same_host" != "" ] ; then
-                echo "found pids: $pids_same_host from our host, checking"
-                local all_stopped=1
-                for pid in $pids_same_host
-                do
-                    kill -0 $pid > /dev/null 2>&1
-                    if [ $? = 0 ] ; then
-                        all_stopped=0
-                        break
-                    fi
-                done
-                if [ $all_stopped = 1 ] ; then
-                    echo "locks from our host found without process, ignoring"
-                    return 0
+            local allStopped=1
+            echo "#local other locks:"
+            printf "%s\n" "${LOCAL_OTHER[@]}"
+            for ll in "${LOCAL_OTHER[@]}"
+            do
+                local localInfo
+                IFS=":" read -ra localInfo <<< "$1l"
+                local pid="${localInfo[1]}"
+                kill -0 $pid > /dev/null 2>&1
+                if [ $? = 0 ] ; then
+                    allStopped=0
+                    break
                 fi
+            done
+            if [ $allStopped = 1 ] ; then
+                echo "locks from our host found without process, ignoring"
+                return 0
             fi
         fi    
         sleep 1
@@ -831,9 +860,12 @@ function acquire_remote_lock {
 }
 
 function release_remote_lock_rsync {
-    rm -rf $REMOTE_LOCK_DIR
-    mkdir -p $REMOTE_LOCK_DIR
     local INFO="$HOSTNAME:$$:$LOCK_TIMESTAMP"
+    if [ -d $REMOTE_LOCK_DIR ] ; then
+        rm -f $REMOTE_LOCK_DIR/$INFO
+    else
+        mkdir -p $REMOTE_LOCK_DIR
+    fi
     echo "#releasing remote lock $REMOTE_LOCK_DIR/$INFO"
     rsync -e "$RSYNC_SSH_OPTIONS" $RSYNC_OPTS -ra --delete --filter="+ $DOT_DIR" --filter="+ $REMOTE_LOCK_DIR" --filter="+ $REMOTE_LOCK_DIR/$INFO" --filter="- *" ./ $REMOTE/
 }

From ba721fc501c9febe0ccf41512c688f0a1c8c88d8 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Sat, 26 Jan 2019 12:21:24 +0100
Subject: [PATCH 07/19] local locks without removing lock dir, allow to always
 remove stale local locks via config

---
 bin/bitpocket | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/bin/bitpocket b/bin/bitpocket
index 29dd249..3606f7a 100755
--- a/bin/bitpocket
+++ b/bin/bitpocket
@@ -1,5 +1,5 @@
 #!/bin/bash
-#set -x
+set -x
 
 LANG=$(locale | grep LANG= | sed 's:LANG=::')
 if [[ -z "$LANG" ]]; then
@@ -109,6 +109,9 @@ parseargs "$@"
 # Load config file
 [[ -f "$CFG_FILE" ]] && . "$CFG_FILE"
 
+if [ "$REMOVE_STALE_LOCKS" = "true" ] ; then
+    OPTIONS+=("force")
+fi
 # Colors
 GREEN=""
 RED=""
@@ -228,6 +231,9 @@ REMOTE_LOCK_WAIT="$REMOTE_LOCK_WAIT"
 #avoid any direct ssh commands, always use rsync
 NO_SSH="$NO_SSH"
 
+#always remove stale local locks (default: false)
+#REMOVE_STALE_LOCKS=true
+
 ## Backups -----------------------------------
 ## Enable file revisioning locally in the pull phase (>false< to disable)
 BACKUPS=true
@@ -656,7 +662,12 @@ function timestamp {
 }
 
 function acquire_lock {
-  if ! mkdir "$LOCK_DIR" 2>/dev/null
+  [ ! -d "$LOCK_DIR" ] && mkdir -p "$LOCK_DIR"
+  if [ ! -d "$LOCK_DIR" ]; then
+      echo -e "${RED}bitpocket error:${CLEAR} unable to create lock dir $LOCK_DIR"
+      exit 2
+  fi
+  if [ -f "$LOCK_DIR/pid" ]
   then
     if kill -0 $(cat "$LOCK_DIR/pid") &>/dev/null
     then
@@ -666,14 +677,14 @@ function acquire_lock {
       if [[ "${OPTIONS[*]}" =~ force ]]
       then
         echo -e "${YELLOW}Removing stale, local lock file${CLEAR}"
-        rm "$LOCK_DIR/pid" && rmdir "$LOCK_DIR" && acquire_lock && return 0
+        rm -f "$LOCK_DIR/pid" && acquire_lock && return 0
       fi
 
       echo -e "${RED}bitpocket error:${CLEAR} Bitpocket found a stale lock directory:"
       echo "  | Root dir: $(pwd)"
       echo "  | Lock dir: $LOCK_DIR"
-      echo "  | Command:  LOCK_PATH=$(pwd)/$LOCK_DIR && rm \$LOCK_PATH/pid && rmdir \$LOCK_PATH"
-      echo "Please remove the lock directory and try again."
+      echo "  | Command:  LOCK_PATH=$(pwd)/$LOCK_DIR && rm \$LOCK_PATH/pid "
+      echo "Please remove the lock file and try again."
       exit 2
     fi
   fi
@@ -682,7 +693,7 @@ function acquire_lock {
 }
 
 function release_lock {
-  rm "$LOCK_DIR/pid" &>/dev/null && rmdir "$LOCK_DIR" &>/dev/null
+  rm -f "$LOCK_DIR/pid" &>/dev/null 
 }
 function filter_outdated(){
     while read -r line

From 93f112cab8870f09b540846357a2608d0f96937b Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Sat, 26 Jan 2019 12:38:46 +0100
Subject: [PATCH 08/19] correct output if other locks

---
 bin/bitpocket | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/bin/bitpocket b/bin/bitpocket
index 3606f7a..fe5e3ae 100755
--- a/bin/bitpocket
+++ b/bin/bitpocket
@@ -1,5 +1,5 @@
 #!/bin/bash
-set -x
+#set -x
 
 LANG=$(locale | grep LANG= | sed 's:LANG=::')
 if [[ -z "$LANG" ]]; then
@@ -809,7 +809,8 @@ function acquire_remote_lock_rsync {
     done    
     local code=3
     echo -e "${YELLOW}Another client is syncing with '$REMOTE'${CLEAR}"
-    cat $REMOTE_STATE | grep -v "$INFO"
+    printf "%s\n" "${REMOTE_OTHER[@]}"
+    printf "%s\n" "${LOCAL_OTHER[@]}"
     cleanup
     exit $code
 }

From f07e375f4e244eb7fd887b5c23ee624aa77d4da1 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Sat, 26 Jan 2019 16:05:04 +0100
Subject: [PATCH 09/19] correctly handle --exclude in RSYNC_OPTIONS

---
 bin/bitpocket | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/bitpocket b/bin/bitpocket
index fe5e3ae..4f270cb 100755
--- a/bin/bitpocket
+++ b/bin/bitpocket
@@ -310,14 +310,14 @@ function pull() {
   # TODO: Consider adding %U and %G to the output format to capture owner and
   # group changes
   prefix "R " < "$TMP_DIR/remote-del" \
-  | rsync -e "$RSYNC_SSH_OPTIONS" -auzx --delete --exclude "/$DOT_DIR" \
+  | rsync -e "$RSYNC_SSH_OPTIONS" -auzx --delete $RSYNC_OPTS --exclude "/$DOT_DIR" \
         --exclude-from="$TMP_DIR/local-del" \
         --exclude-from="$TMP_DIR/local-add-change" \
         --filter=". -" \
         --filter="P **" \
         $DO_BACKUP \
         --out-format=%i:%B:%n \
-        $RSYNC_OPTS $USER_RULES $REMOTE/ . \
+        $USER_RULES $REMOTE/ . \
   | detect_changes \
   | prefix "  | " || die "PULL"
 

From 6adf587cc2e182c9ca8972bca73c6ae375254657 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Sat, 26 Jan 2019 19:53:14 +0100
Subject: [PATCH 10/19] sort trees before comparing

---
 bin/bitpocket | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/bin/bitpocket b/bin/bitpocket
index 4f270cb..1fb1fd8 100755
--- a/bin/bitpocket
+++ b/bin/bitpocket
@@ -1,5 +1,5 @@
 #!/bin/bash
-#set -x
+set -x
 
 LANG=$(locale | grep LANG= | sed 's:LANG=::')
 if [[ -z "$LANG" ]]; then
@@ -467,7 +467,9 @@ function analyse {
           # positives, so run the output back through `grep` to check if there
           # is an exact match in the previous state file. This is much faster
           # than resorting both of the files if they are somewhat large.
-          comm -23 "$STATE_DIR/tree-current" "$STATE_DIR/tree-prev" 2>/dev/null \
+          sort  "$STATE_DIR/tree-current" >  "$STATE_DIR/tree-current.sort" || die "unable to sort tree-current"
+          sort  "$STATE_DIR/tree-prev" >  "$STATE_DIR/tree-prev.sort" || die "unable to sort tree-prev"
+          comm -23 "$STATE_DIR/tree-current.sort" "$STATE_DIR/tree-prev.sort" 2>/dev/null \
           | while read -r line
             do
                 if ! grep -Fxq "$line" "$STATE_DIR/tree-prev"
@@ -506,7 +508,7 @@ function analyse {
   else
       # In the case of a new sync, where no previous tree snapshot is available,
       # assume all the files on the local side should be protected
-      cp "$STATE_DIR/tree-current" "$TMP_DIR/local-add-change"
+      cat "$STATE_DIR/tree-current" | strip_mode > "$TMP_DIR/local-add-change"
       touch "$TMP_DIR/local-del"
       touch "$TMP_DIR/remote-del"
   fi
@@ -695,15 +697,6 @@ function acquire_lock {
 function release_lock {
   rm -f "$LOCK_DIR/pid" &>/dev/null 
 }
-function filter_outdated(){
-    while read -r line
-    do
-        IFS=":" read -ra info <<< "$line"
-        if [ "${info[2]}" -gt "$1" ] ; then
-            echo "$line"
-        fi    
-    done
-}
 #read the remote locks
 #return locks in the variable REMOTE_STATE
 #outdated in REMOTE_OUTDATED, other in REMOTE_OTHER, other from same host in LOCAL_OTHER

From 4b9142eff914561e96036f8e87c1e64cbc597643 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Sat, 26 Jan 2019 19:53:39 +0100
Subject: [PATCH 11/19] disable verbose

---
 bin/bitpocket | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/bitpocket b/bin/bitpocket
index 1fb1fd8..6d979b8 100755
--- a/bin/bitpocket
+++ b/bin/bitpocket
@@ -1,5 +1,5 @@
 #!/bin/bash
-set -x
+#set -x
 
 LANG=$(locale | grep LANG= | sed 's:LANG=::')
 if [[ -z "$LANG" ]]; then

From d032602d14d38a9f2a9c9d438d7895aac9642765 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Sat, 26 Jan 2019 21:29:21 +0100
Subject: [PATCH 12/19] use a simple find to determine which files have been
 changed locally

---
 bin/bitpocket | 48 ++++++------------------------------------------
 1 file changed, 6 insertions(+), 42 deletions(-)

diff --git a/bin/bitpocket b/bin/bitpocket
index 6d979b8..12c362b 100755
--- a/bin/bitpocket
+++ b/bin/bitpocket
@@ -421,9 +421,6 @@ function scrub_rsync_list {
 
 function analyse {
   # Check what has changed
-  touch "$STATE_DIR/tree-prev"
-
-  # Save before-sync state
   # Must be done with rsync itself (rather than find) to respect includes/excludes
   # Order of includes/excludes/filters is EXTREMELY important
   echo "# Capturing current local and remote state"
@@ -458,45 +455,12 @@ function analyse {
       > "$TMP_DIR/local-del"
 
       # Honor local mode changes (link to file, as well as permissions).
-      # These should be protected in the pull phase. Ignore files already
-      # masked as locally added or locally removed.
-      if [[ $STATE_VERSION -gt 1 ]]
-      then
-          # Use comm to detect the differences in the pseudo-sorted files
-          # (they're sorted on column 12). Comm will detect some false
-          # positives, so run the output back through `grep` to check if there
-          # is an exact match in the previous state file. This is much faster
-          # than resorting both of the files if they are somewhat large.
-          sort  "$STATE_DIR/tree-current" >  "$STATE_DIR/tree-current.sort" || die "unable to sort tree-current"
-          sort  "$STATE_DIR/tree-prev" >  "$STATE_DIR/tree-prev.sort" || die "unable to sort tree-prev"
-          comm -23 "$STATE_DIR/tree-current.sort" "$STATE_DIR/tree-prev.sort" 2>/dev/null \
-          | while read -r line
-            do
-                if ! grep -Fxq "$line" "$STATE_DIR/tree-prev"
-                then
-                    echo "$line"
-                fi
-            done \
-          | strip_mode \
-          > "$TMP_DIR/local-add-change"
-      else
-          # In transition to the new state file, ignore the changes in mode
-          comm -23 <(strip_mode < "$STATE_DIR/tree-current") "$STATE_DIR/tree-prev" \
-          > "$TMP_DIR/local-add-change"
-      fi
-
-      # Also protect the folders where files were locally added and removed so
-      # that the modify times of them are not reverted to the remote ones
-      cat "$TMP_DIR/local-del" "$TMP_DIR/local-add-change" \
-      | while read -r line; do
-          # Only parent folders of files--not folders
-          if [[ "${line: -1}" != "/" ]]
-          then
-              echo "${line%/*}/"
-          fi
-        done \
-      | sort -u \
-      >> "$TMP_DIR/local-add-change"
+      # we simply find all files that have been changed after the last check
+      # this leaves a smal gap: files that potentially have been changed remotely during the sync
+      # or if the remote ts does not really fit ours
+      # but if we run the sync repeatedly, this should be fixed with the next run
+      # These should be protected in the pull phase. 
+      find . -type f -newer "$STATE_DIR/tree-prev" | sed 's?^\.??' | grep -v "^/$DOT_DIR" > "$TMP_DIR/local-add-change"
 
       # Prevent deleting local files which were not deleted remotely ie.
       # prevent deleting newly added local files. Compile a list of remotely

From f2b17357235e66d3be2d59a4ad1e9d8d7ae1f91a Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Sun, 27 Jan 2019 17:01:51 +0100
Subject: [PATCH 13/19] 1st simplified version with using find and not
 preserving dir times

---
 bin/bitpocket | 248 +++++++++-----------------------------------------
 1 file changed, 42 insertions(+), 206 deletions(-)

diff --git a/bin/bitpocket b/bin/bitpocket
index 12c362b..7f6c0a6 100755
--- a/bin/bitpocket
+++ b/bin/bitpocket
@@ -1,5 +1,12 @@
 #!/bin/bash
-#set -x
+
+#
+# rsync based sync script - heavily based on bitpocket from
+# https://github.com/sickill/bitpocket
+# simplified by omiting the approach to preserve directory timestamps
+# and fully relying on rsync - no ssh to remote systems
+#
+set -x
 
 LANG=$(locale | grep LANG= | sed 's:LANG=::')
 if [[ -z "$LANG" ]]; then
@@ -28,8 +35,6 @@ RSYNC_RSH="ssh"
 REMOTE_BACKUPS=false
 BACKUPS=true
 LOCAL_MOUNTPOINT=false
-REMOTE_MOUNTPOINT=false
-NO_SSH=false
 
 # Default command-line options and such
 COMMANDS=()
@@ -58,7 +63,6 @@ Options:
    -p, --pretend    Don't really perform the sync or update the current
                     state. Instead, show what would be synchronized.
    -b, --base       base dir
-   -n, --nossh      omit ssh calls - only use rsync (no remote locking)
 
 Note: All commands (apart from help), must be run in the root of a
       new or existing bitpocket directory structure.
@@ -71,7 +75,6 @@ function parseargs() {
             # Switches and configuration
             -p|--pretend)   OPTIONS+=('pretend');;
             -f|--force)     OPTIONS+=('force');;
-            -n|--nossh)     NO_SSH=true;;
         -b|--basedir)   if [[ $# -lt 2 ]] ; then
                     usage
                 exit 128
@@ -140,10 +143,8 @@ fi
 # Decide on runner (ssh / bash -c)
 function setup_remote() {
     if [[ -n "$REMOTE_HOST" ]]; then
-      REMOTE_RUNNER="$RSYNC_RSH $REMOTE_HOST"
       REMOTE="$REMOTE_HOST:$REMOTE_PATH"
     else
-      REMOTE_RUNNER="bash -c"
       REMOTE="$REMOTE_PATH"
     fi
 }
@@ -204,19 +205,11 @@ function init {
 
   REMOTE_PATH="$1"
   setup_remote
-  if [ "$NO_SSH" != true ] ; then
-    if $REMOTE_RUNNER "[ ! -d '$1' ]"; then
-      echo "fatal: '$REMOTE': Remote path is not accessible"
+  rsync -e "$RSYNC_SSH_OPTIONS" $RSYNC_OPTS --list-only --include="." --exclude="**" "$REMOTE/"
+  if [ $? != 0 ] ; then
+      echo "fatal: $REMOTE: remote not accessible"
       exit 128
-    fi
-  else
-    rsync -e "$RSYNC_SSH_OPTIONS" $RSYNC_OPTS --list-only --include="." --exclude="**" "$REMOTE/"
-    if [ $? != 0 ] ; then
-        echo "fatal: $REMOTE: remote not accessible"
-        exit 128
-    fi
   fi
-
   mkdir "$DOT_DIR"
 
   cat <<EOF > "$CFG_FILE"
@@ -224,12 +217,11 @@ function init {
 REMOTE_HOST=$REMOTE_HOST
 REMOTE_PATH="$REMOTE_PATH"
 RSYNC_SSH_OPTIONS="$RSYNC_SSH_OPTIONS"
-#set a timeout after which remote locks are ignored (only with NO_SSH=true)
+#set a timeout after which remote locks are ignored
 REMOTE_LOCK_TIMEOUT="$REMOTE_LOCK_TIMEOUT"
-#set a time to wait for acquiring a remote lock (only with NO_SSH=true)
+#set a time to wait for acquiring a remote lock 
 REMOTE_LOCK_WAIT="$REMOTE_LOCK_WAIT"
 #avoid any direct ssh commands, always use rsync
-NO_SSH="$NO_SSH"
 
 #always remove stale local locks (default: false)
 #REMOVE_STALE_LOCKS=true
@@ -261,7 +253,6 @@ REMOTE_BACKUPS=false
 ## a sync target appears empty because it is not mounted.  Such a sync might
 ## result in all local or remote data disappearing. Give the expected
 ## mountpoint of the local and/or remote target.
-# REMOTE_MOUNTPOINT=/
 EOF
 
   echo "Initialized bitpocket directory at $(pwd)"
@@ -296,25 +287,20 @@ function pull() {
       local DO_BACKUP="--backup --backup-dir=$BACKUP_TARGET"
   fi
 
-  cp "$STATE_DIR/tree-current" "$TMP_DIR/tree-after"
 
   # Determine what will be fetched from server and make backup copies of any
   # local files to be deleted or overwritten.
   #
-  # Only delete locally if deleted remotely. To do this, use the remote-del
-  # file to set the *R*isk filter flag (allow delete), and protect everything
-  # else with the *P*rotect flag.
+  # Only delete locally if deleted remotely.
+  # we preserve files from local-add-change and local-del
   #
   # Order of includes/excludes/filters is EXTREMELY important
   #
   # TODO: Consider adding %U and %G to the output format to capture owner and
   # group changes
-  prefix "R " < "$TMP_DIR/remote-del" \
-  | rsync -e "$RSYNC_SSH_OPTIONS" -auzx --delete $RSYNC_OPTS --exclude "/$DOT_DIR" \
-        --exclude-from="$TMP_DIR/local-del" \
-        --exclude-from="$TMP_DIR/local-add-change" \
-        --filter=". -" \
-        --filter="P **" \
+  rsync -O -e "$RSYNC_SSH_OPTIONS" -auzx --delete $RSYNC_OPTS --exclude "/$DOT_DIR" \
+        --filter=". $TMP_DIR/local-del" \
+        --filter=". $TMP_DIR/local-add-change" \
         $DO_BACKUP \
         --out-format=%i:%B:%n \
         $USER_RULES $REMOTE/ . \
@@ -335,6 +321,7 @@ function pull() {
 }
 
 function detect_changes() {
+    # TODO: we do not need this really....
     # Create a duplicate of STDOUT for logging of backed-up files, and use fd#4
     # for logging of deleted files, which need to be sorted
     exec 3> >(grep -Ff /dev/stdin "$STATE_DIR/tree-current" | sort > "$TMP_DIR/pull-delete")
@@ -378,32 +365,11 @@ function push() {
   fi
 
   # Do not push back remotely deleted files
-  prefix "R " < "$TMP_DIR/local-del" \
-  | rsync -e "$RSYNC_SSH_OPTIONS" -auzxi --delete $RSYNC_OPTS --exclude "/$DOT_DIR" \
-        --exclude-from="$TMP_DIR/remote-del" \
-        --filter=". -" \
-        --filter="P **" \
+  rsync -O -e "$RSYNC_SSH_OPTIONS" -auzxi --delete $RSYNC_OPTS --exclude "/$DOT_DIR" \
         $DO_BACKUP \
         $USER_RULES . $REMOTE/ \
   | prefix "  | " || die "PUSH"
 
-  # Some versions of rsync will create the backup dir, even if it doesn't get
-  # populated with any backups
-  if [[ $REMOTE_BACKUPS == true ]]
-  then
-      $REMOTE_RUNNER "
-      cd '$REMOTE_PATH'
-      if [[ -d '$BACKUP_TARGET' ]]
-      then
-          if (shopt -s nullglob dotglob; f=('$BACKUP_TARGET'/*); ((\${#f[@]})))
-          then
-              echo '  | Some files were backed up to $BACKUP_TARGET'
-          else
-              rmdir '$BACKUP_TARGET'
-          fi
-      fi
-      "
-  fi
 }
 
 function scrub_rsync_list {
@@ -419,63 +385,30 @@ function scrub_rsync_list {
     }'
 }
 
+function create_sync_start {
+  touch "$STATE_DIR/new-sync"
+}
+
+function finish_sync {
+  rm -f "$STATE_DIR/last-sync" "$STATE_DIR/tree-last"
+  mv "$STATE_DIR/new-sync" "$STATE_DIR/last-sync" || die "unable to create sync marker"
+  mv "$STATE_DIR/tree-current" "$STATE_DIR/tree-last"
+}
+
 function analyse {
   # Check what has changed
-  # Must be done with rsync itself (rather than find) to respect includes/excludes
-  # Order of includes/excludes/filters is EXTREMELY important
-  echo "# Capturing current local and remote state"
+  echo "# Capturing current local state "
   echo "  | Root dir: $(pwd)"
 
-  # Collect the current snapshot of the remote tree, if a previous tree
-  # snapshot is available locally
-  if [[ -s "$STATE_DIR/tree-prev" ]]; then
-    echo "  | Root dir: $REMOTE"
-    rsync -e "$RSYNC_SSH_OPTIONS" $RSYNC_OPTS --list-only --recursive --exclude "/$DOT_DIR" $USER_RULES $REMOTE/ \
-      | scrub_rsync_list \
-      | sort -k 1.12 \
-      > "$STATE_DIR/remote-tree-current" &
-    local remote_tree_pid=$!
-  fi
-
   # Collect the current snapshot of the local tree
-  rsync --list-only --recursive --exclude "/$DOT_DIR" $USER_RULES . \
-      | scrub_rsync_list \
-      | sort -k 1.12 \
-      > "$STATE_DIR/tree-current" \
-      || die "SNAPSHOT"
-
-  # Prevent bringing back locally deleted files
-  if [[ -s "$STATE_DIR/tree-prev" ]]
-  then
-      # Compile a list of files added locally and removed locally. These
-      # should be protected in the pull phase. Escape rsync filter wildcard
-      # characters, remove blank lines
-      strip_mode < "$STATE_DIR/tree-prev" \
-      | comm -23 - <(strip_mode < "$STATE_DIR/tree-current") \
-      > "$TMP_DIR/local-del"
-
-      # Honor local mode changes (link to file, as well as permissions).
-      # we simply find all files that have been changed after the last check
-      # this leaves a smal gap: files that potentially have been changed remotely during the sync
-      # or if the remote ts does not really fit ours
-      # but if we run the sync repeatedly, this should be fixed with the next run
-      # These should be protected in the pull phase. 
-      find . -type f -newer "$STATE_DIR/tree-prev" | sed 's?^\.??' | grep -v "^/$DOT_DIR" > "$TMP_DIR/local-add-change"
-
-      # Prevent deleting local files which were not deleted remotely ie.
-      # prevent deleting newly added local files. Compile a list of remotely
-      # deleted files which should be protected in the push phase.
-      wait $remote_tree_pid
-      strip_mode < "$STATE_DIR/tree-prev" \
-      | comm -23 - <(strip_mode < "$STATE_DIR/remote-tree-current") \
-      > "$TMP_DIR/remote-del"
+  find . | sort > "$STATE_DIR/tree-current"
+  if [ -e "$STATE_DIR/last-sync" -a -s "$STATE_DIR/tree-last" ]; then
+    find . -newer "$STATE_DIR/last-sync" | sed -n 's?^\.\([^ ][^ ]*\)?P \1?p' > "$TMP_DIR/local-add-change" || die "unable to create local-add-change"
+    comm -23 "$STATE_DIR/tree-last" "$STATE_DIR/tree-current" |sed -n 's?^\.\([^ ][^ ]*\)?- \1?p'> $TMP_DIR/local-del || die "unable to create local-del"
   else
-      # In the case of a new sync, where no previous tree snapshot is available,
-      # assume all the files on the local side should be protected
-      cat "$STATE_DIR/tree-current" | strip_mode > "$TMP_DIR/local-add-change"
-      touch "$TMP_DIR/local-del"
-      touch "$TMP_DIR/remote-del"
-  fi
+    echo "" > "$TMP_DIR/local-del" || die "unable to create local-del"
+    cp "$STATE_DIR/tree-current" "$TMP_DIR/local-add-change" || die "unable to create local-add-change"
+  fi  
 }
 
 function strip_mode {
@@ -490,13 +423,9 @@ function strip_mode {
 # Do the actual synchronization
 function sync {
   assert_dotdir
-  assert_mountpoints
   acquire_lock
-  if [ "$NO_SSH"  = true ] ; then
-      acquire_remote_lock_rsync
-  else
-      acquire_remote_lock
-  fi
+  acquire_remote_lock_rsync
+  create_sync_start
   check_state_version
   detect_fatfs
 
@@ -517,20 +446,7 @@ function sync {
 
   pull
   push
-
-  if [[ ! "${OPTIONS[*]}" =~ pretend ]]; then
-      # Save after-sync state
-
-      # Generate a incremental snapshot of the local tree including files deleted
-      # and added via the pull()
-      #
-      # Remove pull-deleted files from the tree-after snapshot
-      sort -k1.12 "$TMP_DIR/tree-after" \
-      | comm -23 - "$TMP_DIR/pull-delete" 2> /dev/null \
-      | sed -e "s:/\$::" \
-      > "$STATE_DIR/tree-prev"
-  fi
-  rm "$TMP_DIR/tree-after"
+  finish_sync
 
   # Fire off slow sync stop notifier in background
   on_slow_sync_stop
@@ -771,62 +687,6 @@ function acquire_remote_lock_rsync {
     cleanup
     exit $code
 }
-function acquire_remote_lock {
-    # TODO: Place the local hostname and this PID in a file, which will make
-    # automatic lock file cleanup possible. It will also offer better output if
-    # another host is truly syncing with the remote host.
-    local INFO="$HOSTNAME:$$:$TIMESTAMP"
-    local REMOTE_INFO=$($REMOTE_RUNNER "
-        mkdir -p '$REMOTE_TMP_DIR' && cd '$REMOTE_PATH'
-        [[ -d '$LOCK_DIR' ]] || mkdir '$LOCK_DIR'
-        [[ -e '$LOCK_DIR'/remote ]] || echo '$INFO' > '$LOCK_DIR'/remote
-        cat '$LOCK_DIR'/remote")
-
-    [[ "$INFO" == "$REMOTE_INFO" ]] && return 0
-
-    IFS=":" read -ra INFO <<< "$REMOTE_INFO"
-
-    # From here down, assume the lock could not be acquired
-    local code=3
-    if [[ -z $REMOTE_INFO ]]
-    then
-        echo "Couldn't acquire remote lock or lock file couldn't be created. Exiting."
-    elif [[ "$HOSTNAME" != "${INFO[0]}" ]]
-    then
-        echo -e "${YELLOW}Another client is syncing with '$REMOTE'${CLEAR}"
-        echo ">> Host:    ${INFO[0]}"
-        echo ">> PID:     ${INFO[1]}"
-        echo ">> Started: ${INFO[2]}"
-    elif [[ "$$" != "${INFO[1]}" ]]
-    then
-        # This host is syncing with the remote host. Check if the PID is still running
-        if kill -0 "${INFO[1]}" &>/dev/null
-        then
-            # XXX: This should be handled in the `acquire_lock` function
-            echo "Another instance of Bitpocket is currently syncing this" \
-                 "host with '$REMOTE'"
-            code=1
-        else
-            # In this case, this host is holding the lock with the remote server
-            # but the sync is no longer running. It is perhaps possible to remove
-            # the lock?
-            if [[ "${OPTIONS[*]}" =~ force ]]
-            then
-                echo -e "${YELLOW}Removing stale, remote lock file${CLEAR}"
-                $REMOTE_RUNNER "cd '$REMOTE_PATH' && rm '$LOCK_DIR/remote' && rmdir '$LOCK_DIR'"
-                # Try again
-                acquire_remote_lock && return 0
-            fi
-
-            echo "The remote lock is held by this host and is stale." \
-                 "It should be removed, and the sync should be retried."
-            code=6
-        fi
-    fi
-
-    cleanup
-    exit $code
-}
 
 function release_remote_lock_rsync {
     local INFO="$HOSTNAME:$$:$LOCK_TIMESTAMP"
@@ -838,9 +698,6 @@ function release_remote_lock_rsync {
     echo "#releasing remote lock $REMOTE_LOCK_DIR/$INFO"
     rsync -e "$RSYNC_SSH_OPTIONS" $RSYNC_OPTS -ra --delete --filter="+ $DOT_DIR" --filter="+ $REMOTE_LOCK_DIR" --filter="+ $REMOTE_LOCK_DIR/$INFO" --filter="- *" ./ $REMOTE/
 }
-function release_remote_lock {
-  $REMOTE_RUNNER "cd \"$REMOTE_PATH\" && grep -q '$HOSTNAME:$$' '$LOCK_DIR/remote' && rm '$LOCK_DIR/remote' && rmdir '$LOCK_DIR' &>/dev/null"
-}
 
 function assert_dotdir {
   if [ ! -d "$DOT_DIR" ]; then
@@ -875,31 +732,10 @@ function detect_fatfs {
     # TODO: Consider remote filesystem type?
 }
 
-function assert_mountpoints {
-    if [[ ${REMOTE_MOUNTPOINT} != false ]]
-    then
-        # Sanity check -- ensure mountpoint is a parent of local target
-        if [[ "${REMOTE_PATH:0:${#REMOTE_MOUNTPOINT}}" != "${REMOTE_MOUNTPOINT}" ]]
-        then
-            echo -e "${YELLOW}warning: Remote mount point is not a parent of '${REMOTE_PATH}'${CLEAR}"
-        fi
-
-        $REMOTE_RUNNER "mount | grep -E '\s${REMOTE_MOUNTPOINT}\s'" &> /dev/null
-        if [[ $? != 0 ]]
-        then
-            echo -e "${RED}fatal: Remote sync target is not mounted${CLEAR}"
-            exit 4
-        fi
-    fi
-}
 
 function cleanup {
   release_lock
-  if [ "$NO_SSH" = true ] ; then
-      release_remote_lock_rsync
-  else
-      release_remote_lock
-  fi    
+  release_remote_lock_rsync
 }
 
 ##

From 3a5052ef4084d86d68c0fca5917499235bb5f822 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Sun, 27 Jan 2019 17:25:02 +0100
Subject: [PATCH 14/19] siwtch off debug

---
 bin/bitpocket | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/bin/bitpocket b/bin/bitpocket
index 7f6c0a6..28596e2 100755
--- a/bin/bitpocket
+++ b/bin/bitpocket
@@ -6,7 +6,7 @@
 # simplified by omiting the approach to preserve directory timestamps
 # and fully relying on rsync - no ssh to remote systems
 #
-set -x
+#set -x
 
 LANG=$(locale | grep LANG= | sed 's:LANG=::')
 if [[ -z "$LANG" ]]; then
@@ -397,6 +397,16 @@ function finish_sync {
 
 function analyse {
   # Check what has changed
+  # simpliefied approach:
+  # check for local changes after the last sync start (mtime against last-sync) -> local-add-change, preserve them from deletes
+  # check for local deletes (compare tree-last with tree-current) -> local-del and omit them completely from the pull
+  # remaining issues:
+  # (1) if a remote file gets changed during the sync it will be stored with a mtime > last-sync
+  #     this will prevent it from being deleted on the next sync if afterwards it would be deleted remotely (and this way will be transferred back)
+  #     maybe acceptable as it would only affect deletes (i.e. file would come back)
+  # (2) we completely rely on rsync checking the timestamps - this could cause trouble if local and remote time stamps differ
+  # (3) a file is locally changed/deleted when the sync is running - we don't know which one will finally win - but this problem is always there
+  # all of the problems will finally create backups on bot sides
   echo "# Capturing current local state "
   echo "  | Root dir: $(pwd)"
 

From 19eede2bad7c7f76979221fc28f46c63d626bcd1 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Sun, 27 Jan 2019 17:43:54 +0100
Subject: [PATCH 15/19] correctly handle initial sync, escape wildcards in
 lists

---
 bin/bitpocket | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/bin/bitpocket b/bin/bitpocket
index 28596e2..8fc0f4f 100755
--- a/bin/bitpocket
+++ b/bin/bitpocket
@@ -385,6 +385,11 @@ function scrub_rsync_list {
     }'
 }
 
+#omit the leading . and escape wildcards in filenames
+function find_to_filter {
+  sed -n "s?^\.\([^ ][^ ]*\)?$1 \1?p"| sed 's:\([*?[]\):\\\1:g'
+}
+
 function create_sync_start {
   touch "$STATE_DIR/new-sync"
 }
@@ -413,11 +418,11 @@ function analyse {
   # Collect the current snapshot of the local tree
   find . | sort > "$STATE_DIR/tree-current"
   if [ -e "$STATE_DIR/last-sync" -a -s "$STATE_DIR/tree-last" ]; then
-    find . -newer "$STATE_DIR/last-sync" | sed -n 's?^\.\([^ ][^ ]*\)?P \1?p' > "$TMP_DIR/local-add-change" || die "unable to create local-add-change"
-    comm -23 "$STATE_DIR/tree-last" "$STATE_DIR/tree-current" |sed -n 's?^\.\([^ ][^ ]*\)?- \1?p'> $TMP_DIR/local-del || die "unable to create local-del"
+    find . -newer "$STATE_DIR/last-sync" | find_to_filter P > "$TMP_DIR/local-add-change" || die "unable to create local-add-change"
+    comm -23 "$STATE_DIR/tree-last" "$STATE_DIR/tree-current" | find_to_filter '-' > $TMP_DIR/local-del || die "unable to create local-del"
   else
     echo "" > "$TMP_DIR/local-del" || die "unable to create local-del"
-    cp "$STATE_DIR/tree-current" "$TMP_DIR/local-add-change" || die "unable to create local-add-change"
+    cat "$STATE_DIR/tree-current" | find_to_filter P > "$TMP_DIR/local-add-change" || die "unable to create local-add-change"
   fi  
 }
 

From d43b583dd04c04f8e6df9994d08dcf81ebce2a8a Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Sun, 27 Jan 2019 17:52:16 +0100
Subject: [PATCH 16/19] some code review, remove unused stuff

---
 bin/bitpocket | 40 +++++-----------------------------------
 1 file changed, 5 insertions(+), 35 deletions(-)

diff --git a/bin/bitpocket b/bin/bitpocket
index 8fc0f4f..242525b 100755
--- a/bin/bitpocket
+++ b/bin/bitpocket
@@ -150,16 +150,6 @@ function setup_remote() {
 }
 setup_remote
 
-# Version of the state files. Original version is 1, second version with
-# leading mode is 2.
-#
-# Current state version is 2, and the format of the state file is
-# ":" TYPE MODE "/" PATH "\n"
-# Where TYPE = ("-" | "d" | "l" )
-#       MODE = unix mode (9-character) (like "rw-rw-r--")
-#       PATH = relative full-path of file
-STATE_VERSION=2
-
 REMOTE_TMP_DIR="$REMOTE_PATH/$DOT_DIR/tmp"
 HOSTNAME="$(hostname)"
 
@@ -315,8 +305,11 @@ function pull() {
       then
           echo "  | Some files were backed up to $BACKUP_TARGET"
       else
+          echo "  | No files were backed up to $BACKUP_TARGET, cleaning up"
           rmdir "$BACKUP_TARGET"
       fi
+  else    
+    echo "  | No files were backed up to $BACKUP_TARGET, cleaning up"
   fi
 }
 
@@ -419,7 +412,7 @@ function analyse {
   find . | sort > "$STATE_DIR/tree-current"
   if [ -e "$STATE_DIR/last-sync" -a -s "$STATE_DIR/tree-last" ]; then
     find . -newer "$STATE_DIR/last-sync" | find_to_filter P > "$TMP_DIR/local-add-change" || die "unable to create local-add-change"
-    comm -23 "$STATE_DIR/tree-last" "$STATE_DIR/tree-current" | find_to_filter '-' > $TMP_DIR/local-del || die "unable to create local-del"
+    comm -23 "$STATE_DIR/tree-last" "$STATE_DIR/tree-current" | find_to_filter '-' > "$TMP_DIR/local-del" || die "unable to create local-del"
   else
     echo "" > "$TMP_DIR/local-del" || die "unable to create local-del"
     cat "$STATE_DIR/tree-current" | find_to_filter P > "$TMP_DIR/local-add-change" || die "unable to create local-add-change"
@@ -427,12 +420,7 @@ function analyse {
 }
 
 function strip_mode {
-    if [[ $STATE_VERSION -gt 1 ]]; then
-        cut -c12-
-    else
-        # State file version might be intermixed. Evaluate each line
-        sed -E "s/^:.{10}//"
-    fi
+      cut -c12-
 }
 
 # Do the actual synchronization
@@ -441,7 +429,6 @@ function sync {
   acquire_lock
   acquire_remote_lock_rsync
   create_sync_start
-  check_state_version
   detect_fatfs
 
   echo
@@ -597,7 +584,6 @@ function release_lock {
 #outdated in REMOTE_OUTDATED, other in REMOTE_OTHER, other from same host in LOCAL_OTHER
 #return 0 if our own only or our is oldest, 1 if other, -1 if own not there or other error
 #p1 our own lock file name
-#TODO: handle timeouts for locks
 function read_remote_locks() {
     local old_ts=$(date "+%s")
     let "old_ts=$old_ts-$REMOTE_LOCK_TIMEOUT"
@@ -753,22 +739,6 @@ function cleanup {
   release_remote_lock_rsync
 }
 
-##
-# Inspect the state file tree-prev to see if the format of the file is the
-# current version (2) or the original version (1). This is used in the
-# `strip_mode` function to optimize the sync process when the `tree-prev` file
-# uses the current state version.
-function check_state_version() {
-    # In the original state files, the start of the line was the filename with
-    # a leading slash
-    if [[ -s "$STATE_DIR/tree-prev" ]]; then
-        local first=$(head -1 "$STATE_DIR/tree-prev" 2>/dev/null)
-        if [[ ${first:0:1} == "/" ]]; then
-            STATE_VERSION=1
-        fi
-    fi
-}
-
 function bring_the_children_let_me_kill_them {
   if [ -n "$shell_pid" ]; then
     pkill -P $shell_pid &>/dev/null

From 70386fbfc343149fd53bcc10de80e14db2e1cd41 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Mon, 28 Jan 2019 09:08:42 +0100
Subject: [PATCH 17/19] handle remote deletes correctly on next sync

---
 bin/bitpocket | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/bin/bitpocket b/bin/bitpocket
index 242525b..8409e90 100755
--- a/bin/bitpocket
+++ b/bin/bitpocket
@@ -317,7 +317,7 @@ function detect_changes() {
     # TODO: we do not need this really....
     # Create a duplicate of STDOUT for logging of backed-up files, and use fd#4
     # for logging of deleted files, which need to be sorted
-    exec 3> >(grep -Ff /dev/stdin "$STATE_DIR/tree-current" | sort > "$TMP_DIR/pull-delete")
+    exec 3> "$STATE_DIR/pull-current"
 
     while read -r line
     do
@@ -326,14 +326,10 @@ function detect_changes() {
         filename="${info[*]:2}"
         if [[ "$operation" =~ ^\*deleting ]]
         then
-            echo "/${filename}" >&3
+            echo "d ./${filename}" >&3
         elif [[ "$operation" =~ \+\+\+\+$ ]]
         then
-            # Mark as added locally (with proper mode)
-            mode=${info[1]}
-            filetype="${operation:1:1}"
-            filetype="${filetype/f/-}"
-            echo ":${filetype}${mode}/$filename" >> "$TMP_DIR/tree-after"
+            echo "w ./$filename" >&3
         fi
         echo "$operation $filename"
     done
@@ -387,10 +383,13 @@ function create_sync_start {
   touch "$STATE_DIR/new-sync"
 }
 
+#should be run in background to be as atomic as possible...
 function finish_sync {
-  rm -f "$STATE_DIR/last-sync" "$STATE_DIR/tree-last"
-  mv "$STATE_DIR/new-sync" "$STATE_DIR/last-sync" || die "unable to create sync marker"
-  mv "$STATE_DIR/tree-current" "$STATE_DIR/tree-last"
+  trap '' SIGHUP SIGABRT SIGINT SIGQUIT SIGILL SIGTRAP SIGFPE SIGPIPE SIGTERM SIGSTOP 
+  #the following operations should be somehow atomic	
+  mv -f "$STATE_DIR/new-sync" "$STATE_DIR/last-sync" || die "unable to create sync marker"
+  mv -f "$STATE_DIR/tree-current" "$STATE_DIR/tree-last"
+  mv -f "$STATE_DIR/pull-current" "$STATE_DIR/pull-last"
 }
 
 function analyse {
@@ -398,6 +397,7 @@ function analyse {
   # simpliefied approach:
   # check for local changes after the last sync start (mtime against last-sync) -> local-add-change, preserve them from deletes
   # check for local deletes (compare tree-last with tree-current) -> local-del and omit them completely from the pull
+  # the last deletes from remote (pull-delete) will be substracted from the list 
   # remaining issues:
   # (1) if a remote file gets changed during the sync it will be stored with a mtime > last-sync
   #     this will prevent it from being deleted on the next sync if afterwards it would be deleted remotely (and this way will be transferred back)
@@ -412,7 +412,10 @@ function analyse {
   find . | sort > "$STATE_DIR/tree-current"
   if [ -e "$STATE_DIR/last-sync" -a -s "$STATE_DIR/tree-last" ]; then
     find . -newer "$STATE_DIR/last-sync" | find_to_filter P > "$TMP_DIR/local-add-change" || die "unable to create local-add-change"
-    comm -23 "$STATE_DIR/tree-last" "$STATE_DIR/tree-current" | find_to_filter '-' > "$TMP_DIR/local-del" || die "unable to create local-del"
+    [ ! -f "$STATE_DIR/pull-current" ] && touch "$STATE_DIR/pull-current"
+    [ ! -f "$STATE_DIR/pull-last" ] && touch "$STATE_DIR/pull-last"
+    cat "$STATE_DIR/pull-current" "$STATE_DIR/pull-last" | sed -n 's/^d //p'|sort|uniq > "$TMP_DIR/pull-del"
+    comm -23 "$STATE_DIR/tree-last" "$STATE_DIR/tree-current" | comm -23 - "$TMP_DIR/pull-del" | find_to_filter '-' > "$TMP_DIR/local-del" || die "unable to create local-del"
   else
     echo "" > "$TMP_DIR/local-del" || die "unable to create local-del"
     cat "$STATE_DIR/tree-current" | find_to_filter P > "$TMP_DIR/local-add-change" || die "unable to create local-add-change"
@@ -448,7 +451,8 @@ function sync {
 
   pull
   push
-  finish_sync
+  finish_sync &
+  wait $!
 
   # Fire off slow sync stop notifier in background
   on_slow_sync_stop

From 78e4a9b95221e47af9af419c011a918b777afddc Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Mon, 28 Jan 2019 19:43:36 +0100
Subject: [PATCH 18/19] prevent sync when include/exclude filters have changed

---
 bin/bitpocket | 43 ++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 40 insertions(+), 3 deletions(-)

diff --git a/bin/bitpocket b/bin/bitpocket
index 8409e90..b28f9ca 100755
--- a/bin/bitpocket
+++ b/bin/bitpocket
@@ -1,4 +1,5 @@
 #!/bin/bash
+# vim: et:sw=2:ts=2
 
 #
 # rsync based sync script - heavily based on bitpocket from
@@ -6,7 +7,7 @@
 # simplified by omiting the approach to preserve directory timestamps
 # and fully relying on rsync - no ssh to remote systems
 #
-#set -x
+# set -x
 
 LANG=$(locale | grep LANG= | sed 's:LANG=::')
 if [[ -z "$LANG" ]]; then
@@ -23,6 +24,7 @@ TMP_DIR="$DOT_DIR/tmp"
 STATE_DIR="$DOT_DIR/state"
 REMOTE_LOCK_DIR="$DOT_DIR/$REMOTE_LOCK_NAME"
 LOCK_DIR="$TMP_DIR/lock"  # Use a lock directory for atomic locks. See the Bash FAQ http://mywiki.wooledge.org/BashFAQ/045
+FILTER_FILES=(include exclude filter)
 
 
 # Default settings
@@ -48,6 +50,9 @@ usage:  bitpocket { init [<REMOTE_HOST>] <REMOTE_PATH>
 Available commands:
    sync    Run the sync process. If no command is specified, sync is run by
            default.
+   resync  merge remote and local
+           will not delete any local files
+           always necessary if includes/excludes changed
    init    Initialize a new bitpocket folder. Requires path and optional
            remote host params. Remote path must already exist.
    pack    Pack any existing (automatic) backups into a git repository.
@@ -96,7 +101,7 @@ function parseargs() {
                                 shift;
                             fi
                             shift;;
-            sync|pack|cron|log|list|help)
+            sync|pack|cron|log|list|help|resync)
                             COMMANDS+=("$1");;
             # Anything else
             *)              echo "!!! Invalid command: $1"
@@ -386,7 +391,7 @@ function create_sync_start {
 #should be run in background to be as atomic as possible...
 function finish_sync {
   trap '' SIGHUP SIGABRT SIGINT SIGQUIT SIGILL SIGTRAP SIGFPE SIGPIPE SIGTERM SIGSTOP 
-  #the following operations should be somehow atomic	
+  #the following operations should be somehow atomic  
   mv -f "$STATE_DIR/new-sync" "$STATE_DIR/last-sync" || die "unable to create sync marker"
   mv -f "$STATE_DIR/tree-current" "$STATE_DIR/tree-last"
   mv -f "$STATE_DIR/pull-current" "$STATE_DIR/pull-last"
@@ -426,10 +431,35 @@ function strip_mode {
       cut -c12-
 }
 
+#check all the filter files
+function check_filter_files {
+  #no need to check if there was no sync before
+  [ ! -e "$STATE_DIR/last-sync" ] && return 0
+  for ff in ${FILTER_FILES[@]} 
+  do
+    [ -f "$DOT_DIR/$ff" -a ! -f "$STATE_DIR/$ff" ] && return 1
+    [ ! -f "$DOT_DIR/$ff" -a  -f "$STATE_DIR/$ff" ] && return 1
+    if [ -f "$DOT_DIR/$ff" -a  -f "$STATE_DIR/$ff" ] ; then
+      cmp "$DOT_DIR/$ff" "$STATE_DIR/$ff" || return 1
+    fi
+  done
+  return 0
+}
+
+function copy_filter_files {
+  for ff in ${FILTER_FILES[@]} 
+  do
+    rm -f "$STATE_DIR/$ff"
+    [ -f "$DOT_DIR/$ff" ] && cp "$DOT_DIR/$ff" "$STATE_DIR/$ff"
+  done
+}
+
 # Do the actual synchronization
 function sync {
   assert_dotdir
   acquire_lock
+  check_filter_files || die "include/exclude filters have changed, need to run resync"
+  [ ! -e "$STATE_DIR/last-sync" ] && copy_filter_files
   acquire_remote_lock_rsync
   create_sync_start
   detect_fatfs
@@ -464,6 +494,12 @@ function sync {
 
 }
 
+function resync {
+  echo "#re-syncing"
+  rm -f "$STATE_DIR/last-sync"
+  sync
+}
+
 # Pack backups into a git repository
 function pack {
     assert_dotdir
@@ -786,4 +822,5 @@ case ${COMMANDS[0]} in
     list)       list;;
     help)       usage;;
     sync)       sync;;
+    resync)     resync;;
 esac

From d133afa11a4cac5bc9926a6af58f424a41c3b7f0 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Mon, 28 Jan 2019 19:47:55 +0100
Subject: [PATCH 19/19] more error handling when file lists cannot be created

---
 bin/bitpocket | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/bitpocket b/bin/bitpocket
index b28f9ca..39f185e 100755
--- a/bin/bitpocket
+++ b/bin/bitpocket
@@ -50,7 +50,7 @@ usage:  bitpocket { init [<REMOTE_HOST>] <REMOTE_PATH>
 Available commands:
    sync    Run the sync process. If no command is specified, sync is run by
            default.
-   resync  merge remote and local
+   resync  merge remote and local (like initial sync)
            will not delete any local files
            always necessary if includes/excludes changed
    init    Initialize a new bitpocket folder. Requires path and optional
@@ -419,7 +419,7 @@ function analyse {
     find . -newer "$STATE_DIR/last-sync" | find_to_filter P > "$TMP_DIR/local-add-change" || die "unable to create local-add-change"
     [ ! -f "$STATE_DIR/pull-current" ] && touch "$STATE_DIR/pull-current"
     [ ! -f "$STATE_DIR/pull-last" ] && touch "$STATE_DIR/pull-last"
-    cat "$STATE_DIR/pull-current" "$STATE_DIR/pull-last" | sed -n 's/^d //p'|sort|uniq > "$TMP_DIR/pull-del"
+    cat "$STATE_DIR/pull-current" "$STATE_DIR/pull-last" | sed -n 's/^d //p'|sort|uniq > "$TMP_DIR/pull-del" || die "unable to create pull-del"
     comm -23 "$STATE_DIR/tree-last" "$STATE_DIR/tree-current" | comm -23 - "$TMP_DIR/pull-del" | find_to_filter '-' > "$TMP_DIR/local-del" || die "unable to create local-del"
   else
     echo "" > "$TMP_DIR/local-del" || die "unable to create local-del"