Fixes soundcloud#107 - automatic retry on deadlock

README.md

@@ -212,6 +212,23 @@ Ruby:
 Lhm.cleanup(:run, until: Time.now - 86400)
 ```
 
+## Retry on deadlock
+
+LHM supports automatic retry on deadlock.  By default it retries 10 times with
+a 10 second delay between each retry.  It can be customized using the following
+options:
+
+  1. `retry_on_deadlock` - true / false (true by default)
+  2. `retry_attempts` - Number of retries on deadlock (10 by default)
+  3. `retry_wait_time` - Number of seconds to wait between each retry
+  (10 seconds by default)
+
+```ruby
+Lhm.change_table :users, :retry_on_deadlock => true, :retry_attempts => 5 do |m|
+  m.add_column :arbitrary, "INT(12)"
+end
+```
+
 ## Contributing
 
 First, get set up for local development:

lib/lhm/chunker.rb

@@ -22,14 +22,48 @@ def initialize(migration, connection = nil, options = {})
       @start = options[:start] || select_start
       @limit = options[:limit] || select_limit
       @printer = options[:printer] || Printer::Percentage.new
+      @retry_on_deadlock = retry_on_deadlock(options.with_indifferent_access)
+      @retry_attempts = retry_attempts(options.with_indifferent_access)
+      @retry_wait_time = retry_wait_time(options.with_indifferent_access)
     end
 
     def execute
       return unless @start && @limit
+
+      retries = 0
+
       @next_to_insert = @start
       while @next_to_insert <= @limit
         stride = @throttler.stride
-        affected_rows = @connection.update(copy(bottom, top(stride)))
+
+        begin
+          affected_rows = @connection.update(copy(bottom, top(stride)))
+        rescue ActiveRecord::StatementInvalid => err
+          if err.message.downcase.index('deadlock').nil?
+            raise
+            return
+          end
+
+          if !@retry_on_deadlock
+            raise
+            return
+          end
+
+          retries = retries + 1
+          if retries == (@retry_attempts + 1)
+            puts "Crossed #{@retry_attempts} attempts. Raising exception ..."
+            raise
+            return
+          else
+            puts "Caught exception: #{err.message}."
+            print "Attempt #{retries} of #{@retry_attempts} "
+            puts "after sleeping for #{@retry_wait_time} seconds ..."
+            sleep @retry_wait_time
+            next
+          end
+        end
+
+        retries = 0
 
         if @throttler && affected_rows > 0
           @throttler.run
@@ -108,5 +142,36 @@ def validate
         error('impossible chunk options (limit must be greater than start)')
       end
     end
+
+    def retry_on_deadlock(options)
+      if options.has_key?(:retry_on_deadlock) &&
+         ( options[:retry_on_deadlock].is_a?(TrueClass) ||
+           options[:retry_on_deadlock].is_a?(FalseClass) )
+
+        return options[:retry_on_deadlock]
+      end
+
+      return true
+    end
+
+    def retry_attempts(options)
+      if options.has_key?(:retry_attempts) &&
+         options[:retry_attempts].is_a?(Numeric)
+
+        return options[:retry_attempts]
+      end
+
+      return 10
+    end
+
+    def retry_wait_time(options)
+      if options.has_key?(:retry_wait_time) &&
+         options[:retry_wait_time].is_a?(Numeric)
+
+        return options[:retry_wait_time]
+      end
+
+      return 10
+    end
   end
 end