From 96343787950d35bb6d91ad8f4fddd929496884ac Mon Sep 17 00:00:00 2001
From: Hector Correa <hector_correa@princeton.edu>
Date: Thu, 30 Jan 2025 14:53:39 -0500
Subject: [PATCH] Handle SQL UPDATE errors on long running jobs (#1266)

* Handle the case when the job update request because the job took too long

* Added test
---
 app/jobs/file_inventory_job.rb       | 20 ++++++++++++++------
 spec/jobs/file_inventory_job_spec.rb | 21 +++++++++++++++++++++
 2 files changed, 35 insertions(+), 6 deletions(-)

diff --git a/app/jobs/file_inventory_job.rb b/app/jobs/file_inventory_job.rb
index b5fe62369..f5e78c444 100644
--- a/app/jobs/file_inventory_job.rb
+++ b/app/jobs/file_inventory_job.rb
@@ -33,12 +33,9 @@ def perform(user_id:, project_id:, mediaflux_session:)
     Rails.logger.info "Exporting file list to #{filename} for project #{project_id}"
     project.file_list_to_file(session_id: mediaflux_session, filename: filename)
     Rails.logger.info "Export file generated #{filename} for project #{project_id}"
-    # Make the FileInventoryRequest object
-    inventory_request = FileInventoryRequest.find_by(user_id: user.id, project_id: project.id, job_id: @job_id)
-    request_details = { output_file: filename, project_title: project.title, file_size: File.size(filename) }
-    inventory_request.update(state: UserRequest::COMPLETED, request_details: request_details,
-                             completion_time: Time.current.in_time_zone("America/New_York"))
-    inventory_request
+
+    # Update the job record as completed
+    update_inventory_request(user_id: user.id, project: project, job_id: @job_id, filename: filename)
   end
 
   private
@@ -53,4 +50,15 @@ def filename_for_export
       pathname = Pathname.new(Rails.configuration.mediaflux["shared_files_location"])
       pathname.join("#{job_id}.csv").to_s
     end
+
+    def update_inventory_request(user_id:, project:, job_id:, filename: )
+      inventory_request = FileInventoryRequest.find_by(user_id: user_id, project_id: project.id, job_id: job_id)
+      request_details = { output_file: filename, project_title: project.title, file_size: File.size(filename) }
+      inventory_request.update(state: UserRequest::COMPLETED, request_details: request_details,
+                                completion_time: Time.current.in_time_zone("America/New_York"))
+      inventory_request
+    rescue ActiveRecord::StatementInvalid
+      ActiveRecord::Base.connection_pool.release_connection
+      retry
+    end
 end
diff --git a/spec/jobs/file_inventory_job_spec.rb b/spec/jobs/file_inventory_job_spec.rb
index 3c6ca5cc9..bca7d75ce 100644
--- a/spec/jobs/file_inventory_job_spec.rb
+++ b/spec/jobs/file_inventory_job_spec.rb
@@ -65,5 +65,26 @@
         end.to raise_error(ActiveRecord::RecordNotFound, /Couldn't find User/)
       end
     end
+
+    context "when an ActiveRecord::StatementInvalid error occurs" do
+      before do
+        @error_count = 0
+        allow_any_instance_of(FileInventoryRequest).to receive(:update).and_wrap_original do |original_method, *args|
+          if @error_count == 0
+            # Force an error the first time to make sure the retry is invoked in the code
+            @error_count = 1
+            raise ActiveRecord::StatementInvalid, "error"
+          else
+            original_method.call(*args)
+          end
+        end
+      end
+
+      it "it handles the retry for ActiveRecord::StatementInvalid exception" do
+        described_class.perform_now(user_id: user.id, project_id: project_in_mediaflux.id, mediaflux_session: user.mediaflux_session)
+        file_inventory_request = FileInventoryRequest.first
+        expect(file_inventory_request.state).to eq "completed"
+      end
+    end
   end
 end