Handle case when indexer is not available

geonetwork/src/main/java/au/org/aodn/geonetwork4/handler/GenericEntityListener.java

@@ -7,10 +7,8 @@
 import org.fao.geonet.entitylistener.PersistentEventType;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
-import org.springframework.http.HttpEntity;
-import org.springframework.http.HttpHeaders;
-import org.springframework.http.HttpMethod;
-import org.springframework.http.MediaType;
+import org.springframework.http.*;
+import org.springframework.web.client.HttpServerErrorException;
 import org.springframework.web.client.RestTemplate;
 
 import javax.annotation.PostConstruct;
@@ -67,38 +65,60 @@ public void init() {
             // Noted, our geonetwork setup never use un-publish, therefore it will be always
             // public readable.
             for(String uuid : updateMap.keySet()) {
+                boolean needRemoveFromMap = true;
+
                 try {
                     logger.info("Call indexer on metadata {} after metadata updated.", uuid);
                     Map<String, Object> variables = new HashMap<>();
                     variables.put(UUID, uuid);
 
                     callApiUpdate(indexUrl, variables);
                 }
+                catch(HttpServerErrorException server) {
+                    if(server.getStatusCode() == HttpStatus.SERVICE_UNAVAILABLE) {
+                        // Error may due to indexer reboot, so we just need to keep retry
+                        logger.warn("Indexer not available, will keep retry update operation");
+                        needRemoveFromMap = false;
+                    }
+                }
                 catch (Exception e1) {
                     // Must not throw exception, can only print log and handle it manually
                     logger.error("Fail to call indexer on metadata {} after transaction committed. {}",
                             uuid, e1.getMessage());
                 }
                 finally {
-                    updateMap.remove(uuid);
+                    if(needRemoveFromMap) {
+                        updateMap.remove(uuid);
+                    }
                 }
             }
 
             for(String uuid : deleteMap.keySet()) {
+                boolean needRemoveFromMap = true;
+
                 try {
                     logger.info("Call indexer to delete metadata {} after transaction committed.", uuid);
                     Map<String, Object> variables = new HashMap<>();
                     variables.put(UUID, uuid);
 
                     callApiDelete(indexUrl, variables);
                 }
+                catch(HttpServerErrorException server) {
+                    if(server.getStatusCode() == HttpStatus.SERVICE_UNAVAILABLE) {
+                        // Error may due to indexer reboot, so we just need to keep retry
+                        logger.warn("Indexer not available, will keep retry delete operation");
+                        needRemoveFromMap = false;
+                    }
+                }
                 catch (Exception e1) {
                     // Must not throw exception, can only print log and handle it manually
                     logger.error("Fail to call indexer to delete metadata {} after transaction committed. {}",
                             uuid, e1.getMessage());
                 }
                 finally {
-                    deleteMap.remove(uuid);
+                    if(needRemoveFromMap) {
+                        deleteMap.remove(uuid);
+                    }
                 }
             }
 

geonetwork/src/test/java/au/org/aodn/geonetwork4/handler/GenericEntityListenerTest.java

@@ -5,15 +5,18 @@
 import org.junit.Test;
 import org.mockito.Mockito;
 import org.springframework.http.HttpMethod;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.client.HttpServerErrorException;
 import org.springframework.web.client.RestTemplate;
 
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.ArgumentMatchers.*;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.*;
 
 public class GenericEntityListenerTest {
     /**
@@ -70,5 +73,63 @@ public void verifyUpdateDeleteBehavior() throws InterruptedException {
 
         listener.cleanUp();
     }
+    /**
+     * Make sure if remote service indexer is not available, we will keep retry and not lost the update.
+     * @throws InterruptedException - Should not happen
+     */
+    @Test
+    public void verifyRetryBehavior() throws InterruptedException {
+        CountDownLatch latch = new CountDownLatch(1);
+        GenericEntityListener listener = new GenericEntityListener();
+        RestTemplate template = Mockito.mock(RestTemplate.class);
+
+        // Throw exception on first call
+        when(template.postForEntity(
+                eq("http://localhost/api/v1/indexer/index/{uuid}"),
+                any(),
+                eq(Void.class),
+                anyMap())
+        )
+                .thenThrow(new HttpServerErrorException(HttpStatus.SERVICE_UNAVAILABLE))
+                .thenReturn(ResponseEntity.ok(null));
+
+
+        when(template.exchange(
+                eq("http://localhost/api/v1/indexer/index/{uuid}"),
+                eq(HttpMethod.DELETE),
+                any(),
+                eq(Void.class),
+                anyMap())
+        )
+                .thenThrow(new HttpServerErrorException(HttpStatus.SERVICE_UNAVAILABLE))
+                .thenReturn(ResponseEntity.ok(null));
+
+        // Set of test only, a mock to count what have been called
+        listener.indexUrl = "http://localhost/api/v1/indexer/index/{uuid}";
+        listener.apiKey = "test-key";
+        listener.restTemplate = template;
+
+        listener.init();
 
+        // Test data
+        Metadata metadata1 = new Metadata();
+        metadata1.setUuid("54d0cc03-763c-4393-8796-d79c9979e3f8");
+
+        Metadata metadata2 = new Metadata();
+        metadata2.setUuid("c401f091-fed5-4829-bead-f6cecad3d424");
+
+        listener.handleEvent(PersistentEventType.PostUpdate, metadata1);
+        listener.handleEvent(PersistentEventType.PostRemove, metadata2);
+
+        // There is a delay before the scheduler starts, so we wait 1 more seconds to make
+        // sure the last execution completed.
+        latch.await(listener.delayStart + 1, TimeUnit.SECONDS);
+        assertEquals("Map contains uuid", 1, listener.updateMap.size());
+        assertEquals("Delete contains uuid", 1, listener.deleteMap.size());
+
+        // Wait more time, this time service ok and processed hence map is clear
+        latch.await(listener.delayStart + listener.delayStart , TimeUnit.SECONDS);
+        assertEquals("Map not contains uuid", 0, listener.updateMap.size());
+        assertEquals("Delete not  contains uuid", 0, listener.deleteMap.size());
+    }
 }