Merge pull request #31 from aodn/features/5157-handle-delete

Features/5157 handle delete

Dockerfile

@@ -12,8 +12,8 @@ ENV GN_DIR /opt/geonetwork
 # data and can be located anywhere on the host system. Use volume in docker composes
 #
 
-# Copy log4j
-COPY ./geonetwork/target/classes/log4j*.xml ${GN_DIR}/WEB-INF/classes/
+# Override log4j as we need to add our log appear, by default ROOT level is off which is very strange design
+COPY ./geonetwork/target/classes/log4j-imos.xml ${GN_DIR}/WEB-INF/classes/log4j2.xml
 
 # Copy our jar to the lib folder so that scan can happens
 COPY ./geonetwork/target/geonetwork-*.jar ${GN_DIR}/WEB-INF/lib/

geonetwork-api/src/test/java/au/org/aodn/geonetwork_api/openapi/api/GroupHelperTest.java

@@ -9,11 +9,10 @@
 import org.junit.Test;
 import org.mockito.Mockito;
 import org.springframework.http.ResponseEntity;
-import org.springframework.test.context.TestPropertySource;
 import org.springframework.util.ResourceUtils;
 
 import java.io.IOException;
-import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.Optional;
 
@@ -33,7 +32,7 @@ public void verifyGetHarvestersOwnerGroup() throws IOException {
 
         String json = FileUtils.readFileToString(
                 ResourceUtils.getFile("classpath:catalogue_noaa_owngroup.json"),
-                Charset.forName("UTF-8"));
+                StandardCharsets.UTF_8);
 
         Parser.Parsed parsed = parser.parseHarvestersConfig(json);
         Optional<JSONObject> i = helper.getHarvestersOwnerGroup(parsed.getJsonObject());
@@ -49,7 +48,7 @@ public void verifyUpdateHarvestersOwnerGroup() throws IOException {
 
         String json = FileUtils.readFileToString(
                 ResourceUtils.getFile("classpath:catalogue_noaa.json"),
-                Charset.forName("UTF-8"));
+                StandardCharsets.UTF_8);
 
         Group group = new Group();
         group.setName("ABC");
@@ -58,7 +57,7 @@ public void verifyUpdateHarvestersOwnerGroup() throws IOException {
         Parser.Parsed parsed = parser.parseHarvestersConfig(json);
         Optional<JSONObject> i = helper.getHarvestersOwnerGroup(parsed.getJsonObject());
 
-        assertTrue("No group found", !i.isPresent());
+        assertFalse("No group found", i.isPresent());
 
         JSONObject n = helper.updateHarvestersOwnerGroup(parsed.getJsonObject(), group);
         parsed = parser.parseHarvestersConfig(n.toString());
@@ -72,10 +71,9 @@ public void verifyUpdateHarvestersOwnerGroup() throws IOException {
     }
     /**
      * We do not want to delete any build in group as it cause issues.
-     * @throws IOException
      */
     @Test
-    public void verfiyDeleteGroupKeepBuildInGroup() throws IOException {
+    public void verfiyDeleteGroupKeepBuildInGroup() {
         // Check is equalIgnoreCase, so capital letter or not does not matter
         Group all = new Group().name("All");
         Group intranet = new Group().name("intranet");

geonetwork-api/src/test/java/au/org/aodn/geonetwork_api/openapi/api/ParserTest.java

@@ -8,13 +8,13 @@
 import org.springframework.util.ResourceUtils;
 
 import java.io.IOException;
-import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+
 import org.xmlunit.builder.DiffBuilder;
 import org.xmlunit.diff.DefaultNodeMatcher;
 import org.xmlunit.diff.Diff;
 import org.xmlunit.diff.ElementSelectors;
-
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
 
 public class ParserTest {
 
@@ -24,11 +24,11 @@ public class ParserTest {
     public void verifyConvertToXML() throws IOException {
         String json = FileUtils.readFileToString(
                 ResourceUtils.getFile("classpath:catalogue_cdu_eretmochelys_imbricata.json"),
-                Charset.forName("UTF-8"));
+                StandardCharsets.UTF_8);
 
         String expected = FileUtils.readFileToString(
                 ResourceUtils.getFile("classpath:catalogue_cdu_eretmochelys_imbricata.xml"),
-                Charset.forName("UTF-8"));
+                StandardCharsets.UTF_8);
 
         Parser.Parsed parsed = new Parser().parseHarvestersConfig(json);
 
@@ -41,6 +41,6 @@ public void verifyConvertToXML() throws IOException {
                 .ignoreComments()
                 .build();
 
-        assertTrue("Expected result", !d.hasDifferences());
+        assertFalse("Expected result", d.hasDifferences());
     }
 }

geonetwork-api/src/test/java/au/org/aodn/geonetwork_api/openapi/api/TagsHelperTest.java

@@ -7,7 +7,7 @@
 import org.springframework.util.ResourceUtils;
 
 import java.io.IOException;
-import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.util.Optional;
 
 import static org.junit.Assert.*;
@@ -21,7 +21,7 @@ public void verifyGetHarvestersCategories() throws IOException {
 
         String json = FileUtils.readFileToString(
                 ResourceUtils.getFile("classpath:catalogue_noaa.json"),
-                Charset.forName("UTF-8"));
+                StandardCharsets.UTF_8);
 
         Parser.Parsed parsed = parser.parseHarvestersConfig(json);
         Optional<JSONArray> i = helper.getHarvestersCategories(parsed.getJsonObject());
@@ -30,7 +30,7 @@ public void verifyGetHarvestersCategories() throws IOException {
 
         json = FileUtils.readFileToString(
                 ResourceUtils.getFile("classpath:portal_catalogue_aims.json"),
-                Charset.forName("UTF-8"));
+                StandardCharsets.UTF_8);
 
         parsed = parser.parseHarvestersConfig(json);
         i = helper.getHarvestersCategories(parsed.getJsonObject());

geonetwork/pom.xml

@@ -55,6 +55,11 @@
             <artifactId>stax2-api</artifactId>
             <version>${stax2.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
     <build>
         <plugins>

geonetwork/src/main/java/au/org/aodn/geonetwork4/handler/GenericEntityListener.java

@@ -9,6 +9,7 @@
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.http.HttpEntity;
 import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
 import org.springframework.http.MediaType;
 import org.springframework.web.client.RestTemplate;
 
@@ -26,10 +27,14 @@ public class GenericEntityListener implements GeonetworkEntityListener<Metadata>
     protected Logger logger = LogManager.getLogger(GenericEntityListener.class);
 
     // Expiring map make sure unused item will be discarded
-    protected Map<String, Metadata> maps = new ConcurrentHashMap<>();
+    protected Map<String, Metadata> updateMap = new ConcurrentHashMap<>();
+
+    protected Map<String, Metadata> deleteMap = new ConcurrentHashMap<>();
 
     protected ScheduledExecutorService service = Executors.newSingleThreadScheduledExecutor();
 
+    protected static final String UUID = "uuid";
+
     @Value("${aodn.geonetwork4.esIndexer.urlIndex}")
     protected String indexUrl;
 
@@ -49,53 +54,101 @@ public void cleanUp() {
         service.shutdown();
     }
 
+    protected int delayStart = 5;
+
     @PostConstruct
     public void init() {
         // We pick up the items in map and then post trigger indexer call, this thread keep execute every 5 secs
         service.scheduleWithFixedDelay(() -> {
-            // TODO: If the metadata is unpublished, then indexer cannot read it and hence expect to
-            // get NOT found error, so we can check the status of the metadata before calling indexer.
-            for(String uuid : maps.keySet()) {
+            logger.info("Execute batch of update/delete after time elapsed");
+
+            // If the updateMap contain items that is going do delete, then there is no point to update
+            deleteMap.forEach((key, value) -> updateMap.remove(key));
+
+            // Noted, our geonetwork setup never use un-publish, therefore it will be always
+            // public readable.
+            for(String uuid : updateMap.keySet()) {
                 try {
-                    logger.info("Call indexer on metadata {} after transaction committed.", uuid);
+                    logger.info("Call indexer on metadata {} after metadata updated.", uuid);
                     Map<String, Object> variables = new HashMap<>();
-                    variables.put("uuid", uuid);
+                    variables.put(UUID, uuid);
 
-                    callApi(indexUrl, variables);
+                    callApiUpdate(indexUrl, variables);
                 }
                 catch (Exception e1) {
                     // Must not throw exception, can only print log and handle it manually
                     logger.error("Fail to call indexer on metadata {} after transaction committed. {}",
                             uuid, e1.getMessage());
                 }
                 finally {
-                    maps.remove(uuid);
+                    updateMap.remove(uuid);
+                }
+            }
+
+            for(String uuid : deleteMap.keySet()) {
+                try {
+                    logger.info("Call indexer to delete metadata {} after transaction committed.", uuid);
+                    Map<String, Object> variables = new HashMap<>();
+                    variables.put(UUID, uuid);
+
+                    callApiDelete(indexUrl, variables);
+                }
+                catch (Exception e1) {
+                    // Must not throw exception, can only print log and handle it manually
+                    logger.error("Fail to call indexer to delete metadata {} after transaction committed. {}",
+                            uuid, e1.getMessage());
+                }
+                finally {
+                    deleteMap.remove(uuid);
                 }
             }
-        }, 0,5, TimeUnit.SECONDS);
+
+        }, delayStart,10, TimeUnit.SECONDS);
     }
 
     @Override
     public void handleEvent(PersistentEventType persistentEventType, Metadata metaData) {
         if(persistentEventType == PersistentEventType.PostUpdate) {
             logger.info("{} handler for {}", persistentEventType, metaData);
-            // We see same even fired multiple times, this map will combined the event into one
+            // We see same even fired multiple times, this map will combine the event into one
+            // using a map with same key.
+            updateMap.put(metaData.getUuid(), metaData);
+        }
+        else if(persistentEventType == PersistentEventType.PostRemove) {
+            logger.info("{} handler for {}", persistentEventType, metaData);
+            // We see same even fired multiple times, this map will combine the event into one
             // using a map with same key.
-            maps.put(metaData.getUuid(), metaData);
+            deleteMap.put(metaData.getUuid(), metaData);
         }
     }
-
     /**
-     * All child override function as not always post operation
-     * @param indexUrl
-     * @param variables
+     * Call indexer rest api to update index.
+     *
+     * @param indexUrl - The template URL
+     * @param variables - The variable for the template URL.
      */
-    protected void callApi(String indexUrl, Map<String, Object> variables) {
+    protected void callApiUpdate(String indexUrl, Map<String, Object> variables) {
         HttpHeaders headers = new HttpHeaders();
         headers.set("X-API-Key", apiKey.trim());
         headers.setContentType(MediaType.APPLICATION_JSON);
 
         HttpEntity<Void> request = new HttpEntity<>(null, headers);
+        logger.info("Call indexer to update metadata {}", variables.get(UUID));
         restTemplate.postForEntity(indexUrl, request, Void.class, variables);
     }
+    /**
+     * Call indexer rest api to delete index.
+     *
+     * @param indexUrl - The template URL
+     * @param variables - The variable for the template URL.
+     */
+    protected void callApiDelete(String indexUrl, Map<String, Object> variables) {
+        HttpHeaders headers = new HttpHeaders();
+        headers.set("X-API-Key", apiKey.trim());
+        headers.setContentType(MediaType.APPLICATION_JSON);
+
+        HttpEntity<Void> request = new HttpEntity<>(null, headers);
+        logger.info("Call indexer to delete metadata {}", variables.get(UUID));
+        restTemplate.exchange(indexUrl, HttpMethod.DELETE, request, Void.class, variables);
+    }
 }