From 1951758481146a29fccb17dcb781953e0ed0972d Mon Sep 17 00:00:00 2001 From: utas-raymondng Date: Fri, 5 Jul 2024 18:25:47 +1000 Subject: [PATCH 1/3] no need to call if esindexer not config --- .../handler/GenericEntityListener.java | 36 ++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/geonetwork-core/src/main/java/au/org/aodn/geonetwork4/handler/GenericEntityListener.java b/geonetwork-core/src/main/java/au/org/aodn/geonetwork4/handler/GenericEntityListener.java index 7ee8efb..f4d9353 100644 --- a/geonetwork-core/src/main/java/au/org/aodn/geonetwork4/handler/GenericEntityListener.java +++ b/geonetwork-core/src/main/java/au/org/aodn/geonetwork4/handler/GenericEntityListener.java @@ -128,13 +128,13 @@ public void init() { @Override public void handleEvent(PersistentEventType persistentEventType, Metadata metaData) { if(persistentEventType == PersistentEventType.PostUpdate) { - logger.info("{} handler for {}", persistentEventType, metaData); + logger.info("PostUpdate handler for {}", metaData); // We see same even fired multiple times, this map will combine the event into one // using a map with same key. updateMap.put(metaData.getUuid(), metaData); } else if(persistentEventType == PersistentEventType.PostRemove) { - logger.info("{} handler for {}", persistentEventType, metaData); + logger.info("PostRemove handler for {}", metaData); // We see same even fired multiple times, this map will combine the event into one // using a map with same key. deleteMap.put(metaData.getUuid(), metaData); @@ -147,13 +147,15 @@ else if(persistentEventType == PersistentEventType.PostRemove) { * @param variables - The variable for the template URL. */ protected void callApiUpdate(String indexUrl, Map variables) { - HttpHeaders headers = new HttpHeaders(); - headers.set("X-API-Key", apiKey.trim()); - headers.setContentType(MediaType.APPLICATION_JSON); - - HttpEntity request = new HttpEntity<>(null, headers); - logger.info("Call indexer update {} metadata {}", indexUrl, variables.get(UUID)); - restTemplate.postForEntity(indexUrl, request, Void.class, variables); + if(indexUrl != null) { + HttpHeaders headers = new HttpHeaders(); + headers.set("X-API-Key", apiKey.trim()); + headers.setContentType(MediaType.APPLICATION_JSON); + + HttpEntity request = new HttpEntity<>(null, headers); + logger.info("Call indexer update {} metadata {}", indexUrl, variables.get(UUID)); + restTemplate.postForEntity(indexUrl, request, Void.class, variables); + } } /** * Call indexer rest api to delete index. @@ -162,12 +164,14 @@ protected void callApiUpdate(String indexUrl, Map variables) { * @param variables - The variable for the template URL. */ protected void callApiDelete(String indexUrl, Map variables) { - HttpHeaders headers = new HttpHeaders(); - headers.set("X-API-Key", apiKey.trim()); - headers.setContentType(MediaType.APPLICATION_JSON); - - HttpEntity request = new HttpEntity<>(null, headers); - logger.info("Call indexer delete {} metadata {}", indexUrl, variables.get(UUID)); - restTemplate.exchange(indexUrl, HttpMethod.DELETE, request, Void.class, variables); + if(indexUrl != null) { + HttpHeaders headers = new HttpHeaders(); + headers.set("X-API-Key", apiKey.trim()); + headers.setContentType(MediaType.APPLICATION_JSON); + + HttpEntity request = new HttpEntity<>(null, headers); + logger.info("Call indexer delete {} metadata {}", indexUrl, variables.get(UUID)); + restTemplate.exchange(indexUrl, HttpMethod.DELETE, request, Void.class, variables); + } } } From 47eada7e797165cc46f4fc461acaccaaa28fb76b Mon Sep 17 00:00:00 2001 From: utas-raymondng Date: Fri, 5 Jul 2024 18:30:12 +1000 Subject: [PATCH 2/3] show warning if not config --- .../handler/GenericEntityListener.java | 144 +++++++++--------- 1 file changed, 72 insertions(+), 72 deletions(-) diff --git a/geonetwork-core/src/main/java/au/org/aodn/geonetwork4/handler/GenericEntityListener.java b/geonetwork-core/src/main/java/au/org/aodn/geonetwork4/handler/GenericEntityListener.java index f4d9353..458a4dd 100644 --- a/geonetwork-core/src/main/java/au/org/aodn/geonetwork4/handler/GenericEntityListener.java +++ b/geonetwork-core/src/main/java/au/org/aodn/geonetwork4/handler/GenericEntityListener.java @@ -56,88 +56,88 @@ public void cleanUp() { @PostConstruct public void init() { - // We pick up the items in map and then post trigger indexer call, this thread keep execute every 5 secs - service.scheduleWithFixedDelay(() -> { - - // If the updateMap contain items that is going do delete, then there is no point to update - deleteMap.forEach((key, value) -> updateMap.remove(key)); - - // Noted, our geonetwork setup never use un-publish, therefore it will be always - // public readable. - for(String uuid : updateMap.keySet()) { - boolean needRemoveFromMap = true; - - try { - logger.info("Call indexer on metadata {} after metadata updated.", uuid); - Map variables = new HashMap<>(); - variables.put(UUID, uuid); - - callApiUpdate(indexUrl, variables); - } - catch(HttpServerErrorException server) { - if(server.getStatusCode() == HttpStatus.SERVICE_UNAVAILABLE) { - // Error may due to indexer reboot, so we just need to keep retry - logger.warn("Indexer not available, will keep retry update operation"); - needRemoveFromMap = false; - } - } - catch (Exception e1) { - // Must not throw exception, can only print log and handle it manually - logger.error("Fail to call indexer on metadata {} after transaction committed. {}", - uuid, e1.getMessage()); - } - finally { - if(needRemoveFromMap) { - updateMap.remove(uuid); + if(indexUrl == null) { + logger.warn("Call to es-indexer is off due to config missing"); + } + else { + // We pick up the items in map and then post trigger indexer call, this thread keep execute every 5 secs + service.scheduleWithFixedDelay(() -> { + + // If the updateMap contain items that is going do delete, then there is no point to update + deleteMap.forEach((key, value) -> updateMap.remove(key)); + + // Noted, our geonetwork setup never use un-publish, therefore it will be always + // public readable. + for (String uuid : updateMap.keySet()) { + boolean needRemoveFromMap = true; + + try { + logger.info("Call indexer on metadata {} after metadata updated.", uuid); + Map variables = new HashMap<>(); + variables.put(UUID, uuid); + + callApiUpdate(indexUrl, variables); + } catch (HttpServerErrorException server) { + if (server.getStatusCode() == HttpStatus.SERVICE_UNAVAILABLE) { + // Error may due to indexer reboot, so we just need to keep retry + logger.warn("Indexer not available, will keep retry update operation"); + needRemoveFromMap = false; + } + } catch (Exception e1) { + // Must not throw exception, can only print log and handle it manually + logger.error("Fail to call indexer on metadata {} after transaction committed. {}", + uuid, e1.getMessage()); + } finally { + if (needRemoveFromMap) { + updateMap.remove(uuid); + } } } - } - - for(String uuid : deleteMap.keySet()) { - boolean needRemoveFromMap = true; - try { - logger.info("Call indexer to delete metadata {} after transaction committed.", uuid); - Map variables = new HashMap<>(); - variables.put(UUID, uuid); - - callApiDelete(indexUrl, variables); - } - catch(HttpServerErrorException server) { - if(server.getStatusCode() == HttpStatus.SERVICE_UNAVAILABLE) { - // Error may due to indexer reboot, so we just need to keep retry - logger.warn("Indexer not available, will keep retry delete operation"); - needRemoveFromMap = false; - } - } - catch (Exception e1) { - // Must not throw exception, can only print log and handle it manually - logger.error("Fail to call indexer to delete metadata {} after transaction committed. {}", - uuid, e1.getMessage()); - } - finally { - if(needRemoveFromMap) { - deleteMap.remove(uuid); + for (String uuid : deleteMap.keySet()) { + boolean needRemoveFromMap = true; + + try { + logger.info("Call indexer to delete metadata {} after transaction committed.", uuid); + Map variables = new HashMap<>(); + variables.put(UUID, uuid); + + callApiDelete(indexUrl, variables); + } catch (HttpServerErrorException server) { + if (server.getStatusCode() == HttpStatus.SERVICE_UNAVAILABLE) { + // Error may due to indexer reboot, so we just need to keep retry + logger.warn("Indexer not available, will keep retry delete operation"); + needRemoveFromMap = false; + } + } catch (Exception e1) { + // Must not throw exception, can only print log and handle it manually + logger.error("Fail to call indexer to delete metadata {} after transaction committed. {}", + uuid, e1.getMessage()); + } finally { + if (needRemoveFromMap) { + deleteMap.remove(uuid); + } } } - } - }, delayStart,10, TimeUnit.SECONDS); + }, delayStart, 10, TimeUnit.SECONDS); + } } @Override public void handleEvent(PersistentEventType persistentEventType, Metadata metaData) { - if(persistentEventType == PersistentEventType.PostUpdate) { - logger.info("PostUpdate handler for {}", metaData); - // We see same even fired multiple times, this map will combine the event into one - // using a map with same key. - updateMap.put(metaData.getUuid(), metaData); - } - else if(persistentEventType == PersistentEventType.PostRemove) { - logger.info("PostRemove handler for {}", metaData); - // We see same even fired multiple times, this map will combine the event into one - // using a map with same key. - deleteMap.put(metaData.getUuid(), metaData); + if(indexUrl != null) { + if (persistentEventType == PersistentEventType.PostUpdate) { + logger.info("PostUpdate handler for {}", metaData); + // We see same even fired multiple times, this map will combine the event into one + // using a map with same key. + updateMap.put(metaData.getUuid(), metaData); + } else if (persistentEventType == PersistentEventType.PostRemove) { + logger.info("PostRemove handler for {}", metaData); + // We see same even fired multiple times, this map will combine the event into one + // using a map with same key. + deleteMap.put(metaData.getUuid(), metaData); + } } } /** From 2534c5eb4682f9c93598efdd6746ca47223250a6 Mon Sep 17 00:00:00 2001 From: utas-raymondng Date: Mon, 8 Jul 2024 14:22:49 +1000 Subject: [PATCH 3/3] Fix bug after PR --- .../java/au/org/aodn/geonetwork4/Config.java | 9 ++++- .../handler/GenericEntityListener.java | 12 ++++-- .../src/main/resources/application.properties | 2 +- .../src/main/resources/log4j-imos.xml | 5 +++ .../handler/GenericEntityListenerTest.java | 38 ++++++++++++++----- 5 files changed, 50 insertions(+), 16 deletions(-) diff --git a/geonetwork-core/src/main/java/au/org/aodn/geonetwork4/Config.java b/geonetwork-core/src/main/java/au/org/aodn/geonetwork4/Config.java index 7179b12..c25d59e 100644 --- a/geonetwork-core/src/main/java/au/org/aodn/geonetwork4/Config.java +++ b/geonetwork-core/src/main/java/au/org/aodn/geonetwork4/Config.java @@ -144,8 +144,13 @@ public RestTemplate createRestTemplate() { } @Bean - public GenericEntityListener createGenericEntityListener() { - return new GenericEntityListener(); + public GenericEntityListener createGenericEntityListener( + @Value("${aodn.geonetwork4.esIndexer.apikey}") String apiKey, + @Value("${aodn.geonetwork4.esIndexer.host}") String host, + @Value("${aodn.geonetwork4.esIndexer.urlIndex}") String indexUrl, + RestTemplate restTemplate) { + + return new GenericEntityListener(apiKey, host, indexUrl, restTemplate); } /** * Must use prototype scope as there is a XSRF-TOKEN header for each api, that cannot share diff --git a/geonetwork-core/src/main/java/au/org/aodn/geonetwork4/handler/GenericEntityListener.java b/geonetwork-core/src/main/java/au/org/aodn/geonetwork4/handler/GenericEntityListener.java index 458a4dd..9b0fb2c 100644 --- a/geonetwork-core/src/main/java/au/org/aodn/geonetwork4/handler/GenericEntityListener.java +++ b/geonetwork-core/src/main/java/au/org/aodn/geonetwork4/handler/GenericEntityListener.java @@ -6,7 +6,6 @@ import org.fao.geonet.entitylistener.GeonetworkEntityListener; import org.fao.geonet.entitylistener.PersistentEventType; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.http.*; import org.springframework.web.client.HttpServerErrorException; import org.springframework.web.client.RestTemplate; @@ -33,13 +32,10 @@ public class GenericEntityListener implements GeonetworkEntityListener protected static final String UUID = "uuid"; - @Value("${aodn.geonetwork4.esIndexer.urlIndex}") protected String indexUrl; - @Value("${aodn.geonetwork4.esIndexer.apikey}") protected String apiKey; - @Autowired protected RestTemplate restTemplate; @Override @@ -54,6 +50,14 @@ public void cleanUp() { protected int delayStart = 5; + @Autowired + public GenericEntityListener(String apiKey, String host, String indexUrl, RestTemplate restTemplate) { + + this.apiKey = apiKey; + this.indexUrl = host != null && !host.isEmpty() ? indexUrl : null; + this.restTemplate = restTemplate; + } + @PostConstruct public void init() { if(indexUrl == null) { diff --git a/geonetwork-core/src/main/resources/application.properties b/geonetwork-core/src/main/resources/application.properties index d9d25ac..f7d03bb 100644 --- a/geonetwork-core/src/main/resources/application.properties +++ b/geonetwork-core/src/main/resources/application.properties @@ -1,4 +1,4 @@ -aodn.geonetwork4.esIndexer.host=${INDEXER_HOST:http://localhost} +aodn.geonetwork4.esIndexer.host=${INDEXER_HOST:} aodn.geonetwork4.esIndexer.port=${INDEXER_PORT:80} aodn.geonetwork4.esIndexer.apikey=${INDEXER_APIKEY} aodn.geonetwork4.esIndexer.urlIndex=${aodn.geonetwork4.esIndexer.host}:${aodn.geonetwork4.esIndexer.port}/api/v1/indexer/index/{uuid} diff --git a/geonetwork-core/src/main/resources/log4j-imos.xml b/geonetwork-core/src/main/resources/log4j-imos.xml index 3b89e04..43b7e16 100644 --- a/geonetwork-core/src/main/resources/log4j-imos.xml +++ b/geonetwork-core/src/main/resources/log4j-imos.xml @@ -143,6 +143,11 @@ + + + + + diff --git a/geonetwork-core/src/test/java/au/org/aodn/geonetwork4/handler/GenericEntityListenerTest.java b/geonetwork-core/src/test/java/au/org/aodn/geonetwork4/handler/GenericEntityListenerTest.java index 5546f2a..04a78ec 100644 --- a/geonetwork-core/src/test/java/au/org/aodn/geonetwork4/handler/GenericEntityListenerTest.java +++ b/geonetwork-core/src/test/java/au/org/aodn/geonetwork4/handler/GenericEntityListenerTest.java @@ -25,14 +25,14 @@ public class GenericEntityListenerTest { @Test public void verifyUpdateDeleteBehavior() throws InterruptedException { CountDownLatch latch = new CountDownLatch(1); - GenericEntityListener listener = new GenericEntityListener(); RestTemplate template = Mockito.mock(RestTemplate.class); // Set of test only, a mock to count what have been called - listener.indexUrl = "http://localhost/api/v1/indexer/index/{uuid}"; - listener.apiKey = "test-key"; - listener.restTemplate = template; - + GenericEntityListener listener = new GenericEntityListener( + "test-key", + "localhost", + "http://localhost/api/v1/indexer/index/{uuid}", + template); listener.init(); // Test data @@ -80,7 +80,6 @@ public void verifyUpdateDeleteBehavior() throws InterruptedException { @Test public void verifyRetryBehavior() throws InterruptedException { CountDownLatch latch = new CountDownLatch(1); - GenericEntityListener listener = new GenericEntityListener(); RestTemplate template = Mockito.mock(RestTemplate.class); // Throw exception on first call @@ -105,9 +104,11 @@ public void verifyRetryBehavior() throws InterruptedException { .thenReturn(ResponseEntity.ok(null)); // Set of test only, a mock to count what have been called - listener.indexUrl = "http://localhost/api/v1/indexer/index/{uuid}"; - listener.apiKey = "test-key"; - listener.restTemplate = template; + GenericEntityListener listener = new GenericEntityListener( + "test-key", + "localhost", + "http://localhost/api/v1/indexer/index/{uuid}", + template); listener.init(); @@ -132,4 +133,23 @@ public void verifyRetryBehavior() throws InterruptedException { assertEquals("Map not contains uuid", 0, listener.updateMap.size()); assertEquals("Delete not contains uuid", 0, listener.deleteMap.size()); } + /** + * If host null, then we disable api call to indexer + */ + @Test + public void verifyIndexerCanBeDisabled() { + RestTemplate template = Mockito.mock(RestTemplate.class); + GenericEntityListener listener = new GenericEntityListener( + "test-key", + null, + "http://localhost/api/v1/indexer/index/{uuid}", + template); + + listener.init(); + listener.handleEvent(PersistentEventType.PostUpdate, new Metadata()); + listener.handleEvent(PersistentEventType.PostRemove, new Metadata()); + + assertTrue("Internal update map empty", listener.updateMap.isEmpty()); + assertTrue("Internal delete map empty", listener.deleteMap.isEmpty()); + } }