Skip to content

Commit

Permalink
Merge pull request #56 from aodn/bugs/5688-ignore-indexer-call-if-not…
Browse files Browse the repository at this point in the history 
…-confi

no need to call if esindexer not config
  • Loading branch information
@vietnguyengit
vietnguyengit authored Jul 8, 2024
2 parents a102786 + 2534c5e commit 44b0184
Show file tree
Hide file tree
Showing 5 changed files with 140 additions and 102 deletions.
9 changes: 7 additions & 2 deletions geonetwork-core/src/main/java/au/org/aodn/geonetwork4/Config.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,8 +144,13 @@ public RestTemplate createRestTemplate() {
}

@Bean
public GenericEntityListener createGenericEntityListener() {
return new GenericEntityListener();
public GenericEntityListener createGenericEntityListener(
@Value("${aodn.geonetwork4.esIndexer.apikey}") String apiKey,
@Value("${aodn.geonetwork4.esIndexer.host}") String host,
@Value("${aodn.geonetwork4.esIndexer.urlIndex}") String indexUrl,
RestTemplate restTemplate) {

return new GenericEntityListener(apiKey, host, indexUrl, restTemplate);
}
/**
* Must use prototype scope as there is a XSRF-TOKEN header for each api, that cannot share
Expand Down
188 changes: 98 additions & 90 deletions geonetwork-core/src/main/java/au/org/aodn/geonetwork4/handler/GenericEntityListener.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
import org.fao.geonet.entitylistener.GeonetworkEntityListener;
import org.fao.geonet.entitylistener.PersistentEventType;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.*;
import org.springframework.web.client.HttpServerErrorException;
import org.springframework.web.client.RestTemplate;
Expand All @@ -33,13 +32,10 @@ public class GenericEntityListener implements GeonetworkEntityListener<Metadata>

protected static final String UUID = "uuid";

@Value("${aodn.geonetwork4.esIndexer.urlIndex}")
protected String indexUrl;

@Value("${aodn.geonetwork4.esIndexer.apikey}")
protected String apiKey;

@Autowired
protected RestTemplate restTemplate;

@Override
Expand All @@ -54,90 +50,98 @@ public void cleanUp() {

protected int delayStart = 5;

@PostConstruct
public void init() {
// We pick up the items in map and then post trigger indexer call, this thread keep execute every 5 secs
service.scheduleWithFixedDelay(() -> {

// If the updateMap contain items that is going do delete, then there is no point to update
deleteMap.forEach((key, value) -> updateMap.remove(key));

// Noted, our geonetwork setup never use un-publish, therefore it will be always
// public readable.
for(String uuid : updateMap.keySet()) {
boolean needRemoveFromMap = true;
@Autowired
public GenericEntityListener(String apiKey, String host, String indexUrl, RestTemplate restTemplate) {

try {
logger.info("Call indexer on metadata {} after metadata updated.", uuid);
Map<String, Object> variables = new HashMap<>();
variables.put(UUID, uuid);
this.apiKey = apiKey;
this.indexUrl = host != null && !host.isEmpty() ? indexUrl : null;
this.restTemplate = restTemplate;
}

callApiUpdate(indexUrl, variables);
}
catch(HttpServerErrorException server) {
if(server.getStatusCode() == HttpStatus.SERVICE_UNAVAILABLE) {
// Error may due to indexer reboot, so we just need to keep retry
logger.warn("Indexer not available, will keep retry update operation");
needRemoveFromMap = false;
}
}
catch (Exception e1) {
// Must not throw exception, can only print log and handle it manually
logger.error("Fail to call indexer on metadata {} after transaction committed. {}",
uuid, e1.getMessage());
}
finally {
if(needRemoveFromMap) {
updateMap.remove(uuid);
@PostConstruct
public void init() {
if(indexUrl == null) {
logger.warn("Call to es-indexer is off due to config missing");
}
else {
// We pick up the items in map and then post trigger indexer call, this thread keep execute every 5 secs
service.scheduleWithFixedDelay(() -> {

// If the updateMap contain items that is going do delete, then there is no point to update
deleteMap.forEach((key, value) -> updateMap.remove(key));

// Noted, our geonetwork setup never use un-publish, therefore it will be always
// public readable.
for (String uuid : updateMap.keySet()) {
boolean needRemoveFromMap = true;

try {
logger.info("Call indexer on metadata {} after metadata updated.", uuid);
Map<String, Object> variables = new HashMap<>();
variables.put(UUID, uuid);

callApiUpdate(indexUrl, variables);
} catch (HttpServerErrorException server) {
if (server.getStatusCode() == HttpStatus.SERVICE_UNAVAILABLE) {
// Error may due to indexer reboot, so we just need to keep retry
logger.warn("Indexer not available, will keep retry update operation");
needRemoveFromMap = false;
}
} catch (Exception e1) {
// Must not throw exception, can only print log and handle it manually
logger.error("Fail to call indexer on metadata {} after transaction committed. {}",
uuid, e1.getMessage());
} finally {
if (needRemoveFromMap) {
updateMap.remove(uuid);
}
}
}
}

for(String uuid : deleteMap.keySet()) {
boolean needRemoveFromMap = true;

try {
logger.info("Call indexer to delete metadata {} after transaction committed.", uuid);
Map<String, Object> variables = new HashMap<>();
variables.put(UUID, uuid);

callApiDelete(indexUrl, variables);
}
catch(HttpServerErrorException server) {
if(server.getStatusCode() == HttpStatus.SERVICE_UNAVAILABLE) {
// Error may due to indexer reboot, so we just need to keep retry
logger.warn("Indexer not available, will keep retry delete operation");
needRemoveFromMap = false;
for (String uuid : deleteMap.keySet()) {
boolean needRemoveFromMap = true;

try {
logger.info("Call indexer to delete metadata {} after transaction committed.", uuid);
Map<String, Object> variables = new HashMap<>();
variables.put(UUID, uuid);

callApiDelete(indexUrl, variables);
} catch (HttpServerErrorException server) {
if (server.getStatusCode() == HttpStatus.SERVICE_UNAVAILABLE) {
// Error may due to indexer reboot, so we just need to keep retry
logger.warn("Indexer not available, will keep retry delete operation");
needRemoveFromMap = false;
}
} catch (Exception e1) {
// Must not throw exception, can only print log and handle it manually
logger.error("Fail to call indexer to delete metadata {} after transaction committed. {}",
uuid, e1.getMessage());
} finally {
if (needRemoveFromMap) {
deleteMap.remove(uuid);
}
}
}
catch (Exception e1) {
// Must not throw exception, can only print log and handle it manually
logger.error("Fail to call indexer to delete metadata {} after transaction committed. {}",
uuid, e1.getMessage());
}
finally {
if(needRemoveFromMap) {
deleteMap.remove(uuid);
}
}
}

}, delayStart,10, TimeUnit.SECONDS);
}, delayStart, 10, TimeUnit.SECONDS);
}
}

@Override
public void handleEvent(PersistentEventType persistentEventType, Metadata metaData) {
if(persistentEventType == PersistentEventType.PostUpdate) {
logger.info("{} handler for {}", persistentEventType, metaData);
// We see same even fired multiple times, this map will combine the event into one
// using a map with same key.
updateMap.put(metaData.getUuid(), metaData);
}
else if(persistentEventType == PersistentEventType.PostRemove) {
logger.info("{} handler for {}", persistentEventType, metaData);
// We see same even fired multiple times, this map will combine the event into one
// using a map with same key.
deleteMap.put(metaData.getUuid(), metaData);
if(indexUrl != null) {
if (persistentEventType == PersistentEventType.PostUpdate) {
logger.info("PostUpdate handler for {}", metaData);
// We see same even fired multiple times, this map will combine the event into one
// using a map with same key.
updateMap.put(metaData.getUuid(), metaData);
} else if (persistentEventType == PersistentEventType.PostRemove) {
logger.info("PostRemove handler for {}", metaData);
// We see same even fired multiple times, this map will combine the event into one
// using a map with same key.
deleteMap.put(metaData.getUuid(), metaData);
}
}
}
/**
Expand All @@ -147,13 +151,15 @@ else if(persistentEventType == PersistentEventType.PostRemove) {
* @param variables - The variable for the template URL.
*/
protected void callApiUpdate(String indexUrl, Map<String, Object> variables) {
HttpHeaders headers = new HttpHeaders();
headers.set("X-API-Key", apiKey.trim());
headers.setContentType(MediaType.APPLICATION_JSON);

HttpEntity<Void> request = new HttpEntity<>(null, headers);
logger.info("Call indexer update {} metadata {}", indexUrl, variables.get(UUID));
restTemplate.postForEntity(indexUrl, request, Void.class, variables);
if(indexUrl != null) {
HttpHeaders headers = new HttpHeaders();
headers.set("X-API-Key", apiKey.trim());
headers.setContentType(MediaType.APPLICATION_JSON);

HttpEntity<Void> request = new HttpEntity<>(null, headers);
logger.info("Call indexer update {} metadata {}", indexUrl, variables.get(UUID));
restTemplate.postForEntity(indexUrl, request, Void.class, variables);
}
}
/**
* Call indexer rest api to delete index.
Expand All @@ -162,12 +168,14 @@ protected void callApiUpdate(String indexUrl, Map<String, Object> variables) {
* @param variables - The variable for the template URL.
*/
protected void callApiDelete(String indexUrl, Map<String, Object> variables) {
HttpHeaders headers = new HttpHeaders();
headers.set("X-API-Key", apiKey.trim());
headers.setContentType(MediaType.APPLICATION_JSON);

HttpEntity<Void> request = new HttpEntity<>(null, headers);
logger.info("Call indexer delete {} metadata {}", indexUrl, variables.get(UUID));
restTemplate.exchange(indexUrl, HttpMethod.DELETE, request, Void.class, variables);
if(indexUrl != null) {
HttpHeaders headers = new HttpHeaders();
headers.set("X-API-Key", apiKey.trim());
headers.setContentType(MediaType.APPLICATION_JSON);

HttpEntity<Void> request = new HttpEntity<>(null, headers);
logger.info("Call indexer delete {} metadata {}", indexUrl, variables.get(UUID));
restTemplate.exchange(indexUrl, HttpMethod.DELETE, request, Void.class, variables);
}
}
}
2 changes: 1 addition & 1 deletion geonetwork-core/src/main/resources/application.properties
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
aodn.geonetwork4.esIndexer.host=${INDEXER_HOST:http://localhost}
aodn.geonetwork4.esIndexer.host=${INDEXER_HOST:}
aodn.geonetwork4.esIndexer.port=${INDEXER_PORT:80}
aodn.geonetwork4.esIndexer.apikey=${INDEXER_APIKEY}
aodn.geonetwork4.esIndexer.urlIndex=${aodn.geonetwork4.esIndexer.host}:${aodn.geonetwork4.esIndexer.port}/api/v1/indexer/index/{uuid}
Expand Down
5 changes: 5 additions & 0 deletions geonetwork-core/src/main/resources/log4j-imos.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,11 @@
<AppenderRef ref="File"/>
</Logger>

<Logger name="au.org.aodn.geonetwork4.handler" level="debug" additivity="false">
<AppenderRef ref="Console"/>
<AppenderRef ref="File"/>
</Logger>

<Logger name="au.org.aodn.geonetwork_api" level="debug" additivity="false">
<AppenderRef ref="Console"/>
<AppenderRef ref="File"/>
Expand Down
38 changes: 29 additions & 9 deletions geonetwork-core/src/test/java/au/org/aodn/geonetwork4/handler/GenericEntityListenerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,14 @@ public class GenericEntityListenerTest {
@Test
public void verifyUpdateDeleteBehavior() throws InterruptedException {
CountDownLatch latch = new CountDownLatch(1);
GenericEntityListener listener = new GenericEntityListener();
RestTemplate template = Mockito.mock(RestTemplate.class);

// Set of test only, a mock to count what have been called
listener.indexUrl = "http://localhost/api/v1/indexer/index/{uuid}";
listener.apiKey = "test-key";
listener.restTemplate = template;

GenericEntityListener listener = new GenericEntityListener(
"test-key",
"localhost",
"http://localhost/api/v1/indexer/index/{uuid}",
template);
listener.init();

// Test data
Expand Down Expand Up @@ -80,7 +80,6 @@ public void verifyUpdateDeleteBehavior() throws InterruptedException {
@Test
public void verifyRetryBehavior() throws InterruptedException {
CountDownLatch latch = new CountDownLatch(1);
GenericEntityListener listener = new GenericEntityListener();
RestTemplate template = Mockito.mock(RestTemplate.class);

// Throw exception on first call
Expand All @@ -105,9 +104,11 @@ public void verifyRetryBehavior() throws InterruptedException {
.thenReturn(ResponseEntity.ok(null));

// Set of test only, a mock to count what have been called
listener.indexUrl = "http://localhost/api/v1/indexer/index/{uuid}";
listener.apiKey = "test-key";
listener.restTemplate = template;
GenericEntityListener listener = new GenericEntityListener(
"test-key",
"localhost",
"http://localhost/api/v1/indexer/index/{uuid}",
template);

listener.init();

Expand All @@ -132,4 +133,23 @@ public void verifyRetryBehavior() throws InterruptedException {
assertEquals("Map not contains uuid", 0, listener.updateMap.size());
assertEquals("Delete not contains uuid", 0, listener.deleteMap.size());
}
/**
* If host null, then we disable api call to indexer
*/
@Test
public void verifyIndexerCanBeDisabled() {
RestTemplate template = Mockito.mock(RestTemplate.class);
GenericEntityListener listener = new GenericEntityListener(
"test-key",
null,
"http://localhost/api/v1/indexer/index/{uuid}",
template);

listener.init();
listener.handleEvent(PersistentEventType.PostUpdate, new Metadata());
listener.handleEvent(PersistentEventType.PostRemove, new Metadata());

assertTrue("Internal update map empty", listener.updateMap.isEmpty());
assertTrue("Internal delete map empty", listener.deleteMap.isEmpty());
}
}

0 comments on commit 44b0184

Please sign in to comment.