Merge pull request #32 from aodn/features/5157-handle-delete

Features/5157 handle delete - add more log

README.md

@@ -37,6 +37,9 @@ docker-compose -f docker-gn-compose.yml up --build
 docker-compose -f docker-gn-compose.yml down -v
 ```
 
+Once elastic started you can run ./startGn4Local.sh to start the geonetwork. It is recommend to start
+it like this because it will rebuild your images with the binary that you created from maven build install
+
 ## Ssh to instance
 You can login to the geonetwork4 instance to debug you setting by
 ```shell
@@ -45,6 +48,15 @@ docker exec -it geonetwork4 /bin/bash
 
 Once you in the shell, you can go to /var/lib/jetty/webapps/geonetwork
 
+## Actuator
+We have incorporate actuator to the instance, and you can visit to see the supported endpoints, there is one
+issue is that it is running with ECS then you may not hit the same instance as you want.
+
+http://localhost:8080/geonetwork/srv/api/manage
+
+Geonetwork4 should have config to be a single instance only to avoid this issue. You can use the logfile
+endpoint to view the log file directly as cloud watch is not so easy to use.
+
 ## Schema folder
 
 The schema folder contains an open-api schema file from Genetwork4, you can get it from any instance

docker-gn-compose.yml

@@ -35,6 +35,7 @@ services:
     environment:
       JAVA_OPTS: >-
         -Xms256m -Xmx2g
+        -Dspring.profiles.active=edge
         -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000
         -Dgeonetwork.data.dir=/opt/geonetwork/data
         -Dgeonetwork.config.dir=/opt/geonetwork/config

geonetwork/pom.xml

@@ -39,12 +39,19 @@
             <groupId>org.springframework</groupId>
             <artifactId>spring-aspects</artifactId>
         </dependency>
+        <!-- version inherit from geonetwork-core pom -->
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-configuration-processor</artifactId>
-            <version>2.1.6.RELEASE</version>
+            <version>${springboot.version}</version>
             <optional>true</optional>
         </dependency>
+        <!-- version inherit from geonetwork-core pom -->
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-actuator-autoconfigure</artifactId>
+            <version>${springboot.version}</version>
+        </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.dataformat</groupId>
             <artifactId>jackson-dataformat-xml</artifactId>
@@ -78,15 +85,6 @@
                 </executions>
                 <configuration>
                     <artifactItems>
-                        <!--
-                        <artifactItem>
-                            <groupId>com.amazonaws</groupId>
-                            <artifactId>aws-java-sdk-sts</artifactId>
-                            <version>${aws.sts.version}</version>
-                            <type>jar</type>
-                            <overWrite>true</overWrite>
-                        </artifactItem>
-                        -->
                         <artifactItem>
                             <groupId>com.fasterxml.jackson.dataformat</groupId>
                             <artifactId>jackson-dataformat-xml</artifactId>
@@ -101,6 +99,20 @@
                             <type>jar</type>
                             <overWrite>true</overWrite>
                         </artifactItem>
+                        <artifactItem>
+                            <groupId>org.springframework.boot</groupId>
+                            <artifactId>spring-boot-actuator</artifactId>
+                            <version>${springboot.version}</version>
+                            <type>jar</type>
+                            <overWrite>true</overWrite>
+                        </artifactItem>
+                        <artifactItem>
+                            <groupId>org.springframework.boot</groupId>
+                            <artifactId>spring-boot-actuator-autoconfigure</artifactId>
+                            <version>${springboot.version}</version>
+                            <type>jar</type>
+                            <overWrite>true</overWrite>
+                        </artifactItem>
                     </artifactItems>
                 </configuration>
             </plugin>

geonetwork/src/main/java/au/org/aodn/geonetwork4/ActuatorConfig.java

@@ -0,0 +1,16 @@
+package au.org.aodn.geonetwork4;
+
+import org.springframework.boot.actuate.health.Health;
+import org.springframework.boot.actuate.health.HealthIndicator;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class ActuatorConfig {
+    @Bean
+    public HealthIndicator geonetworkHealthIndicator() {
+        return Health.up()
+                .withDetail("info", "GeoNetwork4")
+                ::build;
+    }
+}

geonetwork/src/main/java/au/org/aodn/geonetwork4/Config.java

@@ -4,12 +4,8 @@
 import au.org.aodn.geonetwork4.ssl.HttpsTrustManager;
 import au.org.aodn.geonetwork_api.openapi.api.*;
 import au.org.aodn.geonetwork_api.openapi.invoker.ApiClient;
-import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.core.LoggerContext;
-import org.apache.logging.log4j.core.config.AppenderRef;
-import org.apache.logging.log4j.core.config.LoggerConfig;
 
 import org.aspectj.lang.ProceedingJoinPoint;
 import org.aspectj.lang.annotation.Around;
@@ -20,18 +16,22 @@
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.beans.factory.config.ConfigurableBeanFactory;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.context.ConfigurableApplicationContext;
 import org.springframework.context.annotation.*;
 import au.org.aodn.geonetwork4.enumeration.Environment;
 
 import javax.annotation.PostConstruct;
+import javax.servlet.ServletContext;
+
 import org.fao.geonet.ApplicationContextHolder;
 
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.ResponseEntity;
 import org.springframework.http.client.support.BasicAuthenticationInterceptor;
 import org.springframework.web.client.HttpClientErrorException;
 import org.springframework.web.client.RestTemplate;
+import org.springframework.web.context.WebApplicationContext;
 import org.springframework.web.util.DefaultUriBuilderFactory;
 
 import java.lang.reflect.Method;
@@ -40,6 +40,16 @@
 
 @Aspect
 @Configuration
+/*
+ * EnableAutoConfiguration so that Actuator can config automatically, however, the
+ * geonetwork itself start a few beans already so auto config will end up with
+ * 2 same bean which is not needed.
+ */
+@EnableAutoConfiguration(exclude = {
+        org.springdoc.core.SpringDocConfiguration.class,
+        org.springdoc.webmvc.core.SpringDocWebMvcConfiguration.class,
+        org.springframework.boot.autoconfigure.jms.JmsAutoConfiguration.class
+})
 @EnableAspectJAutoProxy
 @PropertySources({
         @PropertySource("classpath:application.properties"),
@@ -57,35 +67,35 @@ public class Config {
     @Autowired
     protected GenericEntityListener genericEntityListener;
 
-    /**
-     * Geonetwork set root logger to OFF for most log4j2 profile, hence you miss most of the information,
-     * this make it super hard to debug. The code here is to turn the ROOT logger back to INFO. It will be,
-     * logger dependent and by default log goes to FILE appender only.
-     */
-    protected void resetLoggerLevel(Level level) {
-        LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
-        org.apache.logging.log4j.core.config.Configuration config = ctx.getConfiguration();
-
-        AppenderRef file = AppenderRef.createAppenderRef("File", level, null);
-        AppenderRef console = AppenderRef.createAppenderRef("Console", level, null);
-
-        LoggerConfig c = LoggerConfig.newBuilder()
-                .withLevel(level)
-                .withRefs(new AppenderRef[] {file, console})
-                .withLoggerName("au.org.aodn")
-                .withIncludeLocation("au.org.aodn")
-                .withAdditivity(false)
-                .withConfig(config)
-                .build();
-
-        c.addAppender(config.getAppender("File"), level, null);
-        c.addAppender(config.getAppender("Console"), level, null);
-
-        config.addLogger("au.org.aodn", c);
-        // LoggerConfig loggerConfig = config.getLoggerConfig(LogManager.ROOT_LOGGER_NAME);
-        // loggerConfig.setLevel(Level.INFO);
-        ctx.updateLoggers();
-    }
+//    /**
+//     * Geonetwork set root logger to OFF for most log4j2 profile, hence you miss most of the information,
+//     * this make it super hard to debug. The code here is to turn the ROOT logger back to INFO. It will be,
+//     * logger dependent and by default log goes to FILE appender only.
+//     */
+//    protected void resetLoggerLevel(Level level) {
+//        LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
+//        org.apache.logging.log4j.core.config.Configuration config = ctx.getConfiguration();
+//
+//        AppenderRef file = AppenderRef.createAppenderRef("File", level, null);
+//        AppenderRef console = AppenderRef.createAppenderRef("Console", level, null);
+//
+//        LoggerConfig c = LoggerConfig.newBuilder()
+//                .withLevel(level)
+//                .withRefs(new AppenderRef[] {file, console})
+//                .withLoggerName("au.org.aodn")
+//                .withIncludeLocation("au.org.aodn")
+//                .withAdditivity(false)
+//                .withConfig(config)
+//                .build();
+//
+//        c.addAppender(config.getAppender("File"), level, null);
+//        c.addAppender(config.getAppender("Console"), level, null);
+//
+//        config.addLogger("au.org.aodn", c);
+//        // LoggerConfig loggerConfig = config.getLoggerConfig(LogManager.ROOT_LOGGER_NAME);
+//        // loggerConfig.setLevel(Level.INFO);
+//        ctx.updateLoggers();
+//    }
 
     /**
      * Use aspectJ to intercept all call that ends with WithHttpInfo, we must always use geonetwork api call
@@ -139,6 +149,16 @@ public void init() throws NoSuchAlgorithmException, KeyManagementException {
         ConfigurableApplicationContext jeevesContext = ApplicationContextHolder.get();
         jeevesContext.getBeanFactory().registerSingleton("genericEntityListener", genericEntityListener);
     }
+    /**
+     * The reason we need is to setup the WEB_ROOT context to be use by Actuator. In springboot application it is
+     * set on start, but this geonetwork is a different species that it isn't a springboot app so this setting
+     * is missing
+     * @param sc
+     */
+    @Autowired
+    public void setRootContext(ServletContext sc, ConfigurableApplicationContext context) {
+        sc.setAttribute(WebApplicationContext.ROOT_WEB_APPLICATION_CONTEXT_ATTRIBUTE, context);
+    }
 
     @Bean
     public RestTemplate createRestTemplate() {

geonetwork/src/main/java/au/org/aodn/geonetwork4/handler/GenericEntityListener.java

@@ -60,7 +60,6 @@ public void cleanUp() {
     public void init() {
         // We pick up the items in map and then post trigger indexer call, this thread keep execute every 5 secs
         service.scheduleWithFixedDelay(() -> {
-            logger.info("Execute batch of update/delete after time elapsed");
 
             // If the updateMap contain items that is going do delete, then there is no point to update
             deleteMap.forEach((key, value) -> updateMap.remove(key));
@@ -133,7 +132,7 @@ protected void callApiUpdate(String indexUrl, Map<String, Object> variables) {
         headers.setContentType(MediaType.APPLICATION_JSON);
 
         HttpEntity<Void> request = new HttpEntity<>(null, headers);
-        logger.info("Call indexer to update metadata {}", variables.get(UUID));
+        logger.info("Call indexer update {} metadata {}", indexUrl, variables.get(UUID));
         restTemplate.postForEntity(indexUrl, request, Void.class, variables);
     }
     /**
@@ -148,7 +147,7 @@ protected void callApiDelete(String indexUrl, Map<String, Object> variables) {
         headers.setContentType(MediaType.APPLICATION_JSON);
 
         HttpEntity<Void> request = new HttpEntity<>(null, headers);
-        logger.info("Call indexer to delete metadata {}", variables.get(UUID));
+        logger.info("Call indexer delete {} metadata {}", indexUrl, variables.get(UUID));
         restTemplate.exchange(indexUrl, HttpMethod.DELETE, request, Void.class, variables);
     }
 }

geonetwork/src/main/resources/application-edge.properties

@@ -0,0 +1,8 @@
+management.endpoints.web.base-path=/srv/api/manage
+management.endpoints.web.exposure.include=health,info,env,beans,logfile
+
+management.endpoint.health.show-details=always
+management.health.elasticsearch.enabled=false
+management.health.ldap.enabled=false
+
+management.endpoint.logfile.external-file=/var/lib/jetty/logs/geonetwork.log