Features/5522 new api for source

README.md

@@ -77,19 +77,20 @@ endpoint to view the log file directly as cloud watch is not so easy to use.
 ### Endpoints:
 
 > You need to present X-XSRF-TOKEN in your header to call Setup endpoints, please read comments
-in [Api.java](./geonetwork/src/main/java/au/org/aodn/geonetwork4/controller/Api.java)
-
-| Description                   | Method | Endpoints                                   | Param                            | Environment |
-|-------------------------------|--------|---------------------------------------------|----------------------------------|-------------|
-| Logfile                       | GET    | `/geonetwork/srv/api/manage/logfile`        |                                  | Edge        |
-| Beans info                    | GET    | `/geonetwork/srv/api/manage/beans`          |                                  |Edge   |
-| Env info                      | GET    | `/geonetwork/srv/api/manage/env`            |                                  |Edge  |
-| Info                          | GET    | `/geonetwork/srv/api/manage/info`           |                                  |Edge  |
-| Health check                  | GET    | `/geonetwork/srv/api/manage/health`         |                                  |Edge  |
-| Read Harvester - Config       | GET    | `/geonetwork/srv/api/aodn/setup/harvesters` |                                  |Edge  |
-| Delete All Harvester - Config | DELETE | `/geonetwork/srv/api/aodn/setup/harvesters` |                                  |Edge  |
-| Delete All Category - Config  | DELETE | `/geonetwork/srv/api/aodn/setup/categories` |                                  |Edge  |
-| Setup from github config      | POST   | `/geonetwork/srv/api/aodn/setup`            | source=github |Edge  |
+> in [Api.java](./geonetwork/src/main/java/au/org/aodn/geonetwork4/controller/Api.java)
+
+| Description                   | Method | Endpoints                                      | Param         | Environment   |
+|-------------------------------|--------|------------------------------------------------|---------------|---------------|
+| Logfile                       | GET    | `/geonetwork/srv/api/manage/logfile`           |               | Edge, Staging |
+| Beans info                    | GET    | `/geonetwork/srv/api/manage/beans`             |               | Edge, Staging |
+| Env info                      | GET    | `/geonetwork/srv/api/manage/env`               |               | Edge, Staging |
+| Info                          | GET    | `/geonetwork/srv/api/manage/info`              |               | Edge, Staging |
+| Health check                  | GET    | `/geonetwork/srv/api/manage/health`            |               | Edge, Staging |
+| Read Record Misc Info         | GET    | `/geonetwork/srv/api/aodn/records/{uuid}/info` |               | Edge, Staging |
+| Read Harvester - Config       | GET    | `/geonetwork/srv/api/aodn/setup/harvesters`    |               | Edge, Staging |
+| Delete All Harvester - Config | DELETE | `/geonetwork/srv/api/aodn/setup/harvesters`    |               | Edge, Staging |
+| Delete All Category - Config  | DELETE | `/geonetwork/srv/api/aodn/setup/categories`    |               | Edge, Staging |
+| Setup from github config      | POST   | `/geonetwork/srv/api/aodn/setup`               | source=github | Edge, Staging |
 
 ### How the Setup works?
 
@@ -105,6 +106,9 @@ Given the configuration is store in main branch, that means changes to configura
 > The POST method can carry a body with the same format as the config.json file, if this appears, then the content
 > in the body will be use instead of the config.json in github. Hence, you can run individual setup one by one
 
+### Read Record Misc Info endpoint
+Info not expose in regular api endpoints but needed for indexing, for example logo url of a dataset.
+
 ## Schema folder
 
 The schema folder contains an open-api schema file from Genetwork4, you can get it from any instance

geonetwork-api/src/main/java/au/org/aodn/geonetwork_api/openapi/api/helper/SiteHelper.java

@@ -3,25 +3,27 @@
 import au.org.aodn.geonetwork_api.openapi.api.SiteApi;
 import au.org.aodn.geonetwork_api.openapi.api.Status;
 import au.org.aodn.geonetwork_api.openapi.model.Setting;
-import au.org.aodn.geonetwork_api.openapi.model.SystemInfo;
+
 import org.json.JSONObject;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.client.HttpClientErrorException;
 
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 
 public class SiteHelper {
 
     protected static final String SETTINGS = "settings";
     protected static final String DATA = "data";
 
+    public final static String HOST = "system/server/host";
+    public final static String PROTOCOL = "system/server/protocol";
+    public final static String PORT = "system/server/port";
+
     protected Logger logger = LogManager.getLogger(SiteHelper.class);
     protected SiteApi api;
 
@@ -30,6 +32,24 @@ public SiteHelper(SiteApi api) {
     }
 
     public SiteApi getApi() { return api; }
+    /**
+     * Get the system setting from geonetwork
+     * @return - A map with key as the setting path and value contains the details.
+     */
+    public Map<String, Setting> getAllSettingsDetails() {
+        Map<String, Setting> s = new HashMap<>();
+
+        ResponseEntity<List<Setting>> storedSetting = this.api.getSettingsDetailsWithHttpInfo(null, null);
+
+        if(storedSetting.getStatusCode().is2xxSuccessful()) {
+            // Get the available keys and value
+            s.putAll(Objects.requireNonNull(storedSetting.getBody())
+                    .stream()
+                    .collect(Collectors.toMap(Setting::getName, Function.identity())));
+        }
+
+        return s;
+    }
 
     public List<Status> createSettings(List<String> json) {
         return json.stream()
@@ -42,19 +62,7 @@ public List<Status> createSettings(List<String> json) {
                     if(jsonObject.optJSONObject(SETTINGS) != null
                             && jsonObject.optJSONObject(SETTINGS).optJSONObject(DATA) != null) {
 
-                        Set<String> keys = new HashSet<>();
-
-                        ResponseEntity<List<Setting>> storedSetting = this.api.getSettingsDetailsWithHttpInfo(null, null);
-
-                        if(storedSetting.getStatusCode().is2xxSuccessful()) {
-                            // Get the available keys in the geonetwork4 system.
-                            Set<String> s = storedSetting.getBody()
-                                    .stream()
-                                    .map(k -> k.getName())
-                                    .collect(Collectors.toSet());
-
-                            keys.addAll(s);
-                        }
+                        Set<String> keys = getAllSettingsDetails().keySet();
 
                         // Convert map from <String, Object> to <String, String>
                         Map<String, String> settings = jsonObject.

geonetwork-api/src/test/java/au/org/aodn/geonetwork_api/openapi/api/helper/SiteHelperTest.java

@@ -0,0 +1,52 @@
+package au.org.aodn.geonetwork_api.openapi.api.helper;
+
+import au.org.aodn.geonetwork_api.openapi.api.SiteApi;
+import au.org.aodn.geonetwork_api.openapi.model.Setting;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.springframework.http.ResponseEntity;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.when;
+
+public class SiteHelperTest {
+
+    /**
+     * Function test only, we do not have docker image to run for this test, the
+     * integration test in geonetwork should test it more.
+     */
+    @Test
+    public void verifyGetAllSettingsDetails() {
+        Setting s1 = new Setting();
+        s1.setName(SiteHelper.HOST);
+        s1.setValue("geonetwork-edge.edge.aodn.org.au");
+
+        Setting s2 = new Setting();
+        s2.setName(SiteHelper.PROTOCOL);
+        s2.setValue("https");
+
+        List<Setting> result = new ArrayList<>();
+        result.add(s1);
+        result.add(s2);
+
+        SiteApi api = Mockito.mock(SiteApi.class);
+        when(api.getSettingsDetailsWithHttpInfo(eq(null), eq(null)))
+                .thenReturn(ResponseEntity.ok(result));
+
+        SiteHelper helper = new SiteHelper(api);
+
+        Set<String> v = helper.getAllSettingsDetails().keySet();
+        assertTrue("Contains host", v.contains(SiteHelper.HOST));
+        assertTrue("Contains protocol", v.contains(SiteHelper.PROTOCOL));
+
+        assertEquals("Host equals",
+                "geonetwork-edge.edge.aodn.org.au",
+                helper.getAllSettingsDetails().get(SiteHelper.HOST).getValue());
+    }
+}

geonetwork/src/main/java/au/org/aodn/geonetwork4/Setup.java

@@ -5,6 +5,7 @@
 import au.org.aodn.geonetwork_api.openapi.model.Group;
 import au.org.aodn.geonetwork_api.openapi.model.HarvestersApiLegacyResponse;
 
+import au.org.aodn.geonetwork_api.openapi.model.MeResponse;
 import au.org.aodn.geonetwork_api.openapi.model.MetadataCategory;
 import com.github.underscore.U;
 import org.apache.logging.log4j.LogManager;
@@ -26,6 +27,9 @@
  */
 public class Setup {
 
+    public static final String SYSTEM_INFO = "systemInfo";
+    public static final String SITE_INFO = "siteInfo";
+
     protected Logger logger = LogManager.getLogger(Setup.class);
 
     protected MeApi meApi;
@@ -60,14 +64,14 @@ public Setup(ResourceLoader resourceLoader,
         this.harvestersApi = harvestersApi;
     }
 
-    public void getMe() {
-        logger.info("Login user is {}", meApi.getMeWithHttpInfo().getBody());
+    public ResponseEntity<MeResponse> getMe() {
+        return ResponseEntity.ok(meApi.getMeWithHttpInfo().getBody());
     }
 
     public ResponseEntity<Map<String, Object>> getSystemInfo() {
         return ResponseEntity.ok(Map.of(
-                "systemInfo", Objects.requireNonNull(siteHelper.getApi().getSystemInfoWithHttpInfo().getBody()),
-                "siteInfo", Objects.requireNonNull(siteHelper.getApi().getInformationWithHttpInfo().getBody())
+                SYSTEM_INFO, Objects.requireNonNull(siteHelper.getApi().getSystemInfoWithHttpInfo().getBody()),
+                SITE_INFO, Objects.requireNonNull(siteHelper.getApi().getInformationWithHttpInfo().getBody())
         ));
     }
 
@@ -139,6 +143,10 @@ public ResponseEntity<String> getAllHarvesters() {
     public ResponseEntity<List<HarvestersApiLegacyResponse>> insertHarvester(List<String> config) {
         return ResponseEntity.of(Optional.of(harvestersApiLegacy.createHarvesters(config)));
     }
+
+    public String getSiteSetting(String path) {
+        return siteHelper.getAllSettingsDetails().get(path).getValue();
+    }
     /**
      * TODO: The return type is a bit messy
      * @param config - The json config

geonetwork/src/main/java/au/org/aodn/geonetwork4/controller/Api.java

@@ -2,22 +2,32 @@
 
 import au.org.aodn.geonetwork4.Setup;
 import au.org.aodn.geonetwork4.model.*;
+import au.org.aodn.geonetwork_api.openapi.api.helper.SiteHelper;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectWriter;
 import jeeves.services.ReadWriteController;
+import org.apache.commons.lang.StringUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+
+import org.fao.geonet.domain.Metadata;
+import org.fao.geonet.kernel.harvest.HarvestManagerImpl;
+import org.fao.geonet.kernel.harvest.harvester.csw.CswHarvester;
+import org.fao.geonet.kernel.harvest.harvester.geonet.GeonetHarvester;
+import org.fao.geonet.kernel.harvest.harvester.geonet20.Geonet20Harvester;
+import org.fao.geonet.kernel.harvest.harvester.oaipmh.OaiPmhHarvester;
+import org.fao.geonet.kernel.harvest.harvester.ogcwxs.OgcWxSHarvester;
+import org.fao.geonet.repository.MetadataRepository;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.http.MediaType;
 import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.*;
+import org.springframework.web.servlet.support.ServletUriComponentsBuilder;
 
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
 import java.util.stream.Collectors;
 
@@ -31,21 +41,120 @@
 @RequestMapping(value = {"/{portal}/api/aodn"})
 public class Api {
 
+    public static final String SUGGEST_LOGOS = "suggest_logos";
+
     protected Logger logger = LogManager.getLogger(Api.class);
 
-    @Autowired
     protected Setup setup;
-
-    @Autowired
+    protected MetadataRepository repository;
+    protected HarvestManagerImpl harvestManager;
     protected ObjectMapper objectMapper;
 
     @Autowired
     @Qualifier("remoteSources")
     protected Map<String, GitRemoteConfig> remoteConfigMap;
 
+    public Api(Setup setup, MetadataRepository metadataRepository, HarvestManagerImpl harvestManager, ObjectMapper objectMapper) {
+        this.harvestManager = harvestManager;
+        this.repository = metadataRepository;
+        this.setup = setup;
+        this.objectMapper = objectMapper;
+    }
+
     protected RemoteConfig getRemoteConfig(String type) {
         return remoteConfigMap.get(type);
     }
+    /**
+     * HACK!!
+     * This function is used to expose something not found in the rest api, it used the jpa api to get the
+     * metadata object itself and then expose the additional values, this object contains the sourceId
+     * which is the uuid of the record from the source system being harvested, it different from the
+     * UUID use in this geonetwork, because harvested record get assign a new UUID locally.
+     *
+     * This source id can be useful because the geonetwork may download the log, this all depends on which harvester
+     * you use, for GeonetHarvester, it will download others don't, therefore the logo list will be different
+     *
+     * TODO: We should add suggestion based on group logo.
+     *
+     * @param uuid - UUID of the record use by this geonetwork
+     * @return - A data structure contains the UUID of the record in the source system as well as suggested logo in order of possibility
+     * {
+     *     "sourceId": "dbee258b-8730-4072-96d4-2818a69a4afd",
+     *     "schemaid": "iso19115-3.2018",
+     *     "harvesterUri": "https://catalogue-imos.aodn.org.au/geonetwork",
+     *     "suggest_logos": [
+     *         "http://localhost:8080/geonetwork/images/logos/dbee258b-8730-4072-96d4-2818a69a4afd.png",
+     *         "https://catalogue-imos.aodn.org.au/geonetwork/images/logos/dbee258b-8730-4072-96d4-2818a69a4afd.png"
+     *     ],
+     *     "isHarvested": true,
+     *     "harvesterType": "GeonetHarvester"
+     * }
+     */
+    @GetMapping("/records/{uuid}/info")
+    public ResponseEntity<Map<String, Object>> getRecordExtraInfo(@PathVariable("uuid") String uuid) {
+        Map<String, Object> info = new HashMap<>();
+        info.put(SUGGEST_LOGOS, new ArrayList<String>());
+
+        Metadata metadata = repository.findOneByUuid(uuid);
+        if(metadata != null) {
+            if(metadata.getSourceInfo() != null) {
+                // Here we can get the source id, then we can create the first option for logo
+                // which is extract logo from this host
+                info.put("sourceId", metadata.getSourceInfo().getSourceId());
+
+                if(info.get(SUGGEST_LOGOS) instanceof ArrayList) {
+                    String host = setup.getSiteSetting(SiteHelper.HOST);
+                    String port = setup.getSiteSetting(SiteHelper.PORT);
+                    String protocol = setup.getSiteSetting(SiteHelper.PROTOCOL);
+                    ((ArrayList<String>) info.get(SUGGEST_LOGOS))
+                            .add(String.format("%s://%s:%s/geonetwork/images/logos/%s.png", protocol, host, port, info.get("sourceId")));
+                }
+            }
+
+            // We can also get the harvester uuid, from there we can get the harvester url
+            if(metadata.getHarvestInfo() != null) {
+
+                info.put("isHarvested", metadata.getHarvestInfo().isHarvested());
+
+                Object harvester = harvestManager.getHarvester(metadata.getHarvestInfo().getUuid());
+                if(harvester instanceof GeonetHarvester) {
+                    info.put("harvesterUri", StringUtils.removeEnd(((GeonetHarvester) harvester).getParams().host, "/"));
+                    info.put("harvesterType", "GeonetHarvester");
+                    // The geonetwork store logo under this dir with the uuid name, we provide a list of suggestion
+                    // on where to find the logo
+                    if(info.get(SUGGEST_LOGOS) instanceof ArrayList) {
+                        ((ArrayList<String>) info.get(SUGGEST_LOGOS))
+                                .add(String.format("%s/images/logos/%s.png", info.get("harvesterUri"), info.get("sourceId")));
+                    }
+                }
+                else if(harvester instanceof OaiPmhHarvester) {
+                    // Will have remote link to logo
+                    info.put("harvesterUri", StringUtils.removeEnd(((OaiPmhHarvester) harvester).getParams().url, "/"));
+                    info.put("harvesterType", "OaiPmhHarvester");
+                }
+                else if(harvester instanceof CswHarvester) {
+                    // Will have remote link to logo
+                    info.put("harvesterUri", StringUtils.removeEnd(((CswHarvester) harvester).getParams().capabUrl, "/"));
+                    info.put("harvesterType", "CswHarvester");
+                }
+                else if(harvester instanceof OgcWxSHarvester) {
+                    // Will have remote link to logo
+                    info.put("harvesterUri", StringUtils.removeEnd(((OgcWxSHarvester) harvester).getParams().url, "/"));
+                    info.put("harvesterType", "OgcWxSHarvester");
+                }
+                else if(harvester instanceof Geonet20Harvester) {
+                    // Will have remote link to logo
+                    info.put("harvesterUri", StringUtils.removeEnd(((Geonet20Harvester) harvester).getParams().host, "/"));
+                    info.put("harvesterType", "Geonet20Harvester");
+                }
+            }
+
+            if(metadata.getDataInfo() != null) {
+                info.put("schemaid", metadata.getDataInfo().getSchemaId());
+            }
+        }
+        return ResponseEntity.ok(info);
+    }
 
     @GetMapping("/setup/harvesters")
     public ResponseEntity<String> getAllHarvesters() {