Merge pull request #93 from aodn/features/6008-speed-up-centroid-cal

Minor enhance on remove repeat processing

server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/ElasticSearchConfig.java

@@ -53,7 +53,7 @@ public ElasticsearchClient geoNetworkElasticsearchClient(RestClientTransport tra
     public Search createElasticSearch(ElasticsearchClient client,
                                       ObjectMapper mapper,
                                       @Value("${elasticsearch.index.name}") String indexName,
-                                      @Value("${elasticsearch.index.pageSize:2000}") Integer pageSize,
+                                      @Value("${elasticsearch.index.pageSize:5000}") Integer pageSize,
                                       @Value("${elasticsearch.search_as_you_type.size:10}") Integer searchAsYouTypeSize) {
 
         return new ElasticSearch(client, mapper, indexName, pageSize, searchAsYouTypeSize);

server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/BinaryResponseToBytes.java

@@ -2,6 +2,7 @@
 
 import co.elastic.clients.transport.endpoints.BinaryResponse;
 import org.mapstruct.Mapper;
+import org.opengis.filter.Filter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Service;
@@ -16,7 +17,7 @@ public abstract class BinaryResponseToBytes implements Converter<BinaryResponse,
     protected Logger logger = LoggerFactory.getLogger(BinaryResponseToBytes.class);
 
     @Override
-    public byte[] convert(BinaryResponse from, Param noUse) {
+    public byte[] convert(BinaryResponse from, Filter noUse) {
         logger.debug("Incoming BinaryResponse type is {}", from.contentType());
 
         try (InputStream s = from.content()) {

server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/Converter.java

@@ -6,26 +6,19 @@
 import au.org.aodn.ogcapi.server.core.model.StacCollectionModel;
 import au.org.aodn.ogcapi.server.core.model.enumeration.CQLCrsType;
 import au.org.aodn.ogcapi.server.core.model.enumeration.CollectionProperty;
-import au.org.aodn.ogcapi.server.core.parser.stac.CQLToStacFilterFactory;
 import au.org.aodn.ogcapi.server.core.parser.stac.GeometryVisitor;
 import au.org.aodn.ogcapi.server.core.util.ConstructUtils;
 import au.org.aodn.ogcapi.server.core.util.GeometryUtils;
 import lombok.Builder;
 import lombok.Getter;
 import lombok.Setter;
-import org.geotools.filter.text.commons.CompilerUtil;
-import org.geotools.filter.text.commons.Language;
-import org.geotools.filter.text.cql2.CQLException;
-import org.geotools.geojson.geom.GeometryJSON;
 import org.locationtech.jts.geom.Geometry;
-import org.locationtech.jts.geom.GeometryCollection;
 import org.opengis.filter.Filter;
-import org.opengis.filter.expression.Expression;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.http.MediaType;
 
-import java.lang.Exception;
+import java.util.Map;
 import java.util.stream.Collectors;
 
 import static au.org.aodn.ogcapi.server.core.util.GeometryUtils.createCentroid;
@@ -45,7 +38,7 @@ class Param {
         String filter;
     }
 
-    T convert(F from, Param param);
+    T convert(F from, Filter param);
 
     default au.org.aodn.ogcapi.features.model.Link getSelfCollectionLink(String hostname, String id) {
         au.org.aodn.ogcapi.features.model.Link self = new au.org.aodn.ogcapi.features.model.Link();
@@ -141,8 +134,9 @@ default <D extends StacCollectionModel> Collection getCollection(D m, Filter fil
         }
 
         if(m.getSummaries() != null ) {
-            if (m.getSummaries().getGeometryNoLand() != null) {
-                GeometryUtils.readGeometry(m.getSummaries().getGeometryNoLand())
+            Map<?, ?> noLand = m.getSummaries().getGeometryNoLand();
+            if (noLand != null) {
+                GeometryUtils.readGeometry(noLand)
                         .ifPresent(input -> {
                             Geometry g = filter != null ? (Geometry)filter.accept(visitor, input) : input;
                             collection.getProperties().put(

server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToCollection.java

@@ -3,11 +3,10 @@
 import au.org.aodn.ogcapi.features.model.Collection;
 import au.org.aodn.ogcapi.server.core.model.StacCollectionModel;
 import org.mapstruct.Mapper;
+import org.opengis.filter.Filter;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
-import java.util.Map;
-
 @Service
 @Mapper(componentModel = "spring")
 public abstract class StacToCollection implements Converter<StacCollectionModel, Collection> {
@@ -16,7 +15,7 @@ public abstract class StacToCollection implements Converter<StacCollectionModel,
     protected String hostname;
 
     @Override
-    public Collection convert(StacCollectionModel model, Param param) {
+    public Collection convert(StacCollectionModel model, Filter param) {
         return getCollection(model, null, hostname);
     }
 }

server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToCollections.java

@@ -3,10 +3,7 @@
 import au.org.aodn.ogcapi.features.model.Collection;
 import au.org.aodn.ogcapi.features.model.Collections;
 import au.org.aodn.ogcapi.server.core.model.ExtendedCollections;
-import au.org.aodn.ogcapi.server.core.parser.stac.CQLToStacFilterFactory;
 import au.org.aodn.ogcapi.server.core.service.ElasticSearch;
-import org.geotools.filter.text.commons.CompilerUtil;
-import org.geotools.filter.text.commons.Language;
 import org.mapstruct.Mapper;
 import org.opengis.filter.Filter;
 import org.springframework.beans.factory.annotation.Value;
@@ -23,19 +20,7 @@ public abstract class StacToCollections implements Converter<ElasticSearch.Searc
     protected String hostname;
 
     @Override
-    public Collections convert(ElasticSearch.SearchResult model, Param param) {
-        CQLToStacFilterFactory factory = CQLToStacFilterFactory.builder()
-                .cqlCrsType(param.getCoordinationSystem())
-                .build();
-
-        Filter f = null;
-        try {
-            f = param.getFilter() != null ? CompilerUtil.parseFilter(Language.CQL, param.getFilter(), factory) : null;
-        }
-        catch(Exception ex) {
-            // Do nothing
-        }
-        final Filter filter = f;
+    public Collections convert(ElasticSearch.SearchResult model, Filter filter) {
 
         List<Collection> collections = model.getCollections().parallelStream()
                 .map(m -> getCollection(m, filter, hostname))

server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToInlineResponse2002.java

@@ -1,12 +1,11 @@
 package au.org.aodn.ogcapi.server.core.mapper;
 
-import au.org.aodn.ogcapi.server.core.model.StacCollectionModel;
-
 import au.org.aodn.ogcapi.server.core.model.enumeration.CQLCrsType;
 import au.org.aodn.ogcapi.server.core.service.ElasticSearch;
 import au.org.aodn.ogcapi.tile.model.InlineResponse2002;
 import au.org.aodn.ogcapi.tile.model.TileSetItem;
 import org.mapstruct.Mapper;
+import org.opengis.filter.Filter;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
@@ -21,7 +20,7 @@ public abstract class StacToInlineResponse2002 implements Converter<ElasticSearc
     protected String hostname;
 
     @Override
-    public InlineResponse2002 convert(ElasticSearch.SearchResult model, Param noUse) {
+    public InlineResponse2002 convert(ElasticSearch.SearchResult model, Filter noUse) {
         List<TileSetItem> items = model.getCollections().stream()
                 .map(m -> {
                     TileSetItem item = new TileSetItem();

server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToTileSetWmWGS84Q.java

@@ -6,19 +6,18 @@
 import au.org.aodn.ogcapi.tile.model.TileSet;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import org.mapstruct.Mapper;
+import org.opengis.filter.Filter;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
-import java.util.List;
-
 @Service
 @Mapper(componentModel = "spring")
 public abstract class StacToTileSetWmWGS84Q implements Converter<ElasticSearch.SearchResult, TileSet> {
 
     @Value("${api.host}")
     protected String hostname;
 
-    protected class TileSetWorldMercatorWGS84Quad extends TileSet {
+    protected static class TileSetWorldMercatorWGS84Quad extends TileSet {
 
         @JsonProperty("dataType")
         public String getDataType2() { return "vector"; }
@@ -28,7 +27,7 @@ protected class TileSetWorldMercatorWGS84Quad extends TileSet {
     }
 
     @Override
-    public TileSet convert(ElasticSearch.SearchResult from, Param noUse) {
+    public TileSet convert(ElasticSearch.SearchResult from, Filter noUse) {
         TileSetWorldMercatorWGS84Quad tileSet = new TileSetWorldMercatorWGS84Quad();
 
         tileSet.setTileMatrixSetURI("http://www.opengis.net/def/tilematrixset/OGC/1.0/WorldMercatorWGS84Quad");

server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java

@@ -58,8 +58,6 @@ public class ElasticSearch extends ElasticSearchBase implements Search {
     @Value("${elasticsearch.search_after.split_regex:\\|\\|}")
     protected String searchAfterSplitRegex;
 
-    private final int DATASET_ENTRY_MAX = 2000;
-
     public ElasticSearch(ElasticsearchClient client,
                          ObjectMapper mapper,
                          String indexName,
@@ -444,14 +442,14 @@ public DatasetSearchResult searchDataset(
         Supplier<SearchRequest.Builder> builderSupplier = () -> {
             SearchRequest.Builder builder = new SearchRequest.Builder();
             builder.index(datasetIndexName)
-                    .size(DATASET_ENTRY_MAX)
+                    .size(this.getPageSize())
                     .query(query -> query.bool(createBoolQueryForProperties(queries, null, null)));
 
             return builder;
         };
 
         try {
-            Iterable<Hit<ObjectNode>> response = pagableSearch(builderSupplier, ObjectNode.class, (long) DATASET_ENTRY_MAX);
+            Iterable<Hit<ObjectNode>> response = pagableSearch(builderSupplier, ObjectNode.class, (long) this.getPageSize());
 
             DatasetSearchResult result = new DatasetSearchResult();
 

server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java

@@ -20,6 +20,7 @@
 import lombok.extern.slf4j.Slf4j;
 
 import java.io.IOException;
+import java.time.ZonedDateTime;
 import java.util.*;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
@@ -102,7 +103,6 @@ protected SearchRequest buildSearchAsYouTypeRequest(
      * @param filters - The Query coming from CQL parser
      * @param properties - The fields you want to return in the search, you can search a field but not include in the return
      * @return - The search result from Elastic query and format in StacCollectionModel
-     * @throws IOException
      */
     protected SearchResult searchCollectionBy(final List<Query> queries,
                                                            final List<Query> should,
@@ -183,6 +183,8 @@ protected SearchResult searchCollectionBy(final List<Query> queries,
         };
 
         try {
+            double random = Math.random();
+            log.info("Start search {} {}", ZonedDateTime.now(), random);
             Iterable<Hit<ObjectNode>> response = pagableSearch(builderSupplier, ObjectNode.class, maxSize);
 
             SearchResult result = new SearchResult();
@@ -196,6 +198,7 @@ protected SearchResult searchCollectionBy(final List<Query> queries,
                     lastSortValue = i.sort();
                 }
             }
+            log.info("End search {} {}", ZonedDateTime.now(), random);
             // Return the last sort value if exist
             if(lastSortValue != null && !lastSortValue.isEmpty()) {
                 List<Object> values = new ArrayList<>();

server/src/main/java/au/org/aodn/ogcapi/server/core/service/OGCApiService.java

@@ -1,10 +1,13 @@
 package au.org.aodn.ogcapi.server.core.service;
 
-import au.org.aodn.ogcapi.server.core.mapper.Converter;
 import au.org.aodn.ogcapi.server.core.model.enumeration.CQLCrsType;
 import au.org.aodn.ogcapi.server.core.model.enumeration.OGCMediaTypeMapper;
 import au.org.aodn.ogcapi.server.core.exception.CustomException;
+import au.org.aodn.ogcapi.server.core.parser.stac.CQLToStacFilterFactory;
 import au.org.aodn.ogcapi.server.tile.RestApi;
+import org.geotools.filter.text.commons.CompilerUtil;
+import org.geotools.filter.text.commons.Language;
+import org.opengis.filter.Filter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -13,7 +16,6 @@
 
 import java.util.List;
 import java.util.function.BiFunction;
-import java.util.function.Function;
 
 /**
  *
@@ -27,7 +29,7 @@ public abstract class OGCApiService {
 
     /**
      * You can find conformance id here https://docs.ogc.org/is/19-072/19-072.html#ats_core
-     * @return
+     * @return List of string contains conformance
      */
     public abstract List<String> getConformanceDeclaration();
 
@@ -37,32 +39,37 @@ public <R> ResponseEntity<R> getCollectionList(List<String> keywords,
                                                    String sortBy,
                                                    OGCMediaTypeMapper f,
                                                    CQLCrsType coor,
-                                                   BiFunction<ElasticSearchBase.SearchResult, Converter.Param, R> converter) {
+                                                   BiFunction<ElasticSearchBase.SearchResult, Filter, R> converter) {
         try {
             switch (f) {
                 case json -> {
                     ElasticSearchBase.SearchResult result = search.searchByParameters(keywords, filter, properties, sortBy, coor);
 
-                    Converter.Param param = Converter.Param.builder()
-                            .coordinationSystem(coor)
-                            .filter(filter)
+                    CQLToStacFilterFactory factory = CQLToStacFilterFactory.builder()
+                            .cqlCrsType(coor)
                             .build();
 
+                    Filter cql = null;
+
+                    try {
+                        cql = filter != null ? CompilerUtil.parseFilter(Language.CQL, filter, factory) : null;
+                    }
+                    catch(Exception ex) {
+                        // Do nothing
+                    }
+
                     return ResponseEntity.ok()
-                            .body(converter.apply(result, param));
+                            .body(converter.apply(result, cql));
                 }
                 default -> {
-                    /**
+                    /*
                      * https://opengeospatial.github.io/ogcna-auto-review/19-072.html
-                     *
                      * The OGC API — Common Standard does not mandate a specific encoding or format for
                      * representations of resources. However, both HTML and JSON are commonly used encodings for spatial
                      * data on the web. The HTML and JSON requirements classes specify the encoding of resource
                      * representations using:
-                     *
                      *     HTML
                      *     JSON
-                     *
                      * Neither of these encodings is mandatory. An implementer of the API-Common Standard may decide
                      * to implement other encodings instead of, or in addition to, these two.
                      */

server/src/main/java/au/org/aodn/ogcapi/server/tile/RestService.java

@@ -1,19 +1,18 @@
 package au.org.aodn.ogcapi.server.tile;
 
-import au.org.aodn.ogcapi.server.core.mapper.Converter;
 import au.org.aodn.ogcapi.server.core.model.enumeration.OGCMediaTypeMapper;
 import au.org.aodn.ogcapi.server.core.service.ElasticSearch;
 import au.org.aodn.ogcapi.server.core.service.OGCApiService;
 import au.org.aodn.ogcapi.server.core.exception.CustomException;
 import au.org.aodn.ogcapi.tile.model.TileMatrixSets;
+import org.opengis.filter.Filter;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Service;
 
 import java.io.IOException;
 import java.util.List;
 import java.util.function.BiFunction;
-import java.util.function.Function;
 
 @Service("TileRestService")
 public class RestService extends OGCApiService {
@@ -30,7 +29,7 @@ public <T, R> ResponseEntity<?> getVectorTileOfCollection(
             Integer tileMatrix,
             Integer tileRow,
             Integer tileCol,
-            BiFunction<T, Converter.Param, R> converter) {
+            BiFunction<T, Filter, R> converter) {
 
         // TODO: Implements additional filters
         try {
@@ -52,7 +51,7 @@ public <T, R> ResponseEntity<?> getVectorTileOfCollection(
     }
 
     public <R> ResponseEntity<R> getTileSetsListOfCollection(List<String> id, String sortBy, OGCMediaTypeMapper f,
-                                                             BiFunction<ElasticSearch.SearchResult, Converter.Param, R> converter) {
+                                                             BiFunction<ElasticSearch.SearchResult, Filter, R> converter) {
         try {
             switch (f) {
                 case json -> {
@@ -64,17 +63,14 @@ public <R> ResponseEntity<R> getTileSetsListOfCollection(List<String> id, String
                             .body(converter.apply(result, null));
                 }
                 default -> {
-                    /**
+                    /*
                      * https://opengeospatial.github.io/ogcna-auto-review/19-072.html
-                     *
                      * The OGC API — Common Standard does not mandate a specific encoding or format for
                      * representations of resources. However, both HTML and JSON are commonly used encodings for spatial
                      * data on the web. The HTML and JSON requirements classes specify the encoding of resource
                      * representations using:
-                     *
                      *     HTML
                      *     JSON
-                     *
                      * Neither of these encodings is mandatory. An implementer of the API-Common Standard may decide
                      * to implement other encodings instead of, or in addition to, these two.
                      */