feat(reindex): Extend instance-records reindex endpoint with passing …

…index settings (#681)

* feat(reindex): Extend instance-records reindex endpoint with passing index settings

- Extend full/upload reindex endpoints with index settings
- Recreate indices on publishing merge ranges instead of doing this on upload start in order to use index settings
- Add instance reindexing to tenant service
- Remove instance record type from old reindex endpoint
- Make resourceName required on old reindex endpoint

Implements: MSEARCH-853

NEWS.md

@@ -6,7 +6,7 @@
   * Endpoint `GET /search/consortium/batch/holdings` requires `consortium-search.holdings.batch.collection.get` permission
 
 ### New APIs versions
-* Provides `indices v0.8`
+* Provides `indices v1.0`
 * Provides `search v1.3`
 * Provides `consortium-search v2.1`
 * Provides `browse v1.4`
@@ -49,6 +49,7 @@
 * Implement new re-index flow for instance records ([MSEARCH-793](https://folio-org.atlassian.net/issues/MSEARCH-793), [MSEARCH-794](https://folio-org.atlassian.net/issues/MSEARCH-794), [MSEARCH-796](https://folio-org.atlassian.net/issues/MSEARCH-796), [MSEARCH-797](https://folio-org.atlassian.net/issues/MSEARCH-797), [MSEARCH-798](https://folio-org.atlassian.net/issues/MSEARCH-798), [MSEARCH-799](https://folio-org.atlassian.net/issues/MSEARCH-799), [MSEARCH-800](https://folio-org.atlassian.net/issues/MSEARCH-800), [MSEARCH-801](https://folio-org.atlassian.net/issues/MSEARCH-801), [MSEARCH-802](https://folio-org.atlassian.net/issues/MSEARCH-802))
 * Implement Linked Data HUB index and search API ([MSEARCH-844](https://folio-org.atlassian.net/browse/MSEARCH-844))
 * Extend consortium library, campus, institution API with id param ([MSEARCH-855](https://folio-org.atlassian.net/browse/MSEARCH-855))
+* Extend instance-records reindex endpoint with index settings ([MSEARCH-853](https://folio-org.atlassian.net/browse/MSEARCH-853))
 
 ### Bug fixes
 * Do not delete kafka topics if collection topic is enabled ([MSEARCH-725](https://folio-org.atlassian.net/browse/MSEARCH-725))

README.md

@@ -323,12 +323,13 @@ x-okapi-token: [JWT_TOKEN]
 
 {
   "recreateIndex": true,
-  "resourceName": "instance"
+  "resourceName": "authority"
 }
 ```
 
-* `resourceName` parameter is optional and equal to `instance` by default. Possible values: `instance`, `authority`, `location`,
+* `resourceName` parameter is required. Possible values: `authority`, `location`,
   `linked-data-instance`, `linked-data-work`, `linked-data-hub`. Please note that `location` reindex is synchronous.
+  Refer to [Indexing of Instance Records](#indexing-of-instance-records) section for reindexing of instances
 * `recreateIndex` parameter is optional and equal to `false` by default. If it is equal to `true` then mod-search
   will drop existing indices for tenant and resource, creating them again. Executing request with this parameter
   equal to `true` in query will erase all the tenant data in mod-search.
@@ -341,13 +342,13 @@ There is no end-to-end monitoring implemented yet, however it is possible to mon
 how many records published to Kafka topic use inventory API:
 
 ```http
-GET [OKAPI_URL]/instance-storage/reindex/[reindex job id]
+GET [OKAPI_URL]/authority-storage/reindex/[reindex job id]
 ```
 
 _reindex job id_ - id returned by `/search/index/inventory/reindex` endpoint.
 
 In order to estimate total records that actually added to the index, you can send a "match all" search query and check
-`totalRecords`, e.g. `GET /search/instances?query=id="*"`. Alternatively you can query Elasticsearch directly,
+`totalRecords`, e.g. `GET /search/authorities?query=id="*"`. Alternatively you can query Elasticsearch directly,
 see [ES search API](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-match-all-query.html#query-dsl-match-all-query).
 
 ## Indexing of Instance Records

descriptors/ModuleDescriptor-template.json

@@ -4,7 +4,7 @@
   "provides": [
     {
       "id": "indices",
-      "version": "0.8",
+      "version": "1.0",
       "handlers": [
         {
           "methods": [

src/main/java/org/folio/search/controller/IndexManagementController.java

@@ -6,6 +6,7 @@
 import org.folio.search.domain.dto.CreateIndexRequest;
 import org.folio.search.domain.dto.FolioCreateIndexResponse;
 import org.folio.search.domain.dto.FolioIndexOperationResponse;
+import org.folio.search.domain.dto.IndexSettings;
 import org.folio.search.domain.dto.ReindexJob;
 import org.folio.search.domain.dto.ReindexRequest;
 import org.folio.search.domain.dto.ReindexStatusItem;
@@ -50,15 +51,15 @@ public ResponseEntity<FolioIndexOperationResponse> indexRecords(List<ResourceEve
   }
 
   @Override
-  public ResponseEntity<Void> reindexInstanceRecords(String tenantId) {
+  public ResponseEntity<Void> reindexInstanceRecords(String tenantId, IndexSettings indexSettings) {
     log.info("Attempting to run full-reindex for instance records [tenant: {}]", tenantId);
-    reindexService.submitFullReindex(tenantId);
+    reindexService.submitFullReindex(tenantId, indexSettings);
     return ResponseEntity.ok().build();
   }
 
   @Override
   public ResponseEntity<Void> reindexUploadInstanceRecords(String tenantId, ReindexUploadDto reindexUploadDto) {
-    reindexService.submitUploadReindex(tenantId, reindexUploadDto.getEntityTypes());
+    reindexService.submitUploadReindex(tenantId, reindexUploadDto);
     return ResponseEntity.ok().build();
   }
 

src/main/java/org/folio/search/service/IndexService.java

@@ -271,9 +271,7 @@ private void updateRefreshInterval(ObjectNode settings, Integer refreshInt) {
   }
 
   private static ResourceType getReindexRequestResourceType(ReindexRequest req) {
-    return req == null || req.getResourceName() == null
-           ? ResourceType.INSTANCE
-           : ResourceType.byName(req.getResourceName().getValue());
+    return ResourceType.byName(req.getResourceName().getValue());
   }
 
   private void validateResourceName(ResourceType resourceName, String message) {

src/main/java/org/folio/search/service/SearchTenantService.java

@@ -9,9 +9,11 @@
 import org.folio.search.configuration.properties.SearchConfigurationProperties;
 import org.folio.search.domain.dto.LanguageConfig;
 import org.folio.search.domain.dto.ReindexRequest;
+import org.folio.search.model.types.ReindexEntityType;
 import org.folio.search.service.browse.CallNumberBrowseRangeService;
 import org.folio.search.service.consortium.LanguageConfigServiceDecorator;
 import org.folio.search.service.metadata.ResourceDescriptionService;
+import org.folio.search.service.reindex.ReindexService;
 import org.folio.spring.FolioExecutionContext;
 import org.folio.spring.liquibase.FolioSpringLiquibase;
 import org.folio.spring.service.PrepareSystemUserService;
@@ -32,6 +34,7 @@ public class SearchTenantService extends TenantService {
   private static final String CENTRAL_TENANT_ID_PARAM_NAME = "centralTenantId";
 
   private final IndexService indexService;
+  private final ReindexService reindexService;
   private final KafkaAdminService kafkaAdminService;
   private final PrepareSystemUserService prepareSystemUserService;
   private final LanguageConfigServiceDecorator languageConfigService;
@@ -41,7 +44,7 @@ public class SearchTenantService extends TenantService {
 
   public SearchTenantService(JdbcTemplate jdbcTemplate, FolioExecutionContext context,
                              FolioSpringLiquibase folioSpringLiquibase, KafkaAdminService kafkaAdminService,
-                             IndexService indexService,
+                             IndexService indexService, ReindexService reindexService,
                              PrepareSystemUserService prepareSystemUserService,
                              LanguageConfigServiceDecorator languageConfigService,
                              CallNumberBrowseRangeService callNumberBrowseRangeService,
@@ -50,6 +53,7 @@ public SearchTenantService(JdbcTemplate jdbcTemplate, FolioExecutionContext cont
     super(jdbcTemplate, context, folioSpringLiquibase);
     this.kafkaAdminService = kafkaAdminService;
     this.indexService = indexService;
+    this.reindexService = reindexService;
     this.prepareSystemUserService = prepareSystemUserService;
     this.languageConfigService = languageConfigService;
     this.callNumberBrowseRangeService = callNumberBrowseRangeService;
@@ -167,7 +171,12 @@ private void createIndexesAndReindex(TenantAttributes tenantAttributes) {
       .filter(parameter -> parameter.getKey().equals(REINDEX_PARAM_NAME) && parseBoolean(parameter.getValue()))
       .findFirst()
       .ifPresent(parameter -> resourceNames.forEach(resource -> {
-        if (resourceDescriptionService.get(resource).isReindexSupported()) {
+        if (!resourceDescriptionService.get(resource).isReindexSupported()) {
+          return;
+        }
+        if (resource.getName().equals(ReindexEntityType.INSTANCE.getType())) {
+          reindexService.submitFullReindex(context.getTenantId(), null);
+        } else {
           indexService.reindexInventory(context.getTenantId(),
             new ReindexRequest().resourceName(ReindexRequest.ResourceNameEnum.fromValue(resource.getName())));
         }

src/main/java/org/folio/search/service/reindex/ReindexCommonService.java

@@ -7,6 +7,7 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 import lombok.extern.log4j.Log4j2;
+import org.folio.search.domain.dto.IndexSettings;
 import org.folio.search.model.types.ReindexEntityType;
 import org.folio.search.service.IndexService;
 import org.folio.search.service.reindex.jdbc.ReindexJdbcRepository;
@@ -33,11 +34,11 @@ public void deleteAllRecords() {
     }
   }
 
-  public void recreateIndex(ReindexEntityType reindexEntityType, String tenantId) {
+  public void recreateIndex(ReindexEntityType reindexEntityType, String tenantId, IndexSettings indexSettings) {
     try {
       var resourceType = RESOURCE_NAME_MAP.get(reindexEntityType);
       indexService.dropIndex(resourceType, tenantId);
-      indexService.createIndex(resourceType, tenantId);
+      indexService.createIndex(resourceType, tenantId, indexSettings);
     } catch (Exception e) {
       log.warn("Index cannot be recreated for resource={}, message={}", reindexEntityType, e.getMessage());
     }

src/main/java/org/folio/search/service/reindex/ReindexOrchestrationService.java

@@ -1,15 +1,14 @@
 package org.folio.search.service.reindex;
 
-import java.util.Arrays;
 import java.util.Collection;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.log4j.Log4j2;
 import org.apache.logging.log4j.message.FormattedMessage;
 import org.folio.search.domain.dto.FolioIndexOperationResponse;
-import org.folio.search.domain.dto.ReindexUploadDto;
 import org.folio.search.exception.ReindexException;
 import org.folio.search.model.event.ReindexRangeIndexEvent;
 import org.folio.search.model.event.ReindexRecordsEvent;
+import org.folio.search.model.types.ReindexEntityType;
 import org.folio.search.repository.PrimaryResourceRepository;
 import org.folio.search.service.converter.MultiTenantSearchDocumentConverter;
 import org.folio.spring.FolioExecutionContext;
@@ -64,8 +63,7 @@ public boolean process(ReindexRecordsEvent event) {
         event.getRangeId(), event.getRecordType());
       mergeRangeService.updateFinishDate(entityType, event.getRangeId());
       if (reindexStatusService.isMergeCompleted()) {
-        reindexService.submitUploadReindex(context.getTenantId(),
-          Arrays.asList(ReindexUploadDto.EntityTypesEnum.values()));
+        reindexService.submitUploadReindex(context.getTenantId(), ReindexEntityType.supportUploadTypes());
       }
     }
 

src/main/java/org/folio/search/service/reindex/ReindexService.java

@@ -11,6 +11,7 @@
 import lombok.extern.log4j.Log4j2;
 import org.apache.commons.collections4.CollectionUtils;
 import org.folio.search.converter.ReindexEntityTypeMapper;
+import org.folio.search.domain.dto.IndexSettings;
 import org.folio.search.domain.dto.ReindexUploadDto;
 import org.folio.search.exception.RequestValidationException;
 import org.folio.search.integration.folio.InventoryService;
@@ -62,7 +63,7 @@ public ReindexService(ConsortiumTenantService consortiumService,
     this.reindexCommonService = reindexCommonService;
   }
 
-  public CompletableFuture<Void> submitFullReindex(String tenantId) {
+  public CompletableFuture<Void> submitFullReindex(String tenantId, IndexSettings indexSettings) {
     log.info("submitFullReindex:: for [tenantId: {}]", tenantId);
 
     validateTenant(tenantId);
@@ -81,6 +82,7 @@ public CompletableFuture<Void> submitFullReindex(String tenantId) {
       mergeRangeService.saveMergeRanges(rangesForAllTenants);
     }, reindexFullExecutor)
       .thenRun(() -> publishRecordsRange(tenantId))
+      .thenRun(() -> recreateIndices(tenantId, ReindexEntityType.supportUploadTypes(), indexSettings))
       .handle((unused, throwable) -> {
         if (throwable != null) {
           log.error("initFullReindex:: process failed [tenantId: {}, error: {}]", tenantId, throwable);
@@ -94,20 +96,34 @@ public CompletableFuture<Void> submitFullReindex(String tenantId) {
   }
 
   public CompletableFuture<Void> submitUploadReindex(String tenantId,
-                                                     List<ReindexUploadDto.EntityTypesEnum> entityTypes) {
-    log.info("submitUploadReindex:: for [tenantId: {}, entities: {}]", tenantId, entityTypes);
-    var reindexEntityTypes = entityTypeMapper.convert(entityTypes)
+                                                     ReindexUploadDto reindexUploadDto) {
+    var entityTypes = entityTypeMapper.convert(reindexUploadDto.getEntityTypes())
       .stream().filter(ReindexEntityType::isSupportsUpload).toList();
+    return submitUploadReindex(tenantId, entityTypes, true, reindexUploadDto.getIndexSettings());
+  }
+
+  public CompletableFuture<Void> submitUploadReindex(String tenantId,
+                                                     List<ReindexEntityType> entityTypes) {
+    return submitUploadReindex(tenantId, entityTypes, false, null);
+  }
 
-    validateUploadReindex(tenantId, reindexEntityTypes);
+  private CompletableFuture<Void> submitUploadReindex(String tenantId,
+                                                     List<ReindexEntityType> entityTypes,
+                                                     boolean recreateIndex,
+                                                     IndexSettings indexSettings) {
+    log.info("submitUploadReindex:: for [tenantId: {}, entities: {}]", tenantId, entityTypes);
 
-    for (var reindexEntityType : reindexEntityTypes) {
+    validateUploadReindex(tenantId, entityTypes);
+
+    for (var reindexEntityType : entityTypes) {
       statusService.recreateUploadStatusRecord(reindexEntityType);
-      reindexCommonService.recreateIndex(reindexEntityType, tenantId);
+      if (recreateIndex) {
+        reindexCommonService.recreateIndex(reindexEntityType, tenantId, indexSettings);
+      }
     }
 
     var futures = new ArrayList<>();
-    for (var entityType : reindexEntityTypes) {
+    for (var entityType : entityTypes) {
       var future = CompletableFuture.runAsync(() ->
            uploadRangeService.prepareAndSendIndexRanges(entityType), reindexUploadExecutor)
         .handle((unused, throwable) -> {
@@ -124,6 +140,12 @@ public CompletableFuture<Void> submitUploadReindex(String tenantId,
     return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]));
   }
 
+  private void recreateIndices(String tenantId, List<ReindexEntityType> entityTypes, IndexSettings indexSettings) {
+    for (var reindexEntityType : entityTypes) {
+      reindexCommonService.recreateIndex(reindexEntityType, tenantId, indexSettings);
+    }
+  }
+
   private List<MergeRangeEntity> processForConsortium(String tenantId) {
     List<MergeRangeEntity> mergeRangeEntities = new ArrayList<>();
     var memberTenants = consortiumService.getConsortiumTenants(tenantId);

src/main/resources/swagger.api/examples/request/indexSettings.yaml

@@ -0,0 +1,4 @@
+value:
+  numberOfShards: 1
+  numberOfReplicas: 1
+  refreshInterval: 1

src/main/resources/swagger.api/examples/request/reindexRequest.yaml

@@ -1,3 +1,3 @@
 value:
   recreateIndex: true
-  resourceName: instance
+  resourceName: authority

src/main/resources/swagger.api/paths/reindex-instance-records/reindex-instance-records-full.yaml

@@ -4,6 +4,14 @@ post:
   description: Initiates the full reindex for the inventory instance records
   tags:
     - index-management
+  requestBody:
+    content:
+      application/json:
+        examples:
+          indexSettings:
+            $ref: '../../examples/request/indexSettings.yaml'
+        schema:
+          $ref: '../../schemas/entity/indexSettings.yaml'
   parameters:
     - $ref: '../../parameters/x-okapi-tenant-header.yaml'
   responses:

src/main/resources/swagger.api/paths/search-index/search-index-inventory-reindex.yaml

@@ -5,6 +5,7 @@ post:
   tags:
     - index-management
   requestBody:
+    required: true
     content:
       application/json:
         examples:

src/main/resources/swagger.api/schemas/request/reindexRequest.yaml

@@ -8,9 +8,7 @@ properties:
   resourceName:
     type: string
     description: Resource name to run reindex for
-    default: instance
     enum:
-      - instance
       - authority
       - location
       - linked-data-instance
@@ -19,3 +17,5 @@ properties:
   indexSettings:
     description: Index settings to apply for index
     $ref: "../../schemas/entity/indexSettings.yaml"
+required:
+  - resourceName

src/main/resources/swagger.api/schemas/request/reindexUploadDto.yaml

@@ -11,5 +11,8 @@ properties:
         - contributor
         - classification
     minItems: 1
+  indexSettings:
+    description: Index settings to apply for index
+    $ref: '../entity/indexSettings.yaml'
 required:
   - entityTypes