kit-data-manager · VolkerHartmann · Dec 18, 2023 · Jan 30, 2023 · Feb 13, 2023 · Feb 13, 2023
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -38,11 +38,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -53,7 +53,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -67,4 +67,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3
diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml
@@ -5,14 +5,15 @@ on:
     # Publish `master` as Docker `latest` image.
     branches:
       - main
-      - master
 
     # Publish `v1.2.3` tags as releases.
     tags:
       - v*
 
-  # Run tests for any PRs.
+  # Run tests for PRs on main branch only.
   pull_request:
+    branches:
+      - main
 
 env:
   # TODO: Change variable to your image's name.
@@ -28,9 +29,9 @@ jobs:
 
     steps:
     - name: Checkout repo
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - name: Set up OpenJDK version ...
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: 'zulu'
         java-version: ${{ env.currentBuildVersion }}
@@ -58,7 +59,7 @@ jobs:
 
     steps:
     - name: Checkout repo
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Build image
       run: |
@@ -83,26 +84,26 @@ jobs:
         docker push $IMAGE_ID:$VERSION
     - name: Docker meta
       id: docker_meta
-      uses: crazy-max/ghaction-docker-meta@v4
+      uses: crazy-max/ghaction-docker-meta@v5
       with:
         # list of Docker images to use as base name for tags
         images: |
           kitdm/indexing-service
         # add git short SHA as Docker tag
         tag-sha: true
     - name: Set up QEMU
-      uses: docker/setup-qemu-action@v2
+      uses: docker/setup-qemu-action@v3
 
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2
+      uses: docker/setup-buildx-action@v3
     - name: Login to DockerHub
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       with:
         username: ${{ secrets.DOCKER_USERNAME }}
         password: ${{ secrets.DOCKER_PASSWORD }}
 
     - name: Push to Docker Hub
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         context: .
         push: true

diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml
@@ -23,9 +23,9 @@ jobs:
         jdk: [ 17, 19 ]
     steps:
     - name: Checkout repo
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - name: Set up OpenJDK version ...
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: 'zulu' 
         java-version: ${{ matrix.jdk }}
@@ -57,9 +57,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout repo
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - name: Set up OpenJDK version ...
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: 'zulu' 
         java-version: ${{ env.currentBuildVersion }}

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,9 +9,46 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+### Changed
+
+## [1.0.0] - date 2023-12-18
+### Added
+- Enable authentication for RabbitMQ 
+- Add interface description
+
 ### Changed
 - The mappings directory is now a subdirectory of the installation directory by default.
+- Bump actions/checkout from 3 to 4
+- Bump actions/setup-java from 3 to 4
+- Bump crazy-max/ghaction-docker-meta from 4 to 5
+- Bump docker/build-push-action from 4 to 5
+- Bump docker/setup-buildx-action from 2 to 3
+- Bump docker/setup-qemu-action from 2 to 3
+- Bump github/codeql-action from 2 to 3
+- Bump com.google.guava:guava from 32.1.1-jre to 32.1.3-jre
+- Bump com.h2database:h2 from 2.2.220 to 2.2.224
+- Bump de.codecentric:spring-boot-admin-starter-client from 3.0.4 to 3.1.8
+- Bump edu.kit.datamanager:generic-message-consumer from 1.0.0 to 1.1.1
+- Bump io.freefair.lombok from 8.1.0 to 8.4
+- Bump io.freefair.maven-publish-java from 8.1.0 to 8.4
+- Bump io.spring.dependency-management from 1.1.2 to 1.1.4
+- Bump javersVersion from 7.3.0 to 7.3.6
+- Bump org.apache.tika:tika-core from 2.8.0 to 2.9.1
+- Bump org.json:json from 20230618 to 20231013
+- Bump org.mockito:mockito-core from 5.4.0 to 5.8.0
+- Bump org.owasp.dependencycheck from 8.3.1 to 9.0.6
+- Bump org.postgresql:postgresql from 42.6.0 to 42.7.1
+- Bump org.springframework.boot from 3.1.2 to 3.2.0
+- Bump org.springframework:spring-messaging from 6.0.2 to 6.1.2
+- Bump org.springframework.cloud:spring-cloud-gateway-mvc from 4.0.6 to 4.1.0
+- Bump org.springframework.data:spring-data-elasticsearch from 5.1.2 to 5.2.0
+- Bump org.springframework.restdocs:spring-restdocs-mockmvc from 3.0.0 to 3.0.1
+- Bump springDocVersion from 2.1.0 to 2.3.0
+- Bump elasticsearch instance for test from 7.9.3 to 8.11.1
 
+### Fixed
+- Settings for health check.
+
 ## [0.9.0] - date 2023-07-25
 Update service to Spring Boot 3.
 ### Changed
@@ -98,7 +135,8 @@ and mapping of metadata documents delivered by RabbitMQ
 - Mapping of metadata documents with Gemma
 - Ingest to elasticsearch
 
-[Unreleased]: https://github.com/kit-data-manager/indexing-service/compare/v0.9.0...HEAD
+[Unreleased]: https://github.com/kit-data-manager/indexing-service/compare/v1.0.0...HEAD
+[1.0.0]: https://github.com/kit-data-manager/indexing-service/compare/v0.9.0...v1.0.0
 [0.9.0]: https://github.com/kit-data-manager/indexing-service/compare/v0.1.3...v0.9.0
 [0.1.3]: https://github.com/kit-data-manager/indexing-service/compare/v0.1.2...v0.1.3
 [0.1.2]: https://github.com/kit-data-manager/indexing-service/compare/v0.1.1...v0.1.2

diff --git a/INTERFACE.md b/INTERFACE.md
@@ -0,0 +1,107 @@
+# Interfaces Overview (Indexing-Service)
+
+This document aims to answer questions on how to configure external dependencies and which public interfaces are offered by Indexing-Service in a comprehensive way. It is meant to be used for getting an overview and guidance in addition to the official documentation, which is available at the official [Indexing-Service Web page (not yet available)](https://kit-data-manager.github.io/webpage/indexing-service/).How can this software be connected with other software? This is the question this document aims to answer.
+
+> ℹ️ **Note:** 
+> This document applies to the Indexing-Service version it is shipped with. If you have a specific version running, please refer to `INTERFACE.md` of this particular release.
+
+## TOC
+
+- [Interfaces Overview for Indexing-Service](#interfaces-overview-for-indexing-service)
+   * [External Dependencies](#external-dependencies-) 📤
+      + [Relational Database (mandatory)](#relational-database-mandatory-) ⛁
+      + [Local Filesystem (mandatory)](#local-filesystem-mandatory-) 📂
+      + [Messaging (mandatory)](#messaging-mandatory-) 💬
+      + [Search (mandatory)](#search-mandatory-) 🔍
+      + [Access Control (optional)](#access-control-optional-) 🔐
+   * [Public Interfaces](#public-interfaces-) 📥
+      + [HTTP / REST](#http--rest)
+      + [Monitoring Health/Info](#monitoring-healthinfo) 🔍
+
+## Requirements
+- [Messaging](#messaging-optional-)
+- [Elasticsearch](https://kit-data-manager.github.io/webpage/metastore/documentation/installation/framework/setup-elasticsearch.html)
+Messaging is used to inform indexing service about any updates regarding metadata documents. Indexing service will fetch the document, transform it (if mapping available) and send it to elasticsearch for indexing.
+
+## External Dependencies 📤
+
+External dependencies are third-party services that are required for Indexing-Service to work properly or that can be added optionally to provide additional functionality. Typically, external dependencies require additional software to be installed and configured, before they can be included in the Indexing-Service configuration, which is typically done via the main configuration file `application.properties`. If you do not want to lose your default settings, we recommend that you make a copy of "application.properties" and move it to the "config" subfolder. Remove all properties you want to keep from the new file. **All properties in "config/application.properties" override the settings in "application.properties".**
+
+
+## External Dependencies 📤
+
+External dependencies are third-party services that are required for Indexing-Service to work properly or that can be added optionally to provide additional functionality. Typically, external dependencies require
+additional software to be installed and configured, before they can be included in the Indexing-Service configuration, which is should be done via the  configuration file `config/application.properties`.
+
+### Relational Database (mandatory) ⛁
+A relational database is required by Indexing-Service to store administrative metadata for mapping documents. If not configured properly, Indexing-Service will fail to start.
+
+#### Configuration ⚙️
+ - H2 In-Memory (driver included, used for **testing only**, **not** recommended for **production**) [Example](https://github.com/kit-data-manager/indexing-service/blob/v0.9.0/src/test/resources/test-config/application-test.properties#L14-L18)
+ - H2 File-Based (driver included, used for basic Docker setup or for 'quick and dirty' tests, **not** recommended for **production**) [Example](https://github.com/kit-data-manager/indexing-service/blob/v0.9.0/settings/application-docker.properties#L26-L33)
+ - Postgres (driver included, **requires a running PostgreSQL server**, **recommended for production**) [PostgreSQL](https://www.postgresql.org/), [Example](https://github.com/kit-data-manager/indexing-service/blob/v0.9.0/settings/application-postgres.properties#L34-L47)
+
+> ℹ️ **Note:** 
+> Other relational databases, such as MariaDB, SQLite, or Oracle, may also work, but require additional steps. To allow Indexing-Service to connect, the source code repository must be cloned, an appropriate JDBC driver has to be added to `build.gradle` and Indexing-Service has to be compiled. Proper JDBC drivers are typically provided on the database's web page. Afterwards, the database can be configured in `config/application.properties` similar to PostgreSQL but with database-specific property naming. Please refer to the driver documentation for details.
+
+### Local Filesystem (mandatory) 📂
+Indexing-Service requires access to the local file system in order to store and manage uploaded mapping documents. Indexing-Service requires access to one folder, which can be located on the local hard drive or mounted via NFS. It's recommended to create a subfolder named 'mapping' in the installation directory.
+
+
+#### Configuration ⚙️
+ - see `application.properties` 
+     - [Configure folders](https://github.com/kit-data-manager/indexing-service/blob/v0.9.0/settings/application-default.properties#L71-L74)
+
+> ℹ️ **Note:** 
+> The file path to the folder has to start with three '/'. If you overwrite the default setting we recommend to create or edit 'config/application.properties'. e.g.: 
+> ``` title= config/application.properties
+> metastore.indexer.mappingsLocation:file:///data/indexing/mapping
+>```
+> **If the folder do not exist, it will be created.**
+
+### Messaging (mandatory) 💬
+AMQP-based messaging is a mandatory feature of Indexing-Service, which allows Indexing-Service to **receive** messages about creation, modification, and deletion events inside MetaStore related to **metadata documents**  and process them in an asynchronous way.
+#### Installation
+- [Installing RabbitMQ](https://kit-data-manager.github.io/webpage/metastore/documentation/installation/framework/setup-rabbitMq.html)
+#### Configuration ⚙️
+ - [RabbitMQ](https://www.rabbitmq.com/) (dependencies included, serves as messaging distributor, requires a running RabbitMQ server) 
+     - [Introduction Messaging for MetaStore](https://kit-data-manager.github.io/webpage/metastore/documentation/installation/messaging/messaging-introduction.html)
+     - [Configuration Messaging for Indexing-Service](https://kit-data-manager.github.io/webpage/metastore/documentation/installation/messaging/messaging-configuration.html)
+     - [Example](https://kit-data-manager.github.io/webpage/metastore/documentation/installation/setup-metastore.html#rabbitmq) 
+
+### Enhanced Search (mandatory) 🔍
+Indexing-Service requires an [elasticsearch instance](https://www.elastic.co/de/elasticsearch/) for indexing received metadata documents. (JSON metadata documents are provided by default. XML documents should be transformed to JSON using GEMMA (currently the only available mapping)) (In future versions [mapping-service](https://github.com/kit-data-manager/mapping-service) should be integrated.)
+#### Configuration ⚙️
+- [Configuration Receiver](https://kit-data-manager.github.io/webpage/metastore/documentation/installation/setup-metastore.html#elasticsearch)
+
+### Access Control (optional) 🔐
+By default, Indexing-Service itself is open for all kinds of operations, i.e., read and write, where write access should be restricted on the user interface level, e.g., by a password-protected area for critical operations. Optionally, authentication and authorization via JSON Web Tokens (JWT) issued by a Keycloak instance, can be configured, which allows a fine-granulated access management on document level.
+> ℹ️ **Note:** 
+> As Indexing-Service is not planned to be used by humans, no access control is used by default. Therefor the service should not be accessible from outside KIT. Managing mappings should be done by an administrator.
+
+#### Requirements
+- [Keycloak](https://www.keycloak.org/)
+#### Configuration ⚙️
+- [Setup Indexing-Service](https://kit-data-manager.github.io/webpage/metastore/documentation/installation/setup-metastore.html#keycloak)
+Configuration is similar to MetaStore.
+
+## Public Interfaces 📥
+Public Interfaces are used to access Indexing-Service in order to obtain its contents, typically this happens via HTTP/REST. Depending on the interface, special clients or protocols must be used to access a specific public interface.
+
+
+Interfaces which trigger functionality or return information **on request**.
+
+### HTTP / REST
+A REST interface which allows to access the service functionality, like creating/registering, updating and validating metadata documents/schemas on request.
+#### Documentation 📖
+ - [OpenAPI](http://localhost:8050/swagger-ui.html) via running instance.
+ - [Usage with Examples](https://github.com/kit-data-manager/indexing-service/blob/main/restDocu.md)
+
+#### Services known to connect to this interface:
+-/-
+
+### Monitoring Health/Info
+A REST interface which allows to access the status of the service:
+1. *hostname:port/contextpath*/actuator/info (information about version)
+2. *hostname:port/contextpath*/actuator/health (information about (connected) services)
+
diff --git a/build.gradle b/build.gradle
@@ -1,9 +1,9 @@
 plugins { 
-    id "org.springframework.boot" version "3.1.2"
-    id "io.spring.dependency-management" version "1.1.2" 
-    id "io.freefair.lombok" version "8.1.0"
-    id "io.freefair.maven-publish-java" version "8.1.0"
-    id "org.owasp.dependencycheck" version "8.3.1"
+    id "org.springframework.boot" version "3.2.0"
+    id "io.spring.dependency-management" version "1.1.4" 
+    id "io.freefair.lombok" version "8.4"
+    id "io.freefair.maven-publish-java" version "8.4"
+    id "org.owasp.dependencycheck" version "9.0.6"
     id 'org.asciidoctor.jvm.convert' version '3.3.2'
     id 'net.researchgate.release' version '3.0.2'
     id "com.gorylenko.gradle-git-properties" version "2.4.1"
@@ -13,8 +13,8 @@ plugins {
 
 ext {
   // versions of dependencies
-  javersVersion     = '7.3.0'
-  springDocVersion  = '2.1.0'
+  javersVersion     = '7.3.6'
+  springDocVersion  = '2.3.0'
 }
 
 description = 'Generic indexing service supporting different mapping implementations.'
@@ -51,8 +51,8 @@ if (System.getProperty('profile') == 'minimal') {
 
 dependencies {
     // Spring
-    implementation 'org.springframework:spring-messaging:6.0.2'
-    implementation 'org.springframework.cloud:spring-cloud-gateway-mvc:4.0.6'
+    implementation 'org.springframework:spring-messaging:6.1.2'
+    implementation 'org.springframework.cloud:spring-cloud-gateway-mvc:4.1.0'
 
     // Spring Boot
     // boot starter
@@ -63,49 +63,49 @@ dependencies {
     implementation "org.springframework.boot:spring-boot-starter-security"
     implementation "org.springframework.boot:spring-boot-starter-actuator"
     implementation "org.springframework.boot:spring-boot-starter-data-jpa"
-    implementation 'org.springframework.data:spring-data-elasticsearch:5.1.2'
+    implementation 'org.springframework.data:spring-data-elasticsearch:5.2.1'
 
     // springdoc
     implementation "org.springdoc:springdoc-openapi-starter-webmvc-ui:${springDocVersion}"
     implementation "org.springdoc:springdoc-openapi-starter-common:${springDocVersion}"
     implementation "org.springdoc:springdoc-openapi-starter-webmvc-api:${springDocVersion}"
 
     // driver for postgres
-    implementation "org.postgresql:postgresql:42.6.0"
+    implementation "org.postgresql:postgresql:42.7.1"
     //driver for h2
-    implementation "com.h2database:h2:2.2.220"
+    implementation "com.h2database:h2:2.2.224"
 
     //apache
-    implementation "org.apache.tika:tika-core:2.8.0"
+    implementation "org.apache.tika:tika-core:2.9.1"
     implementation "commons-codec:commons-codec:1.16.0"
 
     implementation "com.github.jknack:handlebars:4.3.1"
-    implementation "org.json:json:20230618"
+    implementation "org.json:json:20231013"
 
     // javers
     implementation "org.javers:javers-spring-boot-starter-sql:${javersVersion}"
     implementation "org.javers:javers-core:${javersVersion}"
 
     // datamanager
-    implementation "edu.kit.datamanager:generic-message-consumer:1.0.0"
+    implementation "edu.kit.datamanager:generic-message-consumer:1.1.1"
     implementation "edu.kit.datamanager:repo-core:1.2.1"
     implementation "edu.kit.datamanager:service-base:1.2.0"
 
     // actuator
-    implementation 'de.codecentric:spring-boot-admin-starter-client:3.0.4'
+    implementation 'de.codecentric:spring-boot-admin-starter-client:3.1.8'
 
     runtimeOnly    'org.apache.httpcomponents:httpclient:4.5.14'
 
     // Additional libraries for tests
-    testImplementation "com.google.guava:guava:32.1.1-jre"
+    testImplementation "com.google.guava:guava:32.1.3-jre"
 
     //Java 11 Support 
-    testImplementation "org.mockito:mockito-core:5.4.0"
+    testImplementation "org.mockito:mockito-core:5.8.0"
     testImplementation "junit:junit:4.13.2"
 
      // boot starter
     testImplementation "org.springframework.boot:spring-boot-starter-test"
-    testImplementation 'org.springframework.restdocs:spring-restdocs-mockmvc:3.0.0'
+    testImplementation 'org.springframework.restdocs:spring-restdocs-mockmvc:3.0.1'
     testImplementation "org.springframework.security:spring-security-test"
     testImplementation "org.springframework:spring-test"
 }
@@ -121,7 +121,7 @@ springBoot {
 
 bootJar {
     manifest {
-        attributes 'Main-Class': 'org.springframework.boot.loader.PropertiesLauncher'
+        attributes 'Main-Class': 'org.springframework.boot.loader.launch.PropertiesLauncher'
     }
     dependsOn asciidoctor
     from ("${asciidoctor.outputDir}/html5") {