From 6c24c708e075b900b00c4ba5111538c512d14684 Mon Sep 17 00:00:00 2001 From: Duckflip Date: Wed, 15 Jan 2025 09:28:32 +0100 Subject: [PATCH] 1741 : solr auto exec complete # Conflicts: # shanoir-ng-datasets/src/main/java/org/shanoir/ng/solr/service/SolrServiceImpl.java # Conflicts: # shanoir-ng-datasets/src/main/java/org/shanoir/ng/solr/service/SolrServiceImpl.java --- docker-compose-dev.yml | 1 + docker-compose/Dockerfile | 15 +++++-- docker-compose/datasets/entrypoint | 4 +- docker-compose/datasets/wait-for-solr.sh | 20 +++++++++ docker-compose/solr/entrypoint | 18 ++++++++ .../shanoir/ng/ShanoirDatasetIndexation.java | 41 +++++++++++++++++++ .../ng/solr/service/SolrServiceImpl.java | 24 ++++++++++- 7 files changed, 118 insertions(+), 5 deletions(-) create mode 100755 docker-compose/datasets/wait-for-solr.sh create mode 100755 docker-compose/solr/entrypoint create mode 100644 shanoir-ng-datasets/src/main/java/org/shanoir/ng/ShanoirDatasetIndexation.java diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 228837e235..fd6879fcba 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -213,6 +213,7 @@ services: - "database" - "rabbitmq" - "users" + - "solr" preclinical: container_name: "${SHANOIR_PREFIX}preclinical" diff --git a/docker-compose/Dockerfile b/docker-compose/Dockerfile index ca260cba8e..3c1e80cda4 100644 --- a/docker-compose/Dockerfile +++ b/docker-compose/Dockerfile @@ -80,7 +80,6 @@ RUN wget -qO dcm4che-bin.zip \ && unzip /dcm4che-bin.zip && mv dcm4che-* dcm4che \ && rm /dcm4che-bin.zip - FROM base-microservice as datasets RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ @@ -91,6 +90,9 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ locales \ locales-all +RUN apt-get install -y iputils-ping + + # take care of path ENV PATH /opt/dcm4che/bin:$PATH ENV LC_ALL en_US.UTF-8 @@ -99,16 +101,21 @@ ENV LANGUAGE en_US.UTF-8 # install the files from the 'dowloader' stage COPY --link --from=datasets-download /target/. / +COPY --link datasets/shanoir-ng-datasets.jar shanoir-ng-datasets.jar + + COPY --link datasets/entrypoint /usr/bin/ -COPY --link datasets/shanoir-ng-datasets.jar shanoir-ng-datasets.jar +COPY --link datasets/wait-for-solr.sh /usr/bin/ -# Use the below line for remote debugging + # Use the below line for remote debugging #CMD ["-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=0.0.0.0:9914", "-jar", "/shanoir-ng-datasets.jar"] CMD ["-jar", "/shanoir-ng-datasets.jar"] + + ################ import #################################################### FROM base-microservice as import @@ -387,6 +394,8 @@ RUN mkdir -p /etc/shanoir-core-template \ USER solr COPY --link solr/core /etc/shanoir-core-template/ +COPY --link solr/entrypoint /usr/bin/ +ENTRYPOINT ["/usr/bin/entrypoint"] CMD ["solr-precreate", "shanoir", "/etc/shanoir-core-template"] diff --git a/docker-compose/datasets/entrypoint b/docker-compose/datasets/entrypoint index 262a1fdea8..8af8638309 100755 --- a/docker-compose/datasets/entrypoint +++ b/docker-compose/datasets/entrypoint @@ -19,4 +19,6 @@ handle_microservice_migration abort_if_error -run_microservice "$@" +/usr/bin/wait-for-solr.sh + +run_microservice "$@" \ No newline at end of file diff --git a/docker-compose/datasets/wait-for-solr.sh b/docker-compose/datasets/wait-for-solr.sh new file mode 100755 index 0000000000..d69aeb41e2 --- /dev/null +++ b/docker-compose/datasets/wait-for-solr.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +WAIT_TIME=300 + +echo "Waiting for the solr container to be reachable..." + +# Try pinging the solr container +while ! ping -c 1 -W 1 "solr" > /dev/null 2>&1; do + echo "Container solr is not reachable, retrying..." + sleep 5 + + # Decrement wait time and exit if it reaches zero + WAIT_TIME=$((WAIT_TIME - 1)) + if [ "$WAIT_TIME" -le 0 ]; then + echo "Error : Timeout waiting for container solr to become reachable." + exit 1 + fi +done + +echo "Container solr reached, launching container datasets." diff --git a/docker-compose/solr/entrypoint b/docker-compose/solr/entrypoint new file mode 100755 index 0000000000..42a512a279 --- /dev/null +++ b/docker-compose/solr/entrypoint @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e +( + # Compare old and new solr scheme.xml + set -e + cd /etc/shanoir-core-template + for file in $(find . -maxdepth 1 -type f); do + cmp -- "$file" "/var/solr/data/shanoir/$file" + done + echo "Solr index is up-to-date, no rebuild needed" +) || ( + # If different, remove last index + echo 'Deleting /var/solr/data (not up-to-date with the template), the index has to be rebuilt' + rm -rf /var/solr/data +) + +$@ \ No newline at end of file diff --git a/shanoir-ng-datasets/src/main/java/org/shanoir/ng/ShanoirDatasetIndexation.java b/shanoir-ng-datasets/src/main/java/org/shanoir/ng/ShanoirDatasetIndexation.java new file mode 100644 index 0000000000..956718b8d5 --- /dev/null +++ b/shanoir-ng-datasets/src/main/java/org/shanoir/ng/ShanoirDatasetIndexation.java @@ -0,0 +1,41 @@ +package org.shanoir.ng; + +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.shanoir.ng.solr.service.SolrServiceImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Component; + +import java.util.Objects; + + +@Component +@Profile("!test") +public class ShanoirDatasetIndexation implements ApplicationRunner { + + @Autowired + private SolrClient solrClient; + + @Autowired + private SolrServiceImpl solrServiceImpl; + + private static final Logger LOG = LoggerFactory.getLogger(ShanoirDatasetIndexation.class); + + @Override + public void run(ApplicationArguments args) throws Exception { + SolrQuery q = new SolrQuery("*:*"); + q.setRows(0); // don't actually request any data + if(Objects.nonNull(solrClient.query(q)) && Objects.equals(0L, solrClient.query(q).getResults().getNumFound())){ + LOG.info("Solr index empty. Re-indexing..."); + solrServiceImpl.indexAllNoAuth(); + LOG.info("Solr indexation complete."); + } else { + LOG.info("Solr index already complete, no re-indexation required."); + } + } +} diff --git a/shanoir-ng-datasets/src/main/java/org/shanoir/ng/solr/service/SolrServiceImpl.java b/shanoir-ng-datasets/src/main/java/org/shanoir/ng/solr/service/SolrServiceImpl.java index e42d13be64..3c3c403d2e 100644 --- a/shanoir-ng-datasets/src/main/java/org/shanoir/ng/solr/service/SolrServiceImpl.java +++ b/shanoir-ng-datasets/src/main/java/org/shanoir/ng/solr/service/SolrServiceImpl.java @@ -56,7 +56,6 @@ import org.springframework.data.domain.Sort; import org.springframework.data.solr.core.query.result.SolrResultPage; import org.springframework.scheduling.annotation.Async; -import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Isolation; import org.springframework.transaction.annotation.Propagation; @@ -195,6 +194,29 @@ protected ShanoirEvent beginIndexationProcess() { return event; } + @Async + @Transactional + public void indexAllNoAuth() { + List documents = new ArrayList<>(); + Map> tags = new HashMap<>(); + ShanoirEvent event; + + try { + event = new ShanoirEvent( + ShanoirEventType.SOLR_INDEX_ALL_EVENT, + null, + 0L, + "Cleaning Solr index...", + ShanoirEvent.IN_PROGRESS, + 0f); + eventService.publishEvent(event); + cleanOldIndex(event); + fetchDatasToIndex(event, documents, tags); + indexDatas(event, documents, tags); + } catch (SolrServerException | IOException ignored) { + } + } + @Transactional @Override public void indexDatasets(List datasetIds) throws SolrServerException, IOException {