From f3e35569c840c94969a274452a25e15c2fdc96a9 Mon Sep 17 00:00:00 2001 From: Markus Weigelt Date: Tue, 16 May 2023 17:52:29 +0200 Subject: [PATCH 1/9] Send task action messages, update active mq client --- Dockerfile | 4 ++-- ocrd_lib.sh | 57 ++++++++++++++++++++++++++++++++++++++++++++--- process_images.sh | 4 +++- process_mets.sh | 4 +++- 4 files changed, 62 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index ace6c57..d11d2ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,7 @@ LABEL \ org.opencontainers.image.revision=$VCS_REF \ org.opencontainers.image.created=$BUILD_DATE -ARG KITODO_MQ_CLIENT_VERSION=0.2 +ARG KITODO_MQ_CLIENT_VERSION=0.3 ENV HOME=/ @@ -48,7 +48,7 @@ COPY kitodo-activemq-client-log4j2.properties /opt/kitodo-activemq-client/log4j2 ENV ACTIVEMQ_CLIENT_LOG4J2 /opt/kitodo-activemq-client/log4j2.properties # add ActiveMQ client library -ADD https://github.com/markusweigelt/kitodo-activemq-client/releases/download/${KITODO_MQ_CLIENT_VERSION}/kitodo-activemq-client-${KITODO_MQ_CLIENT_VERSION}.jar /opt/kitodo-activemq-client +ADD https://github.com/slub/kitodo-production-activemq/releases/download/${KITODO_MQ_CLIENT_VERSION}/kitodo-activemq-client-${KITODO_MQ_CLIENT_VERSION}.jar /opt/kitodo-activemq-client ENV ACTIVEMQ_CLIENT /opt/kitodo-activemq-client/kitodo-activemq-client-${KITODO_MQ_CLIENT_VERSION}.jar RUN chmod go+r $ACTIVEMQ_CLIENT diff --git a/ocrd_lib.sh b/ocrd_lib.sh index 713a93f..99ea482 100644 --- a/ocrd_lib.sh +++ b/ocrd_lib.sh @@ -233,13 +233,64 @@ post_process_to_mets() { # perhaps if URL_PREFIX: mm-update -m "$METS_PATH" validate -u $URL_PREFIX } -close_task() { - if test -n "$ACTIVEMQ" -a -n "$ACTIVEMQ_CLIENT" -a -n "$TASK_ID" -a -n "$PROCESS_ID"; then - java -Dlog4j2.configurationFile=$ACTIVEMQ_CLIENT_LOG4J2 -jar "$ACTIVEMQ_CLIENT" "tcp://$ACTIVEMQ?closeAsync=false" "KitodoProduction.FinalizeStep.Queue" $TASK_ID $PROCESS_ID +kitodo_production_task_action() { + ACTION="" + MESSAGE="${2}" + case ${1} in + 1) + ACTION="COMMENT" + ;; + 2) + ACTION="ERROR_OPEN" + ;; + 3) + ACTION="ERROR_CLOSE" + ;; + 4) + ACTION="PROCESS" + ;; + 5) + ACTION="CLOSE" + ;; + *) + echo -n "Unknown action" + ;; + esac + + if test -n "$ACTIVEMQ" -a -n "$ACTIVEMQ_CLIENT" -a -n "$TASK_ID" -a -n "$ACTION"; then + java -Dlog4j2.configurationFile=$ACTIVEMQ_CLIENT_LOG4J2 -jar "$ACTIVEMQ_CLIENT" "tcp://$ACTIVEMQ?closeAsync=false" "TaskActionQueue" $TASK_ID "$MESSAGE" "$ACTION" fi logret # communicate retval 0 } +kitodo_production_task_action_comment() { + if test -n "${1}"; then + kitodo_production_task_action 1 "${1}" + else + logger -p user.info -t $TASK "Could not send task info cause no message was specified" + fi +} + +kitodo_production_task_action_error_open() { + MESSAGE="${1:-Error occured during the OCR processing}" + kitodo_production_task_action 2 "$MESSAGE" +} + +kitodo_production_task_action_error_close() { + MESSAGE="${1:-OCR processing error has been fixed}" + kitodo_production_task_action 3 "$MESSAGE" +} + +kitodo_production_task_action_process() { + MESSAGE="${1:-OCR processing started}" + kitodo_production_task_action 4 "$MESSAGE" +} + +kitodo_production_task_action_close() { + MESSAGE="${1:-OCR processing completed}" + kitodo_production_task_action 5 "$MESSAGE" +} + # exit in async or sync mode close() { if test -n "$ACTIVEMQ" -a -n "$ACTIVEMQ_CLIENT" -a -n "$TASK_ID" -a -n "$PROCESS_ID"; then diff --git a/process_images.sh b/process_images.sh index dd81067..8034880 100755 --- a/process_images.sh +++ b/process_images.sh @@ -88,6 +88,8 @@ init "$@" pre_sync_workdir + kitodo_production_task_action_process + ocrd_exec ocrd_import_workdir ocrd_validate_workflow ocrd_process_workflow post_sync_workdir @@ -96,7 +98,7 @@ init "$@" post_process_to_procdir - close_task + kitodo_production_task_action_close ) |& tee -a $WORKDIR/ocrd.log 2>&1 | logger -p user.info -t $TASK &>/dev/null & # without output redirect, ssh will not close the connection upon exit, cf. #9 diff --git a/process_mets.sh b/process_mets.sh index 965cc26..2056309 100755 --- a/process_mets.sh +++ b/process_mets.sh @@ -82,6 +82,8 @@ init "$@" pre_sync_workdir + kitodo_production_task_action_process + ocrd_exec ocrd_enter_workdir ocrd_validate_workflow ocrd_process_workflow post_sync_workdir @@ -90,7 +92,7 @@ init "$@" post_process_to_mets - close_task + kitodo_production_task_action_close ) |& tee -a $WORKDIR/ocrd.log 2>&1 | logger -p user.info -t $TASK &>/dev/null & # without output redirect, ssh will not close the connection upon exit, cf. #9 From 550f32ee12770b7941d117f2535c65484109d2ec Mon Sep 17 00:00:00 2001 From: Markus Weigelt Date: Tue, 6 Jun 2023 17:45:13 +0200 Subject: [PATCH 2/9] Support legacy queue --- Dockerfile | 3 +++ docker-compose.yml | 1 + ocrd_lib.sh | 10 +++++++--- process_images.sh | 1 + process_mets.sh | 0 startup.sh | 0 6 files changed, 12 insertions(+), 3 deletions(-) mode change 100755 => 100644 process_images.sh mode change 100755 => 100644 process_mets.sh mode change 100755 => 100644 startup.sh diff --git a/Dockerfile b/Dockerfile index d11d2ec..c0ae792 100644 --- a/Dockerfile +++ b/Dockerfile @@ -52,6 +52,9 @@ ADD https://github.com/slub/kitodo-production-activemq/releases/download/${KITOD ENV ACTIVEMQ_CLIENT /opt/kitodo-activemq-client/kitodo-activemq-client-${KITODO_MQ_CLIENT_VERSION}.jar RUN chmod go+r $ACTIVEMQ_CLIENT +# configure ActiveMQ client queue +ENV ACTIVEMQ_CLIENT_QUEUE FinalizeTaskQueue + # workaround for OCR-D/core#983 RUN pip install ocrd # install mets-mods2tei and page-to-alto diff --git a/docker-compose.yml b/docker-compose.yml index d4f10bc..439047d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,6 +17,7 @@ services: - UMASK=${MANAGER_ENV_UMASK} - "CONTROLLER=${CONTROLLER_HOST}:${CONTROLLER_PORT_SSH}" - "ACTIVEMQ=${MQ_HOST}:${MQ_PORT}" + - ACTIVEMQ_CLIENT_QUEUE=TaskActionQueue ports: - ${MANAGER_PORT_SSH}:22 diff --git a/ocrd_lib.sh b/ocrd_lib.sh index 99ea482..9db323a 100644 --- a/ocrd_lib.sh +++ b/ocrd_lib.sh @@ -257,8 +257,12 @@ kitodo_production_task_action() { ;; esac - if test -n "$ACTIVEMQ" -a -n "$ACTIVEMQ_CLIENT" -a -n "$TASK_ID" -a -n "$ACTION"; then - java -Dlog4j2.configurationFile=$ACTIVEMQ_CLIENT_LOG4J2 -jar "$ACTIVEMQ_CLIENT" "tcp://$ACTIVEMQ?closeAsync=false" "TaskActionQueue" $TASK_ID "$MESSAGE" "$ACTION" + if test -n "$ACTIVEMQ" -a "$ACTIVEMQ" != ":" -a -n "$TASK_ID"; then + if test "$ACTIVEMQ_CLIENT_QUEUE" == "TaskActionQueue" ; then + java -Dlog4j2.configurationFile=$ACTIVEMQ_CLIENT_LOG4J2 -jar "$ACTIVEMQ_CLIENT" "tcp://$ACTIVEMQ?closeAsync=false" "$ACTIVEMQ_CLIENT_QUEUE" $TASK_ID "$MESSAGE" "$ACTION" + elif test "$ACTION" == "CLOSE" + java -Dlog4j2.configurationFile=$ACTIVEMQ_CLIENT_LOG4J2 -jar "$ACTIVEMQ_CLIENT" "tcp://$ACTIVEMQ?closeAsync=false" "$ACTIVEMQ_CLIENT_QUEUE" $TASK_ID "$MESSAGE" + fi fi logret # communicate retval 0 } @@ -293,7 +297,7 @@ kitodo_production_task_action_close() { # exit in async or sync mode close() { - if test -n "$ACTIVEMQ" -a -n "$ACTIVEMQ_CLIENT" -a -n "$TASK_ID" -a -n "$PROCESS_ID"; then + if test -n "$ACTIVEMQ" -a "$ACTIVEMQ" != ":" -a -n "$TASK_ID"; then logger -p user.info -t $TASK "ocr_exit in async mode - immediate termination of the script" # prevent any RETVAL from being written yet trap - EXIT diff --git a/process_images.sh b/process_images.sh old mode 100755 new mode 100644 index 8034880..470111e --- a/process_images.sh +++ b/process_images.sh @@ -53,6 +53,7 @@ ENVIRONMENT VARIABLES: CONTROLLER: host name and port of OCR-D Controller for processing ACTIVEMQ: URL of ActiveMQ server for result callback (optional) + ACTIVEMQ_QUEUE: Queue to handle result callback. Choose between default "FinalizeTaskQueue" or "TaskActionQueue" (optional) ACTIVEMQ_CLIENT: path to ActiveMQ client library JAR file (optional) EOF exit;; diff --git a/process_mets.sh b/process_mets.sh old mode 100755 new mode 100644 diff --git a/startup.sh b/startup.sh old mode 100755 new mode 100644 From 6148ffd8e8b3e4678d4a56214d36a9e615832275 Mon Sep 17 00:00:00 2001 From: Markus Weigelt Date: Wed, 7 Jun 2023 15:21:03 +0200 Subject: [PATCH 3/9] Improve permissions --- process_images.sh | 0 process_mets.sh | 0 startup.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 process_images.sh mode change 100644 => 100755 process_mets.sh mode change 100644 => 100755 startup.sh diff --git a/process_images.sh b/process_images.sh old mode 100644 new mode 100755 diff --git a/process_mets.sh b/process_mets.sh old mode 100644 new mode 100755 diff --git a/startup.sh b/startup.sh old mode 100644 new mode 100755 From fbdd76f5268b63ff0ee9e04dce84c674fed927cd Mon Sep 17 00:00:00 2001 From: Markus Weigelt Date: Wed, 7 Jun 2023 17:13:01 +0200 Subject: [PATCH 4/9] Add env var to config queue and fix implementation --- .env.example | 1 + docker-compose.yml | 2 +- ocrd_lib.sh | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index 06bb568..73edd0a 100644 --- a/.env.example +++ b/.env.example @@ -15,3 +15,4 @@ MANAGER_DATA=~/ MQ_HOST=kitodo-mq MQ_PORT=61616 +MQ_CLIENT_QUEUE=FinalizeTaskQueue \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 439047d..8ae206b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,7 +17,7 @@ services: - UMASK=${MANAGER_ENV_UMASK} - "CONTROLLER=${CONTROLLER_HOST}:${CONTROLLER_PORT_SSH}" - "ACTIVEMQ=${MQ_HOST}:${MQ_PORT}" - - ACTIVEMQ_CLIENT_QUEUE=TaskActionQueue + - ACTIVEMQ_CLIENT_QUEUE=${MQ_CLIENT_QUEUE} ports: - ${MANAGER_PORT_SSH}:22 diff --git a/ocrd_lib.sh b/ocrd_lib.sh index 9db323a..ecc6f93 100644 --- a/ocrd_lib.sh +++ b/ocrd_lib.sh @@ -258,9 +258,9 @@ kitodo_production_task_action() { esac if test -n "$ACTIVEMQ" -a "$ACTIVEMQ" != ":" -a -n "$TASK_ID"; then - if test "$ACTIVEMQ_CLIENT_QUEUE" == "TaskActionQueue" ; then + if test "$ACTIVEMQ_CLIENT_QUEUE" == "TaskActionQueue"; then java -Dlog4j2.configurationFile=$ACTIVEMQ_CLIENT_LOG4J2 -jar "$ACTIVEMQ_CLIENT" "tcp://$ACTIVEMQ?closeAsync=false" "$ACTIVEMQ_CLIENT_QUEUE" $TASK_ID "$MESSAGE" "$ACTION" - elif test "$ACTION" == "CLOSE" + elif test "$ACTION" == "CLOSE"; then java -Dlog4j2.configurationFile=$ACTIVEMQ_CLIENT_LOG4J2 -jar "$ACTIVEMQ_CLIENT" "tcp://$ACTIVEMQ?closeAsync=false" "$ACTIVEMQ_CLIENT_QUEUE" $TASK_ID "$MESSAGE" fi fi From c807cc7bbdcc7a6ec3ae4c8974d5a5eae57c14c7 Mon Sep 17 00:00:00 2001 From: Markus Weigelt Date: Wed, 7 Jun 2023 17:16:40 +0200 Subject: [PATCH 5/9] Update .env.example --- .env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 73edd0a..f59065b 100644 --- a/.env.example +++ b/.env.example @@ -15,4 +15,4 @@ MANAGER_DATA=~/ MQ_HOST=kitodo-mq MQ_PORT=61616 -MQ_CLIENT_QUEUE=FinalizeTaskQueue \ No newline at end of file +MQ_CLIENT_QUEUE=FinalizeTaskQueue From 6b7ff8f20c702dcfb13616df9f2f2de3ed314539 Mon Sep 17 00:00:00 2001 From: Markus Weigelt Date: Wed, 7 Jun 2023 17:23:37 +0200 Subject: [PATCH 6/9] Add task action for error open to logerr function --- ocrd_lib.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ocrd_lib.sh b/ocrd_lib.sh index ecc6f93..8fa0cd7 100644 --- a/ocrd_lib.sh +++ b/ocrd_lib.sh @@ -8,6 +8,8 @@ TASK=$(basename $0) logerr() { logger -p user.info -t $TASK "terminating with error \$?=$? from ${BASH_COMMAND} on line $(caller)" + + kitodo_production_task_action_error_open } stopbg() { From 02c71b5d26f02b27259b091b9e9b6ce1e097ba1a Mon Sep 17 00:00:00 2001 From: Markus Weigelt Date: Wed, 7 Jun 2023 18:00:59 +0200 Subject: [PATCH 7/9] Update ocrd_lib.sh Co-authored-by: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> --- ocrd_lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocrd_lib.sh b/ocrd_lib.sh index 8fa0cd7..b5bf3f5 100644 --- a/ocrd_lib.sh +++ b/ocrd_lib.sh @@ -259,7 +259,7 @@ kitodo_production_task_action() { ;; esac - if test -n "$ACTIVEMQ" -a "$ACTIVEMQ" != ":" -a -n "$TASK_ID"; then + if test -n "$ACTIVEMQ" -a "$ACTIVEMQ" != ":" -a -n "$TASK_ID" -a -n "$ACTION"; then if test "$ACTIVEMQ_CLIENT_QUEUE" == "TaskActionQueue"; then java -Dlog4j2.configurationFile=$ACTIVEMQ_CLIENT_LOG4J2 -jar "$ACTIVEMQ_CLIENT" "tcp://$ACTIVEMQ?closeAsync=false" "$ACTIVEMQ_CLIENT_QUEUE" $TASK_ID "$MESSAGE" "$ACTION" elif test "$ACTION" == "CLOSE"; then From b1cf495608ac175c98f11d4ca8fb8b6ede01f3d2 Mon Sep 17 00:00:00 2001 From: Markus Weigelt Date: Wed, 7 Jun 2023 18:01:20 +0200 Subject: [PATCH 8/9] Update process_images.sh Co-authored-by: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> --- process_images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/process_images.sh b/process_images.sh index 470111e..48de135 100755 --- a/process_images.sh +++ b/process_images.sh @@ -53,7 +53,7 @@ ENVIRONMENT VARIABLES: CONTROLLER: host name and port of OCR-D Controller for processing ACTIVEMQ: URL of ActiveMQ server for result callback (optional) - ACTIVEMQ_QUEUE: Queue to handle result callback. Choose between default "FinalizeTaskQueue" or "TaskActionQueue" (optional) + ACTIVEMQ_QUEUE: Protocol type handling result callbacks. Choose between "FinalizeTaskQueue" (default) or "TaskActionQueue" (optional) ACTIVEMQ_CLIENT: path to ActiveMQ client library JAR file (optional) EOF exit;; From b4e2deb9e3329bdd262bd8a47cfe4bad2e2de6ba Mon Sep 17 00:00:00 2001 From: Markus Weigelt Date: Wed, 7 Jun 2023 18:02:01 +0200 Subject: [PATCH 9/9] Change echo to logger --- ocrd_lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocrd_lib.sh b/ocrd_lib.sh index b5bf3f5..a6cf39d 100644 --- a/ocrd_lib.sh +++ b/ocrd_lib.sh @@ -255,7 +255,7 @@ kitodo_production_task_action() { ACTION="CLOSE" ;; *) - echo -n "Unknown action" + logger -p user.error -t $TASK "Unknown task action type" ;; esac