From 0d9bf0edf4f8e0bd3fce1a64e79624da589ac77a Mon Sep 17 00:00:00 2001
From: joschrew <91774427+joschrew@users.noreply.github.com>
Date: Wed, 13 Mar 2024 11:45:26 +0100
Subject: [PATCH 1/8] Add test for ocrd network

---
 Makefile                         |  13 +++
 core                             |   2 +-
 tests/network/docker-compose.yml | 189 +++++++++++++++++++++++++++++++
 3 files changed, 203 insertions(+), 1 deletion(-)
 create mode 100644 tests/network/docker-compose.yml

diff --git a/Makefile b/Makefile
index a935a330..e0382466 100644
--- a/Makefile
+++ b/Makefile
@@ -747,6 +747,19 @@ testcuda test-cuda: $(ACTIVATE_VENV)
 test-workflow: test-assets core $(BIN)/ocrd $(ACTIVATE_VENV)
 	. $(ACTIVATE_VENV) && cd core/tests/assets/SBB0000F29300010000/data/ && bash -x $(CURDIR)/test-workflow.sh
 
+DOCKER_COMPOSE = docker compose
+INTEGRATION_TEST_IN_DOCKER = docker exec core_test
+
+integration-test: test-assets
+	$(DOCKER_COMPOSE) --file core/tests/network/docker-compose.yml --file tests/network/docker-compose.yml up -d
+	docker cp core/tests/assets/kant_aufklaerung_1784/data/. ocrd_network_processing_server:/data
+	# Queues must exists and it takes time until they are created by the workers. We might need
+	# a mechanism to test if all queues are there but that is not available yet. So sleeping
+	# here for now
+	sleep 10
+	-$(INTEGRATION_TEST_IN_DOCKER) pytest -k 'test_ocrd_all_workflow' -v
+	$(DOCKER_COMPOSE) --file core/tests/network/docker-compose.yml --file tests/network/docker-compose.yml down -v --remove-orphans
+
 test-assets:
 	$(MAKE) -C core assets
 
diff --git a/core b/core
index c5b5580b..db322be4 160000
--- a/core
+++ b/core
@@ -1 +1 @@
-Subproject commit c5b5580ba5c517e6ee151a5b3a1d8fe1b3ba0d88
+Subproject commit db322be4d5dd843941fa7a48dcbea0848214dd55
diff --git a/tests/network/docker-compose.yml b/tests/network/docker-compose.yml
new file mode 100644
index 00000000..48e28fc3
--- /dev/null
+++ b/tests/network/docker-compose.yml
@@ -0,0 +1,189 @@
+volumes:
+  ocrd-test-workspace: {}
+
+services:
+  ocrd_network_processing_server:
+    image: ocrd/core
+    volumes:
+      - "ocrd-test-workspace:/data"
+
+  ocrd_network_core_test:
+    image: my-ocrd-core-test
+    volumes:
+      - "${PWD}/tests/network/test-workflow-1.txt:/ocrd-data/assets/ocrd_all-test-workflow.txt"
+
+  ocrd_dummy_processing_worker:
+    image: my-ocrd-core-test
+
+  ocrd-olena-binarize:
+    build:
+      context: ../../../ocrd_olena
+    container_name: ocrd-olena-binarize
+    depends_on:
+      ocrd_network_processing_server:
+        condition: service_healthy
+    networks:
+      - ${DOCKER_OCRD_NETWORK_NAME}
+    environment:
+      - OCRD_NETWORK_LOGS_ROOT_DIR=/ocrd-data/ocrd_network_logs
+      - OCRD_LOGGING_DEBUG=true
+    volumes:
+      - ${OCRD_NETWORK_LOGS_ROOT}:/ocrd-data/ocrd_network_logs
+      - ${OCRD_NETWORK_SOCKETS_ROOT}:/ocrd-data/ocrd_network_sockets
+      - "ocrd-test-workspace:/data"
+    command: ocrd-olena-binarize worker --database ${MONGODB_URL} --queue ${RABBITMQ_URL}
+
+
+  ocrd-anybaseocr-crop:
+    build:
+      context: ../../../ocrd_anybaseocr
+    container_name: ocrd-anybaseocr-crop
+    depends_on:
+      ocrd_network_processing_server:
+        condition: service_healthy
+    networks:
+      - ${DOCKER_OCRD_NETWORK_NAME}
+    environment:
+      - OCRD_NETWORK_LOGS_ROOT_DIR=/ocrd-data/ocrd_network_logs
+      - OCRD_LOGGING_DEBUG=true
+    volumes:
+      - ${OCRD_NETWORK_LOGS_ROOT}:/ocrd-data/ocrd_network_logs
+      - ${OCRD_NETWORK_SOCKETS_ROOT}:/ocrd-data/ocrd_network_sockets
+      - "ocrd-test-workspace:/data"
+    command: ocrd-anybaseocr-crop worker --database ${MONGODB_URL} --queue ${RABBITMQ_URL}
+
+
+  ocrd-cis-ocropy-denoise:
+    build:
+      context: ../../../ocrd_cis
+    container_name: ocrd-cis-ocropy-denoise
+    depends_on:
+      ocrd_network_processing_server:
+        condition: service_healthy
+    networks:
+      - ${DOCKER_OCRD_NETWORK_NAME}
+    environment:
+      - OCRD_NETWORK_LOGS_ROOT_DIR=/ocrd-data/ocrd_network_logs
+      - OCRD_LOGGING_DEBUG=true
+    volumes:
+      - ${OCRD_NETWORK_LOGS_ROOT}:/ocrd-data/ocrd_network_logs
+      - ${OCRD_NETWORK_SOCKETS_ROOT}:/ocrd-data/ocrd_network_sockets
+      - "ocrd-test-workspace:/data"
+    command: ocrd-cis-ocropy-denoise worker --database ${MONGODB_URL} --queue ${RABBITMQ_URL}
+
+
+  ocrd-cis-ocropy-clip:
+    build:
+      context: ../../../ocrd_cis
+    container_name: ocrd-cis-ocropy-clip
+    depends_on:
+      ocrd_network_processing_server:
+        condition: service_healthy
+    networks:
+      - ${DOCKER_OCRD_NETWORK_NAME}
+    environment:
+      - OCRD_NETWORK_LOGS_ROOT_DIR=/ocrd-data/ocrd_network_logs
+      - OCRD_LOGGING_DEBUG=true
+    volumes:
+      - ${OCRD_NETWORK_LOGS_ROOT}:/ocrd-data/ocrd_network_logs
+      - ${OCRD_NETWORK_SOCKETS_ROOT}:/ocrd-data/ocrd_network_sockets
+      - "ocrd-test-workspace:/data"
+    command: ocrd-cis-ocropy-clip worker --database ${MONGODB_URL} --queue ${RABBITMQ_URL}
+
+
+  ocrd-cis-ocropy-segment:
+    build:
+      context: ../../../ocrd_cis
+    container_name: ocrd-cis-ocropy-segment
+    depends_on:
+      ocrd_network_processing_server:
+        condition: service_healthy
+    networks:
+      - ${DOCKER_OCRD_NETWORK_NAME}
+    environment:
+      - OCRD_NETWORK_LOGS_ROOT_DIR=/ocrd-data/ocrd_network_logs
+      - OCRD_LOGGING_DEBUG=true
+    volumes:
+      - ${OCRD_NETWORK_LOGS_ROOT}:/ocrd-data/ocrd_network_logs
+      - ${OCRD_NETWORK_SOCKETS_ROOT}:/ocrd-data/ocrd_network_sockets
+      - "ocrd-test-workspace:/data"
+    command: ocrd-cis-ocropy-segment worker --database ${MONGODB_URL} --queue ${RABBITMQ_URL}
+
+
+  ocrd-cis-ocropy-dewarp:
+    build:
+      context: ../../../ocrd_cis
+    container_name: ocrd-cis-ocropy-dewarp
+    depends_on:
+      ocrd_network_processing_server:
+        condition: service_healthy
+    networks:
+      - ${DOCKER_OCRD_NETWORK_NAME}
+    environment:
+      - OCRD_NETWORK_LOGS_ROOT_DIR=/ocrd-data/ocrd_network_logs
+      - OCRD_LOGGING_DEBUG=true
+    volumes:
+      - ${OCRD_NETWORK_LOGS_ROOT}:/ocrd-data/ocrd_network_logs
+      - ${OCRD_NETWORK_SOCKETS_ROOT}:/ocrd-data/ocrd_network_sockets
+      - "ocrd-test-workspace:/data"
+    command: ocrd-cis-ocropy-dewarp worker --database ${MONGODB_URL} --queue ${RABBITMQ_URL}
+
+
+  ocrd-tesserocr-segment-region:
+    build:
+      context: ../../../ocrd_tesserocr
+    container_name: ocrd-tesserocr-segment-region
+    depends_on:
+      ocrd_network_processing_server:
+        condition: service_healthy
+    networks:
+      - ${DOCKER_OCRD_NETWORK_NAME}
+    environment:
+      - TESSDATA_PREFIX=/usr/local/share/tessdata
+      - OCRD_NETWORK_LOGS_ROOT_DIR=/ocrd-data/ocrd_network_logs
+      - OCRD_LOGGING_DEBUG=true
+    volumes:
+      - ${OCRD_NETWORK_LOGS_ROOT}:/ocrd-data/ocrd_network_logs
+      - ${OCRD_NETWORK_SOCKETS_ROOT}:/ocrd-data/ocrd_network_sockets
+      - "ocrd-test-workspace:/data"
+    command: ocrd-tesserocr-segment-region worker --database ${MONGODB_URL} --queue ${RABBITMQ_URL}
+
+
+  ocrd-tesserocr-recognize:
+    build:
+      context: ../../../ocrd_tesserocr
+    container_name: ocrd-tesserocr-recognize
+    depends_on:
+      ocrd_network_processing_server:
+        condition: service_healthy
+    networks:
+      - ${DOCKER_OCRD_NETWORK_NAME}
+    environment:
+      - TESSDATA_PREFIX=/usr/local/share/tessdata
+      - OCRD_NETWORK_LOGS_ROOT_DIR=/ocrd-data/ocrd_network_logs
+      - OCRD_LOGGING_DEBUG=true
+    volumes:
+      - ${OCRD_NETWORK_LOGS_ROOT}:/ocrd-data/ocrd_network_logs
+      - ${OCRD_NETWORK_SOCKETS_ROOT}:/ocrd-data/ocrd_network_sockets
+      - "ocrd-test-workspace:/data"
+    command: ocrd-tesserocr-recognize worker --database ${MONGODB_URL} --queue ${RABBITMQ_URL}
+
+
+  ocrd-segment-repair:
+    build:
+      context: ../../../ocrd_segment
+    container_name: ocrd-segment-repair
+    depends_on:
+      ocrd_network_processing_server:
+        condition: service_healthy
+    networks:
+      - ${DOCKER_OCRD_NETWORK_NAME}
+    environment:
+      - OCRD_NETWORK_LOGS_ROOT_DIR=/ocrd-data/ocrd_network_logs
+      - OCRD_LOGGING_DEBUG=true
+    volumes:
+      - ${OCRD_NETWORK_LOGS_ROOT}:/ocrd-data/ocrd_network_logs
+      - ${OCRD_NETWORK_SOCKETS_ROOT}:/ocrd-data/ocrd_network_sockets
+      - "ocrd-test-workspace:/data"
+    command: ocrd-segment-repair worker --database ${MONGODB_URL} --queue ${RABBITMQ_URL}
+

From 78ae4fe1ff678b084b6505e848ced5af20e88707 Mon Sep 17 00:00:00 2001
From: joschrew <91774427+joschrew@users.noreply.github.com>
Date: Wed, 13 Mar 2024 11:45:36 +0100
Subject: [PATCH 2/8] Update network test: change core image usage

---
 tests/network/docker-compose.yml | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/tests/network/docker-compose.yml b/tests/network/docker-compose.yml
index 48e28fc3..c3368723 100644
--- a/tests/network/docker-compose.yml
+++ b/tests/network/docker-compose.yml
@@ -3,17 +3,26 @@ volumes:
 
 services:
   ocrd_network_processing_server:
-    image: ocrd/core
+    image: ocrd/core:v2.62.0
+    build: !reset null
     volumes:
       - "ocrd-test-workspace:/data"
 
   ocrd_network_core_test:
-    image: my-ocrd-core-test
+    build:
+      context: ../../../core
+      args:
+        BASE_IMAGE: ubuntu:20.04
+        SKIP_ASSETS: 1
     volumes:
       - "${PWD}/tests/network/test-workflow-1.txt:/ocrd-data/assets/ocrd_all-test-workflow.txt"
 
   ocrd_dummy_processing_worker:
-    image: my-ocrd-core-test
+    build:
+      context: ../../../core
+      args:
+        BASE_IMAGE: ubuntu:20.04
+        SKIP_ASSETS: 1
 
   ocrd-olena-binarize:
     build:

From 4b6bb365ce6689f979cc310a59b37fffde2c6615 Mon Sep 17 00:00:00 2001
From: joschrew <91774427+joschrew@users.noreply.github.com>
Date: Wed, 13 Mar 2024 11:45:40 +0100
Subject: [PATCH 3/8] Use volume from core for ocrd_all docker tests

---
 tests/network/docker-compose.yml | 27 ++++++++++-----------------
 1 file changed, 10 insertions(+), 17 deletions(-)

diff --git a/tests/network/docker-compose.yml b/tests/network/docker-compose.yml
index c3368723..25dbb438 100644
--- a/tests/network/docker-compose.yml
+++ b/tests/network/docker-compose.yml
@@ -1,12 +1,7 @@
-volumes:
-  ocrd-test-workspace: {}
-
 services:
   ocrd_network_processing_server:
-    image: ocrd/core:v2.62.0
+    image: ocrd/core
     build: !reset null
-    volumes:
-      - "ocrd-test-workspace:/data"
 
   ocrd_network_core_test:
     build:
@@ -14,8 +9,6 @@ services:
       args:
         BASE_IMAGE: ubuntu:20.04
         SKIP_ASSETS: 1
-    volumes:
-      - "${PWD}/tests/network/test-workflow-1.txt:/ocrd-data/assets/ocrd_all-test-workflow.txt"
 
   ocrd_dummy_processing_worker:
     build:
@@ -39,7 +32,7 @@ services:
     volumes:
       - ${OCRD_NETWORK_LOGS_ROOT}:/ocrd-data/ocrd_network_logs
       - ${OCRD_NETWORK_SOCKETS_ROOT}:/ocrd-data/ocrd_network_sockets
-      - "ocrd-test-workspace:/data"
+      - ${OCRD_NETWORK_DATA}:/data
     command: ocrd-olena-binarize worker --database ${MONGODB_URL} --queue ${RABBITMQ_URL}
 
 
@@ -58,7 +51,7 @@ services:
     volumes:
       - ${OCRD_NETWORK_LOGS_ROOT}:/ocrd-data/ocrd_network_logs
       - ${OCRD_NETWORK_SOCKETS_ROOT}:/ocrd-data/ocrd_network_sockets
-      - "ocrd-test-workspace:/data"
+      - ${OCRD_NETWORK_DATA}:/data
     command: ocrd-anybaseocr-crop worker --database ${MONGODB_URL} --queue ${RABBITMQ_URL}
 
 
@@ -77,7 +70,7 @@ services:
     volumes:
       - ${OCRD_NETWORK_LOGS_ROOT}:/ocrd-data/ocrd_network_logs
       - ${OCRD_NETWORK_SOCKETS_ROOT}:/ocrd-data/ocrd_network_sockets
-      - "ocrd-test-workspace:/data"
+      - ${OCRD_NETWORK_DATA}:/data
     command: ocrd-cis-ocropy-denoise worker --database ${MONGODB_URL} --queue ${RABBITMQ_URL}
 
 
@@ -96,7 +89,7 @@ services:
     volumes:
       - ${OCRD_NETWORK_LOGS_ROOT}:/ocrd-data/ocrd_network_logs
       - ${OCRD_NETWORK_SOCKETS_ROOT}:/ocrd-data/ocrd_network_sockets
-      - "ocrd-test-workspace:/data"
+      - ${OCRD_NETWORK_DATA}:/data
     command: ocrd-cis-ocropy-clip worker --database ${MONGODB_URL} --queue ${RABBITMQ_URL}
 
 
@@ -115,7 +108,7 @@ services:
     volumes:
       - ${OCRD_NETWORK_LOGS_ROOT}:/ocrd-data/ocrd_network_logs
       - ${OCRD_NETWORK_SOCKETS_ROOT}:/ocrd-data/ocrd_network_sockets
-      - "ocrd-test-workspace:/data"
+      - ${OCRD_NETWORK_DATA}:/data
     command: ocrd-cis-ocropy-segment worker --database ${MONGODB_URL} --queue ${RABBITMQ_URL}
 
 
@@ -134,7 +127,7 @@ services:
     volumes:
       - ${OCRD_NETWORK_LOGS_ROOT}:/ocrd-data/ocrd_network_logs
       - ${OCRD_NETWORK_SOCKETS_ROOT}:/ocrd-data/ocrd_network_sockets
-      - "ocrd-test-workspace:/data"
+      - ${OCRD_NETWORK_DATA}:/data
     command: ocrd-cis-ocropy-dewarp worker --database ${MONGODB_URL} --queue ${RABBITMQ_URL}
 
 
@@ -154,7 +147,7 @@ services:
     volumes:
       - ${OCRD_NETWORK_LOGS_ROOT}:/ocrd-data/ocrd_network_logs
       - ${OCRD_NETWORK_SOCKETS_ROOT}:/ocrd-data/ocrd_network_sockets
-      - "ocrd-test-workspace:/data"
+      - ${OCRD_NETWORK_DATA}:/data
     command: ocrd-tesserocr-segment-region worker --database ${MONGODB_URL} --queue ${RABBITMQ_URL}
 
 
@@ -174,7 +167,7 @@ services:
     volumes:
       - ${OCRD_NETWORK_LOGS_ROOT}:/ocrd-data/ocrd_network_logs
       - ${OCRD_NETWORK_SOCKETS_ROOT}:/ocrd-data/ocrd_network_sockets
-      - "ocrd-test-workspace:/data"
+      - ${OCRD_NETWORK_DATA}:/data
     command: ocrd-tesserocr-recognize worker --database ${MONGODB_URL} --queue ${RABBITMQ_URL}
 
 
@@ -193,6 +186,6 @@ services:
     volumes:
       - ${OCRD_NETWORK_LOGS_ROOT}:/ocrd-data/ocrd_network_logs
       - ${OCRD_NETWORK_SOCKETS_ROOT}:/ocrd-data/ocrd_network_sockets
-      - "ocrd-test-workspace:/data"
+      - ${OCRD_NETWORK_DATA}:/data
     command: ocrd-segment-repair worker --database ${MONGODB_URL} --queue ${RABBITMQ_URL}
 

From 807f983c4bb8f1f9f9bc66958b34a6c0daea6d71 Mon Sep 17 00:00:00 2001
From: joschrew <91774427+joschrew@users.noreply.github.com>
Date: Wed, 13 Mar 2024 11:45:46 +0100
Subject: [PATCH 4/8] Update cis, segment and tesserocr to latest

---
 .gitmodules | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.gitmodules b/.gitmodules
index e63bf021..aecbc18a 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -30,6 +30,9 @@
 [submodule "cor-asv-ann"]
 	path = cor-asv-ann
 	url = https://github.com/ASVLeipzig/cor-asv-ann.git
+[submodule "tesseract"]
+	path = tesseract
+	url = https://github.com/tesseract-ocr/tesseract.git
 [submodule "format-converters"]
 	path = format-converters
 	url = https://github.com/OCR-D/format-converters.git

From 56ee829b9265f78eaa37a989bc40cb6c609e8da5 Mon Sep 17 00:00:00 2001
From: joschrew <91774427+joschrew@users.noreply.github.com>
Date: Wed, 13 Mar 2024 11:45:52 +0100
Subject: [PATCH 5/8] Update ocrd_all-test-docker-compose

Change service overwrite. Use first images from docker hub instead of
self building.
---
 tests/network/docker-compose.yml | 24 +++++++-----------------
 1 file changed, 7 insertions(+), 17 deletions(-)

diff --git a/tests/network/docker-compose.yml b/tests/network/docker-compose.yml
index 25dbb438..439ac9a5 100644
--- a/tests/network/docker-compose.yml
+++ b/tests/network/docker-compose.yml
@@ -1,20 +1,14 @@
 services:
   ocrd_network_processing_server:
-    image: ocrd/core
-    build: !reset null
-
-  ocrd_network_core_test:
+    image: processing-server
     build:
       context: ../../../core
-      args:
-        BASE_IMAGE: ubuntu:20.04
-        SKIP_ASSETS: 1
+      target: ocrd_core_base
 
-  ocrd_dummy_processing_worker:
+  ocrd_network_core_test:
     build:
       context: ../../../core
       args:
-        BASE_IMAGE: ubuntu:20.04
         SKIP_ASSETS: 1
 
   ocrd-olena-binarize:
@@ -94,8 +88,7 @@ services:
 
 
   ocrd-cis-ocropy-segment:
-    build:
-      context: ../../../ocrd_cis
+    image: ocrd/segment
     container_name: ocrd-cis-ocropy-segment
     depends_on:
       ocrd_network_processing_server:
@@ -132,8 +125,7 @@ services:
 
 
   ocrd-tesserocr-segment-region:
-    build:
-      context: ../../../ocrd_tesserocr
+    image: ocrd/tesserocr
     container_name: ocrd-tesserocr-segment-region
     depends_on:
       ocrd_network_processing_server:
@@ -152,8 +144,7 @@ services:
 
 
   ocrd-tesserocr-recognize:
-    build:
-      context: ../../../ocrd_tesserocr
+    image: ocrd/tesserocr
     container_name: ocrd-tesserocr-recognize
     depends_on:
       ocrd_network_processing_server:
@@ -172,8 +163,7 @@ services:
 
 
   ocrd-segment-repair:
-    build:
-      context: ../../../ocrd_segment
+    image: ocrd/segment
     container_name: ocrd-segment-repair
     depends_on:
       ocrd_network_processing_server:

From 0e3e74c568b4c7a7115d02a2cd421ef34332f57e Mon Sep 17 00:00:00 2001
From: joschrew <91774427+joschrew@users.noreply.github.com>
Date: Wed, 13 Mar 2024 11:45:56 +0100
Subject: [PATCH 6/8] Fix cis-image-usage in ocrd_all-test-dockercompose

---
 tests/network/docker-compose.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/network/docker-compose.yml b/tests/network/docker-compose.yml
index 439ac9a5..38f0aab8 100644
--- a/tests/network/docker-compose.yml
+++ b/tests/network/docker-compose.yml
@@ -88,8 +88,8 @@ services:
 
 
   ocrd-cis-ocropy-segment:
-    image: ocrd/segment
-    container_name: ocrd-cis-ocropy-segment
+    build:
+      context: ../../../ocrd_cis
     depends_on:
       ocrd_network_processing_server:
         condition: service_healthy

From 95b1e5f3d567f64984fc753aa9adb0f89101bc78 Mon Sep 17 00:00:00 2001
From: joschrew <91774427+joschrew@users.noreply.github.com>
Date: Fri, 15 Mar 2024 10:09:37 +0100
Subject: [PATCH 7/8] Update network test docker-compose

No longer depends on / extends core's docker-compose. dry vs kiss
---
 Makefile                          |   6 +-
 core                              |   2 +-
 tests/network/.env                |  24 +++++
 tests/network/docker-compose.yml  | 129 ++++++++++++++++++++-----
 tests/network/ocrd_logging.conf   | 150 ++++++++++++++++++++++++++++++
 tests/network/ps_config.yml       |  17 ++++
 tests/network/test-workflow-1.txt |   9 ++
 7 files changed, 308 insertions(+), 29 deletions(-)
 create mode 100644 tests/network/.env
 create mode 100644 tests/network/ocrd_logging.conf
 create mode 100644 tests/network/ps_config.yml
 create mode 100644 tests/network/test-workflow-1.txt

diff --git a/Makefile b/Makefile
index e0382466..1ac51447 100644
--- a/Makefile
+++ b/Makefile
@@ -751,14 +751,14 @@ DOCKER_COMPOSE = docker compose
 INTEGRATION_TEST_IN_DOCKER = docker exec core_test
 
 integration-test: test-assets
-	$(DOCKER_COMPOSE) --file core/tests/network/docker-compose.yml --file tests/network/docker-compose.yml up -d
+	$(DOCKER_COMPOSE) --file tests/network/docker-compose.yml up -d
 	docker cp core/tests/assets/kant_aufklaerung_1784/data/. ocrd_network_processing_server:/data
 	# Queues must exists and it takes time until they are created by the workers. We might need
 	# a mechanism to test if all queues are there but that is not available yet. So sleeping
 	# here for now
 	sleep 10
-	-$(INTEGRATION_TEST_IN_DOCKER) pytest -k 'test_ocrd_all_workflow' -v
-	$(DOCKER_COMPOSE) --file core/tests/network/docker-compose.yml --file tests/network/docker-compose.yml down -v --remove-orphans
+	-$(INTEGRATION_TEST_IN_DOCKER) pytest 'tests/network/test_ocrd_all_workflow.py' -v
+	$(DOCKER_COMPOSE) --file tests/network/docker-compose.yml down -v --remove-orphans
 
 test-assets:
 	$(MAKE) -C core assets
diff --git a/core b/core
index db322be4..419a5354 160000
--- a/core
+++ b/core
@@ -1 +1 @@
-Subproject commit db322be4d5dd843941fa7a48dcbea0848214dd55
+Subproject commit 419a535455d8e492dba85e094d76f9897b752f02
diff --git a/tests/network/.env b/tests/network/.env
new file mode 100644
index 00000000..3b732822
--- /dev/null
+++ b/tests/network/.env
@@ -0,0 +1,24 @@
+DOCKER_OCRD_NETWORK_NAME=ocrd_network_test
+DOCKER_OCRD_NETWORK_MTU=1450
+
+OCRD_NETWORK_LOGS_ROOT=/tmp/ocrd_network_logs
+OCRD_NETWORK_SOCKETS_ROOT=/tmp/ocrd_network_sockets
+OCRD_NETWORK_DATA=ocrd-test-workspace
+
+OCRD_PS_HOST=ps-docker-host
+OCRD_PS_PORT=8000
+OCRD_PS_URL=http://${OCRD_PS_HOST}.${DOCKER_OCRD_NETWORK_NAME}:${OCRD_PS_PORT}
+
+MONGODB_NAME=ocrd_network_test
+MONGODB_USER=network_test
+MONGODB_PASS=network_test
+MONGODB_HOST=mongodb-docker-host
+MONGODB_PORT=27017
+MONGODB_URL=mongodb://${MONGODB_USER}:${MONGODB_PASS}@${MONGODB_HOST}.${DOCKER_OCRD_NETWORK_NAME}:${MONGODB_PORT}
+
+RABBITMQ_FEATURE_FLAGS=quorum_queue,implicit_default_bindings,classic_mirrored_queue_version
+RABBITMQ_USER=network_test
+RABBITMQ_PASS=network_test
+RABBITMQ_HOST=rabbitmq-docker-host
+RABBITMQ_PORT=5672
+RABBITMQ_URL=amqp://${RABBITMQ_USER}:${RABBITMQ_PASS}@${RABBITMQ_HOST}.${DOCKER_OCRD_NETWORK_NAME}:${RABBITMQ_PORT}
diff --git a/tests/network/docker-compose.yml b/tests/network/docker-compose.yml
index 38f0aab8..96c36a78 100644
--- a/tests/network/docker-compose.yml
+++ b/tests/network/docker-compose.yml
@@ -1,19 +1,111 @@
+networks:
+  ocrd_network_test:
+    name: ${DOCKER_OCRD_NETWORK_NAME}
+    driver: bridge
+    driver_opts:
+      com.docker.network.driver.mtu: ${DOCKER_OCRD_NETWORK_MTU}
+
+volumes:
+  ocrd-test-workspace: {}
+
 services:
   ocrd_network_processing_server:
-    image: processing-server
-    build:
-      context: ../../../core
-      target: ocrd_core_base
+    image: "ocrd/core"
+    hostname: ${OCRD_PS_HOST}
+    container_name: ocrd_network_processing_server
+    depends_on:
+      ocrd_network_mongo_db:
+        condition: service_healthy
+      ocrd_network_rabbit_mq:
+        condition: service_healthy
+    networks:
+      - ${DOCKER_OCRD_NETWORK_NAME}
+    ports:
+      - ${OCRD_PS_PORT}:8000
+    environment:
+      DB_NAME: ${MONGODB_NAME}
+      DB_URL: ${MONGODB_URL}
+      RABBITMQ_URL: ${RABBITMQ_URL}
+      OCRD_NETWORK_LOGS_ROOT_DIR: /ocrd-data/ocrd_network_logs
+      OCRD_NETWORK_SOCKETS_ROOT_DIR: /ocrd-data/ocrd_network_sockets
+    healthcheck:
+      test: curl -f ${OCRD_PS_URL}/
+      interval: 1s
+      timeout: 3s
+      retries: 30
+    volumes:
+      - ${OCRD_NETWORK_LOGS_ROOT}:/ocrd-data/ocrd_network_logs
+      - ${OCRD_NETWORK_SOCKETS_ROOT}:/ocrd-data/ocrd_network_sockets
+      - "./ocrd_logging.conf:/etc/ocrd_logging.conf"
+      - "./ps_config.yml:/ocrd-data/ps_config.yml"
+      - "${OCRD_NETWORK_DATA}:/data"
+    command: ocrd network processing-server -a 0.0.0.0:8000 /ocrd-data/ps_config.yml
+
+  ocrd_network_mongo_db:
+    image: "mongo"
+    hostname: ${MONGODB_HOST}
+    container_name: ocrd_network_mongo_db
+    networks:
+      - ${DOCKER_OCRD_NETWORK_NAME}
+    ports:
+      - ${MONGODB_PORT}:27017
+    environment:
+      - MONGO_INITDB_ROOT_USERNAME=${MONGODB_USER}
+      - MONGO_INITDB_ROOT_PASSWORD=${MONGODB_PASS}
+    healthcheck:
+      test: echo 'db.runCommand("ping").ok' | mongosh localhost:27017/test --quiet
+      interval: 1s
+      timeout: 3s
+      retries: 30
+
+  ocrd_network_rabbit_mq:
+    image: "rabbitmq:3.12-management"
+    hostname: ${RABBITMQ_HOST}
+    container_name: ocrd_network_rabbit_mq
+    networks:
+      - ${DOCKER_OCRD_NETWORK_NAME}
+    ports:
+      - ${RABBITMQ_PORT}:5672
+      - 15672:15672
+      - 25672:25672
+    environment:
+      - RABBITMQ_DEFAULT_USER=${RABBITMQ_USER}
+      - RABBITMQ_DEFAULT_PASS=${RABBITMQ_PASS}
+      - RABBITMQ_FEATURE_FLAGS=${RABBITMQ_FEATURE_FLAGS}
+    healthcheck:
+      test: rabbitmq-diagnostics check_port_connectivity
+      interval: 1s
+      timeout: 3s
+      retries: 30
 
   ocrd_network_core_test:
     build:
-      context: ../../../core
+      context: ../../core
       args:
+        BASE_IMAGE: 'ubuntu:22.04'
         SKIP_ASSETS: 1
+      target: ocrd_core_test
+    container_name: core_test
+    depends_on:
+      ocrd_network_processing_server:
+        condition: service_healthy
+    networks:
+      - ${DOCKER_OCRD_NETWORK_NAME}
+    environment:
+      DB_NAME: ${MONGODB_NAME}
+      DB_URL: ${MONGODB_URL}
+      PROCESSING_SERVER_URL: ${OCRD_PS_URL}
+      RABBITMQ_URL: ${RABBITMQ_URL}
+      OCRD_NETWORK_LOGS_ROOT_DIR: /ocrd-data/ocrd_network_logs
+      OCRD_NETWORK_SOCKETS_ROOT_DIR: /ocrd-data/ocrd_network_sockets
+    volumes:
+      - ${OCRD_NETWORK_LOGS_ROOT}:/ocrd-data/ocrd_network_logs
+      - ${OCRD_NETWORK_SOCKETS_ROOT}:/ocrd-data/ocrd_network_sockets
+      - "./test-workflow-1.txt:/ocrd-data/assets/test-workflow-1.txt"
+      - "./ocrd_logging.conf:/etc/ocrd_logging.conf"
 
   ocrd-olena-binarize:
-    build:
-      context: ../../../ocrd_olena
+    image: ocrd/olena
     container_name: ocrd-olena-binarize
     depends_on:
       ocrd_network_processing_server:
@@ -29,10 +121,8 @@ services:
       - ${OCRD_NETWORK_DATA}:/data
     command: ocrd-olena-binarize worker --database ${MONGODB_URL} --queue ${RABBITMQ_URL}
 
-
   ocrd-anybaseocr-crop:
-    build:
-      context: ../../../ocrd_anybaseocr
+    image: ocrd/anybaseocr
     container_name: ocrd-anybaseocr-crop
     depends_on:
       ocrd_network_processing_server:
@@ -48,10 +138,8 @@ services:
       - ${OCRD_NETWORK_DATA}:/data
     command: ocrd-anybaseocr-crop worker --database ${MONGODB_URL} --queue ${RABBITMQ_URL}
 
-
   ocrd-cis-ocropy-denoise:
-    build:
-      context: ../../../ocrd_cis
+    image: ocrd/cis
     container_name: ocrd-cis-ocropy-denoise
     depends_on:
       ocrd_network_processing_server:
@@ -67,10 +155,8 @@ services:
       - ${OCRD_NETWORK_DATA}:/data
     command: ocrd-cis-ocropy-denoise worker --database ${MONGODB_URL} --queue ${RABBITMQ_URL}
 
-
   ocrd-cis-ocropy-clip:
-    build:
-      context: ../../../ocrd_cis
+    image: ocrd/cis
     container_name: ocrd-cis-ocropy-clip
     depends_on:
       ocrd_network_processing_server:
@@ -86,10 +172,8 @@ services:
       - ${OCRD_NETWORK_DATA}:/data
     command: ocrd-cis-ocropy-clip worker --database ${MONGODB_URL} --queue ${RABBITMQ_URL}
 
-
   ocrd-cis-ocropy-segment:
-    build:
-      context: ../../../ocrd_cis
+    image: ocrd/cis
     depends_on:
       ocrd_network_processing_server:
         condition: service_healthy
@@ -104,10 +188,8 @@ services:
       - ${OCRD_NETWORK_DATA}:/data
     command: ocrd-cis-ocropy-segment worker --database ${MONGODB_URL} --queue ${RABBITMQ_URL}
 
-
   ocrd-cis-ocropy-dewarp:
-    build:
-      context: ../../../ocrd_cis
+    image: ocrd/cis
     container_name: ocrd-cis-ocropy-dewarp
     depends_on:
       ocrd_network_processing_server:
@@ -123,7 +205,6 @@ services:
       - ${OCRD_NETWORK_DATA}:/data
     command: ocrd-cis-ocropy-dewarp worker --database ${MONGODB_URL} --queue ${RABBITMQ_URL}
 
-
   ocrd-tesserocr-segment-region:
     image: ocrd/tesserocr
     container_name: ocrd-tesserocr-segment-region
@@ -142,7 +223,6 @@ services:
       - ${OCRD_NETWORK_DATA}:/data
     command: ocrd-tesserocr-segment-region worker --database ${MONGODB_URL} --queue ${RABBITMQ_URL}
 
-
   ocrd-tesserocr-recognize:
     image: ocrd/tesserocr
     container_name: ocrd-tesserocr-recognize
@@ -161,7 +241,6 @@ services:
       - ${OCRD_NETWORK_DATA}:/data
     command: ocrd-tesserocr-recognize worker --database ${MONGODB_URL} --queue ${RABBITMQ_URL}
 
-
   ocrd-segment-repair:
     image: ocrd/segment
     container_name: ocrd-segment-repair
diff --git a/tests/network/ocrd_logging.conf b/tests/network/ocrd_logging.conf
new file mode 100644
index 00000000..ee081b7b
--- /dev/null
+++ b/tests/network/ocrd_logging.conf
@@ -0,0 +1,150 @@
+# This is a template configuration file which allows customizing
+# format and destination of log messages with OCR-D.
+# It is meant as an example, and should be customized.
+# To get into effect, you must put a copy (under the same name)
+# into your CWD, HOME or /etc. These directories are searched
+# in said order, and the first find wins. When no config file
+# is found, the default logging configuration applies (cf. ocrd.logging.py).
+# 
+# mandatory loggers section
+# configure loggers with corresponding keys "root", ""
+# each logger requires a corresponding configuration section below
+#
+[loggers]
+keys=root,ocrd,ocrd_network,ocrd_tensorflow,ocrd_shapely_geos,ocrd_PIL,uvicorn,uvicorn_access,uvicorn_error,multipart
+
+#
+# mandatory handlers section
+# handle output for each logging "channel"
+# i.e. console, file, smtp, syslog, http, ...
+# each handler requires a corresponding configuration section below
+#
+[handlers]
+keys=consoleHandler,fileHandler,processingServerHandler
+
+#
+# optional custom formatters section
+# format message fields, to be used differently by logging handlers
+# each formatter requires a corresponding formatter section below
+#
+[formatters]
+keys=defaultFormatter,detailedFormatter
+
+#
+# default logger "root" using consoleHandler
+#
+[logger_root]
+level=DEBUG
+handlers=consoleHandler,fileHandler
+
+
+#
+# additional logger configurations can be added
+# as separate configuration sections like below
+#
+# example logger "ocrd_workspace" uses fileHandler and overrides
+# default log level "INFO" with custom level "DEBUG" 
+# "qualname" must match the logger label used in the corresponding 
+# ocrd module
+# see in the module-of-interest (moi)
+#
+#[logger_ocrd_workspace]
+#level=DEBUG
+#handlers=fileHandler
+#qualname=ocrd.workspace
+
+# ocrd loggers
+[logger_ocrd]
+level=DEBUG
+handlers=consoleHandler,fileHandler
+qualname=ocrd
+propagate=0
+
+[logger_ocrd_network]
+level=DEBUG
+handlers=consoleHandler,processingServerHandler
+qualname=ocrd_network
+propagate=0
+
+#
+# logger tensorflow
+#
+[logger_ocrd_tensorflow]
+level=DEBUG
+handlers=consoleHandler
+qualname=tensorflow
+
+#
+# logger shapely.geos
+#
+[logger_ocrd_shapely_geos]
+level=DEBUG
+handlers=consoleHandler
+qualname=shapely.geos
+
+
+#
+# logger PIL
+#
+[logger_ocrd_PIL]
+level=DEBUG
+handlers=consoleHandler
+qualname=PIL
+
+#
+# uvicorn loggers
+#
+[logger_uvicorn]
+level=INFO
+handlers=consoleHandler
+qualname=uvicorn
+[logger_uvicorn_access]
+level=DEBUG
+handlers=consoleHandler
+qualname=uvicorn.access
+[logger_uvicorn_error]
+level=DEBUG
+handlers=consoleHandler
+qualname=uvicorn.error
+[logger_multipart]
+level=DEBUG
+handlers=consoleHandler
+qualname=multipart
+
+
+
+#
+# handle stderr output
+#
+[handler_consoleHandler]
+class=StreamHandler
+formatter=defaultFormatter
+args=(sys.stderr,)
+
+#
+# example logfile handler
+# handle output with logfile
+#
+[handler_fileHandler]
+class=FileHandler
+formatter=defaultFormatter
+args=('ocrd.log','a+')
+
+[handler_processingServerHandler]
+class=FileHandler
+formatter=defaultFormatter
+args=('/ocrd-data/ocrd_processing_server_conf.log','a+')
+
+#
+# default log format conforming to OCR-D (https://ocr-d.de/en/spec/cli#logging)
+#
+[formatter_defaultFormatter]
+format=%(asctime)s.%(msecs)03d %(levelname)s %(name)s - %(message)s
+datefmt=%H:%M:%S
+
+#
+# store more logging context information
+#
+[formatter_detailedFormatter]
+format=%(asctime)s.%(msecs)03d %(levelname)-8s (%(name)s)[%(filename)s:%(lineno)d] - %(message)s
+datefmt=%H:%M:%S
diff --git a/tests/network/ps_config.yml b/tests/network/ps_config.yml
new file mode 100644
index 00000000..655a847b
--- /dev/null
+++ b/tests/network/ps_config.yml
@@ -0,0 +1,17 @@
+# the content of this config file is based on the .env
+internal_callback_url: http://ps-docker-host.ocrd_network_test:8000
+process_queue:
+  address: rabbitmq-docker-host.ocrd_network_test
+  port: 5672
+  skip_deployment: true
+  credentials:
+    username: network_test
+    password: network_test
+database:
+  address: mongodb-docker-host.ocrd_network_test
+  port: 27017
+  skip_deployment: true
+  credentials:
+    username: network_test
+    password: network_test
+hosts: []
diff --git a/tests/network/test-workflow-1.txt b/tests/network/test-workflow-1.txt
new file mode 100644
index 00000000..2e4485da
--- /dev/null
+++ b/tests/network/test-workflow-1.txt
@@ -0,0 +1,9 @@
+olena-binarize           -I OCR-D-IMG -O O-1
+anybaseocr-crop          -I O-1       -O O-2
+cis-ocropy-denoise       -I O-2       -O O-3  -P dpi 300 -P level-of-operation page -P noise_maxsize 3.0
+tesserocr-segment-region -I O-3       -O O-4  -P padding 5 -P find_tables false -P dpi 300
+segment-repair           -I O-4       -O O-5  -P plausibilize true -P plausibilize_merge_min_overlap 0.7
+cis-ocropy-clip          -I O-5       -O O-6
+cis-ocropy-segment       -I O-6       -O O-7  -P spread 2.4 -P dpi 300
+cis-ocropy-dewarp        -I O-7       -O O-8
+tesserocr-recognize      -I O-8       -O PAGE -P textequiv_level word -P model Fraktur

From 1b7a9aff2be08822e2deb1b9dd8637422b164ade Mon Sep 17 00:00:00 2001
From: joschrew <91774427+joschrew@users.noreply.github.com>
Date: Tue, 16 Apr 2024 16:17:54 +0200
Subject: [PATCH 8/8] Update network tests in regard to related core pr

---
 Makefile                                             | 12 +++++++++++-
 core                                                 |  2 +-
 tests/network/docker-compose.yml                     |  2 +-
 ...est-workflow-1.txt => ocrd_all-test-workflow.txt} |  0
 4 files changed, 13 insertions(+), 3 deletions(-)
 rename tests/network/{test-workflow-1.txt => ocrd_all-test-workflow.txt} (100%)

diff --git a/Makefile b/Makefile
index 1ac51447..0ccf360b 100644
--- a/Makefile
+++ b/Makefile
@@ -750,7 +750,7 @@ test-workflow: test-assets core $(BIN)/ocrd $(ACTIVATE_VENV)
 DOCKER_COMPOSE = docker compose
 INTEGRATION_TEST_IN_DOCKER = docker exec core_test
 
-integration-test: test-assets
+network-integration-test: test-assets
 	$(DOCKER_COMPOSE) --file tests/network/docker-compose.yml up -d
 	docker cp core/tests/assets/kant_aufklaerung_1784/data/. ocrd_network_processing_server:/data
 	# Queues must exists and it takes time until they are created by the workers. We might need
@@ -760,6 +760,16 @@ integration-test: test-assets
 	-$(INTEGRATION_TEST_IN_DOCKER) pytest 'tests/network/test_ocrd_all_workflow.py' -v
 	$(DOCKER_COMPOSE) --file tests/network/docker-compose.yml down -v --remove-orphans
 
+network-integration-test-cicd:
+	$(DOCKER_COMPOSE) --file tests/network/docker-compose.yml up -d
+	docker cp core/tests/assets/kant_aufklaerung_1784/data/. ocrd_network_processing_server:/data
+	# Queues must exists and it takes time until they are created by the workers. We might need
+	# a mechanism to test if all queues are there but that is not available yet. So sleeping
+	# here for now
+	sleep 10
+	$(INTEGRATION_TEST_IN_DOCKER) pytest 'tests/network/test_ocrd_all_workflow.py' -v
+	$(DOCKER_COMPOSE) --file tests/network/docker-compose.yml down -v --remove-orphans
+
 test-assets:
 	$(MAKE) -C core assets
 
diff --git a/core b/core
index 419a5354..2a7ef7b4 160000
--- a/core
+++ b/core
@@ -1 +1 @@
-Subproject commit 419a535455d8e492dba85e094d76f9897b752f02
+Subproject commit 2a7ef7b49b8c94efc8e13df77f047e95f62a515d
diff --git a/tests/network/docker-compose.yml b/tests/network/docker-compose.yml
index 96c36a78..3e507d54 100644
--- a/tests/network/docker-compose.yml
+++ b/tests/network/docker-compose.yml
@@ -101,7 +101,7 @@ services:
     volumes:
       - ${OCRD_NETWORK_LOGS_ROOT}:/ocrd-data/ocrd_network_logs
       - ${OCRD_NETWORK_SOCKETS_ROOT}:/ocrd-data/ocrd_network_sockets
-      - "./test-workflow-1.txt:/ocrd-data/assets/test-workflow-1.txt"
+      - "./ocrd_all-test-workflow.txt:/ocrd-data/assets/ocrd_all-test-workflow.txt"
       - "./ocrd_logging.conf:/etc/ocrd_logging.conf"
 
   ocrd-olena-binarize:
diff --git a/tests/network/test-workflow-1.txt b/tests/network/ocrd_all-test-workflow.txt
similarity index 100%
rename from tests/network/test-workflow-1.txt
rename to tests/network/ocrd_all-test-workflow.txt