add rule for ocrd-tool-all.json, reduce image size, fix+update modules, fix CUDA

.circleci/config.yml

@@ -2,16 +2,17 @@ version: 2.1
 jobs:
   build:
     docker:
-      - image: cimg/base:stable-18.04
+      - image: cimg/base:stable-22.04
     steps:
       - checkout
       - setup_remote_docker      # https://circleci.com/docs/2.0/building-docker-images/
       - run:
           name: build image
-          command: make docker-maximum-cuda
+          command: make docker-maximum-cuda GIT_DEPTH=--single-branch
           no_output_timeout: 30m
       - when:
           # takes too long for 1h1m CircleCI timeout overall
+	  # also, storage is limited...
           condition: false
           steps:
             - run:
@@ -26,7 +27,7 @@ jobs:
                 destination: artifacts
   deploy:
     docker:
-      - image: cimg/base:stable-18.04
+      - image: cimg/base:stable-22.04
     environment:
       GIT_DEPTH: "--depth 1"
     parameters:
@@ -38,7 +39,7 @@ jobs:
       - setup_remote_docker      # https://circleci.com/docs/2.0/building-docker-images/
       - run:
           name: Build Docker image
-          command: make docker-<< parameters.variant >>-git
+          command: make docker-<< parameters.variant >>-git GIT_DEPTH=--single-branch
           # fails due to pip races: DOCKER_PARALLEL=-j3
           no_output_timeout: 30m
       - run:

.github/workflows/makedocker.yml

@@ -7,29 +7,10 @@ on:
   # Trigger workflow in GitHub web frontend or from API.
   workflow_dispatch:
     inputs:
-      os:
-        description: 'Operating system'
-        required: true
-        default: 'ubuntu-18.04'
-        type: choice
-        options:
-          - 'ubuntu-18.04'
-          - 'ubuntu-20.04'
-      python-version:
-        description: 'Python version'
-        required: true
-        default: '3.6'
-        type: choice
-        options:
-          - '3.6'
-          - '3.7'
-          - '3.8'
-          - '3.9'
-          - '3.10'
       docker-image:
         description: 'Docker image'
         required: true
-        default: 'docker-minimum'
+        default: 'minimum'
         type: choice
         options:
           - 'minimum'
@@ -44,80 +25,104 @@ on:
           - 'medium-cuda-git'
           - 'maximum-git'
           - 'maximum-cuda-git'
-      upload-docker-image:
-        description: 'Upload Docker image'
+      upload-dockerhub:
+        description: 'Upload Docker image to Dockerhub'
         default: False
         type: boolean
+      upload-github:
+        description: 'Upload Docker image Github Container Registry'
+        default: False
+        type: boolean
+      upterm-session:
+        description: 'Run SSH login server for debugging'
+        default: False
+        type: boolean
+  # not yet:
+  #push:
+  #  branches: [ "master" ]
 
 jobs:
   make:
-    runs-on: ${{ github.event.inputs.os }}
-
-    env:
-      PYTHON_VERSION: ${{ github.event.inputs.python-version }}
+    runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
-      with:
-        python-version: ${{ env.PYTHON_VERSION }}
-    - name: Show Python3 version
-      run: python3 --version
-    - name: Show disk usage of Homebrew, Android and .NET
-      run: sudo du -mscx /home/linuxbrew /usr/local/lib/android /usr/share/dotnet 2>/dev/null || true
     - name: Remove Docker images
       run: |
            df -h
            docker images
-           docker rmi alpine:3.12 alpine:3.13 alpine:3.14
-           docker rmi buildpack-deps:stretch buildpack-deps:buster buildpack-deps:bullseye
-           docker rmi debian:9 debian:10 debian:11
-           docker rmi moby/buildkit:latest
-           docker rmi node:12-alpine node:14-alpine node:16-alpine
-           docker rmi node:12 node:14 node:16
-           if false; then # don't remove Ubuntu images
-           docker rmi ubuntu:16.04 ubuntu:18.04 ubuntu:20.04
-           fi
-           docker images
+           docker rmi $(docker images --filter=reference="alpine:*" -q)
+           docker rmi $(docker images --filter=reference="buildpack-deps:*" -q)
+           docker rmi $(docker images --filter=reference="debian:*" -q)
+           docker rmi $(docker images --filter=reference="node:*" -q)
            df -h /
     - name: Remove unneeded Debian packages
       run: |
-           if false; then # skip time consuming package uninstall
            sudo apt-get install -y deborphan
-           deborphan -a | sort
-           sudo apt-get purge -y $(deborphan -a|fgrep main/cli-mono|while read dummy package; do echo $package; done)
-           sudo apt-get purge -y $(deborphan -a|fgrep main/database|while read dummy package; do echo $package; done)
-           sudo apt-get purge -y $(deborphan -a|fgrep main/devel|while read dummy package; do echo $package; done)
-           sudo apt-get purge -y $(deborphan -a|fgrep main/httpd|while read dummy package; do echo $package; done)
-           sudo apt-get purge -y $(deborphan -a|fgrep main/php|while read dummy package; do echo $package; done)
-           sudo apt-get purge -y $(deborphan -a|fgrep main/vcs|while read dummy package; do echo $package; done)
+           sudo apt-get purge -y $(deborphan -a | fgrep -e main/cli-mono -e main/database -e main/devel -e main/httpd -e main/php -e main/vcs | while read _ pkg; do echo $package; done)
            deborphan | sort
            sudo du -mscx /* 2>/dev/null || true
            sudo du -mscx /opt/* 2>/dev/null || true
            sudo du -mscx /usr/* 2>/dev/null || true
            df -h /
-           fi
     - name: Remove Homebrew, Android and .NET
       run: |
            # https://github.com/actions/virtual-environments/issues/2606#issuecomment-772683150
            # NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/uninstall.sh)"
            sudo rm -rf /home/linuxbrew # will release Homebrew
            sudo rm -rf /usr/local/lib/android # will release about 10 GB if you don't need Android
            sudo rm -rf /usr/share/dotnet # will release about 20GB if you don't need .NET
+           sudo rm -rf /opt/ghc
+           sudo rm -rf /usr/local/share/boost
+           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
            sudo du -mscx /* 2>/dev/null || true
            df -h /
+    - name: Setup upterm session
+      # interactive SSH logins for debugging
+      if: github.event.inputs.upterm-session == 'true'
+      uses: lhotari/action-upterm@v1
     - name: Make Docker image
-      run: make docker-${{ github.event.inputs.docker-image }}
-    - name: Show Docker images
-      run: docker images
-    - name: Login to Docker Hub and push new image(s) to Docker Hub
+      run: make docker-${{ github.event.inputs.docker-image }} GIT_DEPTH=--single-branch
+    - name: Generate ocrd-all-tool.json
+      # the Docker build will set OCRD_MODULES inside the image, which we can re-use
+      # regardless of whether we have /build, we can just use the Makefile from outside again
+      run: |
+           export OCRD_MODULES=$(docker run --rm ocrd/all:${{ github.event.inputs.docker-image }} bash -c 'echo $OCRD_MODULES')
+           make ocrd-all-tool.json
+           wc -l ocrd-all-tool.json
+    - name: Upload ocrd-all-tool.json
+      uses: actions/upload-artifact@v3
+      with:
+        name: ${{ github.event.inputs.docker-image }}_ocrd-all-tool.json
+        path: ./ocrd-all-tool.json
+        # if-no-files-found: error
+    - name: Login to Docker Hub
+      if: github.event.inputs.upload-dockerhub == 'true'
+      run: echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login --username ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin
+    - name: Push to Docker Hub
+      if: github.event.inputs.upload-dockerhub == 'true'
       run: |
-           if ${{ github.event.inputs.upload-docker-image }}; then
-             echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login --username ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin
-             docker push ocrd/all:${{ github.event.inputs.docker-image }}
-             if test ${{ github.event.inputs.docker-image }} = maximum-git; then
-               # Alias Docker image.
-               docker tag ocrd/all:maximum-git ocrd/all:latest
-               docker push ocrd/all:latest
-             fi
+           docker push ocrd/all:${{ github.event.inputs.docker-image }}
+           if test ${{ github.event.inputs.docker-image }} = maximum-git; then
+             # Alias Docker image.
+             docker tag ocrd/all:maximum-git ocrd/all:latest
+             docker push ocrd/all:latest
            fi
+    - name: Login to GitHub Container Registry
+      if: github.event.inputs.upload-github == 'true'
+      uses: docker/login-action@v2
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+    - name: Push to Github Container Registry
+      if: github.event.inputs.upload-github == 'true'
+      run: |
+           docker tag ocrd/all:${{ github.event.inputs.docker-image }} ghcr.io/ocr-d/all:${{ github.event.inputs.docker-image }}
+           docker push ghcr.io/ocr-d/all:${{ github.event.inputs.docker-image }}
+           if test ${{ github.event.inputs.docker-image }} = maximum-git; then
+             # Alias Docker image.
+             docker tag ocrd/all:maximum-git ghcr.io/ocr-d/all:latest
+             docker push ghcr.io/ocr-d/all:latest
+           fi
+