tests.yml

name: tests


on:
  push:
    branches:
      - '**'
    tags-ignore:
      - '**'


concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  # don't cancel on main/master/default
  cancel-in-progress: ${{ format('refs/heads/{0}', github.event.repository.default_branch) != github.ref }}


env:
  PY_COLORS: "1"
  BOT_NAME: wipacdevbot
  BOT_EMAIL: developers@icecube.wisc.edu
  #
  CI_DOCKER_IMAGE_TAG: icecube/skymap_scanner:local
  #
  CI_TEST_RUN_STDOUT_STDERR_DIR: /home/runner/work/skymap_scanner/testrun_outputs
  N_WORKERS: 2
  REALTIME_EVENTS_DIR: /home/runner/work/skymap_scanner/skymap_scanner/tests/data/realtime_events
  CI_SKYSCAN_CACHE_DIR: /home/runner/work/skymap_scanner/skymap_scanner/cache
  CI_SKYSCAN_OUTPUT_DIR: /home/runner/work/skymap_scanner/skymap_scanner/output
  CI_SKYSCAN_DEBUG_DIR: /home/runner/work/skymap_scanner/skymap_scanner/debug
  # see source tests/env-vars.sh


jobs:


  #############################################################################
  # PACKAGING & LINTERS
  #############################################################################


  py-versions:
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.versions.outputs.matrix }}
    steps:
      - uses: actions/checkout@v4
      - id: versions
        uses: WIPACrepo/wipac-dev-py-versions-action@v2.2

  # flake8:
  #   runs-on: ubuntu-latest
  #   steps:
  #     - uses: actions/checkout@v4
  #     - uses: actions/setup-python@v5
  #     - uses: WIPACrepo/wipac-dev-flake8-action@v1.0

  mypy:
    needs: [ py-versions ]
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        py3: ${{ fromJSON(needs.py-versions.outputs.matrix) }}
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.py3 }}
      - uses: WIPACrepo/wipac-dev-mypy-action@v2.0

  py-setup:
    runs-on: ubuntu-latest
    steps:
      - name: checkout
        uses: actions/checkout@v4
        with:
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
      - uses: WIPACrepo/wipac-dev-py-setup-action@v3.1
        if: github.actor != ${{ env.BOT_NAME }}  # no auto-updates for bots # should match all 'git_committer_name' uses
        with:
          git_committer_name: ${{ env.BOT_NAME }}
          git_committer_email: ${{ env.BOT_EMAIL }}
          base-keywords: IceCube

  py-dependencies:
    runs-on: ubuntu-latest
    steps:
      - uses: jlumbroso/free-disk-space@main  # need space for images to build
        with:
          docker-images: false
      - name: checkout
        uses: actions/checkout@v4
        with:
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
      - uses: WIPACrepo/wipac-dev-py-dependencies-action@v1.1
        if: github.actor != ${{ env.BOT_NAME }}  # no auto-updates for bots # should match all 'git_committer_name' uses
        with:
          git_committer_name: ${{ env.BOT_NAME }}
          git_committer_email: ${{ env.BOT_EMAIL }}
          dockerfile_nametags: Dockerfile:icecube/skymap_scanner:latest


  #############################################################################
  # TESTS
  #############################################################################


  test-build-docker:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: docker/setup-buildx-action@v3
      - uses: docker/build-push-action@v6
        with:
          context: .
          file: ./Dockerfile
          tags: ${{ env.CI_DOCKER_IMAGE_TAG }}
          load: true

  test-run-dummy:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        reco_algo: [
          "dummy",
          "crash_dummy"
        ]
        container_platform: [
          "docker",
          "apptainer",
        ]
    env:
      SKYSCAN_MQ_TOCLIENT_BROKER_ADDRESS: user1@localhost/test
      SKYSCAN_MQ_TOCLIENT_AUTH_TOKEN: password  # using this would override password in address
      SKYSCAN_MQ_FROMCLIENT_BROKER_ADDRESS: user1@localhost/test
      SKYSCAN_MQ_FROMCLIENT_AUTH_TOKEN: password  # using this would override password in address
      EWMS_PILOT_STOP_LISTENING_ON_TASK_ERROR: False  # we want many crashes
      EWMS_PILOT_TASK_TIMEOUT: 5
    services:
      rabbitmq:
        # see image.tag -> https://github.com/Observation-Management-Service/path-kubernetes/blob/main/helm-values-rabbitmq-bitnami.yaml (see https://artifacthub.io/packages/helm/bitnami/rabbitmq/11.14.3)
        image: bitnami/rabbitmq:3.11.15-debian-11-r0
        env:
          RABBITMQ_USERNAME: user1
          RABBITMQ_PASSWORD: password
          RABBITMQ_VHOST: test
          BITNAMI_DEBUG: true
        # Note: `--network` option is not supported.
        options: >-
          --name rabbitmq
          --health-cmd "rabbitmqctl node_health_check"
          --health-interval 5s
          --health-timeout 5s
          --health-retries 10
        ports:
          - 5672:5672
          - 15672:15672
    steps:
      - uses: jlumbroso/free-disk-space@main  # need space for mq broker and image
        with:
          docker-images: false
      - uses: actions/checkout@v4
      - uses: docker/setup-buildx-action@v3
      - uses: docker/build-push-action@v6
        with:
          context: .
          file: ./Dockerfile
          tags: ${{ env.CI_DOCKER_IMAGE_TAG }}
          load: true
      - if: ${{ matrix.container_platform == 'apptainer' }}
        uses: eWaterCycle/setup-apptainer@v2
        with:
          apptainer-version: 1.3.2
      - if: ${{ matrix.container_platform == 'apptainer' }}
        name: build apptainer (singularity) image
        run: |
          sudo singularity build skymap_scanner.sif docker-daemon://$CI_DOCKER_IMAGE_TAG
          ls -lh skymap_scanner.sif

      - name: run
        timeout-minutes: 15  # on average ~9min  # yes, `timeout` is used below but this is insurance
        run: |
          set -x
          source tests/env-vars.sh
          export _RECO_ALGO="${{ matrix.reco_algo }}"
          export _EVENTS_FILE=$(realpath $REALTIME_EVENTS_DIR/hese_event_01.json)
          export _NSIDES="1:0 2:12 4:12"
          
          if [ "${{ matrix.container_platform }}" == "apptainer" ]; then
              export _RUN_THIS_SINGULARITY_IMAGE="$(realpath skymap_scanner.sif)"
          else
              export _EWMS_PILOT_APPTAINER_IMAGE_DIRECTORY_MUST_BE_PRESENT=False
          fi
          if [ "${{ matrix.reco_algo }}" == "crash_dummy" ]; then
              export _SKYSCAN_CI_CRASH_DUMMY_PROBABILITY=0.75
          fi
          
          cd ./resources/launch_scripts
          # since _SKYSCAN_CI_CRASH_DUMMY_PROBABILITY<1, this step will go forever
          #   so, stop it after some time and chek that it has stderrfiles
          timeout 240 ./local-scan.sh $N_WORKERS $CI_TEST_RUN_STDOUT_STDERR_DIR  || true

      - if: ${{ matrix.reco_algo == 'crash_dummy' }}
        name: look at stderrfiles
        run: |
          set -x
          # check for fails/errors

          error_type_1='intentional crash-dummy error'
          error_type_2='subprocess timed out after'
          pattern="$error_type_1|$error_type_2"

          if find "$CI_TEST_RUN_STDOUT_STDERR_DIR/worker-"*/pilot.out -type f -exec grep -qE "$pattern" {} +; then
              echo "Match(es) found: PilotSubprocessError and/or TimeoutError occurred."
          else
              echo "No matches found."
              exit 1
          fi
      - if: ${{ matrix.reco_algo == 'dummy' }}
        name: look at results file (.npz)
        run: |
          ls .
          ls $CI_SKYSCAN_OUTPUT_DIR
          outfile=$(ls -d $CI_SKYSCAN_OUTPUT_DIR/*.npz)
          echo $outfile

      - name: central server stdout/stderr
        if: always()
        run: |
          cat $CI_TEST_RUN_STDOUT_STDERR_DIR/server.out
      - name: worker pilot \#1 stdout/stderr
        if: always()
        run: |
          more $CI_TEST_RUN_STDOUT_STDERR_DIR/worker-1/pilot.out | cat
      - name: worker clients / reco-icetray instances \#1 stdouts/stderrs
        if: always()
        run: |
          find $CI_TEST_RUN_STDOUT_STDERR_DIR/worker-1/pilot-* -name "stderrfile" -o -name "stdoutfile" | xargs more | cat
          echo "::::::::::::::" && tree $CI_TEST_RUN_STDOUT_STDERR_DIR/worker-1/pilot-*
      - name: worker pilot \#2 stdout/stderr
        if: always() && env.N_WORKERS == '2'
        run: |
          more $CI_TEST_RUN_STDOUT_STDERR_DIR/worker-2/pilot.out | cat
      - name: worker clients / reco-icetray instances \#2 stdouts/stderrs
        if: always() && env.N_WORKERS == '2'
        run: |
          find $CI_TEST_RUN_STDOUT_STDERR_DIR/worker-2/pilot-* -name "stderrfile" -o -name "stdoutfile" | xargs more | cat
          echo "::::::::::::::" && tree $CI_TEST_RUN_STDOUT_STDERR_DIR/worker-2/pilot-*

  test-run-singularity-dummy-reco:
    runs-on: ubuntu-latest
    env:
      SKYSCAN_MQ_TOCLIENT_BROKER_ADDRESS: user1:password@localhost/test
      SKYSCAN_MQ_FROMCLIENT_BROKER_ADDRESS: user1:password@localhost/test
      # ..._AUTH_TOKEN: user1  # using this would override password in address
      _EWMS_PILOT_CONTAINER_PLATFORM: apptainer
    services:
      rabbitmq:
        # see image.tag -> https://github.com/Observation-Management-Service/path-kubernetes/blob/main/helm-values-rabbitmq-bitnami.yaml (see https://artifacthub.io/packages/helm/bitnami/rabbitmq/11.14.3)
        image: bitnami/rabbitmq:3.11.15-debian-11-r0
        env:
          RABBITMQ_USERNAME: user1
          RABBITMQ_PASSWORD: password
          RABBITMQ_VHOST: test
          BITNAMI_DEBUG: true
        # Note: `--network` option is not supported.
        options: >-
          --name rabbitmq
          --health-cmd "rabbitmqctl node_health_check"
          --health-interval 5s
          --health-timeout 5s
          --health-retries 10
        ports:
          - 5672:5672
          - 15672:15672
    steps:
      - uses: jlumbroso/free-disk-space@main  # need space for mq broker and image
        with:
          docker-images: false
      - uses: actions/checkout@v4
      - uses: docker/setup-buildx-action@v3
      - uses: docker/build-push-action@v6
        with:
          context: .
          file: ./Dockerfile
          tags: ${{ env.CI_DOCKER_IMAGE_TAG }}
          load: true
      - uses: eWaterCycle/setup-apptainer@v2
        with:
          apptainer-version: 1.3.2
      - name: build singularity image
        run: |
          sudo singularity build skymap_scanner.sif docker-daemon://$CI_DOCKER_IMAGE_TAG
          ls -lh skymap_scanner.sif
      - name: run singularity container
        run: |
          set -x
          source tests/env-vars.sh
          export _RUN_THIS_SINGULARITY_IMAGE="$(realpath skymap_scanner.sif)"
          export _RECO_ALGO=dummy
          export _EVENTS_FILE=$(realpath $REALTIME_EVENTS_DIR/hese_event_01.json)
          export _NSIDES="1:0"
          
          cd ./resources/launch_scripts
          ./local-scan.sh $N_WORKERS $CI_TEST_RUN_STDOUT_STDERR_DIR

      - name: look at results file (.npz)
        run: |
          ls .
          ls $CI_SKYSCAN_OUTPUT_DIR
          outfile=$(ls -d $CI_SKYSCAN_OUTPUT_DIR/*.npz)
          echo $outfile

      - name: central server stdout/stderr
        if: always()
        run: |
          cat $CI_TEST_RUN_STDOUT_STDERR_DIR/server.out
      - name: worker pilot \#1 stdout/stderr
        if: always()
        run: |
          more $CI_TEST_RUN_STDOUT_STDERR_DIR/worker-1/pilot.out | cat
      - name: worker clients / reco-icetray instances \#1 stdouts/stderrs
        if: always()
        run: |
          find $CI_TEST_RUN_STDOUT_STDERR_DIR/worker-1/pilot-* -name "stderrfile" -o -name "stdoutfile" | xargs more | cat
          echo "::::::::::::::" && tree $CI_TEST_RUN_STDOUT_STDERR_DIR/worker-1/pilot-*
      - name: worker pilot \#2 stdout/stderr
        if: always() && env.N_WORKERS == '2'
        run: |
          more $CI_TEST_RUN_STDOUT_STDERR_DIR/worker-2/pilot.out | cat
      - name: worker clients / reco-icetray instances \#2 stdouts/stderrs
        if: always() && env.N_WORKERS == '2'
        run: |
          find $CI_TEST_RUN_STDOUT_STDERR_DIR/worker-2/pilot-* -name "stderrfile" -o -name "stdoutfile" | xargs more | cat
          echo "::::::::::::::" && tree $CI_TEST_RUN_STDOUT_STDERR_DIR/worker-2/pilot-*

      - name: rabbitmq logs
        if: always()
        run: |
          docker logs rabbitmq


  test-run-nsides-thresholds-dummy:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        nsides: [
          "1:0",
          "1:0 2:12",
          "1:0 2:12 4:12"
        ]
        predictive_scanning_threshold: [
          1.0,
          0.65,
        ]
    env:
      SKYSCAN_MQ_TOCLIENT_BROKER_ADDRESS: user1@localhost/test
      SKYSCAN_MQ_TOCLIENT_AUTH_TOKEN: password  # using this would override password in address
      SKYSCAN_MQ_FROMCLIENT_BROKER_ADDRESS: user1@localhost/test
      SKYSCAN_MQ_FROMCLIENT_AUTH_TOKEN: password  # using this would override password in address
    services:
      rabbitmq:
        # see image.tag -> https://github.com/Observation-Management-Service/path-kubernetes/blob/main/helm-values-rabbitmq-bitnami.yaml (see https://artifacthub.io/packages/helm/bitnami/rabbitmq/11.14.3)
        image: bitnami/rabbitmq:3.11.15-debian-11-r0
        env:
          RABBITMQ_USERNAME: user1
          RABBITMQ_PASSWORD: password
          RABBITMQ_VHOST: test
          BITNAMI_DEBUG: true
        # Note: `--network` option is not supported.
        options: >-
          --name rabbitmq
          --health-cmd "rabbitmqctl node_health_check"
          --health-interval 5s
          --health-timeout 5s
          --health-retries 10
        ports:
          - 5672:5672
          - 15672:15672
    steps:
      - uses: jlumbroso/free-disk-space@main  # need space for mq broker and image
        with:
          docker-images: false
      - uses: actions/checkout@v4
      - uses: docker/setup-buildx-action@v3
      - uses: docker/build-push-action@v6
        with:
          context: .
          file: ./Dockerfile
          tags: ${{ env.CI_DOCKER_IMAGE_TAG }}
          load: true
      - name: run
        timeout-minutes: 12  # on average max~=8.5min
        run: |
          set -x
          source tests/env-vars.sh
          export _RECO_ALGO=dummy
          export _EVENTS_FILE=$(realpath $REALTIME_EVENTS_DIR/hese_event_01.json)
          export _NSIDES="${{ matrix.nsides }}"
          export _PREDICTIVE_SCANNING_THRESHOLD=${{ matrix.predictive_scanning_threshold }}

          cd ./resources/launch_scripts
          ./local-scan.sh $N_WORKERS $CI_TEST_RUN_STDOUT_STDERR_DIR

      - name: check no nsides skipped
        run: |
          ls $CI_SKYSCAN_OUTPUT_DIR
          # get newest run*.json
          export outfile=$(find $CI_SKYSCAN_OUTPUT_DIR -type f -name "run*.json" -exec stat -c '%y %n'  {} + | sort | tail -1 | awk '{print $4}')
          echo $outfile
          python3 -c '
          import json
          import os
          with open(os.getenv("outfile")) as f:
            pydict = json.load(f)
          nsides = "${{ matrix.nsides }}"
          assert len(pydict) == nsides.count(":")
          '

      - name: central server stdout/stderr
        if: always()
        run: |
          cat $CI_TEST_RUN_STDOUT_STDERR_DIR/server.out
      - name: worker pilot \#1 stdout/stderr
        if: always()
        run: |
          more $CI_TEST_RUN_STDOUT_STDERR_DIR/worker-1/pilot.out | cat
      - name: worker clients / reco-icetray instances \#1 stdouts/stderrs
        if: always()
        run: |
          find $CI_TEST_RUN_STDOUT_STDERR_DIR/worker-1/pilot-* -name "stderrfile" -o -name "stdoutfile" | xargs more | cat
          echo "::::::::::::::" && tree $CI_TEST_RUN_STDOUT_STDERR_DIR/worker-1/pilot-*
      - name: worker pilot \#2 stdout/stderr
        if: always() && env.N_WORKERS == '2'
        run: |
          more $CI_TEST_RUN_STDOUT_STDERR_DIR/worker-2/pilot.out | cat
      - name: worker clients / reco-icetray instances \#2 stdouts/stderrs
        if: always() && env.N_WORKERS == '2'
        run: |
          find $CI_TEST_RUN_STDOUT_STDERR_DIR/worker-2/pilot-* -name "stderrfile" -o -name "stdoutfile" | xargs more | cat
          echo "::::::::::::::" && tree $CI_TEST_RUN_STDOUT_STDERR_DIR/worker-2/pilot-*


      - name: rabbitmq logs
        if: always()
        run: |
          docker logs rabbitmq


  test-run-crash-dummy:
    runs-on: ubuntu-latest
    env:
      SKYSCAN_MQ_TOCLIENT_BROKER_ADDRESS: user1@localhost/test
      SKYSCAN_MQ_TOCLIENT_AUTH_TOKEN: password  # using this would override password in address
      SKYSCAN_MQ_FROMCLIENT_BROKER_ADDRESS: user1@localhost/test
      SKYSCAN_MQ_FROMCLIENT_AUTH_TOKEN: password  # using this would override password in address
      EWMS_PILOT_STOP_LISTENING_ON_TASK_ERROR: False  # we want many crashes
    services:
      rabbitmq:
        # see image.tag -> https://github.com/Observation-Management-Service/path-kubernetes/blob/main/helm-values-rabbitmq-bitnami.yaml (see https://artifacthub.io/packages/helm/bitnami/rabbitmq/11.14.3)
        image: bitnami/rabbitmq:3.11.15-debian-11-r0
        env:
          RABBITMQ_USERNAME: user1
          RABBITMQ_PASSWORD: password
          RABBITMQ_VHOST: test
          BITNAMI_DEBUG: true
        # Note: `--network` option is not supported.
        options: >-
          --name rabbitmq
          --health-cmd "rabbitmqctl node_health_check"
          --health-interval 5s
          --health-timeout 5s
          --health-retries 10
        ports:
          - 5672:5672
          - 15672:15672
    steps:
      - uses: jlumbroso/free-disk-space@main  # need space for mq broker and image
        with:
          docker-images: false
      - uses: actions/checkout@v4
      - uses: docker/setup-buildx-action@v3
      - uses: docker/build-push-action@v6
        with:
          context: .
          file: ./Dockerfile
          tags: ${{ env.CI_DOCKER_IMAGE_TAG }}
          load: true
      - name: run
        timeout-minutes: 12  # on average ~9min  # yes, `timeout` is used below but this is insurance
        run: |
          set -x
          source tests/env-vars.sh
          export _RECO_ALGO=crash_dummy
          export _EVENTS_FILE=$(realpath $REALTIME_EVENTS_DIR/hese_event_01.json)
          export _NSIDES="1:0 2:12 4:12"
          export _SKYSCAN_CI_CRASH_DUMMY_PROBABILITY=0.75
          export EWMS_PILOT_TASK_TIMEOUT=15

          cd ./resources/launch_scripts
          # since _SKYSCAN_CI_CRASH_DUMMY_PROBABILITY<1, this step will go forever
          #   so, stop it after some time and chek that it has stderrfiles
          timeout 240 ./local-scan.sh $N_WORKERS $CI_TEST_RUN_STDOUT_STDERR_DIR  || true
      - name: look at stderrfiles
        run: |
          set -x
          # check for fails/errors
          
          error_type_1='intentional crash-dummy error'
          error_type_2='subprocess timed out after'
          pattern="$error_type_1|$error_type_2"
          
          if find "$CI_TEST_RUN_STDOUT_STDERR_DIR/worker-"*/pilot.out -type f -exec grep -qE "$pattern" {} +; then
              echo "Match(es) found: PilotSubprocessError and/or TimeoutError occurred."
          else
              echo "No matches found."
              exit 1
          fi

      - name: central server stdout/stderr
        if: always()
        run: |
          cat $CI_TEST_RUN_STDOUT_STDERR_DIR/server.out
      - name: worker pilot \#1 stdout/stderr
        if: always()
        run: |
          more $CI_TEST_RUN_STDOUT_STDERR_DIR/worker-1/pilot.out | cat
      - name: worker clients / reco-icetray instances \#1 stdouts/stderrs
        if: always()
        run: |
          find $CI_TEST_RUN_STDOUT_STDERR_DIR/worker-1/pilot-* -name "stderrfile" -o -name "stdoutfile" | xargs more | cat
          echo "::::::::::::::" && tree $CI_TEST_RUN_STDOUT_STDERR_DIR/worker-1/pilot-*
      - name: worker pilot \#2 stdout/stderr
        if: always() && env.N_WORKERS == '2'
        run: |
          more $CI_TEST_RUN_STDOUT_STDERR_DIR/worker-2/pilot.out | cat
      - name: worker clients / reco-icetray instances \#2 stdouts/stderrs
        if: always() && env.N_WORKERS == '2'
        run: |
          find $CI_TEST_RUN_STDOUT_STDERR_DIR/worker-2/pilot-* -name "stderrfile" -o -name "stdoutfile" | xargs more | cat
          echo "::::::::::::::" && tree $CI_TEST_RUN_STDOUT_STDERR_DIR/worker-2/pilot-*


  test-run-realistic:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        reco_algo: [
          millipede_original,
          millipede_wilks,
          splinempe,
          splinempe_pointed
        ]
        eventfile: [
          hese_event_01.json,
          run00136766-evt000007637140-GOLD.pkl,
          run00136662-evt000035405932-BRONZE.pkl,
          138632_31747601.json
        ]
        exclude:
          # splinempe should not run on HESE
          - reco_algo: splinempe
            eventfile: hese_event_01.json
          - reco_algo: splinempe_pointed
            eventfile: hese_event_01.json
    env:
      SKYSCAN_MQ_TOCLIENT_BROKER_ADDRESS: user1@localhost/test
      SKYSCAN_MQ_TOCLIENT_AUTH_TOKEN: password  # using this would override password in address
      SKYSCAN_MQ_FROMCLIENT_BROKER_ADDRESS: user1@localhost/test
      SKYSCAN_MQ_FROMCLIENT_AUTH_TOKEN: password  # using this would override password in address
    services:
      rabbitmq:
        # see image.tag -> https://github.com/Observation-Management-Service/path-kubernetes/blob/main/helm-values-rabbitmq-bitnami.yaml (see https://artifacthub.io/packages/helm/bitnami/rabbitmq/11.14.3)
        image: bitnami/rabbitmq:3.11.15-debian-11-r0
        env:
          RABBITMQ_USERNAME: user1
          RABBITMQ_PASSWORD: password
          RABBITMQ_VHOST: test
          BITNAMI_DEBUG: true
        # Note: `--network` option is not supported.
        options: >-
          --name rabbitmq
          --health-cmd "rabbitmqctl node_health_check"
          --health-interval 5s
          --health-timeout 5s
          --health-retries 10
        ports:
          - 5672:5672
          - 15672:15672
    steps:
      - uses: jlumbroso/free-disk-space@main  # need space for mq broker and image
        with:
          docker-images: false
      - uses: actionhippie/swap-space@v1
        with:
          size: 10G
      - uses: actions/checkout@v4
      - uses: docker/setup-buildx-action@v3
      - uses: docker/build-push-action@v6
        with:
          context: .
          file: ./Dockerfile
          tags: ${{ env.CI_DOCKER_IMAGE_TAG }}
          load: true

      - name: run
        timeout-minutes: 45  # on average max~=35min
        run: |
          set -x
          lscpu
          python3 --version
          source tests/env-vars.sh
          export _RECO_ALGO=${{ matrix.reco_algo }}
          export _EVENTS_FILE=$(realpath $REALTIME_EVENTS_DIR/${{ matrix.eventfile }})
          export _NSIDES="1:0"

          cd ./resources/launch_scripts
          ./local-scan.sh $N_WORKERS $CI_TEST_RUN_STDOUT_STDERR_DIR

      - name: test output against known result (.json)
        run: |
          ls $CI_SKYSCAN_OUTPUT_DIR
          # get newest run*.json
          outfile=$(find $CI_SKYSCAN_OUTPUT_DIR -type f -name "run*.json" -exec stat -c '%y %n'  {} + | sort | tail -1 | awk '{print $4}')
          echo $outfile
          cat $outfile
          pip install .  # don't need icecube, so no docker container needed
          python tests/compare_scan_results.py \
            --actual $outfile \
            --expected tests/data/results_json/${{ matrix.reco_algo }}/$(basename $outfile) \
            --assert \
            || (cat $(ls *.diff.json) && false)

      - name: central server stdout/stderr
        if: always()
        run: |
          cat $CI_TEST_RUN_STDOUT_STDERR_DIR/server.out
      - name: worker pilot \#1 stdout/stderr
        if: always()
        run: |
          more $CI_TEST_RUN_STDOUT_STDERR_DIR/worker-1/pilot.out | cat
      - name: worker clients / reco-icetray instances \#1 stdouts/stderrs
        if: always()
        run: |
          find $CI_TEST_RUN_STDOUT_STDERR_DIR/worker-1/pilot-* -name "stderrfile" -o -name "stdoutfile" | xargs more | cat
          echo "::::::::::::::" && tree $CI_TEST_RUN_STDOUT_STDERR_DIR/worker-1/pilot-*
      - name: worker pilot \#2 stdout/stderr
        if: always() && env.N_WORKERS == '2'
        run: |
          more $CI_TEST_RUN_STDOUT_STDERR_DIR/worker-2/pilot.out | cat
      - name: worker clients / reco-icetray instances \#2 stdouts/stderrs
        if: always() && env.N_WORKERS == '2'
        run: |
          find $CI_TEST_RUN_STDOUT_STDERR_DIR/worker-2/pilot-* -name "stderrfile" -o -name "stdoutfile" | xargs more | cat
          echo "::::::::::::::" && tree $CI_TEST_RUN_STDOUT_STDERR_DIR/worker-2/pilot-*

      - name: rabbitmq logs
        if: always()
        run: |
          docker logs rabbitmq


  test-file-staging:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: docker/setup-buildx-action@v3
      - uses: docker/build-push-action@v6
        with:
          context: .
          file: ./Dockerfile
          tags: ${{ env.CI_DOCKER_IMAGE_TAG }}
          load: true
      - name: run
        run: |
          source tests/env-vars.sh
          
          docker run --rm -i \
            $(env | grep -E '^(SKYSCAN_|_SKYSCAN_)' | cut -d'=' -f1 | sed 's/^/--env /') \
            $CI_DOCKER_IMAGE_TAG \
            python tests/file_staging.py


  test-run-single-pixel:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        dir: [
          "BRONZE",
          "GOLD",
          "JSON",
        ]
        reco_algo:
          [
            millipede_original,
            millipede_wilks,
            splinempe,
            splinempe_pointed
          ]
        exclude:
          # splinempe should not run on HESE
          - reco_algo: splinempe
            dir: "JSON"
          - reco_algo: splinempe_pointed
            dir: "JSON"
    steps:
      - uses: actions/checkout@v4
      - uses: docker/setup-buildx-action@v3
      - uses: docker/build-push-action@v6
        with:
          context: .
          file: ./Dockerfile
          tags: ${{ env.CI_DOCKER_IMAGE_TAG }}
          load: true
      - name: run
        timeout-minutes: 10  # on average max~=5min
        run: |
          set -e
          source tests/env-vars.sh
          
          # run reco directly
          docker run --network="host" --rm -i \
            --shm-size=6gb \
            --mount type=bind,source=$(readlink -f tests/data/reco_pixel_single/${{ matrix.reco_algo }}/${{ matrix.dir }}),target=/local/test-data \
            --env PY_COLORS=1 \
            $(env | grep -E '^(SKYSCAN_|_SKYSCAN_)' | cut -d'=' -f1 | sed 's/^/--env /') \
            $CI_DOCKER_IMAGE_TAG \
            python -m skymap_scanner.client \
            --infile /local/test-data/in.json \
            --client-startup-json /local/test-data/startup.json \
            --outfile /local/test-data/out-actual.json

      - name: test output against known result
        run: |
          source tests/env-vars.sh
          
          ls tests/data/reco_pixel_pkls/${{ matrix.reco_algo }}/${{ matrix.dir }}
          ls tests/data/reco_pixel_single/${{ matrix.reco_algo }}/${{ matrix.dir }}

          # need icecube for depickling, so docker container needed
          docker run --network="host" --rm -i \
            --shm-size=6gb \
            --mount type=bind,source=$(readlink -f tests/data/reco_pixel_single/${{ matrix.reco_algo }}/${{ matrix.dir }}),target=/local/test-data \
            --env PY_COLORS=1 \
            $(env | grep -E '^(SKYSCAN_|_SKYSCAN_)' | cut -d'=' -f1 | sed 's/^/--env /') \
            $CI_DOCKER_IMAGE_TAG \
            python tests/compare_reco_pixel_single.py \
            --actual /local/test-data/out-actual.json \
            --expected /local/test-data/out.json \
            --diff-out-dir /local/test-data/ \
            --assert \
            || (cat $(ls tests/data/reco_pixel_single/${{ matrix.reco_algo }}/${{ matrix.dir }}/*.diff.json) && false)


  #############################################################################
  # GITHUB RELEASE
  #############################################################################


  release:
    # only run on main/master/default
    if: format('refs/heads/{0}', github.event.repository.default_branch) == github.ref
    # have to wait for tests so python-semantic-release can push (branch protection on main don't work w/ the bot)
    needs: [
      mypy,
      py-setup,
      py-dependencies,
      test-build-docker,
      test-run-dummy,
      test-run-singularity-dummy-reco,
      test-file-staging,
      test-run-nsides-thresholds-dummy,
      test-run-crash-dummy,
      test-run-realistic,
      test-run-single-pixel,
    ]
    runs-on: ubuntu-latest
    concurrency: release
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
      - name: Python Semantic Release
        uses: python-semantic-release/python-semantic-release@v7.34.6
        with:
          git_committer_name: ${{ env.BOT_NAME }}
          git_committer_email: ${{ env.BOT_EMAIL }}
          github_token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
          # repository_username: __token__
          # repository_password: ${{ secrets.PYPI_TOKEN }}