tests.yml

name: tests


on:
  push:
    branches:
      - '**'
    tags-ignore:
      - '**'


concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  # don't cancel on main/master/default
  cancel-in-progress: ${{ format('refs/heads/{0}', github.event.repository.default_branch) != github.ref }}


env:
  PY_COLORS: "1"
  BOT_NAME: wipacdevbot
  BOT_EMAIL: developers@icecube.wisc.edu
  CI_TEST_RUN_STDOUT_STDERR_DIR: /home/runner/work/skymap_scanner/testrun_outputs
  REALTIME_EVENTS_DIR: /home/runner/work/skymap_scanner/skymap_scanner/tests/data/realtime_events
  SKYSCAN_CACHE_DIR: /home/runner/work/skymap_scanner/skymap_scanner/cache
  SKYSCAN_OUTPUT_DIR: /home/runner/work/skymap_scanner/skymap_scanner/output
  SKYSCAN_DEBUG_DIR: /home/runner/work/skymap_scanner/skymap_scanner/debug
  EWMS_PILOT_DUMP_SUBPROC_OUTPUT: False  # get logs in "reco-icetray logs" step instead
  EWMS_PILOT_STOP_LISTENING_ON_TASK_ERROR: False  # allow a reco to fail (if it keeps failing on redelivery/ies, expect it to affect the scanner server's timeout eventually)
  # see source tests/env-vars.sh


jobs:


  #############################################################################
  # PACKAGING & LINTERS
  #############################################################################


  py-versions:
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.versions.outputs.matrix }}
    steps:
      - uses: actions/checkout@v3
      - id: versions
        uses: WIPACrepo/wipac-dev-py-versions-action@v2.2

  # flake8:
  #   runs-on: ubuntu-latest
  #   steps:
  #     - uses: actions/checkout@v3
  #     - uses: actions/setup-python@v3
  #     - uses: WIPACrepo/wipac-dev-flake8-action@v1.0

  mypy:
    needs: [ py-versions ]
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        py3: ${{ fromJSON(needs.py-versions.outputs.matrix) }}
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v3
        with:
          python-version: ${{ matrix.py3 }}
      - uses: WIPACrepo/wipac-dev-mypy-action@v2.0

  py-setup:
    runs-on: ubuntu-latest
    steps:
      - name: checkout
        uses: actions/checkout@v3
        with:
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
      - uses: WIPACrepo/wipac-dev-py-setup-action@v3.1
        if: github.actor != ${{ env.BOT_NAME }}  # no auto-updates for bots # should match all 'git_committer_name' uses
        with:
          git_committer_name: ${{ env.BOT_NAME }}
          git_committer_email: ${{ env.BOT_EMAIL }}
          base-keywords: IceCube

  py-dependencies:
    runs-on: ubuntu-latest
    steps:
      - uses: jlumbroso/free-disk-space@main  # need space for images to build
        with:
          docker-images: false
      - name: checkout
        uses: actions/checkout@v3
        with:
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
      - uses: WIPACrepo/wipac-dev-py-dependencies-action@v1.1
        if: github.actor != ${{ env.BOT_NAME }}  # no auto-updates for bots # should match all 'git_committer_name' uses
        with:
          git_committer_name: ${{ env.BOT_NAME }}
          git_committer_email: ${{ env.BOT_EMAIL }}
          dockerfile_nametags: Dockerfile:icecube/skymap_scanner:latest


  #############################################################################
  # TESTS
  #############################################################################


  test-build-docker:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: docker/setup-buildx-action@v2
      - uses: docker/build-push-action@v3
        with:
          context: .
          cache-from: type=gha
          cache-to: type=gha,mode=min
          file: Dockerfile
          tags: icecube/skymap_scanner:local


  test-run-singularity-dummy-reco:
    needs: test-build-docker
    runs-on: ubuntu-latest
    env:
      SKYSCAN_BROKER_ADDRESS: user1:password@localhost/test
      # SKYSCAN_BROKER_AUTH: user1  # using this would override password in address
    services:
      rabbitmq:
        # see image.tag -> https://github.com/Observation-Management-Service/path-kubernetes/blob/main/helm-values-rabbitmq-bitnami.yaml (see https://artifacthub.io/packages/helm/bitnami/rabbitmq/11.14.3)
        image: bitnami/rabbitmq:3.11.15-debian-11-r0
        env:
          RABBITMQ_USERNAME: user1
          RABBITMQ_PASSWORD: password
          RABBITMQ_VHOST: test
          BITNAMI_DEBUG: true
        # Note: `--network` option is not supported.
        options: >-
          --name rabbitmq
          --health-cmd "rabbitmqctl node_health_check"
          --health-interval 5s
          --health-timeout 5s
          --health-retries 10
        ports:
          - 5672:5672
          - 15672:15672
    steps:
      - uses: jlumbroso/free-disk-space@main  # need space for mq broker and image
        with:
          docker-images: false
      - uses: actions/checkout@v3
      - uses: docker/setup-buildx-action@v2
      - uses: docker/build-push-action@v3
        with:
          context: .
          cache-from: type=gha
          # cache-to: type=gha,mode=min
          file: Dockerfile
          tags: icecube/skymap_scanner:local
          load: true
      - uses: eWaterCycle/setup-apptainer@v2
        with:
          apptainer-version: 1.3.2
      - name: build singularity image
        run: |
          sudo singularity build skymap_scanner.sif docker-daemon://icecube/skymap_scanner:local
          ls -lh skymap_scanner.sif
      - name: run singularity container
        run: |
          source tests/env-vars.sh

          mkdir $SKYSCAN_CACHE_DIR
          mkdir $SKYSCAN_OUTPUT_DIR

          # Launch Server
          singularity run skymap_scanner.sif \
            python -m skymap_scanner.server \
            --reco-algo dummy \
            --event-file $REALTIME_EVENTS_DIR/hese_event_01.json \
            --cache-dir $SKYSCAN_CACHE_DIR \
            --output-dir $SKYSCAN_OUTPUT_DIR \
            --client-startup-json ./startup.json \
            --nsides 1:0 \
            --simulated-event \
            &

          ./resources/launch_scripts/wait_for_file.sh ./startup.json $CLIENT_STARTER_WAIT_FOR_STARTUP_JSON

          # Launch Clients
          nclients=2
          echo "Launching $nclients clients"
          mkdir $SKYSCAN_DEBUG_DIR
          export EWMS_PILOT_TASK_TIMEOUT=1800  # 30 mins
          for i in $( seq 1 $nclients ); do
            singularity run skymap_scanner.sif \
              python -m skymap_scanner.client \
              --client-startup-json ./startup.json \
              --debug-directory $SKYSCAN_DEBUG_DIR \
              &
            echo -e "\tclient #$i launched"
          done

          wait -n  # for server
          for i in $( seq 1 $nclients ); do
            wait -n  # for client
          done
      - name: look at results file (.npz)
        run: |
          ls .
          ls $SKYSCAN_OUTPUT_DIR
          outfile=$(ls -d $SKYSCAN_OUTPUT_DIR/*.npz)
          echo $outfile
      - name: broker docker logs
        if: always()
        run: |
          docker logs rabbitmq


  test-run-nsides-thresholds-dummy:
    needs: test-build-docker
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        nsides: [
          "1:0",
          "1:0 2:12",
          "1:0 2:12 4:12"
        ]
        predictive_scanning_threshold: [
          1.0,
          0.65,
        ]
    env:
      SKYSCAN_BROKER_ADDRESS: user1@localhost/test
      SKYSCAN_BROKER_AUTH: password  # using this would override password in address
    services:
      rabbitmq:
        # see image.tag -> https://github.com/Observation-Management-Service/path-kubernetes/blob/main/helm-values-rabbitmq-bitnami.yaml (see https://artifacthub.io/packages/helm/bitnami/rabbitmq/11.14.3)
        image: bitnami/rabbitmq:3.11.15-debian-11-r0
        env:
          RABBITMQ_USERNAME: user1
          RABBITMQ_PASSWORD: password
          RABBITMQ_VHOST: test
          BITNAMI_DEBUG: true
        # Note: `--network` option is not supported.
        options: >-
          --name rabbitmq
          --health-cmd "rabbitmqctl node_health_check"
          --health-interval 5s
          --health-timeout 5s
          --health-retries 10
        ports:
          - 5672:5672
          - 15672:15672
    steps:
      - uses: jlumbroso/free-disk-space@main  # need space for mq broker and image
        with:
          docker-images: false
      - uses: actions/checkout@v3
      - uses: docker/setup-buildx-action@v2
      - uses: docker/build-push-action@v3
        with:
          context: .
          cache-from: type=gha
          # cache-to: type=gha,mode=min
          file: Dockerfile
          tags: icecube/skymap_scanner:local
          load: true
      - name: run
        timeout-minutes: 10  # on average max~=5min
        run: |
          set -x
          source tests/env-vars.sh
          export _RECO_ALGO=dummy
          export _EVENTS_FILE=$(realpath $REALTIME_EVENTS_DIR/hese_event_01.json)
          export _NSIDES="${{ matrix.nsides }}"
          export _PREDICTIVE_SCANNING_THRESHOLD=${{ matrix.predictive_scanning_threshold }}

          cd ./resources/launch_scripts
          ./local-scan.sh 2 $CI_TEST_RUN_STDOUT_STDERR_DIR

      - name: check no nsides skipped
        run: |
          ls $SKYSCAN_OUTPUT_DIR
          # get newest run*.json
          export outfile=$(find $SKYSCAN_OUTPUT_DIR -type f -name "run*.json" -exec stat -c '%y %n'  {} + | sort | tail -1 | awk '{print $4}')
          echo $outfile
          python3 -c '
          import json
          import os
          with open(os.getenv("outfile")) as f:
            pydict = json.load(f)
          nsides = "${{ matrix.nsides }}"
          assert len(pydict) == nsides.count(":")
          '

      - name: central server stdout/stderr
        if: always()
        run: |
          cat $CI_TEST_RUN_STDOUT_STDERR_DIR/server.out
      - name: client worker \#1 stdout/stderr
        if: always()
        run: |
          cat $CI_TEST_RUN_STDOUT_STDERR_DIR/client-1.out
      - name: client worker \#2 stdout/stderr
        if: always()
        run: |
          cat $CI_TEST_RUN_STDOUT_STDERR_DIR/client-2.out

      - name: reco-icetray logs
        if: always()
        run: |
          sudo apt install tree
          tree $SKYSCAN_DEBUG_DIR
          find $SKYSCAN_DEBUG_DIR -type f -not -name "*.i3" -not -name "*.pkl" -exec "more" {} + | cat  # recursively cats with filenames (delimited by :::::::)

      - name: broker docker logs
        if: always()
        run: |
          docker logs rabbitmq


  test-run-crash-dummy:
    needs: test-build-docker
    runs-on: ubuntu-latest
    env:
      SKYSCAN_BROKER_ADDRESS: user1@localhost/test
      SKYSCAN_BROKER_AUTH: password  # using this would override password in address
    services:
      rabbitmq:
        # see image.tag -> https://github.com/Observation-Management-Service/path-kubernetes/blob/main/helm-values-rabbitmq-bitnami.yaml (see https://artifacthub.io/packages/helm/bitnami/rabbitmq/11.14.3)
        image: bitnami/rabbitmq:3.11.15-debian-11-r0
        env:
          RABBITMQ_USERNAME: user1
          RABBITMQ_PASSWORD: password
          RABBITMQ_VHOST: test
          BITNAMI_DEBUG: true
        # Note: `--network` option is not supported.
        options: >-
          --name rabbitmq
          --health-cmd "rabbitmqctl node_health_check"
          --health-interval 5s
          --health-timeout 5s
          --health-retries 10
        ports:
          - 5672:5672
          - 15672:15672
    steps:
      - uses: jlumbroso/free-disk-space@main  # need space for mq broker and image
        with:
          docker-images: false
      - uses: actions/checkout@v3
      - uses: docker/setup-buildx-action@v2
      - uses: docker/build-push-action@v3
        with:
          context: .
          cache-from: type=gha
          # cache-to: type=gha,mode=min
          file: Dockerfile
          tags: icecube/skymap_scanner:local
          load: true
      - name: run
        timeout-minutes: 10  # on average ~6min  # yes, `timeout` is used below but this is insurance
        run: |
          set -x
          source tests/env-vars.sh
          export _RECO_ALGO=crash_dummy
          export _EVENTS_FILE=$(realpath $REALTIME_EVENTS_DIR/hese_event_01.json)
          export _NSIDES="1:0 2:12 4:12"
          export SKYSCAN_CRASH_DUMMY_PROBABILITY=0.75
          export EWMS_PILOT_TASK_TIMEOUT=15

          cd ./resources/launch_scripts
          # since SKYSCAN_CRASH_DUMMY_PROBABILITY<1, this step will go forever
          #   so, stop it after some time and chek that it has stderrfiles
          timeout 240 ./local-scan.sh 2 $CI_TEST_RUN_STDOUT_STDERR_DIR  || true
      - name: look at stderrfiles
        run: |
          set -x
          # check for fails
          stderrfiles=$(find $SKYSCAN_DEBUG_DIR/ -name "stderrfile" -type f -exec sh -c "tail -1 {} | sed -e 's/^.*DEBUG //p' " \; | sort | uniq -c)
          echo $stderrfiles
          if [ -z "${stderrfiles}" ]; then
              exit 1
          fi

      - name: central server stdout/stderr
        if: always()
        run: |
          cat $CI_TEST_RUN_STDOUT_STDERR_DIR/server.out
      - name: client worker \#1 stdout/stderr
        if: always()
        run: |
          cat $CI_TEST_RUN_STDOUT_STDERR_DIR/client-1.out
      - name: client worker \#2 stdout/stderr
        if: always()
        run: |
          cat $CI_TEST_RUN_STDOUT_STDERR_DIR/client-2.out


  test-run-realistic:
    needs: test-build-docker
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        reco_algo: [
          millipede_original,
          millipede_wilks,
          splinempe,
          splinempe_pointed
        ]
        eventfile: [
          hese_event_01.json,
          run00136766-evt000007637140-GOLD.pkl,
          run00136662-evt000035405932-BRONZE.pkl,
          138632_31747601.json
        ]
        exclude:
          # splinempe should not run on HESE
          - reco_algo: splinempe
            eventfile: hese_event_01.json
          - reco_algo: splinempe_pointed
            eventfile: hese_event_01.json
    env:
      SKYSCAN_BROKER_ADDRESS: user1@localhost/test
      SKYSCAN_BROKER_AUTH: password  # using this would override password in address
    services:
      rabbitmq:
        # see image.tag -> https://github.com/Observation-Management-Service/path-kubernetes/blob/main/helm-values-rabbitmq-bitnami.yaml (see https://artifacthub.io/packages/helm/bitnami/rabbitmq/11.14.3)
        image: bitnami/rabbitmq:3.11.15-debian-11-r0
        env:
          RABBITMQ_USERNAME: user1
          RABBITMQ_PASSWORD: password
          RABBITMQ_VHOST: test
          BITNAMI_DEBUG: true
        # Note: `--network` option is not supported.
        options: >-
          --name rabbitmq
          --health-cmd "rabbitmqctl node_health_check"
          --health-interval 5s
          --health-timeout 5s
          --health-retries 10
        ports:
          - 5672:5672
          - 15672:15672
    steps:
      - uses: jlumbroso/free-disk-space@main  # need space for mq broker and image
        with:
          docker-images: false
      - uses: actionhippie/swap-space@v1
        with:
          size: 10G
      - uses: actions/checkout@v3
      - uses: docker/setup-buildx-action@v2
      - uses: docker/build-push-action@v3
        with:
          context: .
          cache-from: type=gha
          # cache-to: type=gha,mode=min
          file: Dockerfile
          tags: icecube/skymap_scanner:local
          load: true
      - name: run
        timeout-minutes: 35  # on average max~=26min
        run: |
          set -x
          lscpu
          python3 --version
          source tests/env-vars.sh
          export _RECO_ALGO=${{ matrix.reco_algo }}
          export _EVENTS_FILE=$(realpath $REALTIME_EVENTS_DIR/${{ matrix.eventfile }})
          export _NSIDES="1:0"

          cd ./resources/launch_scripts
          ./local-scan.sh 2 $CI_TEST_RUN_STDOUT_STDERR_DIR

      - name: test output against known result (.json)
        run: |
          ls $SKYSCAN_OUTPUT_DIR
          # get newest run*.json
          outfile=$(find $SKYSCAN_OUTPUT_DIR -type f -name "run*.json" -exec stat -c '%y %n'  {} + | sort | tail -1 | awk '{print $4}')
          echo $outfile
          cat $outfile
          pip install .  # don't need icecube, so no docker container needed
          python tests/compare_scan_results.py \
            --actual $outfile \
            --expected tests/data/results_json/${{ matrix.reco_algo }}/$(basename $outfile) \
            --assert \
            || (cat $(ls *.diff.json) && false)

      - name: central server stdout/stderr
        if: always()
        run: |
          cat $CI_TEST_RUN_STDOUT_STDERR_DIR/server.out
      - name: client worker \#1 stdout/stderr
        if: always()
        run: |
          cat $CI_TEST_RUN_STDOUT_STDERR_DIR/client-1.out
      - name: client worker \#2 stdout/stderr
        if: always()
        run: |
          cat $CI_TEST_RUN_STDOUT_STDERR_DIR/client-2.out

      - name: reco-icetray logs
        if: always()
        run: |
          sudo apt install tree
          tree $SKYSCAN_DEBUG_DIR
          find $SKYSCAN_DEBUG_DIR -type f -not -name "*.i3" -not -name "*.pkl" -exec "more" {} + | cat  # recursively cats with filenames (delimited by :::::::)

      - name: broker docker logs
        if: always()
        run: |
          docker logs rabbitmq


  test-file-staging:
    needs: test-build-docker
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: docker/setup-buildx-action@v2
      - uses: docker/build-push-action@v3
        with:
          context: .
          cache-from: type=gha
          # cache-to: type=gha,mode=min
          file: Dockerfile
          tags: icecube/skymap_scanner:local
          load: true
      - name: run
        run: |
          docker run --rm -i \
            icecube/skymap_scanner:local \
            python tests/file_staging.py


  test-run-single-pixel:
    needs: test-build-docker
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        dir: [
          "BRONZE",
          "GOLD",
          "JSON",
        ]
        reco_algo:
          [
            millipede_original,
            millipede_wilks,
            splinempe,
            splinempe_pointed
          ]
        exclude:
          # splinempe should not run on HESE
          - reco_algo: splinempe
            dir: "JSON"
          - reco_algo: splinempe_pointed
            dir: "JSON"
    steps:
      - uses: actions/checkout@v3
      - uses: docker/setup-buildx-action@v2
      - uses: docker/build-push-action@v3
        with:
          context: .
          cache-from: type=gha
          # cache-to: type=gha,mode=min
          file: Dockerfile
          tags: icecube/skymap_scanner:local
          load: true
      - name: run
        timeout-minutes: 15  # on average max~=7min
        run: |
          set -e
          source tests/env-vars.sh

          # grab the GCDQp_packet key and throw into a file
          jq '.GCDQp_packet' \
            tests/data/reco_pixel_single/${{ matrix.reco_algo }}/${{ matrix.dir }}/startup.json > \
            tests/data/reco_pixel_single/${{ matrix.reco_algo }}/${{ matrix.dir }}//GCDQp_packet.json
          
          # run reco
          docker run --network="host" --rm -i \
            --shm-size=6gb \
            --mount type=bind,source=$(readlink -f tests/data/reco_pixel_single/${{ matrix.reco_algo }}/${{ matrix.dir }}),target=/local/test-data \
            --env PY_COLORS=1 \
            $(env | grep '^SKYSCAN_' | awk '$0="--env "$0') \
            icecube/skymap_scanner:local \
            python -m skymap_scanner.client.reco_icetray \
            --infile /local/test-data/in.json \
            --gcdqp-packet-json /local/test-data/GCDQp_packet.json \
            --baseline-gcd-file $(jq -r '.baseline_GCD_file' tests/data/reco_pixel_single/${{ matrix.reco_algo }}/${{ matrix.dir }}/startup.json) \
            --outfile /local/test-data/out-actual.json

      - name: test output against known result
        run: |
          ls tests/data/reco_pixel_single/${{ matrix.reco_algo }}/${{ matrix.dir }}

          # need icecube for depickling, so docker container needed
          docker run --network="host" --rm -i \
            --shm-size=6gb \
            --mount type=bind,source=$(readlink -f tests/data/reco_pixel_single/${{ matrix.reco_algo }}/${{ matrix.dir }}),target=/local/test-data \
            --env PY_COLORS=1 \
            $(env | grep '^SKYSCAN_' | awk '$0="--env "$0') \
            icecube/skymap_scanner:local \
            python tests/compare_reco_pixel_single.py \
            --actual /local/test-data/out-actual.json \
            --expected /local/test-data/out.json \
            --diff-out-dir /local/test-data/ \
            --assert \
            || (cat $(ls tests/data/reco_pixel_single/${{ matrix.reco_algo }}/${{ matrix.dir }}/*.diff.json) && false)


  #############################################################################
  # GITHUB RELEASE
  #############################################################################


  release:
    # only run on main/master/default
    if: format('refs/heads/{0}', github.event.repository.default_branch) == github.ref
    # have to wait for tests so python-semantic-release can push (branch protection on main don't work w/ the bot)
    needs: [
      mypy,
      py-setup,
      py-dependencies,
      test-build-docker,
      test-run-singularity-dummy-reco,
      test-file-staging,
      test-run-nsides-thresholds-dummy,
      test-run-crash-dummy,
      test-run-realistic,
      test-run-single-pixel
    ]
    runs-on: ubuntu-latest
    concurrency: release
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
      - name: Python Semantic Release
        uses: python-semantic-release/python-semantic-release@v7.34.6
        with:
          git_committer_name: ${{ env.BOT_NAME }}
          git_committer_email: ${{ env.BOT_EMAIL }}
          github_token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
          # repository_username: __token__
          # repository_password: ${{ secrets.PYPI_TOKEN }}