icd_tests_linux.yml

name: Linux ICD tests
on:
  push:
  schedule:
    - cron: '0 0 * * *' # UTC time
env:
  ICD_RXJS_BRANCH_NAME: ajm/icd-test-stages-lists

jobs:

  Build:
    name: Build ${{ matrix.os_version }}
    runs-on: ${{ matrix.runner }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - os_version: ubuntu-20.04
            runner: [self-hosted, Linux, Apptainer, ICD1]
            image: /opt/apptainer/ubuntu-2004-dec2023.sif
          - os_version: ubuntu-22.04
            runner: [self-hosted, Linux, Apptainer, ICD1]
            image: /opt/apptainer/ubuntu-2204-dec2023.sif
          - os_version: rhel-7
            runner: [self-hosted, linux, Apptainer, ICD2]
            image: /opt/apptainer/centos7-dec2023.sif              
          - os_version: rhel-8
            runner: [self-hosted, linux, Apptainer, ICD2]
            image: /opt/apptainer/almalinux8-dec2023.sif
          - os_version: rhel-9
            runner: [self-hosted, linux, Apptainer, ICD3]
            image: /opt/apptainer/almalinux9-dec2023.sif
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          path: source

      - name: Build backend
        shell: bash
        run: |
          SRC_DIR=$GITHUB_WORKSPACE/source
          BUILD_DIR=$GITHUB_WORKSPACE/build-${{ matrix.os_version }}
          rm -rf $BUILD_DIR
          mkdir -p $BUILD_DIR
          apptainer exec --bind $GITHUB_WORKSPACE:$GITHUB_WORKSPACE --pwd $SRC_DIR ${{ matrix.image }} /bin/bash -c "\
            git submodule update --init --recursive && \
            cd $BUILD_DIR && \
            cmake $SRC_DIR \
              -Dtest=on \
              -DCMAKE_BUILD_TYPE=Debug \
              -DDevSuppressExternalWarnings=ON \
              -DCMAKE_CXX_FLAGS='-O0 -g -fsanitize=address -fno-omit-frame-pointer' \
              -DCMAKE_EXE_LINKER_FLAGS='-fsanitize=address' && \
            make -j 16"

      - name: Check backend runs
        shell: bash
        run: |
          BUILD_DIR=$GITHUB_WORKSPACE/build-${{ matrix.os_version }}
          apptainer exec --bind $GITHUB_WORKSPACE:$GITHUB_WORKSPACE --pwd $BUILD_DIR ${{ matrix.image }} /bin/bash -c "./carta_backend --version"          

  Prepare-ICD-RxJS:
    name: Prepare-ICD-RxJS ${{ matrix.os_version }}
    runs-on: ${{ matrix.runner }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - os_version: ubuntu-20.04
            runner: [self-hosted, Linux, Apptainer, ICD1]
            image: /opt/apptainer/ubuntu-2004-dec2023.sif
            port: 9001
          - os_version: ubuntu-22.04
            runner: [self-hosted, Linux, Apptainer, ICD1]
            image: /opt/apptainer/ubuntu-2204-dec2023.sif
            port: 9002
          - os_version: rhel-7
            runner: [self-hosted, linux, Apptainer, ICD2]
            image: /opt/apptainer/centos7-dec2023.sif
            port: 9003
          - os_version: rhel-8
            runner: [self-hosted, linux, Apptainer, ICD2]
            image: /opt/apptainer/almalinux8-dec2023.sif
            port: 9004
          - os_version: rhel-9
            runner: [self-hosted, linux, Apptainer, ICD3]
            image: /opt/apptainer/almalinux9-dec2023.sif
            port: 9005
    needs: Build
    steps:
      - name: Prepare ICD-RxJS
        shell: bash
        run: |
          BUILD_DIR=$GITHUB_WORKSPACE/build-${{ matrix.os_version }}
          ICD_RXJS_DIR=$BUILD_DIR/ICD-RxJS
          apptainer exec --bind $GITHUB_WORKSPACE:$GITHUB_WORKSPACE --pwd $BUILD_DIR ${{ matrix.image }} /bin/bash -c "\
            if [ -d '$ICD_RXJS_DIR' ]; then
                cd $ICD_RXJS_DIR && git pull
              else
                echo 'Cloning ICD-RxJS repository...'
                git clone https://github.com/CARTAvis/ICD-RxJS.git
              cd ICD-RxJS
            fi && \
            git checkout ${{ env.ICD_RXJS_BRANCH_NAME }} && \
            git submodule init && git submodule update && npm install && \
            cd protobuf && \
            ./build_proto.sh && \
            cd ../src/test && \
            perl -p -i -e 's/3002/${{ matrix.port }}/' config.json"

  File-Browser-ICD-Tests:
    name: File-Browser ${{ matrix.os_version }}
    runs-on: ${{ matrix.runner }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - os_version: ubuntu-20.04
            runner: [self-hosted, Linux, Apptainer, ICD1]
            image: /opt/apptainer/ubuntu-2004-dec2023.sif
            port: 9001
          - os_version: ubuntu-22.04
            runner: [self-hosted, Linux, Apptainer, ICD1]
            image: /opt/apptainer/ubuntu-2204-dec2023.sif
            port: 9002
          - os_version: rhel-7
            runner: [self-hosted, linux, Apptainer, ICD2]
            image: /opt/apptainer/centos7-dec2023.sif
            port: 9003
          - os_version: rhel-8
            runner: [self-hosted, linux, Apptainer, ICD2]
            image: /opt/apptainer/almalinux8-dec2023.sif
            port: 9004
          - os_version: rhel-9
            runner: [self-hosted, linux, Apptainer, ICD3]
            image: /opt/apptainer/almalinux9-dec2023.sif
            port: 9005
    needs: [Build, Prepare-ICD-RxJS]
    steps:
      - name: File Browser ICD tests
        shell: bash
        run: |
          SRC_DIR=$GITHUB_WORKSPACE/source
          BUILD_DIR=$GITHUB_WORKSPACE/build-${{ matrix.os_version }}
          TEST_STAGE="$BUILD_DIR/ICD-RxJS/ICD_test_stages/file_browser.tests"
          LOG_FILE="/tmp/carta_icd_${{ matrix.os_version }}_file_browser.log"
          apptainer exec \
            --bind $GITHUB_WORKSPACE:$GITHUB_WORKSPACE \
            --bind /images:/images \
            --pwd $BUILD_DIR \
            ${{ matrix.image }} /bin/bash -c "\
            # Start the carta_backend
            ASAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan.supp \
            LSAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan-leaks.supp \
            ASAN_SYMBOLIZER_PATH=llvm-symbolizer \
            ./carta_backend /images \
              --top_level_folder /images \
              --port ${{ matrix.port }} \
              --omp_threads=4 \
              --debug_no_auth \
              --no_frontend \
              --no_database \
              --no_log \
              --verbosity=5 >> $LOG_FILE 2>&1 & \
            CARTA_BACKEND_PID=\$(pgrep -f 'carta_backend.*${{ matrix.port }}' | head -n 1) && \
            echo 'carta_backend started with PID' \$CARTA_BACKEND_PID && \
            # Run the ICD tests
            cd $BUILD_DIR/ICD-RxJS && \
            pwd && \
            cat $TEST_STAGE && \
            while IFS= read -r test_file || [[ -n "\$test_file" ]]; do
              CI=true npm test -- "\$test_file"
            done < $TEST_STAGE"

  Animator-ICD-Tests:
    name: Animator ${{ matrix.os_version }}
    runs-on: ${{ matrix.runner }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - os_version: ubuntu-20.04
            runner: [self-hosted, Linux, Apptainer, ICD1]
            image: /opt/apptainer/ubuntu-2004-dec2023.sif
            port: 9001
          - os_version: ubuntu-22.04
            runner: [self-hosted, Linux, Apptainer, ICD1]
            image: /opt/apptainer/ubuntu-2204-dec2023.sif
            port: 9002
          - os_version: rhel-7
            runner: [self-hosted, linux, Apptainer, ICD2]
            image: /opt/apptainer/centos7-dec2023.sif
            port: 9003
          - os_version: rhel-8
            runner: [self-hosted, linux, Apptainer, ICD2]
            image: /opt/apptainer/almalinux8-dec2023.sif
            port: 9004
          - os_version: rhel-9
            runner: [self-hosted, linux, Apptainer, ICD3]
            image: /opt/apptainer/almalinux9-dec2023.sif
            port: 9005          
    needs: [Prepare-ICD-RxJS]
    steps:
      - name: Animator ICD tests
        shell: bash
        run: |
          SRC_DIR=$GITHUB_WORKSPACE/source
          BUILD_DIR=$GITHUB_WORKSPACE/build-${{ matrix.os_version }}
          TEST_STAGE="$BUILD_DIR/ICD-RxJS/ICD_test_stages/animator.tests"
          LOG_FILE="/tmp/carta_icd_${{ matrix.os_version }}_animator.log"
          PORT_NUMBER=${{ matrix.port }}
          apptainer exec \
            --bind $GITHUB_WORKSPACE:$GITHUB_WORKSPACE \
            --bind /images:/images \
            --pwd $BUILD_DIR \
            ${{ matrix.image }} /bin/bash -c "\
            # Start the carta_backend
            ASAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan.supp \
            LSAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan-leaks.supp \
            ASAN_SYMBOLIZER_PATH=llvm-symbolizer \
            ./carta_backend /images \
              --top_level_folder /images \
              --port ${{ matrix.port }} \
              --omp_threads=4 \
              --debug_no_auth \
              --no_frontend \
              --no_database \
              --no_log \
              --verbosity=5 >> $LOG_FILE 2>&1 & \
            CARTA_BACKEND_PID=\$(pgrep -f 'carta_backend.*${{ matrix.port }}' | head -n 1) && \
            echo 'carta_backend started with PID' \$CARTA_BACKEND_PID && \
            # Run the ICD tests
            cd $BUILD_DIR/ICD-RxJS && \
            pwd && \
            cat $TEST_STAGE && \
            while IFS= read -r test_file || [[ -n "\$test_file" ]]; do
              CI=true npm test -- "\$test_file"
            done < $TEST_STAGE"

  Region-Statistics-ICD-Tests:
    name: Region-Statistics ${{ matrix.os_version }}
    runs-on: ${{ matrix.runner }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - os_version: ubuntu-20.04
            runner: [self-hosted, Linux, Apptainer, ICD1]
            image: /opt/apptainer/ubuntu-2004-dec2023.sif
            port: 9001
          - os_version: ubuntu-22.04
            runner: [self-hosted, Linux, Apptainer, ICD1]
            image: /opt/apptainer/ubuntu-2204-dec2023.sif
            port: 9002
          - os_version: rhel-7
            runner: [self-hosted, linux, Apptainer, ICD2]
            image: /opt/apptainer/centos7-dec2023.sif
            port: 9003
          - os_version: rhel-8
            runner: [self-hosted, linux, Apptainer, ICD2]
            image: /opt/apptainer/almalinux8-dec2023.sif
            port: 9004
          - os_version: rhel-9
            runner: [self-hosted, linux, Apptainer, ICD3]
            image: /opt/apptainer/almalinux9-dec2023.sif
            port: 9005          
    needs: [Prepare-ICD-RxJS]
    steps:
      - name: Region Statistics ICD tests
        shell: bash
        run: |
          SRC_DIR=$GITHUB_WORKSPACE/source
          BUILD_DIR=$GITHUB_WORKSPACE/build-${{ matrix.os_version }}
          TEST_STAGE="$BUILD_DIR/ICD-RxJS/ICD_test_stages/region_statistics.tests"
          LOG_FILE="/tmp/carta_icd_${{ matrix.os_version }}_region_statistics.log"
          PORT_NUMBER=${{ matrix.port }}
          apptainer exec \
            --bind $GITHUB_WORKSPACE:$GITHUB_WORKSPACE \
            --bind /images:/images \
            --pwd $BUILD_DIR \
            ${{ matrix.image }} /bin/bash -c "\
            # Start the carta_backend
            ASAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan.supp \
            LSAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan-leaks.supp \
            ASAN_SYMBOLIZER_PATH=llvm-symbolizer \
            ./carta_backend /images \
              --top_level_folder /images \
              --port ${{ matrix.port }} \
              --omp_threads=4 \
              --debug_no_auth \
              --no_frontend \
              --no_database \
              --no_log \
              --verbosity=5 >> $LOG_FILE 2>&1 & \
            CARTA_BACKEND_PID=\$(pgrep -f 'carta_backend.*${{ matrix.port }}' | head -n 1) && \
            echo 'carta_backend started with PID' \$CARTA_BACKEND_PID && \
            # Run the ICD tests
            cd $BUILD_DIR/ICD-RxJS && \
            pwd && \
            cat $TEST_STAGE && \
            while IFS= read -r test_file || [[ -n "\$test_file" ]]; do
              CI=true npm test -- "\$test_file"
            done < $TEST_STAGE"

  Region-Manipulation-ICD-Tests:
    name: Region-Manipulation ${{ matrix.os_version }}
    runs-on: ${{ matrix.runner }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - os_version: ubuntu-20.04
            runner: [self-hosted, Linux, Apptainer, ICD1]
            image: /opt/apptainer/ubuntu-2004-dec2023.sif
            port: 9001
          - os_version: ubuntu-22.04
            runner: [self-hosted, Linux, Apptainer, ICD1]
            image: /opt/apptainer/ubuntu-2204-dec2023.sif
            port: 9002
          - os_version: rhel-7
            runner: [self-hosted, linux, Apptainer, ICD2]
            image: /opt/apptainer/centos7-dec2023.sif
            port: 9003
          - os_version: rhel-8
            runner: [self-hosted, linux, Apptainer, ICD2]
            image: /opt/apptainer/almalinux8-dec2023.sif
            port: 9004
          - os_version: rhel-9
            runner: [self-hosted, linux, Apptainer, ICD3]
            image: /opt/apptainer/almalinux9-dec2023.sif
            port: 9005
    needs: [Prepare-ICD-RxJS]
    steps:
      - name: Region Manipulation ICD tests
        shell: bash
        run: |
          SRC_DIR=$GITHUB_WORKSPACE/source
          BUILD_DIR=$GITHUB_WORKSPACE/build-${{ matrix.os_version }}
          TEST_STAGE="$BUILD_DIR/ICD-RxJS/ICD_test_stages/region_manipulation.tests"
          LOG_FILE="/tmp/carta_icd_${{ matrix.os_version }}_region_manipulation.log"
          PORT_NUMBER=${{ matrix.port }}
          apptainer exec \
            --bind $GITHUB_WORKSPACE:$GITHUB_WORKSPACE \
            --bind /images:/images \
            --pwd $BUILD_DIR \
            ${{ matrix.image }} /bin/bash -c "\
            # Start the carta_backend
            ASAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan.supp \
            LSAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan-leaks.supp \
            ASAN_SYMBOLIZER_PATH=llvm-symbolizer \
            ./carta_backend /images \
              --top_level_folder /images \
              --port ${{ matrix.port }} \
              --omp_threads=4 \
              --debug_no_auth \
              --no_frontend \
              --no_database \
              --no_log \
              --verbosity=5 >> $LOG_FILE 2>&1 & \
            CARTA_BACKEND_PID=\$(pgrep -f 'carta_backend.*${{ matrix.port }}' | head -n 1) && \
            echo 'carta_backend started with PID' \$CARTA_BACKEND_PID && \
            # Run the ICD tests
            cd $BUILD_DIR/ICD-RxJS && \
            pwd && \
            cat $TEST_STAGE && \
            while IFS= read -r test_file || [[ -n "\$test_file" ]]; do
              CI=true npm test -- "\$test_file"
            done < $TEST_STAGE"

  Cube-Histogram-ICD-Tests:
    name: Cube-Histogram ${{ matrix.os_version }}
    runs-on: ${{ matrix.runner }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - os_version: ubuntu-20.04
            runner: [self-hosted, Linux, Apptainer, ICD1]
            image: /opt/apptainer/ubuntu-2004-dec2023.sif
            port: 9001
          - os_version: ubuntu-22.04
            runner: [self-hosted, Linux, Apptainer, ICD1]
            image: /opt/apptainer/ubuntu-2204-dec2023.sif
            port: 9002
          - os_version: rhel-7
            runner: [self-hosted, linux, Apptainer, ICD2]
            image: /opt/apptainer/centos7-dec2023.sif
            port: 9003
          - os_version: rhel-8
            runner: [self-hosted, linux, Apptainer, ICD2]
            image: /opt/apptainer/almalinux8-dec2023.sif
            port: 9004
          - os_version: rhel-9
            runner: [self-hosted, linux, Apptainer, ICD3]
            image: /opt/apptainer/almalinux9-dec2023.sif
            port: 9005
    needs: [Prepare-ICD-RxJS]
    steps:
      - name: Cube Histogram ICD tests
        shell: bash
        run: |
          SRC_DIR=$GITHUB_WORKSPACE/source
          BUILD_DIR=$GITHUB_WORKSPACE/build-${{ matrix.os_version }}
          TEST_STAGE="$BUILD_DIR/ICD-RxJS/ICD_test_stages/cube_histogram.tests"
          LOG_FILE="/tmp/carta_icd_${{ matrix.os_version }}_cube_histogram.log"
          PORT_NUMBER=${{ matrix.port }}
          apptainer exec \
            --bind $GITHUB_WORKSPACE:$GITHUB_WORKSPACE \
            --bind /images:/images \
            --pwd $BUILD_DIR \
            ${{ matrix.image }} /bin/bash -c "\
            # Start the carta_backend
            ASAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan.supp \
            LSAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan-leaks.supp \
            ASAN_SYMBOLIZER_PATH=llvm-symbolizer \
            ./carta_backend /images \
              --top_level_folder /images \
              --port ${{ matrix.port }} \
              --omp_threads=4 \
              --debug_no_auth \
              --no_frontend \
              --no_database \
              --no_log \
              --verbosity=5 >> $LOG_FILE 2>&1 & \
            CARTA_BACKEND_PID=\$(pgrep -f 'carta_backend.*${{ matrix.port }}' | head -n 1) && \
            echo 'carta_backend started with PID' \$CARTA_BACKEND_PID && \
            # Run the ICD tests
            cd $BUILD_DIR/ICD-RxJS && \
            pwd && \
            cat $TEST_STAGE && \
            while IFS= read -r test_file || [[ -n "\$test_file" ]]; do
              CI=true npm test -- "\$test_file"
            done < $TEST_STAGE"

  PV-Generator-ICD-Tests:
    name: PV-Generator ${{ matrix.os_version }}
    runs-on: ${{ matrix.runner }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - os_version: ubuntu-20.04
            runner: [self-hosted, Linux, Apptainer, ICD1]
            image: /opt/apptainer/ubuntu-2004-dec2023.sif
            port: 9001
          - os_version: ubuntu-22.04
            runner: [self-hosted, Linux, Apptainer, ICD1]
            image: /opt/apptainer/ubuntu-2204-dec2023.sif
            port: 9002
          - os_version: rhel-7
            runner: [self-hosted, linux, Apptainer, ICD2]
            image: /opt/apptainer/centos7-dec2023.sif
            port: 9003
          - os_version: rhel-8
            runner: [self-hosted, linux, Apptainer, ICD2]
            image: /opt/apptainer/almalinux8-dec2023.sif
            port: 9004
          - os_version: rhel-9
            runner: [self-hosted, linux, Apptainer, ICD3]
            image: /opt/apptainer/almalinux9-dec2023.sif
            port: 9005
    needs: [Prepare-ICD-RxJS]
    steps:
      - name: PV Generator ICD tests
        shell: bash
        run: |
          SRC_DIR=$GITHUB_WORKSPACE/source
          BUILD_DIR=$GITHUB_WORKSPACE/build-${{ matrix.os_version }}
          TEST_STAGE="$BUILD_DIR/ICD-RxJS/ICD_test_stages/pv_generator.tests"
          LOG_FILE="/tmp/carta_icd_${{ matrix.os_version }}_pv_generator.log"
          PORT_NUMBER=${{ matrix.port }}
          apptainer exec \
            --bind $GITHUB_WORKSPACE:$GITHUB_WORKSPACE \
            --bind /images:/images \
            --pwd $BUILD_DIR \
            ${{ matrix.image }} /bin/bash -c "\
            # Start the carta_backend
            ASAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan.supp \
            LSAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan-leaks.supp \
            ASAN_SYMBOLIZER_PATH=llvm-symbolizer \
            ./carta_backend /images \
              --top_level_folder /images \
              --port ${{ matrix.port }} \
              --omp_threads=4 \
              --debug_no_auth \
              --no_frontend \
              --no_database \
              --no_log \
              --verbosity=5 >> $LOG_FILE 2>&1 & \
            CARTA_BACKEND_PID=\$(pgrep -f 'carta_backend.*${{ matrix.port }}' | head -n 1) && \
            echo 'carta_backend started with PID' \$CARTA_BACKEND_PID && \
            # Run the ICD tests
            cd $BUILD_DIR/ICD-RxJS && \
            pwd && \
            cat $TEST_STAGE && \
            while IFS= read -r test_file || [[ -n "\$test_file" ]]; do
              CI=true npm test -- "\$test_file"
            done < $TEST_STAGE"

  Raster-Tiles-ICD-Tests:
    name: Raster-Tiles ${{ matrix.os_version }}
    runs-on: ${{ matrix.runner }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - os_version: ubuntu-20.04
            runner: [self-hosted, Linux, Apptainer, ICD1]
            image: /opt/apptainer/ubuntu-2004-dec2023.sif
            port: 9001
          - os_version: ubuntu-22.04
            runner: [self-hosted, Linux, Apptainer, ICD1]
            image: /opt/apptainer/ubuntu-2204-dec2023.sif
            port: 9002
          - os_version: rhel-7
            runner: [self-hosted, linux, Apptainer, ICD2]
            image: /opt/apptainer/centos7-dec2023.sif
            port: 9003
          - os_version: rhel-8
            runner: [self-hosted, linux, Apptainer, ICD2]
            image: /opt/apptainer/almalinux8-dec2023.sif
            port: 9004
          - os_version: rhel-9
            runner: [self-hosted, linux, Apptainer, ICD3]
            image: /opt/apptainer/almalinux9-dec2023.sif
            port: 9005
    needs: [Prepare-ICD-RxJS]
    steps:
      - name: Raster Tiles ICD tests
        shell: bash
        run: |
          SRC_DIR=$GITHUB_WORKSPACE/source
          BUILD_DIR=$GITHUB_WORKSPACE/build-${{ matrix.os_version }}
          TEST_STAGE="$BUILD_DIR/ICD-RxJS/ICD_test_stages/raster_tiles.tests"
          LOG_FILE="/tmp/carta_icd_${{ matrix.os_version }}_raster_tiles.log"
          PORT_NUMBER=${{ matrix.port }}
          apptainer exec \
            --bind $GITHUB_WORKSPACE:$GITHUB_WORKSPACE \
            --bind /images:/images \
            --pwd $BUILD_DIR \
            ${{ matrix.image }} /bin/bash -c "\
            # Start the carta_backend
            ASAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan.supp \
            LSAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan-leaks.supp \
            ASAN_SYMBOLIZER_PATH=llvm-symbolizer \
            ./carta_backend /images \
              --top_level_folder /images \
              --port ${{ matrix.port }} \
              --omp_threads=4 \
              --debug_no_auth \
              --no_frontend \
              --no_database \
              --no_log \
              --verbosity=5 >> $LOG_FILE 2>&1 & \
            CARTA_BACKEND_PID=\$(pgrep -f 'carta_backend.*${{ matrix.port }}' | head -n 1) && \
            echo 'carta_backend started with PID' \$CARTA_BACKEND_PID && \
            # Run the ICD tests
            cd $BUILD_DIR/ICD-RxJS && \
            pwd && \
            cat $TEST_STAGE && \
            while IFS= read -r test_file || [[ -n "\$test_file" ]]; do
              CI=true npm test -- "\$test_file"
            done < $TEST_STAGE"

  Catalog-ICD-Tests:
    name: Catalog ${{ matrix.os_version }}
    runs-on: ${{ matrix.runner }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - os_version: ubuntu-20.04
            runner: [self-hosted, Linux, Apptainer, ICD1]
            image: /opt/apptainer/ubuntu-2004-dec2023.sif
            port: 9001
          - os_version: ubuntu-22.04
            runner: [self-hosted, Linux, Apptainer, ICD1]
            image: /opt/apptainer/ubuntu-2204-dec2023.sif
            port: 9002
          - os_version: rhel-7
            runner: [self-hosted, linux, Apptainer, ICD2]
            image: /opt/apptainer/centos7-dec2023.sif
            port: 9003
          - os_version: rhel-8
            runner: [self-hosted, linux, Apptainer, ICD2]
            image: /opt/apptainer/almalinux8-dec2023.sif
            port: 9004
          - os_version: rhel-9
            runner: [self-hosted, linux, Apptainer, ICD3]
            image: /opt/apptainer/almalinux9-dec2023.sif
            port: 9005
    needs: [Prepare-ICD-RxJS]
    steps:
      - name: Catalog ICD tests
        shell: bash
        run: |
          SRC_DIR=$GITHUB_WORKSPACE/source
          BUILD_DIR=$GITHUB_WORKSPACE/build-${{ matrix.os_version }}
          TEST_STAGE="$BUILD_DIR/ICD-RxJS/ICD_test_stages/catalog.tests"
          LOG_FILE="/tmp/carta_icd_${{ matrix.os_version }}_catalog.log"
          PORT_NUMBER=${{ matrix.port }}
          apptainer exec \
            --bind $GITHUB_WORKSPACE:$GITHUB_WORKSPACE \
            --bind /images:/images \
            --pwd $BUILD_DIR \
            ${{ matrix.image }} /bin/bash -c "\
            # Start the carta_backend
            ASAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan.supp \
            LSAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan-leaks.supp \
            ASAN_SYMBOLIZER_PATH=llvm-symbolizer \
            ./carta_backend /images \
              --top_level_folder /images \
              --port ${{ matrix.port }} \
              --omp_threads=4 \
              --debug_no_auth \
              --no_frontend \
              --no_database \
              --no_log \
              --verbosity=5 >> $LOG_FILE 2>&1 & \
            CARTA_BACKEND_PID=\$(pgrep -f 'carta_backend.*${{ matrix.port }}' | head -n 1) && \
            echo 'carta_backend started with PID' \$CARTA_BACKEND_PID && \
            # Run the ICD tests
            cd $BUILD_DIR/ICD-RxJS && \
            pwd && \
            cat $TEST_STAGE && \
            while IFS= read -r test_file || [[ -n "\$test_file" ]]; do
              CI=true npm test -- "\$test_file"
            done < $TEST_STAGE"

  Moment-ICD-Tests:
    name: Moment ${{ matrix.os_version }}
    runs-on: ${{ matrix.runner }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - os_version: ubuntu-20.04
            runner: [self-hosted, Linux, Apptainer, ICD1]
            image: /opt/apptainer/ubuntu-2004-dec2023.sif
            port: 9001
          - os_version: ubuntu-22.04
            runner: [self-hosted, Linux, Apptainer, ICD1]
            image: /opt/apptainer/ubuntu-2204-dec2023.sif
            port: 9002
          - os_version: rhel-7
            runner: [self-hosted, linux, Apptainer, ICD2]
            image: /opt/apptainer/centos7-dec2023.sif
            port: 9003     
          - os_version: rhel-8
            runner: [self-hosted, linux, Apptainer, ICD2]
            image: /opt/apptainer/almalinux8-dec2023.sif
            port: 9004
          - os_version: rhel-9
            runner: [self-hosted, linux, Apptainer, ICD3]
            image: /opt/apptainer/almalinux9-dec2023.sif
            port: 9005
    needs: [Prepare-ICD-RxJS]
    steps:
      - name: Moment ICD tests
        shell: bash
        run: |
          SRC_DIR=$GITHUB_WORKSPACE/source     
          BUILD_DIR=$GITHUB_WORKSPACE/build-${{ matrix.os_version }}
          TEST_STAGE="$BUILD_DIR/ICD-RxJS/ICD_test_stages/moment.tests"
          LOG_FILE="/tmp/carta_icd_${{ matrix.os_version }}_moment.log"
          PORT_NUMBER=${{ matrix.port }}
          apptainer exec \
            --bind $GITHUB_WORKSPACE:$GITHUB_WORKSPACE \
            --bind /images:/images \
            --pwd $BUILD_DIR \
            ${{ matrix.image }} /bin/bash -c "\
            # Start the carta_backend
            ASAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan.supp \
            LSAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan-leaks.supp \
            ASAN_SYMBOLIZER_PATH=llvm-symbolizer \
            ./carta_backend /images \
              --top_level_folder /images \
              --port ${{ matrix.port }} \
              --omp_threads=4 \
              --debug_no_auth \
              --no_frontend \
              --no_database \
              --no_log \
              --verbosity=5 >> $LOG_FILE 2>&1 & \
            CARTA_BACKEND_PID=\$(pgrep -f 'carta_backend.*${{ matrix.port }}' | head -n 1) && \
            echo 'carta_backend started with PID' \$CARTA_BACKEND_PID && \
            # Run the ICD tests
            cd $BUILD_DIR/ICD-RxJS && \
            pwd && \
            cat $TEST_STAGE && \
            while IFS= read -r test_file || [[ -n "\$test_file" ]]; do
              CI=true npm test -- "\$test_file"
            done < $TEST_STAGE"

  Match-ICD-Tests:
    name: Match ${{ matrix.os_version }}
    runs-on: ${{ matrix.runner }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - os_version: ubuntu-20.04
            runner: [self-hosted, Linux, Apptainer, ICD1]
            image: /opt/apptainer/ubuntu-2004-dec2023.sif
            port: 9001
          - os_version: ubuntu-22.04
            runner: [self-hosted, Linux, Apptainer, ICD1]
            image: /opt/apptainer/ubuntu-2204-dec2023.sif
            port: 9002
          - os_version: rhel-7
            runner: [self-hosted, linux, Apptainer, ICD2]
            image: /opt/apptainer/centos7-dec2023.sif
            port: 9003
          - os_version: rhel-8
            runner: [self-hosted, linux, Apptainer, ICD2]
            image: /opt/apptainer/almalinux8-dec2023.sif
            port: 9004
          - os_version: rhel-9
            runner: [self-hosted, linux, Apptainer, ICD3]
            image: /opt/apptainer/almalinux9-dec2023.sif
            port: 9005
    needs: [Prepare-ICD-RxJS]
    steps:
      - name: Match ICD tests
        shell: bash
        run: |
          SRC_DIR=$GITHUB_WORKSPACE/source
          BUILD_DIR=$GITHUB_WORKSPACE/build-${{ matrix.os_version }}
          TEST_STAGE="$BUILD_DIR/ICD-RxJS/ICD_test_stages/match.tests"
          LOG_FILE="/tmp/carta_icd_${{ matrix.os_version }}_match.log"
          PORT_NUMBER=${{ matrix.port }}
          apptainer exec \
            --bind $GITHUB_WORKSPACE:$GITHUB_WORKSPACE \
            --bind /images:/images \
            --pwd $BUILD_DIR \
            ${{ matrix.image }} /bin/bash -c "\
            # Start the carta_backend
            ASAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan.supp \
            LSAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan-leaks.supp \
            ASAN_SYMBOLIZER_PATH=llvm-symbolizer \
            ./carta_backend /images \
              --top_level_folder /images \
              --port ${{ matrix.port }} \
              --omp_threads=4 \
              --debug_no_auth \
              --no_frontend \
              --no_database \
              --no_log \
              --verbosity=5 >> $LOG_FILE 2>&1 & \  
            CARTA_BACKEND_PID=\$(pgrep -f 'carta_backend.*${{ matrix.port }}' | head -n 1) && \
            echo 'carta_backend started with PID' \$CARTA_BACKEND_PID && \
            # Run the ICD tests   
            cd $BUILD_DIR/ICD-RxJS && \
            pwd && \
            cat $TEST_STAGE && \
            while IFS= read -r test_file || [[ -n "\$test_file" ]]; do
              CI=true npm test -- "\$test_file"
            done < $TEST_STAGE"

  Close-File-Tests:
    name: Close-File ${{ matrix.os_version }}
    runs-on: ${{ matrix.runner }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - os_version: ubuntu-20.04   
            runner: [self-hosted, Linux, Apptainer, ICD1]
            image: /opt/apptainer/ubuntu-2004-dec2023.sif
            port: 9001
          - os_version: ubuntu-22.04
            runner: [self-hosted, Linux, Apptainer, ICD1]
            image: /opt/apptainer/ubuntu-2204-dec2023.sif
            port: 9002
          - os_version: rhel-7
            runner: [self-hosted, linux, Apptainer, ICD2]
            image: /opt/apptainer/centos7-dec2023.sif
            port: 9003
          - os_version: rhel-8
            runner: [self-hosted, linux, Apptainer, ICD2]
            image: /opt/apptainer/almalinux8-dec2023.sif
            port: 9004
          - os_version: rhel-9
            runner: [self-hosted, linux, Apptainer, ICD3]
            image: /opt/apptainer/almalinux9-dec2023.sif
            port: 9005
    needs: [Prepare-ICD-RxJS]
    steps:      
      - name: Close File ICD tests
        shell: bash
        run: |
          SRC_DIR=$GITHUB_WORKSPACE/source
          BUILD_DIR=$GITHUB_WORKSPACE/build-${{ matrix.os_version }}
          TEST_STAGE="$BUILD_DIR/ICD-RxJS/ICD_test_stages/close_file.tests"
          LOG_FILE="/tmp/carta_icd_${{ matrix.os_version }}_close_file.log"
          PORT_NUMBER=${{ matrix.port }}
          apptainer exec \
            --bind $GITHUB_WORKSPACE:$GITHUB_WORKSPACE \
            --bind /images:/images \
            --pwd $BUILD_DIR \
            ${{ matrix.image }} /bin/bash -c "\
            # Start the carta_backend
            ASAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan.supp \
            LSAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan-leaks.supp \
            ASAN_SYMBOLIZER_PATH=llvm-symbolizer \
            ./carta_backend /images \
              --top_level_folder /images \
              --port ${{ matrix.port }} \
              --omp_threads=4 \
              --debug_no_auth \
              --no_frontend \
              --no_database \
              --no_log \
              --verbosity=5 >> $LOG_FILE 2>&1 & \  
            CARTA_BACKEND_PID=\$(pgrep -f 'carta_backend.*${{ matrix.port }}' | head -n 1) && \
            echo 'carta_backend started with PID' \$CARTA_BACKEND_PID && \
            # Run the ICD tests   
            cd $BUILD_DIR/ICD-RxJS && \
            pwd && \
            cat $TEST_STAGE && \
            while IFS= read -r test_file || [[ -n "\$test_file" ]]; do
              CI=true npm test -- "\$test_file"
            done < $TEST_STAGE"

  Image-Fitting-Tests:
    name: Image-Fitting ${{ matrix.os_version }}
    runs-on: ${{ matrix.runner }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - os_version: ubuntu-20.04
            runner: [self-hosted, Linux, Apptainer, ICD1]
            image: /opt/apptainer/ubuntu-2004-dec2023.sif
            port: 9001
          - os_version: ubuntu-22.04
            runner: [self-hosted, Linux, Apptainer, ICD1]
            image: /opt/apptainer/ubuntu-2204-dec2023.sif
            port: 9002
          - os_version: rhel-7
            runner: [self-hosted, linux, Apptainer, ICD2]
            image: /opt/apptainer/centos7-dec2023.sif
            port: 9003
          - os_version: rhel-8
            runner: [self-hosted, linux, Apptainer, ICD2]
            image: /opt/apptainer/almalinux8-dec2023.sif
            port: 9004
          - os_version: rhel-9
            runner: [self-hosted, linux, Apptainer, ICD3]
            image: /opt/apptainer/almalinux9-dec2023.sif
            port: 9005
    needs: [Prepare-ICD-RxJS]
    steps:
      - name: Image Fitting ICD tests
        shell: bash
        run: |
          SRC_DIR=$GITHUB_WORKSPACE/source
          BUILD_DIR=$GITHUB_WORKSPACE/build-${{ matrix.os_version }}
          TEST_STAGE="$BUILD_DIR/ICD-RxJS/ICD_test_stages/image_fitting.tests"
          LOG_FILE="/tmp/carta_icd_${{ matrix.os_version }}_image_fitting.log"
          PORT_NUMBER=${{ matrix.port }}
          apptainer exec \
            --bind $GITHUB_WORKSPACE:$GITHUB_WORKSPACE \
            --bind /images:/images \
            --pwd $BUILD_DIR \
            ${{ matrix.image }} /bin/bash -c "\
            # Start the carta_backend   
            ASAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan.supp \
            LSAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan-leaks.supp \
            ASAN_SYMBOLIZER_PATH=llvm-symbolizer \
            ./carta_backend /images \
              --top_level_folder /images \
              --port ${{ matrix.port }} \
              --omp_threads=4 \
              --debug_no_auth \
              --no_frontend \
              --no_database \
              --no_log \
              --verbosity=5 >> $LOG_FILE 2>&1 & \
            CARTA_BACKEND_PID=\$(pgrep -f 'carta_backend.*${{ matrix.port }}' | head -n 1) && \
            echo 'carta_backend started with PID' \$CARTA_BACKEND_PID && \
            # Run the ICD tests   
            cd $BUILD_DIR/ICD-RxJS && \
            pwd && \
            cat $TEST_STAGE && \
            while IFS= read -r test_file || [[ -n "\$test_file" ]]; do
              CI=true npm test -- "\$test_file"
            done < $TEST_STAGE"

  Vector-Overlay-Tests:
    name: Vector-Overlay ${{ matrix.os_version }}
    runs-on: ${{ matrix.runner }}
    strategy:
      fail-fast: false      
      matrix:
        include:
          - os_version: ubuntu-20.04
            runner: [self-hosted, Linux, Apptainer, ICD1]
            image: /opt/apptainer/ubuntu-2004-dec2023.sif
            port: 9001
          - os_version: ubuntu-22.04
            runner: [self-hosted, Linux, Apptainer, ICD1]
            image: /opt/apptainer/ubuntu-2204-dec2023.sif
            port: 9002
          - os_version: rhel-7
            runner: [self-hosted, linux, Apptainer, ICD2]
            image: /opt/apptainer/centos7-dec2023.sif
            port: 9003
          - os_version: rhel-8
            runner: [self-hosted, linux, Apptainer, ICD2]
            image: /opt/apptainer/almalinux8-dec2023.sif
            port: 9004
          - os_version: rhel-9
            runner: [self-hosted, linux, Apptainer, ICD3]
            image: /opt/apptainer/almalinux9-dec2023.sif
            port: 9005
    needs: [Prepare-ICD-RxJS]
    steps:
      - name: Vector Overlay ICD tests
        shell: bash
        run: |
          SRC_DIR=$GITHUB_WORKSPACE/source
          BUILD_DIR=$GITHUB_WORKSPACE/build-${{ matrix.os_version }}
          TEST_STAGE="$BUILD_DIR/ICD-RxJS/ICD_test_stages/vector_overlay.tests"
          LOG_FILE="/tmp/carta_icd_${{ matrix.os_version }}_vector_overlay.log"
          PORT_NUMBER=${{ matrix.port }}
          apptainer exec \
            --bind $GITHUB_WORKSPACE:$GITHUB_WORKSPACE \
            --bind /images:/images \
            --pwd $BUILD_DIR \
            ${{ matrix.image }} /bin/bash -c "\
            # Start the carta_backend   
            ASAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan.supp \
            LSAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan-leaks.supp \
            ASAN_SYMBOLIZER_PATH=llvm-symbolizer \
            ./carta_backend /images \
              --top_level_folder /images \
              --port ${{ matrix.port }} \
              --omp_threads=4 \
              --debug_no_auth \
              --no_frontend \
              --no_database \
              --no_log \
              --verbosity=5 >> $LOG_FILE 2>&1 & \
            CARTA_BACKEND_PID=\$(pgrep -f 'carta_backend.*${{ matrix.port }}' | head -n 1) && \
            echo 'carta_backend started with PID' \$CARTA_BACKEND_PID && \
            # Run the ICD tests   
            cd $BUILD_DIR/ICD-RxJS && \
            pwd && \
            cat $TEST_STAGE && \
            while IFS= read -r test_file || [[ -n "\$test_file" ]]; do
              CI=true npm test -- "\$test_file"
            done < $TEST_STAGE"