.gitlab-ci.yml

stages:
  - build   # Builds all projects
  - run     # Runs the tests across architectures
  - test    # Runs various tests of the software
  - publish # Publishes the results of the tests and runs in channels and grafana

check-formatting:
  stage: build
  image: ubuntu:latest
  script:
    - apt update && apt install -y clang-format-8 python-pip git curl
    - pip install yapf==0.24.0 whichcraft
    - if [ ! -e .clang-format ] ; then
    -   curl -o .clang-format "https://gitlab.cern.ch/lhcb-parallelization/Allen/raw/master/.clang-format?inline=false"
    -   echo '.clang-format' >> .gitignore
    -   git add .gitignore
    - fi
    - curl -o lb-format "https://gitlab.cern.ch/lhcb-core/LbDevTools/raw/master/LbDevTools/SourceTools.py?inline=false"
    - python lb-format --format-patch apply-formatting.patch origin/master
  artifacts:
    paths:
      - apply-formatting.patch
    when: on_failure
    expire_in: 1 week
  allow_failure: true

.build_job: &build_job_def
  only:
    refs:
      - master
      - schedules
      - web
      - merge_requests
  stage: build
  script:
    - declare -A DEVICE_NUMBERS_MAP=${DEVICE_NUMBERS}
    - PREVIOUS_IFS=${IFS}
    - IFS=':' read -ra JOB_NAME_SPLIT <<< "${CI_JOB_NAME}"
    - IFS=${PREVIOUS_IFS}
    - TARGET=${JOB_NAME_SPLIT[0]}
    - SEQUENCE=${JOB_NAME_SPLIT[1]}
    - BUILD_TYPE=${JOB_NAME_SPLIT[2]}
    - ADDITIONAL_OPTIONS=${JOB_NAME_SPLIT[3]}
    - BUILD_ARCH_FLAG="-gencode arch=compute_75,code=sm_75 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_61,code=sm_61 -gencode arch=compute_52,code=sm_52 -gencode arch=compute_30,code=sm_30"
    - source /cvmfs/sft.cern.ch/lcg/views/setupViews.sh LCG_95apython3 x86_64-centos7-gcc8-opt
    - export PATH=/usr/local/cuda/bin:/cvmfs/sft.cern.ch/lcg/contrib/CMake/3.14.2/Linux-x86_64/bin:$PATH
    - mkdir build_${TARGET}
    - cd build_${TARGET}
    - cmake -DTARGET_${TARGET}=ON -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DSEQUENCE=${SEQUENCE} -DOVERRIDE_ARCH_FLAG="${BUILD_ARCH_FLAG}" -DCPU_ARCH="ivybridge" ${ADDITIONAL_OPTIONS} ..
    - make -j
  artifacts:
    expire_in: 2 hrs
    paths:
      - build*/*Allen*
      - input
  retry: 1

.build_job: &build_test_job_def
  only:
    refs:
      - master
      - schedules
      - web
      - merge_requests
  stage: build
  script:
    - declare -A DEVICE_NUMBERS_MAP=${DEVICE_NUMBERS}
    - PREVIOUS_IFS=${IFS}
    - IFS=':' read -ra JOB_NAME_SPLIT <<< "${CI_JOB_NAME}"
    - IFS=${PREVIOUS_IFS}
    - TARGET=${JOB_NAME_SPLIT[0]}
    - SEQUENCE=${JOB_NAME_SPLIT[1]}
    - BUILD_TYPE=${JOB_NAME_SPLIT[2]}
    - ADDITIONAL_OPTIONS=${JOB_NAME_SPLIT[3]}
    - BUILD_ARCH_FLAG="-gencode arch=compute_75,code=sm_75 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_61,code=sm_61 -gencode arch=compute_52,code=sm_52 -gencode arch=compute_30,code=sm_30"
    - source /cvmfs/sft.cern.ch/lcg/views/setupViews.sh LCG_95apython3 x86_64-centos7-gcc8-opt
    - export PATH=/usr/local/cuda/bin:/cvmfs/sft.cern.ch/lcg/contrib/CMake/3.14.2/Linux-x86_64/bin:$PATH
    - mkdir build_${TARGET}
    - cd build_${TARGET}
    - cmake -DBUILD_TESTS=ON -DTARGET_${TARGET}=ON -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DSEQUENCE=${SEQUENCE} -DOVERRIDE_ARCH_FLAG="${BUILD_ARCH_FLAG}" -DCPU_ARCH="ivybridge" ${ADDITIONAL_OPTIONS} ..
    - make -j
  artifacts:
    expire_in: 2 hrs
    paths:
      - build*/*
      - input
  retry: 1

.build_clang_job: &build_clang_job_def
  only:
    refs:
      - master
      - schedules
      - web
      - merge_requests
  stage: build
  script:
    - declare -A DEVICE_NUMBERS_MAP=${DEVICE_NUMBERS}
    - PREVIOUS_IFS=${IFS}
    - IFS=':' read -ra JOB_NAME_SPLIT <<< "${CI_JOB_NAME}"
    - IFS=${PREVIOUS_IFS}
    - TARGET=${JOB_NAME_SPLIT[0]}
    - SEQUENCE=${JOB_NAME_SPLIT[1]}
    - BUILD_TYPE=${JOB_NAME_SPLIT[2]}
    - ADDITIONAL_OPTIONS=${JOB_NAME_SPLIT[3]}
    - BUILD_ARCH_FLAG="-gencode arch=compute_75,code=sm_75 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_61,code=sm_61 -gencode arch=compute_52,code=sm_52 -gencode arch=compute_30,code=sm_30"
    - source /cvmfs/sft.cern.ch/lcg/views/setupViews.sh LCG_95apython3 x86_64-centos7-gcc8-opt
    - source /cvmfs/sft.cern.ch/lcg/releases/clang/8.0.0.1/x86_64-centos7/setup.sh
    - export PATH=/usr/local/cuda/bin:/cvmfs/sft.cern.ch/lcg/contrib/CMake/3.14.2/Linux-x86_64/bin:$PATH
    - mkdir build_${TARGET}
    - cd build_${TARGET}
    - cmake -DTARGET_${TARGET}=ON -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DSEQUENCE=${SEQUENCE} -DOVERRIDE_ARCH_FLAG="${BUILD_ARCH_FLAG}" -DCPU_ARCH="ivybridge" ${ADDITIONAL_OPTIONS} ..
    - make -j
  artifacts:
    expire_in: 2 hrs
    paths:
      - build*/*Allen*
      - input
  retry: 1

.run_physics_efficiency_job: &run_physics_efficiency_job_def
  only:
    refs:
      - master
      - schedules
      - web
      - merge_requests
  stage: run
  script:
    - TOPLEVEL=${PWD}
    - declare -A DEVICE_NUMBERS_MAP=${DEVICE_NUMBERS}
    - declare -A DEVICE_MEMORY_MAP=${DEVICE_MEMORY}
    - PREVIOUS_IFS=${IFS}
    - IFS=':' read -ra JOB_NAME_SPLIT <<< "${CI_JOB_NAME}"
    - IFS=${PREVIOUS_IFS}
    - DEVICE_ID=${JOB_NAME_SPLIT[0]}
    - TARGET=${JOB_NAME_SPLIT[1]}
    - SEQUENCE=${JOB_NAME_SPLIT[2]}
    - D_NUMBER=${DEVICE_NUMBERS_MAP[${DEVICE_ID}]}
    - RUN_OPTIONS="-n 1000 -m 700"
    - source /cvmfs/sft.cern.ch/lcg/views/setupViews.sh LCG_95apython3 x86_64-centos7-gcc8-opt
    - export PATH=/usr/local/cuda/bin:/cvmfs/sft.cern.ch/lcg/contrib/CMake/3.14.2/Linux-x86_64/bin:$PATH
    - mkdir validation_output
    - ln -s validation_output output # Needed by Root build
    - cd build_${TARGET}
    - ls
    - export LD_LIBRARY_PATH=${PWD}:$LD_LIBRARY_PATH
    - CUDA_VISIBLE_DEVICES=${D_NUMBER} ./Allen -f /scratch/dcampora/allen_data/201907/bsphiphi_mag_down ${RUN_OPTIONS} 2>&1 | tee ../validation_output/bsphiphi_${DEVICE_ID}.txt
  artifacts:
    expire_in: 2 hrs
    paths:
      - validation_output/*
  allow_failure: true
  retry: 1

.run_throughput_job_no_profiling: &run_throughput_job_no_profiling_def
  only:
    refs:
      - master
      - schedules
      - web
      - merge_requests
  stage: run
  script:
    - TOPLEVEL=${PWD}
    - declare -A DEVICE_NUMBERS_MAP=${DEVICE_NUMBERS}
    - declare -A DEVICE_MEMORY_MAP=${DEVICE_MEMORY}
    - PREVIOUS_IFS=${IFS}
    - IFS=':' read -ra JOB_NAME_SPLIT <<< "${CI_JOB_NAME}"
    - IFS=${PREVIOUS_IFS}
    - DEVICE_ID=${JOB_NAME_SPLIT[0]}
    - TARGET=${JOB_NAME_SPLIT[1]}
    - SEQUENCE=${JOB_NAME_SPLIT[2]}
    - D_NUMBER=${DEVICE_NUMBERS_MAP[${DEVICE_ID}]}
    - D_MEMORY=${DEVICE_MEMORY_MAP[${DEVICE_ID}]}
    - RUN_OPTIONS="-n 1000 -m 700 -r 100 -t 12 -c 0"
    - if [ "${D_MEMORY}" = "LOW" ]; then
    -   RUN_OPTIONS="-n 1000 -r 100 -t 2 -m 700 -c 0"
    - fi
    - source /cvmfs/sft.cern.ch/lcg/views/setupViews.sh LCG_95apython3 x86_64-centos7-gcc8-opt
    - export PATH=/usr/local/cuda/bin:/cvmfs/sft.cern.ch/lcg/contrib/CMake/3.14.2/Linux-x86_64/bin:$PATH
    - mkdir output_${DEVICE_ID}
    - cd build_${TARGET}
    - ls
    - export LD_LIBRARY_PATH=${PWD}:$LD_LIBRARY_PATH
    - CUDA_VISIBLE_DEVICES=${D_NUMBER} ./Allen -f /localdisk/shared/201907/minbias_mag_down ${RUN_OPTIONS} 2>&1 | tee ../output_${DEVICE_ID}/output.txt
  artifacts:
    expire_in: 2 hrs
    paths:
      - output_*
  allow_failure: true
  retry: 1

.run_throughput_job_no_profiling_cpu: &run_throughput_job_no_profiling_def_cpu
  only:
    refs:
      - master
      - schedules
      - web
      - merge_requests
  stage: run
  script:
    - TOPLEVEL=${PWD}
    - PREVIOUS_IFS=${IFS}
    - IFS=':' read -ra JOB_NAME_SPLIT <<< "${CI_JOB_NAME}"
    - IFS=${PREVIOUS_IFS}
    - DEVICE_ID=${JOB_NAME_SPLIT[0]}
    - TARGET=${JOB_NAME_SPLIT[1]}
    - SEQUENCE=${JOB_NAME_SPLIT[2]}
    - THREADS=$(lscpu | egrep "^CPU\(s\):.*[0-9]+$" --color=none | awk '{ print $2; }')
    - RUN_OPTIONS="-n 1000 -m 700 -r 10 -t ${THREADS} -c 0" # If we ever execute on other CPUs we'll have to set this configurable
    - source /cvmfs/sft.cern.ch/lcg/views/setupViews.sh LCG_95apython3 x86_64-centos7-gcc8-opt
    - export PATH=/usr/local/cuda/bin:/cvmfs/sft.cern.ch/lcg/contrib/CMake/3.14.2/Linux-x86_64/bin:$PATH
    - mkdir output_${DEVICE_ID}
    - cd build_${TARGET}
    - ls
    - export LD_LIBRARY_PATH=${PWD}:$LD_LIBRARY_PATH
    - ./Allen -f /localdisk/shared/201907/minbias_mag_down ${RUN_OPTIONS} 2>&1 | tee ../output_${DEVICE_ID}/output.txt
  artifacts:
    expire_in: 2 hrs
    paths:
      - output_*
  allow_failure: true
  retry: 1

.run_throughput_job: &run_throughput_job_def
  only:
    refs:
      - master
      - schedules
      - web
      - merge_requests
  stage: run
  script:
    - TOPLEVEL=${PWD}
    - declare -A DEVICE_NUMBERS_MAP=${DEVICE_NUMBERS}
    - declare -A DEVICE_MEMORY_MAP=${DEVICE_MEMORY}
    - PREVIOUS_IFS=${IFS}
    - IFS=':' read -ra JOB_NAME_SPLIT <<< "${CI_JOB_NAME}"
    - IFS=${PREVIOUS_IFS}
    - DEVICE_ID=${JOB_NAME_SPLIT[0]}
    - TARGET=${JOB_NAME_SPLIT[1]}
    - SEQUENCE=${JOB_NAME_SPLIT[2]}
    - D_NUMBER=${DEVICE_NUMBERS_MAP[${DEVICE_ID}]}
    - D_MEMORY=${DEVICE_MEMORY_MAP[${DEVICE_ID}]}
    - RUN_OPTIONS="-n 1000 -m 700 -r 100 -t 12 -c 0"
    - if [ "${D_MEMORY}" = "LOW" ]; then
    -   RUN_OPTIONS="-n 1000 -r 100 -t 2 -m 700 -c 0"
    - fi
    - source /cvmfs/sft.cern.ch/lcg/views/setupViews.sh LCG_95apython3 x86_64-centos7-gcc8-opt
    - export PATH=/usr/local/cuda/bin:/cvmfs/sft.cern.ch/lcg/contrib/CMake/3.14.2/Linux-x86_64/bin:$PATH
    - mkdir output_${DEVICE_ID}
    - cd build_${TARGET}
    - ls
    - export LD_LIBRARY_PATH=${PWD}:$LD_LIBRARY_PATH
    - CUDA_VISIBLE_DEVICES=${D_NUMBER} ./Allen -f /localdisk/shared/201907/minbias_mag_down ${RUN_OPTIONS} 2>&1 | tee ../output_${DEVICE_ID}/output.txt
    - CUDA_VISIBLE_DEVICES=${D_NUMBER} nvprof ./Allen -f /localdisk/shared/201907/minbias_mag_down ${RUN_OPTIONS} 2>&1 | tee ../output_${DEVICE_ID}/profiler_output.txt
    - python3 ${TOPLEVEL}/checker/plotting/extract_algo_breakdown.py -d ${TOPLEVEL}
  artifacts:
    expire_in: 2 hrs
    paths:
      - output_*
  allow_failure: true
  retry: 1

.throughput_cli_plot_job: &publish_algo_breakdown_plot_def
  only:
    refs:
      - master
      - schedules
      - web
      - merge_requests
  stage: publish
  script:
    - declare -A DEVICE_NUMBERS_MAP=${DEVICE_NUMBERS}
    - PREVIOUS_IFS=${IFS}
    - IFS=':' read -ra JOB_NAME_SPLIT <<< "${CI_JOB_NAME}"
    - IFS=${PREVIOUS_IFS}
    - DEVICE_ID=${JOB_NAME_SPLIT[0]}
    - SEQUENCE=${JOB_NAME_SPLIT[1]}
    - D_NUMBER=${DEVICE_NUMBERS_MAP[${DEVICE_ID}]}
    - source /cvmfs/sft.cern.ch/lcg/views/setupViews.sh LCG_95apython3 x86_64-centos7-gcc8-opt
    - export PATH=/usr/local/cuda/bin:/cvmfs/sft.cern.ch/lcg/contrib/CMake/3.14.2/Linux-x86_64/bin:$PATH
    - python3 checker/plotting/csv_plotter.py -t "Algorithm Breakdown of sequence __${SEQUENCE}__, branch _${CI_COMMIT_REF_NAME}_" -u "%" -x 30 -m ${MATTERMOST_KEY} output_${DEVICE_ID}/algo_breakdown.csv
    # - python3 checker/plotting/csv_plotter.py -t "Algorithm Groups of sequence __${SEQUENCE}__, branch _${CI_COMMIT_REF_NAME}_" -u "%" -m ${MATTERMOST_KEY} output_${DEVICE_ID}/algo_summary.csv

test_physics_efficiency:
  only:
    refs:
      - master
      - schedules
      - web
      - merge_requests
  stage: test
  script:
    - TOPLEVEL=${PWD}
    - ls validation_output
    - ls ${TOPLEVEL}/test/reference
    - cd validation_output
    - for i in $( ls ); do echo "Checking ${i}"; tail -n133 ${i} | head -n129 > efficiency_${i}; colordiff ${TOPLEVEL}/test/reference/${i} efficiency_${i} | tee ${i}_diff || true; done
    - cat *_diff > alldiffs
    - if [ -s alldiffs ]; then echo "Differences were found against reference files."; exit 1; else echo "No differences found against reference files."; exit 0; fi
  dependencies:
    - t4:CUDA:DefaultSequence:run_physics_efficiency
    - x862630v4:CPU:DefaultSequence:run_physics_efficiency
  tags:
    - rtx6000 # Note: colordiff must be installed
  allow_failure: true

# run_built_tests:
#   only:
#     refs:
#       - master
#       - schedules
#       - web
#       - merge_requests
#   stage: test
#   script:
#     - cd build_CUDA
#     - ctest -V
#   dependencies:
#     - CUDA:DefaultSequence:Debug::build
#   allow_failure: true

.publish_throughput_job: &publish_throughput_job_def
  only:
    refs:
      - master
      - schedules
      - web
      - merge_requests
  stage: publish
  script:
    - PREVIOUS_IFS=${IFS}
    - IFS=':' read -ra JOB_NAME_SPLIT <<< "${CI_JOB_NAME}"
    - IFS=${PREVIOUS_IFS}
    - SEQUENCE=${JOB_NAME_SPLIT[1]}
    - BREAKDOWN_DEVICE_ID=${JOB_NAME_SPLIT[2]}
    - cat output_*/output.txt | grep --color=none "device" | sed 's/.*:\ [0-9]*\,\ //' > devices.txt
    - cat output_*/output.txt | grep --color=none "events/s" | awk '{ print $1; }' > throughputs.txt
    - cat devices.txt
    - cat throughputs.txt
    - paste -d, devices.txt throughputs.txt > devices_throughputs.csv
    - cat devices_throughputs.csv
    - python3 checker/plotting/post_combined_message.py -l "Throughput of [sequence __${SEQUENCE}__, branch _${CI_COMMIT_REF_NAME}_](https://gitlab.cern.ch/lhcb/Allen/pipelines/${CI_PIPELINE_ID})" -m ${MATTERMOST_KEY} -t devices_throughputs.csv -b output_${BREAKDOWN_DEVICE_ID}/algo_breakdown.csv
    # - python3 checker/plotting/csv_plotter.py -t "Throughput of sequence __${SEQUENCE}__, branch _${CI_COMMIT_REF_NAME}_" -u "kHz" -x 70 -s 1e-3 -m ${MATTERMOST_KEY} devices_throughputs.csv
    - python3 checker/plotting/post_telegraf.py -d . -s ${SEQUENCE} -b ${CI_COMMIT_REF_NAME}

.throughput_speedup_job: &publish_speedup_job_def
  only:
    refs:
      - master
      - schedules
      - web
      - merge_requests
  stage: publish
  script:
    - cat output_*/output.txt | grep --color=none "device" | sed 's/.*:\ [0-9]*\,\ //' > devices.txt
    - cat output_*/output.txt | grep --color=none "events/s" | awk '{ print $1; }' > throughputs.txt
    - cat devices.txt
    - cat throughputs.txt
    - paste -d, devices.txt throughputs.txt > devices_throughputs.csv
    - cat devices_throughputs.csv
    - python3 checker/plotting/csv_plotter.py -n -t "Speedup across GPUs, branch _${CI_COMMIT_REF_NAME}_" -u "x" -x 30 -m ${MATTERMOST_KEY} devices_throughputs.csv

# Build for all platforms
CUDA:DefaultSequence:RelWithDebInfo::build:
  <<: *build_job_def
  tags:
    - cuda

CUDA:DefaultSequence:Debug::build:
  <<: *build_test_job_def
  tags:
    - cuda

CUDA:DefaultSequence:Debug:-DUSE_ROOT=ON:build:
  <<: *build_job_def
  tags:
    - cuda

CPU:DefaultSequence:RelWithDebInfo::build_x86:
  <<: *build_job_def
  tags:
    - x86

CPU:DefaultSequence:RelWithDebInfo::build_x86_clang:
  <<: *build_clang_job_def
  tags:
    - x86

# Run on all platforms
# Performance runs
rtx2080tifan:CUDA:DefaultSequence:run_throughput:
  <<: *run_throughput_job_def
  tags:
    - rtx2080tifan
  dependencies:
    - CUDA:DefaultSequence:RelWithDebInfo::build

rtx6000:CUDA:DefaultSequence:run_throughput:
  <<: *run_throughput_job_no_profiling_def
  tags:
    - rtx6000
  dependencies:
    - CUDA:DefaultSequence:RelWithDebInfo::build

v100:CUDA:DefaultSequence:run_throughput:
  <<: *run_throughput_job_no_profiling_def
  tags:
    - v100
  dependencies:
    - CUDA:DefaultSequence:RelWithDebInfo::build

t4:CUDA:DefaultSequence:run_throughput:
  <<: *run_throughput_job_no_profiling_def
  tags:
    - t4
  dependencies:
    - CUDA:DefaultSequence:RelWithDebInfo::build

# gtx10606g:CUDA:DefaultSequence:run_throughput:
#   <<: *run_throughput_job_no_profiling_def
#   tags:
#     - gtx10606g
#   dependencies:
#     - CUDA:DefaultSequence:RelWithDebInfo::build

# gtx680:CUDA:DefaultSequence:run_throughput:
#   <<: *run_throughput_job_no_profiling_def
#   tags:
#     - gtx680
#   dependencies:
#     - CUDA:DefaultSequence:RelWithDebInfo::build

# gtxtitanx:CUDA:DefaultSequence:run_throughput:
#   <<: *run_throughput_job_no_profiling_def
#   tags:
#     - gtxtitanx
#   dependencies:
#     - CUDA:DefaultSequence:RelWithDebInfo::build

x862630v4:CPU:DefaultSequence:run_throughput_cpu:
  <<: *run_throughput_job_no_profiling_def_cpu
  tags:
    - x862630v4
  dependencies:
    - CPU:DefaultSequence:RelWithDebInfo::build_x86

# gtx670:CUDA:DefaultSequence:run_throughput:
#   <<: *run_throughput_job_no_profiling_def
#   tags:
#     - gtx670
#   dependencies:
#     - CUDA:DefaultSequence:RelWithDebInfo::build

# gtx1080ti:CUDA:DefaultSequence:run_throughput:
#   <<: *run_throughput_job_no_profiling_def
#   tags:
#     - gtx1080ti
#   dependencies:
#     - CUDA:DefaultSequence:RelWithDebInfo::build

# Physics runs

t4:CUDA:DefaultSequence:run_physics_efficiency:
  <<: *run_physics_efficiency_job_def
  tags:
    - t4
  dependencies:
    - CUDA:DefaultSequence:RelWithDebInfo::build

x862630v4:CPU:DefaultSequence:run_physics_efficiency:
  <<: *run_physics_efficiency_job_def
  tags:
    - x862630v4
  dependencies:
    - CPU:DefaultSequence:RelWithDebInfo::build_x86

gpu0:CUDA:DefaultSequence:run_physics_efficiency_debug:
  <<: *run_physics_efficiency_job_def
  tags:
    - cuda
  dependencies:
    - CUDA:DefaultSequence:Debug::build

gpu0:CUDA:DefaultSequence:run_physics_efficiency_debug_root:
  <<: *run_physics_efficiency_job_def
  tags:
    - cuda
  dependencies:
    - CUDA:DefaultSequence:Debug:-DUSE_ROOT=ON:build

# Publish

# Plain results to mattermost
# rtx2080tifan:DefaultSequence:publish_algo_breakdown_plot:
#   <<: *publish_algo_breakdown_plot_def
#   tags:
#     - gpu
#   dependencies:
#     - rtx2080tifan:CUDA:DefaultSequence:run_throughput

throughput:DefaultSequence:rtx2080tifan:publish_throughput:
  <<: *publish_throughput_job_def
  tags:
    - gpu
  dependencies:
    - rtx2080tifan:CUDA:DefaultSequence:run_throughput
    - rtx6000:CUDA:DefaultSequence:run_throughput
    - v100:CUDA:DefaultSequence:run_throughput
    - t4:CUDA:DefaultSequence:run_throughput
    - x862630v4:CPU:DefaultSequence:run_throughput_cpu
    # - gtx1080ti:CUDA:DefaultSequence:run_throughput
    # - gtxtitanx:CUDA:DefaultSequence:run_throughput
    # - gtx670:CUDA:DefaultSequence:run_throughput
    # - gtx10606g:CUDA:DefaultSequence:run_throughput
    # - gtx680:CUDA:DefaultSequence:run_throughput