gitlab-ci-frontier.yml

# Ad-hoc build that runs in the ECP Hardware, concretely in OLCF Frontier.

default:
  id_tokens:
    OLCF_ID_TOKEN:
      aud: https://code.olcf.ornl.gov

stages:
  - pre
  - setup
  - build
  - post

.setup_env_ecpci: &setup_env_ecpci |
  git fetch
  source scripts/ci/gitlab-ci/setup-vars.sh
  git checkout "$CI_COMMIT_REF"
  module purge
  module load ${JOB_MODULES}
  module list
  export PATH="${CCACHE_INSTALL_DIR}/ccache:$PATH"

.install_ccache: &install_ccache |
  mkdir -p "$CCACHE_INSTALL_DIR"
  cmake --version
  cmake -VV -P .gitlab/config/ccache.cmake
  ccache -z
  ccache -s

.frontier-common:
  variables:
    CUSTOM_CI_BUILDS_DIR: "/lustre/orion/ums029/scratch/ums029_auser/ci/adios2/runtime"
    OLCF_SERVICE_ACCOUNT: "ums029_auser"

.frontier-cmake-common:
  rules:
    - if: $CI_PIPELINE_SOURCE =~ /parent_pipeline|web/
  interruptible: true
  variables:
    CCACHE_BASEDIR:       "/lustre/orion/ums029/scratch/"
    CCACHE_DIR:           "/lustre/orion/ums029/scratch/ums029_auser/ci/adios2/ccache"

    # -isystem= is not affected by CCACHE_BASEDIR, thus we must ignore it
    CCACHE_IGNOREOPTIONS: "-isystem=*"
    CCACHE_NOHASHDIR:     "true"
    CCACHE_INSTALL_DIR:   "$CI_PROJECT_DIR/deps/ccache_install"

    CMAKE_BUILD_TYPE:     "RelWithDebInfo"
    CMAKE_GENERATOR:      "Ninja"
    CMAKE_PREFIX_PATH:    "$CI_PROJECT_DIR/deps/kokkos_install"

    # We do not want to use the user's ~/.gitconfig
    GIT_CONFIG_GLOBAL:    "true"
    GITLAB_SITE:          "OLCF Frontier"
    CI_BIN_DIR:           "$CI_PROJECT_DIR/build"

  extends:
    .frontier-common

.setup-common:
  stage: setup
  tags: [ shell ]
  before_script:
    - *setup_env_ecpci
    - *install_ccache
  script:
    - bash scripts/ci/gitlab-ci/run.sh update
  artifacts:
    expire_in: 24 hours
    when: always
    paths:
      - deps/*install/
      - build/

.build-common:
  stage: build
  tags: [frontier, slurm]
  variables:
    SCHEDULER_PARAMETERS: "-AUMS029 -pbatch -t 00:30:00 --nice=0 -c48 --threads-per-core=2 --gpus=8 -N 1"
  before_script:
    - *setup_env_ecpci
  script:
    - bash scripts/ci/gitlab-ci/run.sh configure
    - bash scripts/ci/gitlab-ci/run.sh build
    - bash scripts/ci/gitlab-ci/run.sh test
  after_script:
    - *setup_env_ecpci
    - bash scripts/ci/gitlab-ci/run.sh submit
    - ccache -s

.kokkos-hip-common:
  variables:
    Kokkos_DIR: "$CI_PROJECT_DIR/deps/kokkos_install"
    # Cmake would not install a RPATH inside the source dir
    LD_LIBRARY_PATH: "$Kokkos_DIR/lib64/:$LD_LIBRARY_PATH"
    # Order matters
    JOB_MODULES: >-
      Core/24.00
      lfs-wrapper
      hsi
      PrgEnv-gnu
      gcc-native/12
      craype-accel-amd-gfx90a
      rocm/5.7.1
      cmake
      git
      ninja
      libffi
      hdf5
      zstd

setup:frontier-kokkos-hip:
  variables:
    KOKKOS_VER: 4.4.01
    KOKKOS_OPTS: >-
      -DCMAKE_INSTALL_PREFIX:PATH=$Kokkos_DIR
      -DCMAKE_CXX_COMPILER:FILEPATH=/opt/rocm-5.7.1/hip/bin/hipcc
      -DKokkos_ARCH_VEGA90A:BOOL=ON
      -DKokkos_ENABLE_HIP:BOOL=ON
      -DKokkos_ENABLE_HIP_RELOCATABLE_DEVICE_CODE:BOOL=OFF
      -DKokkos_ENABLE_SERIAL:BOOL=ON
  extends:
    - .frontier-cmake-common
    - .setup-common
    - .kokkos-hip-common
  before_script:
    - *setup_env_ecpci
    - *install_ccache
    - .gitlab/config/kokkos.sh "$CI_PROJECT_DIR/deps" "$KOKKOS_VER" $KOKKOS_OPTS

build:frontier-kokkos-hip:
  extends:
    - .frontier-cmake-common
    - .build-common
    - .kokkos-hip-common
  before_script:
    - *setup_env_ecpci
  needs:
    - setup:frontier-kokkos-hip
  dependencies:
    - setup:frontier-kokkos-hip

.cray-common:
  variables:
    # Order matters
    JOB_MODULES: >-
      Core/24.00
      PrgEnv-cray
      cmake
      git
      ninja
      libffi
      zstd
  extends:
    - .frontier-cmake-common

setup:frontier-cray:
  extends:
    - .setup-common
    - .cray-common

build:frontier-cray:
  extends:
    - .build-common
    - .cray-common
  needs:
    - setup:frontier-cray
  dependencies:
    - setup:frontier-cray

.report-status:
  rules:
    - if: $CI_PIPELINE_SOURCE =~ /parent_pipeline|web/
  tags: [ shell ]
  extends:
    .frontier-common
  variables:
    STATUS_PROJECT: ornladios/ADIOS2
    STATUS_NAME: OLCF Frontier
  before_script: |
    git fetch
    source scripts/ci/gitlab-ci/setup-vars.sh
    git checkout "$CI_COMMIT_REF"
  script: >
    curl -X POST -H "${GITHUB_CURL_HEADERS}"
    "https://api.github.com/repos/${STATUS_PROJECT}/statuses/${CI_ORIGINAL_SHA}"
    -d "{\"state\":\"${CI_JOB_NAME}\", \"context\":\"${STATUS_NAME}\",\"target_url\":\"${CI_PIPELINE_URL}\",\"description\":\"${STATUS_DESC}\"}"
  environment:
    name: report-$DOWNSTREAM_COMMIT_SHA

pending:
  stage: pre
  variables:
    STATUS_DESC: Pipeline is running
  extends:
    - .report-status
success:
  stage: post
  variables:
    STATUS_DESC: Pipeline succeeded
  extends:
    - .report-status
  dependencies:
    - build:frontier-kokkos-hip
    - build:frontier-cray
failure:
  stage: post
  rules:
    - if: $CI_PIPELINE_SOURCE =~ /parent_pipeline|web/
      when: on_failure
  variables:
    STATUS_DESC: Pipeline failed
  extends:
    - .report-status
  dependencies:
    - build:frontier-kokkos-hip
    - build:frontier-cray

generate_pipelines:
  stage: setup
  extends:
    .frontier-common
  tags: [frontier, shell]
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
  variables:
    GL_PROJECT_ID: 105
    # This the commit where the frontier CI was added, we want to avoid testing
    # commits before this.
    ROOT_COMMIT_SHA: 30368427fc41d57e428cdd15e7629dded2f71728
  script:
    - pip install --user urllib3 requests GitPython
    - >
      .gitlab/config/generate_pipelines.py
      -u "https://code.olcf.ornl.gov/"
      -r "${ROOT_COMMIT_SHA}"
      -p "${GL_PROJECT_ID}"
      -n ornladios/ADIOS2
      -f .gitlab/config/dynamic_pipeline.yml.in
      > generated_pipelines.yml

  artifacts:
    paths:
      - generated_pipelines.yml

launch_pipelines:
  stage: build
  extends:
    .frontier-common
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
  trigger:
    include:
      - artifact: generated_pipelines.yml
        job: generate_pipelines