CI updates (#493)

* update developer.yml * update Intel.yml * update Spack.yml * update Linux_options.yml * update Linux_versions.yml * update MacOS.yml * update windows.yml * debug MacOS.yml * debug MacOS.yml * debug MacOS.yml * debug MacOS.yml * fix windows.yml * Update developer.yml
NOAA-EMC · Jul 1, 2024 · 7b38144 · 7b38144
1 parent 364d89d
commit 7b38144
Show file tree

Hide file tree

Showing 7 changed files with 57 additions and 168 deletions.
diff --git a/.github/workflows/Intel.yml b/.github/workflows/Intel.yml
@@ -7,88 +7,52 @@ on:
     branches:
     - develop
 
-# Use custom shell with -l so .bash_profile is sourced which loads intel/oneapi/setvars.sh
-# without having to do it in manually every step
-defaults:
-  run:
-    shell: bash -leo pipefail {0}
-
 jobs:
   Intel:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        compilers: ["CC=icc FC=ifort CXX=icpc", "CC=icx FC=ifx CXX=icpx"]
+        compilers: ["oneapi", "classic"]
 
     steps:
 
-      # See https://software.intel.com/content/www/us/en/develop/articles/oneapi-repo-instructions.html
     - name: install-dependencies
       run: |
         sudo apt-get update
         sudo apt-get install libaec-dev libpng-dev zlib1g-dev libjpeg-dev doxygen
-        cd /tmp
-        wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
-        sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
-        rm GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
-        echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
-        sudo apt-get update
-        sudo apt-get install intel-oneapi-dev-utilities intel-oneapi-openmp intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic
-        echo "source /opt/intel/oneapi/setvars.sh" >> ~/.bash_profile
 
-    - name: checkout-jasper
-      uses: actions/checkout@v3
+    - name: "Install Intel"
+      uses: NOAA-EMC/ci-install-intel-toolkit@develop
       with:
-        repository: jasper-software/jasper
-        path: jasper
-        ref: version-2.0.25
+        compiler-setup: ${{ matrix.compilers }}
 
-    - name: cache-jasper
-      id: cache-jasper
-      uses: actions/cache@v3
+    - name: "Build dependencies"
+      uses: NOAA-EMC/ci-build-nceplibs@develop
       with:
-        path: ~/Jasper
-        key: jasper-${{ runner.os }}-${{ hashFiles('jasper/VERSION') }}-${{ matrix.compilers }}
-
-    - name: build-jasper
-      if: steps.cache-jasper.outputs.cache-hit != 'true'
-      run: |
-        cd jasper
-        mkdir cmake_build
-        cd cmake_build
-        ${{ matrix.compilers }} cmake .. -DCMAKE_INSTALL_PREFIX=~/Jasper
-        make -j2
-        make install
+        jasper-version: version-2.0.25
 
     - name: checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         path: g2c
 
     - name: Initialize CodeQL
-      if: ${{ matrix.compilers == 'CC=icc FC=ifort CXX=icpc' }}
+      if: ${{ matrix.compilers == 'classic' }}
       uses: github/codeql-action/init@v2
       with:
         languages: cpp
         queries: +security-and-quality
 
     - name: build
       run: |
-        cd g2c
-        mkdir build
-        cd build
-        ${{ matrix.compilers }} cmake -DJasper_ROOT=~/Jasper ..
-        make -j2 VERBOSE=1
+        cmake -S g2c -B g2c/build -DJasper_ROOT=$GITHUB_WORKSPACE/nceplibs/jasper
+        cmake --build g2c/build --parallel 2 --verbose
 
     - name: Perform CodeQL Analysis
-      if: ${{ matrix.compilers == 'CC=icc FC=ifort CXX=icpc' }}
+      if: ${{ matrix.compilers == 'classic' }}
       uses: github/codeql-action/analyze@v2
       with:
         category: "/language:cpp"
 
     - name: test
-      run: |
-        cd $GITHUB_WORKSPACE/g2c/build
-        ctest --verbose --output-on-failure --rerun-failed
-
-      
+      run: ctest --test-dir g2c/build --verbose --output-on-failure --rerun-failed
diff --git a/.github/workflows/Linux_options.yml b/.github/workflows/Linux_options.yml
@@ -85,15 +85,15 @@ jobs:
         sudo apt-get install libaec-dev libpng-dev zlib1g-dev libjpeg-dev libopenjp2-7-dev
 
     - name: checkout-jasper
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         repository: jasper-software/jasper
         path: jasper
         ref: version-2.0.25
 
     - name: cache-jasper
       id: cache-jasper
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ~/Jasper
         key: jasper-${{ runner.os }}-${{ hashFiles('jasper/VERSION') }}
@@ -109,7 +109,7 @@ jobs:
         make install
 
     - name: checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         path: g2c
 

diff --git a/.github/workflows/Linux_versions.yml b/.github/workflows/Linux_versions.yml
@@ -30,14 +30,14 @@ jobs:
 
     - name: cache-jasper
       id: cache-jasper
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ~/jasper
         key: jasper-${{ runner.os }}-${{ matrix.jasper-version }}
 
     - name: checkout-jasper
       if: steps.cache-jasper.outputs.cache-hit != 'true'
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         repository: jasper-software/jasper
         path: jasper
@@ -59,7 +59,7 @@ jobs:
         make install
 
     - name: checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         path: g2c
 

diff --git a/.github/workflows/MacOS.yml b/.github/workflows/MacOS.yml
@@ -35,14 +35,14 @@ jobs:
 
     - name: cache-jasper
       id: cache-jasper
-      uses: actions/cache@v2
+      uses: actions/cache@v4
       with:
         path: ~/jasper
         key: jasper-${{ runner.os }}-4.0.0-macOS-2
 
     - name: checkout-jasper
       if: steps.cache-jasper.outputs.cache-hit != 'true'
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         repository: jasper-software/jasper
         path: jasper
@@ -63,7 +63,7 @@ jobs:
 
     - name: cache-png
       id: cache-png
-      uses: actions/cache@v2
+      uses: actions/cache@v4
       with:
         path: ~/png
         key: png-${{ runner.os }}-1.6.37-macOS
@@ -76,12 +76,12 @@ jobs:
         cd libpng-1.6.37
         mkdir build
         cd build
-        cmake .. -DCMAKE_INSTALL_PREFIX=~/png -DPNG_ARM_NEON=off
+        cmake .. -DCMAKE_INSTALL_PREFIX=~/png -DCMAKE_C_FLAGS="-Wno-implicit-function-declaration" -DPNG_ARM_NEON=off
         make -j2
         make install
 
     - name: checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         path: g2c
 

diff --git a/.github/workflows/Spack.yml b/.github/workflows/Spack.yml
@@ -16,7 +16,7 @@ on:
     - develop
 
 jobs:
-  # This job builds with Spack using every combination of variants and runs the CTest suite each time
+  # This job builds with Spack, exercising all variants, and runs the CTest suite each time
   Spack:
 
     strategy:
@@ -27,72 +27,25 @@ jobs:
 
     steps:
 
-    - name: checkout-g2c
-      uses: actions/checkout@v4
+    - name: "Build Spack package"
+      uses: NOAA-EMC/ci-test-spack-package@develop
       with:
-        path: g2c
-
-    - name: cache-spack
-      id: cache-spack
-      uses: actions/cache@v3
-      with:
-        path: ~/spack-build-cache
-        key: spack-build-cache-${{ matrix.os }}-${{ matrix.variants }}-1
-
-    - name: spack-build-and-test
-      run: |
-        set -x
-        git clone -c feature.manyFiles=true https://github.com/spack/spack
-        . spack/share/spack/setup-env.sh
-        spack env create g2c-env
-        spack env activate g2c-env
-        cp $GITHUB_WORKSPACE/g2c/spack/package.py $SPACK_ROOT/var/spack/repos/builtin/packages/g2c/package.py
-        spack develop --no-clone --path $GITHUB_WORKSPACE/g2c g2c@develop
-        spack add g2c@develop%gcc@11 ${{ matrix.variants }}
-        spack external find cmake gmake
-        spack mirror add spack-build-cache ~/spack-build-cache
-        spack concretize
-        # Run installation and run CTest suite
-        spack install --fail-fast --no-check-signature --test root
-        # Print test results
-        cat $(spack location -i g2c)/.spack/install-time-test-log.txt
-        # Run 'spack load' to check for obvious errors in setup_run_environment
-        echo 'Loading g2c through Spack and checking $G2C_LIB ...'
-        spack load g2c
-        if [[ "${{ matrix.variants }}" =~ "libs=shared" ]]; then suffix="so" ; else suffix="a"; fi
-        ls $G2C_LIB | grep -cE '/libg2c\.'$suffix'$'
-        if [[ "${{ matrix.variants }}" == *"+png"* && "${{ matrix.variants }}" == "libs=shared"* ]]; then ldd $G2C_LIB | grep -c libpng; fi
-        if [[ "${{ matrix.variants }}" == *"+utils"* ]]; then
-          ls $(spack location -i g2c)/bin/{g2c_compare,g2c_degrib2,g2c_index}
-        else
-          if [ -f $(spack location -i g2c)/bin/g2c_compare ]; then echo "utils were built but shouldn't have been!"; exit 1; fi
-        fi
-        # Setup build cache (automatically skipped if it already exists)
-        spack buildcache push --only dependencies --unsigned ~/spack-build-cache g2c
-
-    - name: Upload test results
-      uses: actions/upload-artifact@v3
-      if: ${{ failure() }}
-      with:
-        name: spackci-ctest-output-${{ matrix.os }}-${{ matrix.variants }}
-        path: ${{ github.workspace }}/g2c/spack-build-*/Testing/Temporary/LastTest.log
+        package-name: g2c
+        package-variants: ${{ matrix.variants }}
+        custom-recipe: spack/package.py
+        use-repo-cache: true
+        spack-compiler: gcc
+        repo-cache-key-suffix: ${{ matrix.os }}-${{ matrix.variants }}-1
 
   # This job validates the Spack recipe by making sure each cmake build option is represented
   recipe-check:
     runs-on: ubuntu-latest
 
     steps:
 
-    - name: checkout-g2c
-      uses: actions/checkout@v4
-      with:
-        path: g2c
-
     - name: recipe-check
-      run: |
-        echo "If this jobs fails, look at the most recently output CMake option below and make sure that option appears in spack/package.py"
-        excl="ENABLE_DOCS|FTP_TEST_FILES|FTP_LARGE_TEST_FILES|FTP_EXTRA_TEST_FILES|LOGGING"
-        for opt in $(grep -ioP "^option\(\K(?!($excl))[^ ]+" $GITHUB_WORKSPACE/g2c/CMakeLists.txt) ; do
-          echo "Checking for presence of '$opt' CMake option in package.py"
-          grep -cP "define.+\b${opt}\b" $GITHUB_WORKSPACE/g2c/spack/package.py
-        done
+      uses: NOAA-EMC/ci-check-spack-recipe@develop
+      with:
+        recipe-file: package/spack/package.py
+        cmakelists-txt: package/CMakeLists.txt
+        ignore-list: ENABLE_DOCS,FTP_TEST_FILES,FTP_LARGE_TEST_FILES,FTP_EXTRA_TEST_FILES,LOGGING
diff --git a/.github/workflows/developer.yml b/.github/workflows/developer.yml
@@ -31,40 +31,19 @@ jobs:
         sudo apt-get install libpng-dev zlib1g-dev libjpeg-dev doxygen libopenjp2-7-dev valgrind libaec-dev
         python3 -m pip install gcovr
 
-    - name: cache-jasper
-      id: cache-jasper
-      uses: actions/cache@v3
+    - name: "Build dependencies"
+      uses: NOAA-EMC/ci-build-nceplibs@develop
       with:
-        path: ~/jasper
-        key: jasper-${{ runner.os }}-4.0.0
-
-    - name: checkout-jasper
-      if: steps.cache-jasper.outputs.cache-hit != 'true'
-      uses: actions/checkout@v3
-      with:
-        repository: jasper-software/jasper
-        path: jasper
-        ref: version-4.0.0
-
-    - name: build-jasper
-      if: steps.cache-jasper.outputs.cache-hit != 'true'
-      run: |
-        export CC=gcc
-        cd jasper
-        mkdir cmake_build
-        cd cmake_build
-        cmake -DCMAKE_INSTALL_PREFIX=~/jasper ..
-        make -j2
-        make install
+        jasper-version: version-4.0.0
 
     - name: checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         path: g2c
 
     - name: cache-data
       id: cache-data
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ~/data
         key: data-2
@@ -74,28 +53,21 @@ jobs:
       run: |
         set -x
         export CC=gcc
-        export CFLAGS='-Wall -Werror -g -fprofile-abs-path -fprofile-arcs -ftest-coverage -O0 -fsanitize=address'
-        cd g2c
-        mkdir build
-        cd build
-        cmake -DUSE_AEC=ON -DJasper_ROOT=~/jasper -DBUILD_G2C=ON -DLOGGING=On -DENABLE_DOCS=On -DPTHREADS=ON -DFTP_TEST_FILES=ON -DFTP_LARGE_TEST_FILES=OFF -DTEST_FILE_DIR=/home/runner/data -DCMAKE_BUILD_TYPE=Debug .. 
-        make -j2 VERBOSE=1
-        ctest --verbose --output-on-failure --rerun-failed
-        gcovr --root .. -v  --html-details --exclude ../tests --exclude CMakeFiles --print-summary -o test-coverage.html
+        cmake -S g2c -B g2c/build -DUSE_AEC=ON -DJasper_ROOT=$GITHUB_WORKSPACE/nceplibs/jasper -DBUILD_G2C=ON -DLOGGING=On -DENABLE_DOCS=On -DPTHREADS=ON -DFTP_TEST_FILES=ON -DFTP_LARGE_TEST_FILES=OFF -DTEST_FILE_DIR=/home/runner/data -DCMAKE_BUILD_TYPE=Debug
+        cmake --build g2c/build --parallel 2 --verbose
+        ctest --test-dir g2c/build --verbose --output-on-failure --rerun-failed
+        gcovr --root g2c -v  --html-details --exclude g2c/tests --exclude g2c/build/CMakeFiles --print-summary -o test-coverage.html
         ls -l
         
     - name: memcheck
       if: matrix.config == 'memcheck'
       run: |
         set -x
         export LD_LIBRARY_PATH="/home/runner/jasper/lib:$LD_LIBRARY_PATH"
-        cd g2c
-        mkdir build
-        cd build
-        cmake -DUSE_AEC=ON -DJasper_ROOT=~/jasper -DBUILD_G2C=ON -DLOGGING=On -DENABLE_DOCS=On -DPTHREADS=ON -DFTP_TEST_FILES=ON -DTEST_FILE_DIR=/home/runner/data -DCMAKE_BUILD_TYPE=Debug .. 
-        make -j2 VERBOSE=1
-        ctest
-        ls -l
+        cmake -S g2c -B g2c/build -DUSE_AEC=ON -DJasper_ROOT=$GITHUB_WORKSPACE/nceplibs/jasper -DBUILD_G2C=ON -DLOGGING=On -DENABLE_DOCS=On -DPTHREADS=ON -DFTP_TEST_FILES=ON -DTEST_FILE_DIR=/home/runner/data -DCMAKE_BUILD_TYPE=Debug
+        cmake --build g2c/build --parallel 2 --verbose
+        ctest --test-dir g2c/build
+        ls -l g2c/build
         
     - name: cache-data
       if: steps.cache-data.outputs.cache-hit != 'true'
@@ -105,9 +77,9 @@ jobs:
         
     - name: upload-test-coverage
       if: matrix.config == 'asan/code coverage'      
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: g2c-test-coverage
         path: |
-          g2c/build/*.html
-          g2c/build/*.css
+          *.html
+          *.css