diff --git a/.github/workflows/macos-ci-x86_64.yaml b/.github/workflows/macos-ci-x86_64.yaml deleted file mode 100644 index a5a216a29..000000000 --- a/.github/workflows/macos-ci-x86_64.yaml +++ /dev/null @@ -1,141 +0,0 @@ -name: macos-ci-x86_64-build -on: - pull_request: - paths-ignore: - - 'doc/**' - - '**.md' - - '.github/ISSUE_TEMPLATE/*' - - '.gitignore' - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -defaults: - run: - shell: bash - -jobs: - macos-ci-x86_64-build: - runs-on: [macos-ci-x86_64] - - steps: - - name: cleanup - run: | - pwd - ls -lart - rm -fr * - - - name: checkout - uses: actions/checkout@v2 - with: - submodules: true - - - name: create-env - run: | - source ./setup.sh - - # DH* 20230302 - to avoid using padded_length for build caches, - # always build in the same environment so that the length of the - # path doesn't change - see also other 'DH* 20230302' changes below - #export ENVNAME=${{ inputs.template || 'unified-dev' }}.macos-ci-x86_64 - export ENVNAME=ci-env.macos-ci-x86_64 - # *DH - export ENVDIR=$PWD/envs/${ENVNAME} - spack stack create env --site macos.default --template ${{ inputs.template || 'unified-dev' }} --name ${ENVNAME} - spack env activate ${ENVDIR} - spack add ${{ inputs.specs || '' }} - export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site" - - # Find external packages - spack external find --scope system \ - --exclude bison --exclude openssl \ - --exclude python --exclude gettext - # Need to find gettext outside of default (presumed to be a system) path for krb5 - spack external find --path=/usr/local/Cellar/gettext/0.21.1 gettext - spack external find --scope system perl - spack external find --scope system wget - PATH="/usr/local/opt/curl/bin:$PATH" \ - spack external find --scope system curl - PATH="/usr/local/opt/qt5/bin:$PATH" \ - spack external find --scope system qt - spack external find --scope system texlive - spack external find --scope system mysql - - # Find compilers - spack compiler find --scope system - - export -n SPACK_SYSTEM_CONFIG_PATH - - # DH* 20230302 - Don't use yet, too many problems - ## For buildcaches - #spack config add config:install_tree:padded_length:true - # *DH - - # Set compiler and MPI - spack config add "packages:all:providers:mpi:[openmpi@5.0.1]" - spack config add "packages:all:compiler:[apple-clang@14.0.0]" - sed -i '' "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%apple-clang'\]/g" $ENVDIR/spack.yaml - - # Add additional variants for MET packages, different from config/common/packages.yaml - spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" - - # Concretize and check for duplicates - spack concretize 2>&1 | tee log.concretize.apple-clang-14.0.0 - ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.apple-clang-14.0.0 -i fms -i crtm -i esmf -i mapl - - # Add and update source cache - spack mirror add local-source file:///Users/ec2-user/spack-stack/source-cache/ - spack mirror create -a -d /Users/ec2-user/spack-stack/source-cache/ - - # Add binary cache and reindex it - spack mirror add local-binary file:///Users/ec2-user/spack-stack/build-cache/ - spack buildcache update-index local-binary - echo "Packages in combined spack build caches:" - spack buildcache list - - # Break installation up in pieces and create build caches in between - # This allows us to "spin up" builds that altogether take longer than - # six hours, and/or fail later in the build process. - - # base-env - echo "base-env ..." - spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.apple-clang-14.0.0.base-env - spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ base-env - - # the rest - echo "${{ inputs.template || 'unified-dev' }} ..." - spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.apple-clang-14.0.0.${{ inputs.template || 'unified-dev' }} - spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ - - # Next steps: synchronize source and build cache to a central/combined mirror? - echo "Next steps ..." - spack clean -a - spack module lmod refresh -y - spack stack setup-meta-modules - spack env deactivate - - - name: test-env - run: | - source /usr/local/opt/lmod/init/profile - - # DH* 20230302 - to avoid using padded_length for build caches, - # always build in the same environment so that the length of the - # path doesn't change - see also other 'DH* 20230302' changes below - #export ENVNAME=${{ inputs.template || 'unified-dev' }}.macos-ci-x86_64 - export ENVNAME=ci-env.macos-ci-x86_64 - # *DH - export ENVDIR=$PWD/envs/${ENVNAME} - ls -l ${ENVDIR}/install/modulefiles/Core - - module use ${ENVDIR}/install/modulefiles/Core - module load stack-apple-clang/14.0.0 - module load stack-openmpi/5.0.1 - module load stack-python/3.10.13 - module available - - module load jedi-ufs-env/1.0.0 - module load ewok-env/1.0.0 - module load soca-env/1.0.0 - module list diff --git a/.github/workflows/ubuntu-rnd-x86_64.yaml b/.github/workflows/ubuntu-rnd-x86_64.yaml deleted file mode 100644 index 0b32d4f24..000000000 --- a/.github/workflows/ubuntu-rnd-x86_64.yaml +++ /dev/null @@ -1,235 +0,0 @@ -name: ubuntu-rnd-x86_64-build -on: - schedule: - - cron: '0 0 * * *' - workflow_dispatch: - -defaults: - run: - shell: bash - -jobs: - ubuntu-rnd-x86_64-build: - runs-on: [ubuntu-rnd-x86_64] - timeout-minutes: 720 # 12 hours - - steps: - - name: cleanup - run: | - pwd - ls -lart - rm -fr * - rm -fr /home/ubuntu/spack-stack/CI/tmp - mkdir /home/ubuntu/spack-stack/CI/tmp - - TODAY=$(date +%Y%m%d) - LAST_TO_KEEP=$(date --date='-1 days' +%Y%m%d) - echo "TODAY is ${TODAY}" - echo "LAST_TO_KEEP is ${LAST_TO_KEEP}" - - cd /home/ubuntu/spack-stack/CI/unified-env - for dir in `ls -1`; do echo ${dir}; if [[ ${dir} < ${LAST_TO_KEEP} ]]; then echo "Deleting ${dir}"; rm -fr ${dir}; fi; done - - - name: checkout - uses: actions/checkout@v3 - with: - submodules: true - - - name: create-buildcache - run: | - source ./setup.sh - - # Get day of week to set default container for scheduled builds - DOW=$(date +%u) - # Monday is 1 ... Sunday is 7 - if [[ $DOW == 7 ]]; then - export USE_BINARY_CACHE=false - echo "Ignore existing binary cache for creating buildcache environment" - else - export USE_BINARY_CACHE=true - echo "Use existing binary cache for creating buildcache environment" - fi - - # Make environment for creating/updating binary cache - export TEMPLATE=${{ inputs.template || 'unified-dev' }} - export ENVNAME=buildcache-${TEMPLATE}.ubuntu-rnd-x86_64 - export ENVDIR=$PWD/envs/${ENVNAME} - spack stack create env --site aws-pcluster --template ${TEMPLATE} --name ${ENVNAME} --prefix=/home/ubuntu/spack-stack/CI/tmp/spack-stack-dev-buildcache - spack env activate ${ENVDIR} - spack add ${{ inputs.specs || '' }} - - # For buildcaches - spack config add config:install_tree:padded_length:200 - - # Add additional variants for MET packages, different from config/common/packages.yaml - spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" - - # Overrides for spack build and staging areas to speed up builds, - # separate from the default site config locations to avoid conflicts - spack config add "config:build_stage:/home/ubuntu/spack-stack/CI/tmp/build_stage" - spack config add "config:test_stage:/home/ubuntu/spack-stack/CI/tmp/test_stage" - spack config add "config:source_cache:/home/ubuntu/spack-stack/CI/tmp/source_cache" - spack config add "config:misc_cache:/home/ubuntu/spack-stack/CI/tmp/misc_cache" - - # Loop over compilers - declare -a compilers=("gcc" "intel") - cp ${ENVDIR}/spack.yaml ${ENVDIR}/spack.yaml.original - - for compiler in "${compilers[@]}" - do - # Set compiler and MPI - cp ${ENVDIR}/spack.yaml.original ${ENVDIR}/spack.yaml - sed -i "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%${compiler}'\]/g" ${ENVDIR}/spack.yaml - cat ${ENVDIR}/spack.yaml - - # Concretize and check for duplicates - spack concretize --force --fresh 2>&1 | tee log.concretize.${ENVNAME}.${compiler} - ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.${ENVNAME}.${compiler} -i fms -i crtm -i esmf -i mapl - - # Update spack source cache - spack mirror create -a -d /mnt/experiments-efs/spack-stack/source-cache - - # Add binary cache if requested - if [ "$USE_BINARY_CACHE" = true ] ; then - spack mirror add local-binary file:///mnt/experiments-efs/spack-stack/build-cache/ - spack buildcache update-index local-binary - echo "Packages in spack binary cache:" - spack buildcache list - fi - - # Create/update binary cache - echo "Create/update build cache for environment ${TEMPLATE} and compiler ${compiler}..." - spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.${TEMPLATE}.${compiler} - spack buildcache create -a -u /mnt/experiments-efs/spack-stack/build-cache/ - - # Next steps: synchronize source and build cache to a central/combined mirror? - #echo "Next steps ..." - - # Remove binary cache for next round of concretization - if [ "$USE_BINARY_CACHE" = true ] ; then - spack mirror rm local-binary - fi - done - - spack env deactivate - rm -fr /home/ubuntu/spack-stack/CI/tmp/spack-stack-dev-buildcache - - - name: create-env - run: | - source ./setup.sh - TODAY=$(date +%Y%m%d) - - # Create actual environment from binary cache - export TEMPLATE=${{ inputs.template || 'unified-dev' }} - export ENVNAME=${TEMPLATE}.ubuntu-rnd-x86_64 - export ENVDIR=$PWD/envs/${ENVNAME} - spack stack create env --site aws-pcluster --template ${TEMPLATE} --name ${ENVNAME} --prefix=/home/ubuntu/spack-stack/CI/unified-env/${TODAY} - spack env activate ${ENVDIR} - spack add ${{ inputs.specs || '' }} - - # Add additional variants for MET packages, different from config/common/packages.yaml - spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" - - # Overrides for spack build and staging areas to speed up builds, - # separate from the default site config locations to avoid conflicts - spack config add "config:build_stage:/home/ubuntu/spack-stack/CI/tmp/build_stage" - spack config add "config:test_stage:/home/ubuntu/spack-stack/CI/tmp/test_stage" - spack config add "config:source_cache:/home/ubuntu/spack-stack/CI/tmp/source_cache" - spack config add "config:misc_cache:/home/ubuntu/spack-stack/CI/tmp/misc_cache" - - # Loop over compilers - declare -a compilers=("gcc" "intel") - cp ${ENVDIR}/spack.yaml ${ENVDIR}/spack.yaml.original - - for compiler in "${compilers[@]}" - do - # Set compiler and MPI - cp ${ENVDIR}/spack.yaml.original ${ENVDIR}/spack.yaml - sed -i "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%${compiler}'\]/g" ${ENVDIR}/spack.yaml - cat ${ENVDIR}/spack.yaml - - # Concretize and check for duplicates - spack concretize --force --fresh 2>&1 | tee log.concretize.${ENVNAME}.${compiler} - ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.${ENVNAME}.${compiler} -i fms -i crtm -i esmf -i mapl - - # Update spack source cache - spack mirror create -a -d /mnt/experiments-efs/spack-stack/source-cache - - # Add binary cache - spack mirror add local-binary file:///mnt/experiments-efs/spack-stack/build-cache/ - spack buildcache update-index local-binary - echo "Packages in spack binary cache:" - spack buildcache list - - # Install from binary cache - echo "Install environment ${TEMPLATE} for compiler ${compiler} from binary cache ..." - spack install --fail-fast --no-check-signature 2>&1 | tee log.install.${TEMPLATE}.${compiler} - spack clean -a - - # Create modules - spack module lmod refresh -y - spack stack setup-meta-modules - - # Remove binary cache for next round of concretization - spack mirror rm local-binary - done - - spack env deactivate - - - name: test-env - run: | - source /etc/profile.d/z00_lmod.sh - module use /usr/share/modules/modulefiles - module use /opt/intel/mpi/2021.6.0/modulefiles - module use /home/ubuntu/jedi/modulefiles - - TODAY=$(date +%Y%m%d) - echo "TODAY is ${TODAY}" - ls -l /home/ubuntu/spack-stack/CI/unified-env/${TODAY}/modulefiles/Core - - module use /home/ubuntu/spack-stack/CI/unified-env/${TODAY}/modulefiles/Core - module load stack-intel/2021.6.0 - module load stack-intel-oneapi-mpi/2021.6.0 - module load stack-python/3.10.13 - module available - - module load jedi-ufs-env/1.0.0 - module load ewok-env/1.0.0 - module load soca-env/1.0.0 - module list - - module purge - module load stack-gcc/9.4.0 - module load stack-openmpi/4.1.4 - module load stack-python/3.10.13 - module available - - module load jedi-ufs-env/1.0.0 - module load ewok-env/1.0.0 - module load soca-env/1.0.0 - module list - - # Create empty file to signal that this environment is ready for testing - touch /home/ubuntu/spack-stack/CI/unified-env/${TODAY}.ready - - # Report status to JCSDA CI slack channel for nightly runs only - - name: Report Status - if: always() - uses: ravsamhq/notify-slack-action@v1 - env: - SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }} - with: - # https://www.ravsam.in/blog/send-slack-notification-when-github-actions-fails/#getting-a-webhook-url - # https://www.ravsam.in/blog/collect-form-responses-using-google-apps-script - # https://github.com/marketplace/actions/notify-slack-action - status: ${{ job.status }} - notify_when: 'success,failure,warnings' - notification_title: '{workflow} has {status_message}' - message_format: '{emoji} *{workflow}* {status_message} for branch {branch} in <{repo_url}|{repo}>' - footer: ${{ github.event.pull_request.number || github.event_name || 'workflow dispatched manually' }} - # For testing: only notify user Dom - #mention_users: 'U02NLGXF5HV' - #mention_users_when: 'failure,warnings' - # Default: notify channel - mention_groups: '!channel' - mention_groups_when: 'failure,warnings' diff --git a/configs/common/modules_lmod.yaml b/configs/common/modules_lmod.yaml index 383071612..6b2b9af75 100644 --- a/configs/common/modules_lmod.yaml +++ b/configs/common/modules_lmod.yaml @@ -71,8 +71,6 @@ modules: - openssl - perl - pkgconf - - protobuf - - py-protobuf - qt - randrproto - readline diff --git a/configs/common/modules_tcl.yaml b/configs/common/modules_tcl.yaml index 58fd733c7..8011b215f 100644 --- a/configs/common/modules_tcl.yaml +++ b/configs/common/modules_tcl.yaml @@ -73,8 +73,6 @@ modules: - openssl - perl - pkgconf - - protobuf - - py-protobuf - qt - randrproto - readline diff --git a/configs/sites/narwhal/compilers.yaml b/configs/sites/narwhal/compilers.yaml index d279a5213..4102aa0cd 100644 --- a/configs/sites/narwhal/compilers.yaml +++ b/configs/sites/narwhal/compilers.yaml @@ -11,7 +11,6 @@ compilers:: modules: - PrgEnv-intel/8.3.2 - intel-classic/2021.4.0 - - node.js/20.10.0 environment: prepend_path: PATH: '/opt/cray/pe/gcc/10.3.0/snos/bin' @@ -32,7 +31,6 @@ compilers:: modules: - PrgEnv-gnu/8.3.2 - gcc/10.3.0 - - node.js/20.10.0 environment: prepend_path: LD_LIBRARY_PATH: '/opt/cray/pe/libsci/22.08.1.1/GNU/9.1/x86_64/lib' diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index ebe307ce8..ffb5e788e 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -259,6 +259,10 @@ For ``spack-stack-1.6.0`` with Intel, proceed with loading the following modules .. code-block:: console + # These extra steps are required for performance reason, ofi is about 30% slower than ucx + # Note we can't load craype-network-ucx for building spack-stack environments, must do here + module unload craype-network-ofi + module load craype-network-ucx module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/unified-env-intel-2021.4.0/install/modulefiles/Core module load stack-intel/2021.4.0 module load stack-cray-mpich/8.1.14 @@ -287,6 +291,10 @@ For ``spack-stack-1.6.0`` with GNU, proceed with loading the following modules: .. code-block:: console + # These extra steps are required for performance reason, ofi is about 30% slower than ucx + # Note we can't load craype-network-ucx for building spack-stack environments, must do here + module unload craype-network-ofi + module load craype-network-ucx module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/unified-env-gcc-10.3.0/install/modulefiles/Core module load stack-gcc/10.3.0 module load stack-cray-mpich/8.1.14 @@ -393,7 +401,7 @@ The following is required for building new spack environments and for using spac module use /glade/work/epicufsrt/contrib/spack-stack/derecho/modulefiles module load ecflow/5.8.4 -For ``spack-stack-1.6.0`` with Intel, proceed with loading the following modules:: +For ``spack-stack-1.6.0`` with Intel, proceed with loading the following modules: .. code-block:: console diff --git a/setup.sh b/setup.sh index 3f033d620..23c3b7ed1 100755 --- a/setup.sh +++ b/setup.sh @@ -7,7 +7,9 @@ echo "Setting environment variable SPACK_STACK_DIR to ${SPACK_STACK_DIR}" source ${SPACK_STACK_DIR:?}/spack/share/spack/setup-env.sh echo "Sourcing spack environment ${SPACK_STACK_DIR}/spack/share/spack/setup-env.sh" -export SPACK_USER_CACHE_PATH=$SPACK_ROOT/user_cache +if [ "$(uname)" != "Darwin" ]; then + export SPACK_USER_CACHE_PATH=$SPACK_ROOT/user_cache +fi # Get the current hash of the spack-stack code export SPACK_STACK_HASH=`git rev-parse --short HEAD` diff --git a/spack b/spack index b40323ef1..4fafe3ead 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit b40323ef10659795b66e8e165a6b16c7ca1c0066 +Subproject commit 4fafe3eadbe330e64423b8ee16c88f8309f48d5b