Update cee configs (#445)

* Use spack-manager built compilers on CEE * Update tests * Leave [email protected] in place for a soft switch * Fix typo in file name * Add key packages as externals * Remove file accidently added * Update boost
sandialabs · Nov 28, 2022 · 365c80d · 365c80d
1 parent a649561
commit 365c80d
Show file tree

Hide file tree

Showing 5 changed files with 74 additions and 79 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,6 @@
 # directories
 spack
+stage
 environments
 golds
 logs

diff --git a/configs/cee/compilers.yaml b/configs/cee/compilers.yaml
@@ -1,20 +1,47 @@
 compilers:
+- compiler:
+    spec: [email protected]
+    paths:
+      cc: /projects/wind/core-spack-manager/scripts/admin-system-setup/compilers_store/linux-rhel7-x86_64/gcc-12.2.0/llvm-13.0.1-bzlmdor3cuyofkkukjhqfyts5ningcmc/bin/clang
+      cxx: /projects/wind/core-spack-manager/scripts/admin-system-setup/compilers_store/linux-rhel7-x86_64/gcc-12.2.0/llvm-13.0.1-bzlmdor3cuyofkkukjhqfyts5ningcmc/bin/clang++
+      f77: /projects/wind/core-spack-manager/scripts/admin-system-setup/compilers_store/linux-rhel7-x86_64/gcc-12.2.0/gcc-10.3.0-lfhoe53uf43qust4kyb32b3w453dod5o/bin/gfortran
+      fc: /projects/wind/core-spack-manager/scripts/admin-system-setup/compilers_store/linux-rhel7-x86_64/gcc-12.2.0/gcc-10.3.0-lfhoe53uf43qust4kyb32b3w453dod5o/bin/gfortran
+    flags: {}
+    operating_system: rhel7
+    target: x86_64
+    modules: []
+    environment:
+      prepend_path:
+        PATH: /projects/wind/core-spack-manager/scripts/admin-system-setup/compilers_store/linux-rhel7-x86_64/gcc-12.2.0/binutils-2.38-parduyka7dxmn5h53cj5pag7kk6ilisb/bin
+    extra_rpaths: []
 - compiler:
     spec: [email protected]
     paths:
-      cc: /projects/cde/v3/cee/spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5/gcc-10.3.0-bxx4ufppf7l3xlbvdinnusa4e6ykqok5/bin/gcc
-      cxx: /projects/cde/v3/cee/spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5/gcc-10.3.0-bxx4ufppf7l3xlbvdinnusa4e6ykqok5/bin/g++
-      f77: /projects/cde/v3/cee/spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5/gcc-10.3.0-bxx4ufppf7l3xlbvdinnusa4e6ykqok5/bin/gfortran
-      fc: /projects/cde/v3/cee/spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5/gcc-10.3.0-bxx4ufppf7l3xlbvdinnusa4e6ykqok5/bin/gfortran
+      cc: /projects/wind/core-spack-manager/scripts/admin-system-setup/compilers_store/linux-rhel7-x86_64/gcc-12.2.0/gcc-10.3.0-lfhoe53uf43qust4kyb32b3w453dod5o/bin/gcc
+      cxx: /projects/wind/core-spack-manager/scripts/admin-system-setup/compilers_store/linux-rhel7-x86_64/gcc-12.2.0/gcc-10.3.0-lfhoe53uf43qust4kyb32b3w453dod5o/bin/g++
+      f77: /projects/wind/core-spack-manager/scripts/admin-system-setup/compilers_store/linux-rhel7-x86_64/gcc-12.2.0/gcc-10.3.0-lfhoe53uf43qust4kyb32b3w453dod5o/bin/gfortran
+      fc: /projects/wind/core-spack-manager/scripts/admin-system-setup/compilers_store/linux-rhel7-x86_64/gcc-12.2.0/gcc-10.3.0-lfhoe53uf43qust4kyb32b3w453dod5o/bin/gfortran
     flags: {}
     operating_system: rhel7
     target: x86_64
     modules: []
     environment:
-      set:
-        http_proxy: http://proxy.sandia.gov:80/
-        https_proxy: http://proxy.sandia.gov:80/
+      prepend_path:
+        PATH: /projects/wind/core-spack-manager/scripts/admin-system-setup/compilers_store/linux-rhel7-x86_64/gcc-12.2.0/binutils-2.38-kocidchyuf644svn4zbsqnmctze7ffvs/bin
     extra_rpaths: []
+- compiler:
+     spec: [email protected]
+     paths:
+       cc: /projects/wind/system-spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5/gcc-9.3.0-tg7stm4b2owxiisgsu4rltm5im7pe7wz/bin/gcc
+       cxx: /projects/wind/system-spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5/gcc-9.3.0-tg7stm4b2owxiisgsu4rltm5im7pe7wz/bin/g++
+       f77: /projects/wind/system-spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5/gcc-9.3.0-tg7stm4b2owxiisgsu4rltm5im7pe7wz/bin/gfortran
+       fc: /projects/wind/system-spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5/gcc-9.3.0-tg7stm4b2owxiisgsu4rltm5im7pe7wz/bin/gfortran
+     flags: {}
+     operating_system: rhel7
+     target: x86_64
+     modules: []
+     environment: {}
+     extra_rpaths: []
 - compiler:
     spec: [email protected]
     paths:
@@ -43,59 +70,3 @@ compilers:
         I_MPI_CXX: /projects/cde/v3/cee/spack/opt/spack/linux-rhel7-x86_64/gcc-10.3.0/intel-oneapi-compilers-2021.1.2-lufoj3442adjwmyj2djuozq5aec3ofn2/compiler/2021.1.2/linux/bin/intel64/icpc
         I_MPI_FC: /projects/cde/v3/cee/spack/opt/spack/linux-rhel7-x86_64/gcc-10.3.0/intel-oneapi-compilers-2021.1.2-lufoj3442adjwmyj2djuozq5aec3ofn2/compiler/2021.1.2/linux/bin/intel64/ifort
     extra_rpaths: []
-- compiler:
-    spec: [email protected]
-    paths:
-      cc: /projects/cde/v3/cee/spack/opt/spack/linux-rhel7-x86_64/gcc-10.3.0/llvm-12.0.1-4ld5bciw4oetcntm2hgn23pklymf55ou/bin/clang
-      cxx: /projects/cde/v3/cee/spack/opt/spack/linux-rhel7-x86_64/gcc-10.3.0/llvm-12.0.1-4ld5bciw4oetcntm2hgn23pklymf55ou/bin/clang++
-      f77: /projects/cde/v3/cee/spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5/gcc-10.3.0-bxx4ufppf7l3xlbvdinnusa4e6ykqok5/bin/gfortran
-      fc: /projects/cde/v3/cee/spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5/gcc-10.3.0-bxx4ufppf7l3xlbvdinnusa4e6ykqok5/bin/gfortran
-    flags: {}
-    operating_system: rhel7
-    target: x86_64
-    modules: []
-    environment: {}
-    extra_rpaths: []
-- compiler:
-    spec: [email protected]
-    paths:
-      cc: /projects/wind/system-spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5/gcc-9.3.0-tg7stm4b2owxiisgsu4rltm5im7pe7wz/bin/gcc
-      cxx: /projects/wind/system-spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5/gcc-9.3.0-tg7stm4b2owxiisgsu4rltm5im7pe7wz/bin/g++
-      f77: /projects/wind/system-spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5/gcc-9.3.0-tg7stm4b2owxiisgsu4rltm5im7pe7wz/bin/gfortran
-      fc: /projects/wind/system-spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5/gcc-9.3.0-tg7stm4b2owxiisgsu4rltm5im7pe7wz/bin/gfortran
-    flags: {}
-    operating_system: rhel7
-    target: x86_64
-    modules: []
-    environment: {}
-    extra_rpaths: []
-- compiler:
-    spec: [email protected]
-    environment: {}
-    extra_rpaths:
-    - /opt/rh/devtoolset-8/root/usr/lib64
-    modules: []
-    flags: {}
-    operating_system: rhel7
-    paths:
-      cc: /opt/rh/devtoolset-8/root/usr/bin/gcc
-      cxx: /opt/rh/devtoolset-8/root/usr/bin/g++
-      f77: /opt/rh/devtoolset-8/root/usr/bin/gfortran
-      fc: /opt/rh/devtoolset-8/root/usr/bin/gfortran
-- compiler:
-    spec: [email protected]
-    environment: {}
-    extra_rpaths:
-    - /projects/sierra/linux_rh7/SDK/compilers/gcc/7.2.0-RHEL6/lib64
-    - /projects/sierra/linux_rh7/SDK/compilers/gcc/7.2.0-RHEL6/lib
-    - /projects/sierra/linux_rh7/SDK/compilers/gcc/7.2.0-RHEL6/libexec
-    flags: {}
-    modules: []
-    operating_system: rhel7
-    paths:
-      cc: /projects/sierra/linux_rh7/SDK/compilers/gcc/7.2.0-RHEL6/bin/gcc
-      cxx: /projects/sierra/linux_rh7/SDK/compilers/gcc/7.2.0-RHEL6/bin/g++
-      f77: /projects/sierra/linux_rh7/SDK/compilers/gcc/7.2.0-RHEL6/bin/gfortran
-      fc: /projects/sierra/linux_rh7/SDK/compilers/gcc/7.2.0-RHEL6/bin/gfortran
-    target: x86_64
-    extra_rpaths: []
diff --git a/configs/cee/packages.yaml b/configs/cee/packages.yaml
@@ -1,27 +1,27 @@
 packages:
   all:
-    compiler: [gcc, intel]
+    compiler: [gcc, clang, intel]
     providers:
-      mpi: [mpich, openmpi]
+      mpi: [mpich, openmpi, mvapich2]
     variants: +mpi build_type=Release cuda_arch=70
     target: [x86_64]
+  boost:
+    version: [1.80.0]
   cmake:
     externals:
     - spec: [email protected]
       prefix: /projects/wind/system-spack/opt/spack/linux-rhel7-x86_64/gcc-9.3.0/cmake-3.22.2-s67tgoaqaw4ky5kmpuzxdg5une7xouvh/
+    - spec: [email protected]%[email protected]~doc+ncurses+ownlibs~qt build_type=Release arch=linux-rhel7-x86_64
+      prefix: /projects/wind/core-spack-manager/scripts/admin-system-setup/packages/.spack-env/view/clang/13.0.1/cmake/3.25.0/flff6w2l
+    - spec: [email protected]%[email protected]~doc+ncurses+ownlibs~qt build_type=Release arch=linux-rhel7-x86_64
+      prefix: /projects/wind/core-spack-manager/scripts/admin-system-setup/packages/.spack-env/view/gcc/10.3.0/cmake/3.25.0/vfumwtcx
     buildable: false
   cuda:
     externals:
     - spec: [email protected]%[email protected]
       prefix: /projects/wind/system-spack/opt/spack/linux-rhel7-x86_64/gcc-9.3.0/cuda-11.2.2-wlah7an4q7rej4uylqlimczaa6z3zlq7/
-    - spec: [email protected]%[email protected]
-      prefix: /projects/cde/v3/cee/spack/opt/spack/linux-rhel7-x86_64/gcc-10.3.0/cuda-11.2.2-5msjxkf5yufbjndzanbavtnyxslnxnkv
-    buildable: false
-  openmpi:
-    version: [4.0.5]
-    externals:
-    - spec: [email protected]%[email protected]
-      prefix: /projects/sierra/linux_rh7/SDK/mpi/openmpi/4.0.5-gcc-7.2.0-RHEL7/
+    - spec: [email protected]%[email protected]~allow-unsupported-compilers~dev arch=linux-rhel7-x86_64
+      prefix: /projects/wind/core-spack-manager/scripts/admin-system-setup/packages/.spack-env/view/gcc/10.3.0/cuda/11.2.2/pveyd7vw
     buildable: false
   pkg-config:
     buildable: false
@@ -50,3 +50,21 @@ packages:
     - spec: [email protected]%[email protected]
       prefix: /projects/wind/system-spack/opt/spack/linux-rhel7-x86_64/gcc-9.3.0/mpich-3.4.2-4h2muy6jlgfdahehxmxa4ybndfdb6gx2/
     buildable: false
+  mvapich2:
+    externals:
+    - spec: [email protected]%[email protected]~alloca~cuda~debug+regcache+wrapperrpath ch3_rank_bits=32
+        fabrics=mrail file_systems=auto process_managers=auto threads=multiple arch=linux-rhel7-x86_64
+      prefix: /projects/wind/core-spack-manager/scripts/admin-system-setup/packages/.spack-env/view/gcc/10.3.0/mvapich2/2.3.7/lygp3cww
+    buildable: false
+  openmpi:
+    externals:
+    - spec: [email protected]%[email protected]~atomics~cuda~cxx~cxx_exceptions~gpfs~internal-hwloc~java+legacylaunchers~lustre~memchecker+romio+rsh~singularity+static+vt+wrapper-rpath
+        fabrics=auto schedulers=auto arch=linux-rhel7-x86_64
+      prefix: /projects/wind/core-spack-manager/scripts/admin-system-setup/packages/.spack-env/view/clang/13.0.1/openmpi/4.0.2/hjdjmvpl
+    - spec: [email protected]%[email protected]~atomics~cuda~cxx~cxx_exceptions~gpfs~internal-hwloc~java+legacylaunchers~lustre~memchecker+romio+rsh~singularity+static+vt+wrapper-rpath
+        fabrics=auto schedulers=auto arch=linux-rhel7-x86_64
+      prefix: /projects/wind/core-spack-manager/scripts/admin-system-setup/packages/.spack-env/view/gcc/10.3.0/openmpi/4.0.2/zcrfmvuz
+    - spec: [email protected]%[email protected]~atomics~cuda~cxx~cxx_exceptions~gpfs~internal-hwloc~java+legacylaunchers~lustre~memchecker+romio+rsh~singularity+static+vt+wrapper-rpath
+        fabrics=auto schedulers=auto arch=linux-rhel7-x86_64
+      prefix: /projects/wind/core-spack-manager/scripts/admin-system-setup/packages/.spack-env/view/gcc/10.3.0/openmpi/4.1.4/jhvqmwf6
+    buildable: false
diff --git a/configs/cee/upstreams.yaml b/configs/cee/upstreams.yaml
@@ -0,0 +1,4 @@
+upstreams:
+  packages:
+    install_tree: /projects/wind/core-spack-manager/scripts/admin-system-setup/packages_store/
+
diff --git a/env-templates/exawind_cee_tests.yaml b/env-templates/exawind_cee_tests.yaml
@@ -3,18 +3,19 @@ spack:
   # CPU tests run on ascic
   - nalus:
     - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast %[email protected] build_type=Release ^[email protected]'
+    - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast %[email protected] build_type=Release ^[email protected]'
     - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast %[email protected] build_type=Release ^[email protected]'
-    - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast %clang@12.0.1 build_type=Release ^[email protected]'
-    - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast+asan %clang@12.0.1 build_type=Debug ^[email protected]'
+    - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast %clang@13.0.1 build_type=Release ^[email protected]'
+    - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast+asan %clang@13.0.1 build_type=Debug ^[email protected]'
     - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast %[email protected] build_type=Release ^trilinos@develop'
     - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast %[email protected] build_type=Release ^trilinos@develop'
-    - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast %clang@12.0.1 build_type=Release ^trilinos@develop'
-    - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast+asan %clang@12.0.1 build_type=Debug ^trilinos@develop'
-    - 'nalu-wind-nightly+snl ~fftw~tioga~hypre~openfast %clang@12.0.1 build_type=Release ^trilinos@develop'
+    - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast %clang@13.0.1 build_type=Release ^trilinos@develop'
+    - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast+asan %clang@13.0.1 build_type=Debug ^trilinos@develop'
+    - 'nalu-wind-nightly+snl ~fftw~tioga~hypre~openfast %clang@13.0.1 build_type=Release ^trilinos@develop'
     when: '"gpu" not in hostname'
   # GPU tests run on ascicgpu
   - nalus:
-    - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast+cuda %[email protected] build_type=Release ^trilinos@develop~uvm'
+    - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast+cuda %[email protected] build_type=Release ^trilinos@develop+uvm'
     - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast+cuda %[email protected] build_type=Release ^trilinos@develop+uvm'
     - 'nalu-wind-nightly+snl ~fftw~tioga~hypre~openfast+cuda %[email protected] build_type=Release ^trilinos@develop~uvm'
     - 'nalu-wind-nightly+snl ~fftw~tioga~hypre~openfast+cuda %[email protected] build_type=Release ^trilinos@develop+uvm'