diff --git a/.gitignore b/.gitignore index e72bb35b..9b5f440b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # directories spack +stage environments golds logs diff --git a/configs/cee/compilers.yaml b/configs/cee/compilers.yaml index 840b4f5a..4da5f86d 100644 --- a/configs/cee/compilers.yaml +++ b/configs/cee/compilers.yaml @@ -1,20 +1,47 @@ compilers: +- compiler: + spec: clang@13.0.1 + paths: + cc: /projects/wind/core-spack-manager/scripts/admin-system-setup/compilers_store/linux-rhel7-x86_64/gcc-12.2.0/llvm-13.0.1-bzlmdor3cuyofkkukjhqfyts5ningcmc/bin/clang + cxx: /projects/wind/core-spack-manager/scripts/admin-system-setup/compilers_store/linux-rhel7-x86_64/gcc-12.2.0/llvm-13.0.1-bzlmdor3cuyofkkukjhqfyts5ningcmc/bin/clang++ + f77: /projects/wind/core-spack-manager/scripts/admin-system-setup/compilers_store/linux-rhel7-x86_64/gcc-12.2.0/gcc-10.3.0-lfhoe53uf43qust4kyb32b3w453dod5o/bin/gfortran + fc: /projects/wind/core-spack-manager/scripts/admin-system-setup/compilers_store/linux-rhel7-x86_64/gcc-12.2.0/gcc-10.3.0-lfhoe53uf43qust4kyb32b3w453dod5o/bin/gfortran + flags: {} + operating_system: rhel7 + target: x86_64 + modules: [] + environment: + prepend_path: + PATH: /projects/wind/core-spack-manager/scripts/admin-system-setup/compilers_store/linux-rhel7-x86_64/gcc-12.2.0/binutils-2.38-parduyka7dxmn5h53cj5pag7kk6ilisb/bin + extra_rpaths: [] - compiler: spec: gcc@10.3.0 paths: - cc: /projects/cde/v3/cee/spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5/gcc-10.3.0-bxx4ufppf7l3xlbvdinnusa4e6ykqok5/bin/gcc - cxx: /projects/cde/v3/cee/spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5/gcc-10.3.0-bxx4ufppf7l3xlbvdinnusa4e6ykqok5/bin/g++ - f77: /projects/cde/v3/cee/spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5/gcc-10.3.0-bxx4ufppf7l3xlbvdinnusa4e6ykqok5/bin/gfortran - fc: /projects/cde/v3/cee/spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5/gcc-10.3.0-bxx4ufppf7l3xlbvdinnusa4e6ykqok5/bin/gfortran + cc: /projects/wind/core-spack-manager/scripts/admin-system-setup/compilers_store/linux-rhel7-x86_64/gcc-12.2.0/gcc-10.3.0-lfhoe53uf43qust4kyb32b3w453dod5o/bin/gcc + cxx: /projects/wind/core-spack-manager/scripts/admin-system-setup/compilers_store/linux-rhel7-x86_64/gcc-12.2.0/gcc-10.3.0-lfhoe53uf43qust4kyb32b3w453dod5o/bin/g++ + f77: /projects/wind/core-spack-manager/scripts/admin-system-setup/compilers_store/linux-rhel7-x86_64/gcc-12.2.0/gcc-10.3.0-lfhoe53uf43qust4kyb32b3w453dod5o/bin/gfortran + fc: /projects/wind/core-spack-manager/scripts/admin-system-setup/compilers_store/linux-rhel7-x86_64/gcc-12.2.0/gcc-10.3.0-lfhoe53uf43qust4kyb32b3w453dod5o/bin/gfortran flags: {} operating_system: rhel7 target: x86_64 modules: [] environment: - set: - http_proxy: http://proxy.sandia.gov:80/ - https_proxy: http://proxy.sandia.gov:80/ + prepend_path: + PATH: /projects/wind/core-spack-manager/scripts/admin-system-setup/compilers_store/linux-rhel7-x86_64/gcc-12.2.0/binutils-2.38-kocidchyuf644svn4zbsqnmctze7ffvs/bin extra_rpaths: [] +- compiler: + spec: gcc@9.3.0 + paths: + cc: /projects/wind/system-spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5/gcc-9.3.0-tg7stm4b2owxiisgsu4rltm5im7pe7wz/bin/gcc + cxx: /projects/wind/system-spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5/gcc-9.3.0-tg7stm4b2owxiisgsu4rltm5im7pe7wz/bin/g++ + f77: /projects/wind/system-spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5/gcc-9.3.0-tg7stm4b2owxiisgsu4rltm5im7pe7wz/bin/gfortran + fc: /projects/wind/system-spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5/gcc-9.3.0-tg7stm4b2owxiisgsu4rltm5im7pe7wz/bin/gfortran + flags: {} + operating_system: rhel7 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] - compiler: spec: intel@2021.1.2 paths: @@ -43,59 +70,3 @@ compilers: I_MPI_CXX: /projects/cde/v3/cee/spack/opt/spack/linux-rhel7-x86_64/gcc-10.3.0/intel-oneapi-compilers-2021.1.2-lufoj3442adjwmyj2djuozq5aec3ofn2/compiler/2021.1.2/linux/bin/intel64/icpc I_MPI_FC: /projects/cde/v3/cee/spack/opt/spack/linux-rhel7-x86_64/gcc-10.3.0/intel-oneapi-compilers-2021.1.2-lufoj3442adjwmyj2djuozq5aec3ofn2/compiler/2021.1.2/linux/bin/intel64/ifort extra_rpaths: [] -- compiler: - spec: clang@12.0.1 - paths: - cc: /projects/cde/v3/cee/spack/opt/spack/linux-rhel7-x86_64/gcc-10.3.0/llvm-12.0.1-4ld5bciw4oetcntm2hgn23pklymf55ou/bin/clang - cxx: /projects/cde/v3/cee/spack/opt/spack/linux-rhel7-x86_64/gcc-10.3.0/llvm-12.0.1-4ld5bciw4oetcntm2hgn23pklymf55ou/bin/clang++ - f77: /projects/cde/v3/cee/spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5/gcc-10.3.0-bxx4ufppf7l3xlbvdinnusa4e6ykqok5/bin/gfortran - fc: /projects/cde/v3/cee/spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5/gcc-10.3.0-bxx4ufppf7l3xlbvdinnusa4e6ykqok5/bin/gfortran - flags: {} - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: gcc@9.3.0 - paths: - cc: /projects/wind/system-spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5/gcc-9.3.0-tg7stm4b2owxiisgsu4rltm5im7pe7wz/bin/gcc - cxx: /projects/wind/system-spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5/gcc-9.3.0-tg7stm4b2owxiisgsu4rltm5im7pe7wz/bin/g++ - f77: /projects/wind/system-spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5/gcc-9.3.0-tg7stm4b2owxiisgsu4rltm5im7pe7wz/bin/gfortran - fc: /projects/wind/system-spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5/gcc-9.3.0-tg7stm4b2owxiisgsu4rltm5im7pe7wz/bin/gfortran - flags: {} - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: gcc@8.3.1 - environment: {} - extra_rpaths: - - /opt/rh/devtoolset-8/root/usr/lib64 - modules: [] - flags: {} - operating_system: rhel7 - paths: - cc: /opt/rh/devtoolset-8/root/usr/bin/gcc - cxx: /opt/rh/devtoolset-8/root/usr/bin/g++ - f77: /opt/rh/devtoolset-8/root/usr/bin/gfortran - fc: /opt/rh/devtoolset-8/root/usr/bin/gfortran -- compiler: - spec: gcc@7.2.0 - environment: {} - extra_rpaths: - - /projects/sierra/linux_rh7/SDK/compilers/gcc/7.2.0-RHEL6/lib64 - - /projects/sierra/linux_rh7/SDK/compilers/gcc/7.2.0-RHEL6/lib - - /projects/sierra/linux_rh7/SDK/compilers/gcc/7.2.0-RHEL6/libexec - flags: {} - modules: [] - operating_system: rhel7 - paths: - cc: /projects/sierra/linux_rh7/SDK/compilers/gcc/7.2.0-RHEL6/bin/gcc - cxx: /projects/sierra/linux_rh7/SDK/compilers/gcc/7.2.0-RHEL6/bin/g++ - f77: /projects/sierra/linux_rh7/SDK/compilers/gcc/7.2.0-RHEL6/bin/gfortran - fc: /projects/sierra/linux_rh7/SDK/compilers/gcc/7.2.0-RHEL6/bin/gfortran - target: x86_64 - extra_rpaths: [] diff --git a/configs/cee/packages.yaml b/configs/cee/packages.yaml index 2c66bbb4..0b35b2da 100644 --- a/configs/cee/packages.yaml +++ b/configs/cee/packages.yaml @@ -1,27 +1,27 @@ packages: all: - compiler: [gcc, intel] + compiler: [gcc, clang, intel] providers: - mpi: [mpich, openmpi] + mpi: [mpich, openmpi, mvapich2] variants: +mpi build_type=Release cuda_arch=70 target: [x86_64] + boost: + version: [1.80.0] cmake: externals: - spec: cmake@3.22.2 prefix: /projects/wind/system-spack/opt/spack/linux-rhel7-x86_64/gcc-9.3.0/cmake-3.22.2-s67tgoaqaw4ky5kmpuzxdg5une7xouvh/ + - spec: cmake@3.25.0%clang@13.0.1~doc+ncurses+ownlibs~qt build_type=Release arch=linux-rhel7-x86_64 + prefix: /projects/wind/core-spack-manager/scripts/admin-system-setup/packages/.spack-env/view/clang/13.0.1/cmake/3.25.0/flff6w2l + - spec: cmake@3.25.0%gcc@10.3.0~doc+ncurses+ownlibs~qt build_type=Release arch=linux-rhel7-x86_64 + prefix: /projects/wind/core-spack-manager/scripts/admin-system-setup/packages/.spack-env/view/gcc/10.3.0/cmake/3.25.0/vfumwtcx buildable: false cuda: externals: - spec: cuda@11.2.2%gcc@9.3.0 prefix: /projects/wind/system-spack/opt/spack/linux-rhel7-x86_64/gcc-9.3.0/cuda-11.2.2-wlah7an4q7rej4uylqlimczaa6z3zlq7/ - - spec: cuda@11.2.152%gcc@10.3.0 - prefix: /projects/cde/v3/cee/spack/opt/spack/linux-rhel7-x86_64/gcc-10.3.0/cuda-11.2.2-5msjxkf5yufbjndzanbavtnyxslnxnkv - buildable: false - openmpi: - version: [4.0.5] - externals: - - spec: openmpi@4.0.5%gcc@7.2.0 - prefix: /projects/sierra/linux_rh7/SDK/mpi/openmpi/4.0.5-gcc-7.2.0-RHEL7/ + - spec: cuda@11.2.2%gcc@10.3.0~allow-unsupported-compilers~dev arch=linux-rhel7-x86_64 + prefix: /projects/wind/core-spack-manager/scripts/admin-system-setup/packages/.spack-env/view/gcc/10.3.0/cuda/11.2.2/pveyd7vw buildable: false pkg-config: buildable: false @@ -50,3 +50,21 @@ packages: - spec: mpich@3.4.2%gcc@9.3.0 prefix: /projects/wind/system-spack/opt/spack/linux-rhel7-x86_64/gcc-9.3.0/mpich-3.4.2-4h2muy6jlgfdahehxmxa4ybndfdb6gx2/ buildable: false + mvapich2: + externals: + - spec: mvapich2@2.3.7%gcc@10.3.0~alloca~cuda~debug+regcache+wrapperrpath ch3_rank_bits=32 + fabrics=mrail file_systems=auto process_managers=auto threads=multiple arch=linux-rhel7-x86_64 + prefix: /projects/wind/core-spack-manager/scripts/admin-system-setup/packages/.spack-env/view/gcc/10.3.0/mvapich2/2.3.7/lygp3cww + buildable: false + openmpi: + externals: + - spec: openmpi@4.0.2%clang@13.0.1~atomics~cuda~cxx~cxx_exceptions~gpfs~internal-hwloc~java+legacylaunchers~lustre~memchecker+romio+rsh~singularity+static+vt+wrapper-rpath + fabrics=auto schedulers=auto arch=linux-rhel7-x86_64 + prefix: /projects/wind/core-spack-manager/scripts/admin-system-setup/packages/.spack-env/view/clang/13.0.1/openmpi/4.0.2/hjdjmvpl + - spec: openmpi@4.0.2%gcc@10.3.0~atomics~cuda~cxx~cxx_exceptions~gpfs~internal-hwloc~java+legacylaunchers~lustre~memchecker+romio+rsh~singularity+static+vt+wrapper-rpath + fabrics=auto schedulers=auto arch=linux-rhel7-x86_64 + prefix: /projects/wind/core-spack-manager/scripts/admin-system-setup/packages/.spack-env/view/gcc/10.3.0/openmpi/4.0.2/zcrfmvuz + - spec: openmpi@4.1.4%gcc@10.3.0~atomics~cuda~cxx~cxx_exceptions~gpfs~internal-hwloc~java+legacylaunchers~lustre~memchecker+romio+rsh~singularity+static+vt+wrapper-rpath + fabrics=auto schedulers=auto arch=linux-rhel7-x86_64 + prefix: /projects/wind/core-spack-manager/scripts/admin-system-setup/packages/.spack-env/view/gcc/10.3.0/openmpi/4.1.4/jhvqmwf6 + buildable: false diff --git a/configs/cee/upstreams.yaml b/configs/cee/upstreams.yaml new file mode 100644 index 00000000..9b4bd275 --- /dev/null +++ b/configs/cee/upstreams.yaml @@ -0,0 +1,4 @@ +upstreams: + packages: + install_tree: /projects/wind/core-spack-manager/scripts/admin-system-setup/packages_store/ + diff --git a/env-templates/exawind_cee_tests.yaml b/env-templates/exawind_cee_tests.yaml index b88d0ea1..c9098974 100644 --- a/env-templates/exawind_cee_tests.yaml +++ b/env-templates/exawind_cee_tests.yaml @@ -3,18 +3,19 @@ spack: # CPU tests run on ascic - nalus: - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast %gcc@9.3.0 build_type=Release ^trilinos@13.4.0.2022.10.27' + - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast %gcc@10.3.0 build_type=Release ^trilinos@13.4.0.2022.10.27' - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast %intel@2021.1.2 build_type=Release ^trilinos@13.4.0.2022.10.27' - - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast %clang@12.0.1 build_type=Release ^trilinos@13.4.0.2022.10.27' - - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast+asan %clang@12.0.1 build_type=Debug ^trilinos@13.4.0.2022.10.27' + - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast %clang@13.0.1 build_type=Release ^trilinos@13.4.0.2022.10.27' + - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast+asan %clang@13.0.1 build_type=Debug ^trilinos@13.4.0.2022.10.27' - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast %gcc@9.3.0 build_type=Release ^trilinos@develop' - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast %intel@2021.1.2 build_type=Release ^trilinos@develop' - - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast %clang@12.0.1 build_type=Release ^trilinos@develop' - - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast+asan %clang@12.0.1 build_type=Debug ^trilinos@develop' - - 'nalu-wind-nightly+snl ~fftw~tioga~hypre~openfast %clang@12.0.1 build_type=Release ^trilinos@develop' + - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast %clang@13.0.1 build_type=Release ^trilinos@develop' + - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast+asan %clang@13.0.1 build_type=Debug ^trilinos@develop' + - 'nalu-wind-nightly+snl ~fftw~tioga~hypre~openfast %clang@13.0.1 build_type=Release ^trilinos@develop' when: '"gpu" not in hostname' # GPU tests run on ascicgpu - nalus: - - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast+cuda %gcc@10.3.0 build_type=Release ^trilinos@develop~uvm' + - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast+cuda %gcc@10.3.0 build_type=Release ^trilinos@develop+uvm' - 'nalu-wind-nightly+snl +fftw+tioga+hypre+openfast+cuda %gcc@9.3.0 build_type=Release ^trilinos@develop+uvm' - 'nalu-wind-nightly+snl ~fftw~tioga~hypre~openfast+cuda %gcc@9.3.0 build_type=Release ^trilinos@develop~uvm' - 'nalu-wind-nightly+snl ~fftw~tioga~hypre~openfast+cuda %gcc@9.3.0 build_type=Release ^trilinos@develop+uvm'