From 22e6424c60c7dd80706b234d9517b159e2acd53b Mon Sep 17 00:00:00 2001 From: eap Date: Tue, 17 Dec 2024 15:44:09 -0700 Subject: [PATCH 01/17] add changes to intel container --- .../containers/docker-ubuntu-intel-impi.yaml | 192 ++++++++++++++---- 1 file changed, 151 insertions(+), 41 deletions(-) diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index 6dd5ad15d..672004eb3 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -9,21 +9,47 @@ spack: compilers: - compiler: - spec: intel@2021.6.0 + spec: intel@2021.10.0 paths: cc: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/icc cxx: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/icpc f77: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/ifort fc: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/ifort flags: {} - operating_system: ubuntu22.04 + operating_system: ubuntu24.04 modules: [] environment: prepend_path: PATH: /usr/bin - LD_LIBRARY_PATH: /usr/lib64:/opt/intel/oneapi/compiler/2022.1.0/linux/compiler/lib/intel64_lin + LD_LIBRARY_PATH: /opt/intel/oneapi/compiler/2023.2.3/linux/compiler/lib/intel64_lin CPATH: /usr/include extra_rpaths: [] + - compiler: + spec: gcc@=11.4.0 + paths: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + flags: {} + operating_system: ubuntu24.04 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + - compiler: + spec: oneapi@=2023.2.3 + paths: + cc: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/icx + cxx: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/icpx + f77: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/ifx + fc: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/ifx + flags: {} + operating_system: ubuntu24.04 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] # Basic package config from configs/common/packages.yaml # Additional package config for container @@ -32,29 +58,50 @@ spack: require: '%intel' target: [core2] providers: - mpi: [intel-oneapi-mpi@2021.6.0] - compiler: [intel@2021.6.0] - intel: - buildable: false + mpi: [intel-oneapi-mpi@2021.10.0] + compiler: [intel@2021.10.0, gcc@11.4.0] + autoconf: externals: - - spec: intel@2021.6.0 - prefix: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64 - intel-oneapi-mpi: - buildable: false + - spec: autoconf@2.71 + prefix: /usr + automake: externals: - - spec: intel-oneapi-mpi@2021.6.0 - prefix: /opt/intel/oneapi - # Comment out for now so that fftw-api uses fftw and not mkl - #intel-oneapi-mkl: - # buildable: false - # externals: - # - spec: intel-oneapi-mkl@2022.1.0 - # prefix: /opt/intel/oneapi + - spec: automake@1.16.5 + prefix: /usr + binutils: + externals: + - spec: binutils@2.38 + prefix: /usr + coreutils: + externals: + - spec: coreutils@8.32 + prefix: /usr diffutils: buildable: false externals: - spec: diffutils@3.8 prefix: /usr + ectrans: + require:: + - '@1.2.0 ~mkl +fftw' + findutils: + externals: + - spec: findutils@4.8.0 + prefix: /usr + # DO Not submit - figure out what is going on - MLK is not installed so it must be built but might cause errors + #intel-oneapi-mkl: + # buildable: false + #externals: + #- spec: intel-oneapi-mkl@2022.1.0 + # prefix: /opt/intel/oneapi + flex: + externals: + - spec: flex@2.6.4+lex + prefix: /usr + gawk: + externals: + - spec: gawk@5.1.0 + prefix: /usr git: buildable: false externals: @@ -65,36 +112,89 @@ spack: externals: - spec: git-lfs@3.0.2 prefix: /usr - mysql: - buildable: false + gmake: externals: - - spec: mysql@8.0.39 + - spec: gmake@4.3 prefix: /usr - qt: + groff: + externals: + - spec: groff@1.22.4 + prefix: /usr + gsibec: + require:: + - '@1.2.1 ~mkl' + intel: buildable: false externals: - - spec: qt@5.15.3 + - spec: intel@2021.10.0 + prefix: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64 + intel-oneapi-mpi: + buildable: false + externals: + - spec: intel-oneapi-mpi@2021.10.0 + prefix: /opt/intel/oneapi + libtool: + externals: + - spec: libtool@2.4.6 prefix: /usr - version: [5.15.3] - wget: + mpi: + buildable: False + m4: + externals: + - spec: m4@1.4.18 + prefix: /usr + lcms: + externals: + - spec: lcms@2.13.1 + prefix: /usr + mysql: buildable: false externals: - - spec: wget@1.21.2 + - spec: mysql@8.0.37 prefix: /usr - version: [1.21.2] - gettext: + perl: externals: - - spec: gettext@0.21 + - spec: perl@5.34.0~cpanm+opcode+open+shared+threads + prefix: /usr + pkg-config: + externals: + - spec: pkg-config@0.29.2 prefix: /usr py-pynacl: buildable: false externals: - spec: py-pynacl@1.5.0 prefix: /usr + py-numpy: + require:: + - '@:1.23 ^openblas' # Turn off crypt, because libxcrypt doesn't # build with Intel. python: variants: ~crypt + qt: + buildable: false + externals: + - spec: qt@5.15.3 + prefix: /usr + version: [5.15.3] + sed: + externals: + - spec: sed@4.8 + prefix: /usr + tar: + externals: + - spec: tar@1.34 + prefix: /usr + wget: + buildable: false + externals: + - spec: wget@1.21.2 + prefix: /usr + gettext: + externals: + - spec: gettext@0.21 + prefix: /usr specs: [] @@ -116,7 +216,7 @@ spack: # Sets the base images for the stages where Spack builds the # software or where the software gets installed after being built.. images: - os: ubuntu:22.04 + os: ubuntu:24.04 spack: url: https://github.com/jcsda/spack ref: spack-stack-dev @@ -130,39 +230,43 @@ spack: build: - autopoint - bc - - cpp - - g++ - - gcc + - cpp-11 + - g++-11 + - gcc-11 - gettext - - gfortran + - gfortran-11 - git - git-lfs - gpg + - libgomp1 - make - mysql-server - qtbase5-dev - qt5-qmake - libqt5svg5-dev - qt5dxcb-plugin + - libcurl4-openssl-dev - wget final: - autopoint - bc - - cpp - - g++ - - gcc + - cpp-11 + - g++-11 + - gcc-11 - gettext - - gfortran + - gfortran-11 - git - git-lfs - gpg + - libgomp1 - make - mysql-server - qtbase5-dev - qt5-qmake - libqt5svg5-dev - qt5dxcb-plugin + - libcurl4-openssl-dev - wget # Implicitly included in build step - build-essential @@ -192,7 +296,10 @@ spack: wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \ echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \ apt update && \ - apt install intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-2022.1.0 intel-oneapi-compiler-fortran-2022.1.0 intel-oneapi-mkl-devel-2022.1.0 intel-oneapi-mpi-devel-2021.6.0 -y && \ + apt install intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-2023.2.3 intel-oneapi-compiler-fortran-2023.2.3 intel-oneapi-mkl-devel-2023.2.0 intel-oneapi-mpi-devel-2021.10.0 -y && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \ + update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-11 100 && \ rm -rf /var/lib/apt/lists/* pre_final: | # Set environment variables for installing tzdata @@ -212,7 +319,10 @@ spack: wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \ echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \ apt update && \ - apt install intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-2022.1.0 intel-oneapi-compiler-fortran-2022.1.0 intel-oneapi-mkl-devel-2022.1.0 intel-oneapi-mpi-devel-2021.6.0 -y && \ + apt install intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-2023.2.3 intel-oneapi-compiler-fortran-2023.2.3 intel-oneapi-mkl-devel-2023.2.0 intel-oneapi-mpi-devel-2021.10.0 -y && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \ + update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-11 100 && \ rm -rf /var/lib/apt/lists/* && \ /opt/views/view/bin/pip3 install pynacl # Copy spack find output from builder From 89665b69c1fee4fcd01590eecc014389bf7d9c8c Mon Sep 17 00:00:00 2001 From: eap Date: Tue, 17 Dec 2024 23:02:12 -0700 Subject: [PATCH 02/17] Several jedi-ci spec changes --- configs/containers/specs/jedi-ci.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/configs/containers/specs/jedi-ci.yaml b/configs/containers/specs/jedi-ci.yaml index 6f93c5432..d5d4c423d 100644 --- a/configs/containers/specs/jedi-ci.yaml +++ b/configs/containers/specs/jedi-ci.yaml @@ -37,11 +37,11 @@ py-eccodes@1.5.0, py-f90nml@1.4.3, py-gitpython@3.1.40, - py-h5py@3.11.0, + py-h5py@3.12.1, py-numpy@1.23.5, py-pandas@1.5.3, py-pip, - py-pyyaml@6.0, + py-pyyaml@6.0.2, py-scipy@1.12.0, py-shapely@1.8.0, py-xarray@2023.7.0, @@ -51,11 +51,11 @@ nco@5.1.6, esmf@8.6.1, mapl@2.46.3, - zlib-ng@2.1.5, + zlib-ng@2.2.1, zstd@1.5.2, odc@1.5.2, shumlib@macos_clang_linux_intel_port, - awscli-v2@2.13.22, + awscli-v2@2.15.53, py-globus-cli@3.16.0, py-fortranformat@2.0.0, # Added for new CI system 2024/04/30 From e2b9ffdd69ed76480e877fd965f8133a00e16491 Mon Sep 17 00:00:00 2001 From: Evan Parker Date: Wed, 18 Dec 2024 22:56:16 +0000 Subject: [PATCH 03/17] several updates --- configs/containers/docker-ubuntu-intel-impi.yaml | 16 ++++------------ configs/containers/specs/jedi-ci.yaml | 6 +++++- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index 672004eb3..eff6e99a0 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -11,10 +11,10 @@ spack: - compiler: spec: intel@2021.10.0 paths: - cc: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/icc - cxx: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/icpc - f77: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/ifort - fc: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/ifort + cc: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/icc + cxx: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/icpc + f77: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/ifort + fc: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/ifort flags: {} operating_system: ubuntu24.04 modules: [] @@ -81,9 +81,6 @@ spack: externals: - spec: diffutils@3.8 prefix: /usr - ectrans: - require:: - - '@1.2.0 ~mkl +fftw' findutils: externals: - spec: findutils@4.8.0 @@ -123,11 +120,6 @@ spack: gsibec: require:: - '@1.2.1 ~mkl' - intel: - buildable: false - externals: - - spec: intel@2021.10.0 - prefix: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64 intel-oneapi-mpi: buildable: false externals: diff --git a/configs/containers/specs/jedi-ci.yaml b/configs/containers/specs/jedi-ci.yaml index d5d4c423d..4fb1600e9 100644 --- a/configs/containers/specs/jedi-ci.yaml +++ b/configs/containers/specs/jedi-ci.yaml @@ -15,7 +15,7 @@ eckit@1.28.3, ecmwf-atlas@0.40.0 +fckit +trans +tesselation +fftw, fiat@1.4.1, - ectrans@1.5.0 +fftw, + ectrans@1.5.0, eigen@3.4.0, fckit@0.13.2, fms@2024.02, @@ -32,8 +32,12 @@ netcdf-c@4.9.2, netcdf-cxx4@4.3.1, netcdf-fortran@4.6.1, + openssl@3.4.0, parallelio@2.6.2 +pnetcdf, parallel-netcdf@1.12.3, + py-boto3@1.34.44, + py-cartopy@0.24.1, + py-cryptography@42.0.8, py-eccodes@1.5.0, py-f90nml@1.4.3, py-gitpython@3.1.40, From 90eb8629f271cec4b1ec451b66acd05ad20af2a0 Mon Sep 17 00:00:00 2001 From: eap Date: Wed, 18 Dec 2024 19:48:04 -0700 Subject: [PATCH 04/17] add gcc require for cross compiler packages --- configs/containers/docker-ubuntu-intel-impi.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index eff6e99a0..766101d40 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -72,6 +72,9 @@ spack: externals: - spec: binutils@2.38 prefix: /usr + boost: + require: + - '%gcc' coreutils: externals: - spec: coreutils@8.32 @@ -81,6 +84,9 @@ spack: externals: - spec: diffutils@3.8 prefix: /usr + ecflow: + require: + - '%gcc' findutils: externals: - spec: findutils@4.8.0 @@ -144,6 +150,9 @@ spack: externals: - spec: mysql@8.0.37 prefix: /usr + pixman: + require: + - '%gcc' perl: externals: - spec: perl@5.34.0~cpanm+opcode+open+shared+threads From 1e0906e27a6f3e5e9aa7d79ff63a0dd8fe3cf156 Mon Sep 17 00:00:00 2001 From: eap Date: Wed, 18 Dec 2024 20:27:43 -0700 Subject: [PATCH 05/17] unpin fixes from #1429 --- configs/containers/specs/jedi-ci.yaml | 32 +++++++++++++-------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/configs/containers/specs/jedi-ci.yaml b/configs/containers/specs/jedi-ci.yaml index 4fb1600e9..7fb01480b 100644 --- a/configs/containers/specs/jedi-ci.yaml +++ b/configs/containers/specs/jedi-ci.yaml @@ -38,25 +38,25 @@ py-boto3@1.34.44, py-cartopy@0.24.1, py-cryptography@42.0.8, - py-eccodes@1.5.0, - py-f90nml@1.4.3, - py-gitpython@3.1.40, - py-h5py@3.12.1, - py-numpy@1.23.5, - py-pandas@1.5.3, + py-eccodes, + py-f90nml, + py-gitpython, + py-h5py, + py-numpy, + py-pandas, py-pip, - py-pyyaml@6.0.2, - py-scipy@1.12.0, - py-shapely@1.8.0, - py-xarray@2023.7.0, + py-pyyaml, + py-scipy, + py-shapely, + py-xarray, sp@2.5.0, udunits@2.2.28, w3emc@2.10.0, - nco@5.1.6, + nco, esmf@8.6.1, mapl@2.46.3, - zlib-ng@2.2.1, - zstd@1.5.2, + zlib-ng, + zstd, odc@1.5.2, shumlib@macos_clang_linux_intel_port, awscli-v2@2.15.53, @@ -68,6 +68,6 @@ jq@1.6 ] # Notes: # 1. Don't build CRTM by default so that it gets built in the JEDI bundles - # 2. Comment out for now until build problems are solved - # https://github.com/jcsda/spack-stack/issues/522 - # py-mysql-connector-python@8.0.32 + # 2. Avoid pinning packages to specific versions/variants unless absolutely + # needed, because config/common/packages.yaml does that already (removal + # of pinned versions in the above list is work in progress, DH 2024/12/18) \ No newline at end of file From a51c344c654364e11bd3dce44258b52402af3dd2 Mon Sep 17 00:00:00 2001 From: eap Date: Wed, 18 Dec 2024 23:31:13 -0700 Subject: [PATCH 06/17] add build essentials --- configs/containers/docker-ubuntu-intel-impi.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index 766101d40..e08c8ebaf 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -229,9 +229,13 @@ spack: ## Additional system packages that are needed at runtime os_packages: build: + - autoconf + - automake - autopoint - bc + - build-essential - cpp-11 + - cmake - g++-11 - gcc-11 - gettext From 9081611177c042719c0c8d7b8764fc77488e5bac Mon Sep 17 00:00:00 2001 From: eap Date: Thu, 19 Dec 2024 15:14:10 -0700 Subject: [PATCH 07/17] Many fixes --- .../containers/docker-ubuntu-intel-impi.yaml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index e08c8ebaf..118e650d4 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -4,7 +4,7 @@ spack: config: checksum: false - build_jobs: 2 + build_jobs: 5 connect_timeout: 60 compilers: @@ -233,9 +233,9 @@ spack: - automake - autopoint - bc - - build-essential - cpp-11 - cmake + - flex - g++-11 - gcc-11 - gettext @@ -244,8 +244,8 @@ spack: - git-lfs - gpg - libgomp1 - - make - mysql-server + - pkg-config - qtbase5-dev - qt5-qmake - libqt5svg5-dev @@ -267,6 +267,7 @@ spack: - libgomp1 - make - mysql-server + - pkg-config - qtbase5-dev - qt5-qmake - libqt5svg5-dev @@ -305,6 +306,7 @@ spack: update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 && \ update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \ update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-11 100 && \ + rm /usr/bin/cpp && ln -s /usr/bin/cpp-11 /usr/bin/cpp && \ rm -rf /var/lib/apt/lists/* pre_final: | # Set environment variables for installing tzdata @@ -315,7 +317,8 @@ spack: ENV FC=ifort build: | # Put output of spack find into a file - RUN cd /opt/spack-environment && \ + RUN source /opt/intel/oneapi/setvars.sh && \ + cd /opt/spack-environment && \ spack env activate -d . && \ spack find 2>&1 | tee /root/spack_find.out final: | @@ -328,6 +331,7 @@ spack: update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 && \ update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \ update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-11 100 && \ + rm /usr/bin/cpp && ln -s /usr/bin/cpp-11 /usr/bin/cpp && \ rm -rf /var/lib/apt/lists/* && \ /opt/views/view/bin/pip3 install pynacl # Copy spack find output from builder @@ -339,8 +343,7 @@ spack: echo "export CC=icc" >> /home/nonroot/.bashrc && \ echo "export CXX=icpc" >> /home/nonroot/.bashrc && \ echo "export FC=ifort" >> /home/nonroot/.bashrc && \ - echo "source /opt/intel/oneapi/compiler/latest/env/vars.sh" >> /home/nonroot/.bashrc && \ - echo "source /opt/intel/oneapi/mpi/latest/env/vars.sh" >> /home/nonroot/.bashrc && \ + echo "source /opt/intel/oneapi/setvars.sh" > /etc/profile.d/02-intel-vars.sh && \ printf "[credential]\n helper = cache --timeout=7200\n" >> /home/nonroot/.gitconfig && \ chown -R nonroot:nonroot /home/nonroot/.gitconfig # Replicate settings for root user @@ -350,8 +353,6 @@ spack: echo "export CXX=icpc" >> /root/.bashrc && \ echo "export FC=ifort" >> /root/.bashrc && \ echo "export jedi_cmake_ROOT=$(ls -d /opt/software/linux-ubuntu*/intel-*/jedi-cmake-*)" >> /root/.bashrc && \ - echo "source /opt/intel/oneapi/compiler/latest/env/vars.sh" >> /root/.bashrc && \ - echo "source /opt/intel/oneapi/mpi/latest/env/vars.sh" >> /root/.bashrc && \ printf "[credential]\n helper = cache --timeout=7200\n" >> /root/.gitconfig # Labels for the image From 578375bc9b4602cee66f4ef44ed03012be1022d3 Mon Sep 17 00:00:00 2001 From: eap Date: Fri, 20 Dec 2024 12:54:20 -0700 Subject: [PATCH 08/17] llvm fixes --- .../containers/docker-ubuntu-intel-impi.yaml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index 118e650d4..5ed76d910 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -129,7 +129,8 @@ spack: intel-oneapi-mpi: buildable: false externals: - - spec: intel-oneapi-mpi@2021.10.0 + # This spec is required in order set paths allowing execution of mpiicx. + - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 +classic-names prefix: /opt/intel/oneapi libtool: externals: @@ -145,6 +146,11 @@ spack: externals: - spec: lcms@2.13.1 prefix: /usr + llvm: + buildable: false + externals: + - spec: llvm@14.0.6 + prefix: /usr mysql: buildable: false externals: @@ -244,6 +250,7 @@ spack: - git-lfs - gpg - libgomp1 + - llvm-14 - mysql-server - pkg-config - qtbase5-dev @@ -265,6 +272,7 @@ spack: - git-lfs - gpg - libgomp1 + - llvm-14 - make - mysql-server - pkg-config @@ -307,6 +315,9 @@ spack: update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \ update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-11 100 && \ rm /usr/bin/cpp && ln -s /usr/bin/cpp-11 /usr/bin/cpp && \ + # Needed by py-llvmlite + update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 100 && \ + echo "source /opt/intel/oneapi/setvars.sh" > /etc/profile.d/02-intel-vars.sh && \ rm -rf /var/lib/apt/lists/* pre_final: | # Set environment variables for installing tzdata @@ -317,8 +328,7 @@ spack: ENV FC=ifort build: | # Put output of spack find into a file - RUN source /opt/intel/oneapi/setvars.sh && \ - cd /opt/spack-environment && \ + RUN cd /opt/spack-environment && \ spack env activate -d . && \ spack find 2>&1 | tee /root/spack_find.out final: | @@ -332,6 +342,8 @@ spack: update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \ update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-11 100 && \ rm /usr/bin/cpp && ln -s /usr/bin/cpp-11 /usr/bin/cpp && \ + # Needed by py-llvmlite + update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 100 && \ rm -rf /var/lib/apt/lists/* && \ /opt/views/view/bin/pip3 install pynacl # Copy spack find output from builder From 90a4c263f125ab56142e1116822d55249dc3dcb4 Mon Sep 17 00:00:00 2001 From: eap Date: Fri, 20 Dec 2024 22:30:56 -0700 Subject: [PATCH 09/17] add tbb and mkl --- configs/containers/docker-ubuntu-intel-impi.yaml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index 5ed76d910..856e3a585 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -126,12 +126,22 @@ spack: gsibec: require:: - '@1.2.1 ~mkl' + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mpi@2023.2.0 + prefix: /opt/intel/oneapi intel-oneapi-mpi: buildable: false externals: # This spec is required in order set paths allowing execution of mpiicx. - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 +classic-names prefix: /opt/intel/oneapi + intel-oneapi-tbb: + buildable: false + externals: + - spec: intel-oneapi-mpi@2021.10.0 + prefix: /opt/intel/oneapi libtool: externals: - spec: libtool@2.4.6 @@ -317,7 +327,6 @@ spack: rm /usr/bin/cpp && ln -s /usr/bin/cpp-11 /usr/bin/cpp && \ # Needed by py-llvmlite update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 100 && \ - echo "source /opt/intel/oneapi/setvars.sh" > /etc/profile.d/02-intel-vars.sh && \ rm -rf /var/lib/apt/lists/* pre_final: | # Set environment variables for installing tzdata @@ -355,7 +364,7 @@ spack: echo "export CC=icc" >> /home/nonroot/.bashrc && \ echo "export CXX=icpc" >> /home/nonroot/.bashrc && \ echo "export FC=ifort" >> /home/nonroot/.bashrc && \ - echo "source /opt/intel/oneapi/setvars.sh" > /etc/profile.d/02-intel-vars.sh && \ + echo "source /opt/intel/oneapi/setvars.sh" >> /etc/bash.bashrc && \ printf "[credential]\n helper = cache --timeout=7200\n" >> /home/nonroot/.gitconfig && \ chown -R nonroot:nonroot /home/nonroot/.gitconfig # Replicate settings for root user From 2d3dea5f15775f4485b95127808d04d61bd08c3a Mon Sep 17 00:00:00 2001 From: eap Date: Fri, 20 Dec 2024 22:38:41 -0700 Subject: [PATCH 10/17] gcc updates --- .../containers/docker-ubuntu-gcc-openmpi.yaml | 43 ++++++++----------- .../containers/docker-ubuntu-intel-impi.yaml | 2 +- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/configs/containers/docker-ubuntu-gcc-openmpi.yaml b/configs/containers/docker-ubuntu-gcc-openmpi.yaml index 43b5b2fb8..197583b63 100644 --- a/configs/containers/docker-ubuntu-gcc-openmpi.yaml +++ b/configs/containers/docker-ubuntu-gcc-openmpi.yaml @@ -4,19 +4,19 @@ spack: config: checksum: false - build_jobs: 2 + build_jobs: 5 connect_timeout: 60 compilers: - compiler: - spec: gcc@12.3.0 + spec: gcc@13.3.0 paths: - cc: /usr/bin/gcc-12 - cxx: /usr/bin/g++-12 - f77: /usr/bin/gfortran-12 - fc: /usr/bin/gfortran-12 + cc: /usr/bin/gcc-13 + cxx: /usr/bin/g++-13 + f77: /usr/bin/gfortran-13 + fc: /usr/bin/gfortran-13 flags: {} - operating_system: ubuntu22.04 + operating_system: ubuntu24.04 modules: [] environment: {} extra_rpaths: [] @@ -29,16 +29,16 @@ spack: target: [x86_64] providers: mpi: [openmpi@5.0.5] - compiler: [gcc@12.3.0] + compiler: [gcc@13.3.0] gcc: buildable: false externals: - - spec: gcc@12.3.0 + - spec: gcc@13.3.0 prefix: /usr gcc-runtime: buildable: false externals: - - spec: gcc-runtime@12.3.0 + - spec: gcc-runtime@13.3.0 prefix: /usr diffutils: buildable: false @@ -58,7 +58,7 @@ spack: mysql: buildable: false externals: - - spec: mysql@8.0.39 + - spec: mysql@8.0.40 prefix: /usr qt: buildable: false @@ -71,7 +71,6 @@ spack: externals: - spec: wget@1.21.2 prefix: /usr - version: [1.21.2] specs: [] @@ -106,10 +105,10 @@ spack: os_packages: build: - bc - - cpp-12 - - g++-12 - - gcc-12 - - gfortran-12 + - cpp-13 + - g++-13 + - gcc-13 + - gfortran-13 - git - git-lfs - make @@ -122,10 +121,10 @@ spack: final: - bc - - cpp-12 - - g++-12 - - gcc-12 - - gfortran-12 + - cpp-13 + - g++-13 + - gcc-13 + - gfortran-13 - git - git-lfs - make @@ -154,10 +153,6 @@ spack: # Register spack-stack extension repo RUN spack repo add $SPACK_ROOT/var/spack/repos/spack-stack --scope defaults # Set the GNU 12.x toolchain as the default compilers. - RUN ln -svf /usr/bin/gfortran-12 /usr/bin/gfortran && \ - ln -svf /usr/bin/cpp-12 /usr/bin/cpp && \ - ln -svf /usr/bin/gcc-12 /usr/bin/gcc && \ - ln -svf /usr/bin/g++-12 /usr/bin/g++ pre_final: | # Set environment variables for installing tzdata ENV DEBIAN_FRONTEND=noninteractive diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index 856e3a585..60e7bfe66 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -164,7 +164,7 @@ spack: mysql: buildable: false externals: - - spec: mysql@8.0.37 + - spec: mysql@8.0.40 prefix: /usr pixman: require: From bdff3ac3b6c33db25a1df0c8de36852b2851b795 Mon Sep 17 00:00:00 2001 From: eap Date: Fri, 20 Dec 2024 22:47:49 -0700 Subject: [PATCH 11/17] Fix ubuntu version for gcc --- configs/containers/docker-ubuntu-gcc-openmpi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/containers/docker-ubuntu-gcc-openmpi.yaml b/configs/containers/docker-ubuntu-gcc-openmpi.yaml index 197583b63..af25f97e8 100644 --- a/configs/containers/docker-ubuntu-gcc-openmpi.yaml +++ b/configs/containers/docker-ubuntu-gcc-openmpi.yaml @@ -92,7 +92,7 @@ spack: # Sets the base images for the stages where Spack builds the # software or where the software gets installed after being built.. images: - os: ubuntu:22.04 + os: ubuntu:24.04 spack: url: https://github.com/jcsda/spack ref: spack-stack-dev From 21e8d58552fb0763f6d1b581c010a4aa0bafe5dc Mon Sep 17 00:00:00 2001 From: eap Date: Sat, 21 Dec 2024 15:48:30 -0700 Subject: [PATCH 12/17] fixes for intel gcc --- .../containers/docker-ubuntu-gcc-openmpi.yaml | 17 +++++++++++------ .../containers/docker-ubuntu-intel-impi.yaml | 4 ++-- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/configs/containers/docker-ubuntu-gcc-openmpi.yaml b/configs/containers/docker-ubuntu-gcc-openmpi.yaml index af25f97e8..c2a635431 100644 --- a/configs/containers/docker-ubuntu-gcc-openmpi.yaml +++ b/configs/containers/docker-ubuntu-gcc-openmpi.yaml @@ -55,6 +55,11 @@ spack: externals: - spec: git-lfs@3.0.2 prefix: /usr + llvm: + buildable: false + externals: + - spec: llvm@14.0.6 + prefix: /usr mysql: buildable: false externals: @@ -111,6 +116,7 @@ spack: - gfortran-13 - git - git-lfs + - llvm-14 - make - mysql-server - qtbase5-dev @@ -127,6 +133,7 @@ spack: - gfortran-13 - git - git-lfs + - llvm-14 - make - mysql-server - qtbase5-dev @@ -152,7 +159,8 @@ spack: COPY spack-ext-SPACK_STACK_HASH/repos/spack-stack $SPACK_ROOT/var/spack/repos/spack-stack # Register spack-stack extension repo RUN spack repo add $SPACK_ROOT/var/spack/repos/spack-stack --scope defaults - # Set the GNU 12.x toolchain as the default compilers. + # Needed by py-llvmlite + update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 100 pre_final: | # Set environment variables for installing tzdata ENV DEBIAN_FRONTEND=noninteractive @@ -168,11 +176,6 @@ spack: final: | # Copy spack find output from builder COPY --from=builder /root/spack_find.out /root/spack_find.out - # Set the GNU 12.x toolchain as the default compilers. - RUN ln -svf /usr/bin/gfortran-12 /usr/bin/gfortran && \ - ln -svf /usr/bin/cpp-12 /usr/bin/cpp && \ - ln -svf /usr/bin/gcc-12 /usr/bin/gcc && \ - ln -svf /usr/bin/g++-12 /usr/bin/g++ # Make a non-root user:nonroot / group:nonroot for running MPI RUN useradd -U -k /etc/skel -s /bin/bash -d /home/nonroot -m nonroot --uid 43891 && \ echo "ulimit -s unlimited" >> /home/nonroot/.bashrc && \ @@ -180,6 +183,8 @@ spack: echo "export CC=gcc" >> /home/nonroot/.bashrc && \ echo "export CXX=g++" >> /home/nonroot/.bashrc && \ echo "export FC=gfortran" >> /home/nonroot/.bashrc && \ + # Needed by py-llvmlite + update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 100 && \ printf "[credential]\n helper = cache --timeout=7200\n" >> /home/nonroot/.gitconfig && \ mkdir /home/nonroot/.pmix && \ echo "rmaps_default_mapping_policy=:oversubscribe" >> /home/nonroot/.pmix/mca-params.conf && \ diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index 60e7bfe66..f552987e0 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -129,7 +129,7 @@ spack: intel-oneapi-mkl: buildable: false externals: - - spec: intel-oneapi-mpi@2023.2.0 + - spec: intel-oneapi-mkl@2023.2.0 prefix: /opt/intel/oneapi intel-oneapi-mpi: buildable: false @@ -140,7 +140,7 @@ spack: intel-oneapi-tbb: buildable: false externals: - - spec: intel-oneapi-mpi@2021.10.0 + - spec: intel-oneapi-tbb@2021.10.0 prefix: /opt/intel/oneapi libtool: externals: From 14495cb51c5d38943f2cc71718042cba26dda88f Mon Sep 17 00:00:00 2001 From: eap Date: Sat, 21 Dec 2024 15:51:34 -0700 Subject: [PATCH 13/17] fix llvm alternative --- configs/containers/docker-ubuntu-gcc-openmpi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/containers/docker-ubuntu-gcc-openmpi.yaml b/configs/containers/docker-ubuntu-gcc-openmpi.yaml index c2a635431..7469d904d 100644 --- a/configs/containers/docker-ubuntu-gcc-openmpi.yaml +++ b/configs/containers/docker-ubuntu-gcc-openmpi.yaml @@ -158,7 +158,7 @@ spack: # Copy the spack-stack extension repo into the spack code COPY spack-ext-SPACK_STACK_HASH/repos/spack-stack $SPACK_ROOT/var/spack/repos/spack-stack # Register spack-stack extension repo - RUN spack repo add $SPACK_ROOT/var/spack/repos/spack-stack --scope defaults + RUN spack repo add $SPACK_ROOT/var/spack/repos/spack-stack --scope defaults && \ # Needed by py-llvmlite update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 100 pre_final: | From f982b7f1b6c411fd2e902c268f2c93154e5b517c Mon Sep 17 00:00:00 2001 From: eap Date: Sat, 21 Dec 2024 21:11:30 -0700 Subject: [PATCH 14/17] remove unnecessary requirements --- configs/containers/docker-ubuntu-intel-impi.yaml | 4 ++-- configs/containers/specs/jedi-ci.yaml | 7 +------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index f2843e266..15fa3703d 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -187,8 +187,8 @@ spack: - spec: py-pynacl@1.5.0 prefix: /usr py-numpy: - require:: - - '@:1.23 ^openblas' + require: + - '^intel-oneapi-mkl' # Turn off crypt, because libxcrypt doesn't # build with Intel. python: diff --git a/configs/containers/specs/jedi-ci.yaml b/configs/containers/specs/jedi-ci.yaml index fdf5b8708..dcf3ce744 100644 --- a/configs/containers/specs/jedi-ci.yaml +++ b/configs/containers/specs/jedi-ci.yaml @@ -32,17 +32,12 @@ netcdf-c@4.9.2, netcdf-cxx4@4.3.1, netcdf-fortran@4.6.1, - openssl@3.4.0, parallelio@2.6.2 +pnetcdf, parallel-netcdf@1.12.3, - py-boto3@1.34.44, - py-cartopy@0.24.1, - py-cryptography@42.0.8, py-eccodes, py-f90nml, py-gitpython, py-h5py, - py-numpy, py-pandas, py-pip, py-pyyaml, @@ -58,7 +53,7 @@ zstd, odc@1.5.2, shumlib@macos_clang_linux_intel_port, - awscli-v2@2.15.53, + awscli-v2, py-globus-cli@3.16.0, py-fortranformat@2.0.0, # Added for new CI system 2024/04/30 From 941bd4ae24a254f0e374dd0259f5f40eda319faf Mon Sep 17 00:00:00 2001 From: eap Date: Sat, 21 Dec 2024 21:13:26 -0700 Subject: [PATCH 15/17] remove old oneapi-mkl with 'DO NOT SUBMIT' --- configs/containers/docker-ubuntu-intel-impi.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index 15fa3703d..a6206e5bc 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -91,12 +91,6 @@ spack: externals: - spec: findutils@4.8.0 prefix: /usr - # DO Not submit - figure out what is going on - MLK is not installed so it must be built but might cause errors - #intel-oneapi-mkl: - # buildable: false - #externals: - #- spec: intel-oneapi-mkl@2022.1.0 - # prefix: /opt/intel/oneapi flex: externals: - spec: flex@2.6.4+lex From b24b879ad6bcfcd74567e52a01a87d877704a98c Mon Sep 17 00:00:00 2001 From: eap Date: Sat, 21 Dec 2024 21:17:08 -0700 Subject: [PATCH 16/17] re-order 'all' packages --- configs/containers/docker-ubuntu-intel-impi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index a6206e5bc..8f2bd3c8a 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -57,9 +57,9 @@ spack: all: require: '%intel' target: [core2] + compiler: [intel@2021.10.0, gcc@11.4.0] providers: mpi: [intel-oneapi-mpi@2021.10.0] - compiler: [intel@2021.10.0, gcc@11.4.0] autoconf: externals: - spec: autoconf@2.71 From 0d1dfcde609534ef2b32a5c3a992a5c6074e23af Mon Sep 17 00:00:00 2001 From: eap Date: Mon, 23 Dec 2024 10:12:48 -0700 Subject: [PATCH 17/17] fix build jobs for intel and gcc --- configs/containers/docker-ubuntu-gcc-openmpi.yaml | 2 +- configs/containers/docker-ubuntu-intel-impi.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/containers/docker-ubuntu-gcc-openmpi.yaml b/configs/containers/docker-ubuntu-gcc-openmpi.yaml index 7469d904d..9a1ed0eb9 100644 --- a/configs/containers/docker-ubuntu-gcc-openmpi.yaml +++ b/configs/containers/docker-ubuntu-gcc-openmpi.yaml @@ -4,7 +4,7 @@ spack: config: checksum: false - build_jobs: 5 + build_jobs: 2 connect_timeout: 60 compilers: diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index 8f2bd3c8a..c2a418083 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -4,7 +4,7 @@ spack: config: checksum: false - build_jobs: 5 + build_jobs: 2 connect_timeout: 60 compilers: