From c91d438e15b8a0bbf332e2fb5dcf190c7bc3805c Mon Sep 17 00:00:00 2001 From: Weston Ortiz Date: Thu, 4 Apr 2024 16:46:09 -0600 Subject: [PATCH] update scalapack in build script, use int_int for tpetra build script --- CMakeLists.txt | 10 + scripts/build-goma-dependencies.sh | 1457 +--------------------------- src/bc/rotate.c | 8 +- 3 files changed, 37 insertions(+), 1438 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9afcf222..28edee38 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -117,6 +117,16 @@ list(FIND Trilinos_PACKAGE_LIST Tpetra tpetra_package_index) if(${tpetra_package_index} GREATER_EQUAL 0) option(ENABLE_TPETRA "ENABLE_TPETRA" ON) if(ENABLE_TPETRA) + option(GOMA_MATRIX_GO_LONG_LONG "GOMA_MATRIX_GO_LONG_LONG" OFF) + if (GOMA_MATRIX_GO_LONG_LONG) + message(WARNING "GOMA_MATRIX_GO_LONG_LONG is enabled, using long long for Tpetra this is experimental") + list(APPEND GOMA_COMPILE_DEFINITIONS GOMA_MATRIX_GO_LONG_LONG) + endif() + if ((NOT ${HAVE_TPETRA_INST_INT_INT}) AND (NOT ${GOMA_MATRIX_GO_LONG_LONG})) + message(FATAL_ERROR "Trilinos not built with Tpetra with int int, " + "please disable Tpetra, or enable GOMA_MATRIX_GO_LONG_LONG, " + "or rebuild Trilinos with Tpetra int int") + endif() message(STATUS "TRILINOS: Tpetra found, enabling in Goma") list(APPEND GOMA_COMPILE_DEFINITIONS GOMA_ENABLE_TPETRA) list(FIND Trilinos_PACKAGE_LIST Amesos2 amesos2_package_index) diff --git a/scripts/build-goma-dependencies.sh b/scripts/build-goma-dependencies.sh index 7404b82e..a0588438 100755 --- a/scripts/build-goma-dependencies.sh +++ b/scripts/build-goma-dependencies.sh @@ -247,22 +247,24 @@ MUMPS_VERSION="5.5.1" MUMPS_URL="https://graal.ens-lyon.fr/MUMPS/MUMPS_$MUMPS_VERSION.tar.gz" MUMPS_MD5="da26c4b43d53a9a6096775245cee847f" -OPENMPI_VERSION="4.1.4" -OPENMPI_MD5="f057e12aabaf7dd5a6a658180fca404e" +OPENMPI_VERSION="4.1.6" +OPENMPI_MD5="c9b1c974cfc23c77c0fbdb965cd58a1c" OPENMPI_ARCHIVE_URL="https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-$OPENMPI_VERSION.tar.bz2" OPENMPI_EXTRA_CONFIGURE_FLAGS="" CMAKE_VERSION="3.24.2" CMAKE_MD5="f6f79ec66c91bbc075757a70205128ca" +#SUITESPARSE_VERSION="7.7.0" +#SUITESPARSE_MD5="e659373ed5e9b961d2fcb6d67d250783" SUITESPARSE_VERSION="5.13.0" SUITESPARSE_MD5="e9e7bc594b77ae4b58d943cdc286d679" MATIO_VERSION="1.5.21" MATIO_MD5="afeb5d21b234699fd5b9dc4564afe1ca" -SCALAPACK_VERSION="2.2.0" -SCALAPACK_MD5="2397d36790d1445383bc3cdb1e18ca5f" +SCALAPACK_VERSION="0234af94c6578c53ac4c19f2925eb6e5c4ad6f0f" +SCALAPACK_MD5="221dab094055b73fa7bed645c426548c" LAPACK_VERSION="3.8.0" LAPACK_MD5="96591affdbf58c450d45c1daa540dbd2" @@ -299,7 +301,6 @@ ARCHIVE_NAMES=("arpack-ng-$ARPACK_NG_VERSION.tar.gz" \ "parmetis-$PARMETIS_VERSION.tar.gz" \ "sparse.tar.gz" \ "superlu_dist-$SUPERLU_DIST_VERSION.tar.gz" \ -"y12m.tar.gz" \ "Trilinos-trilinos-release-$TRILINOS_VERSION_DASH.tar.gz" \ "MUMPS_$MUMPS_VERSION.tar.gz" \ "SuiteSparse-$SUITESPARSE_VERSION.tar.gz" \ @@ -317,7 +318,6 @@ ARCHIVE_MD5SUMS=("$ARPACK_NG_MD5" \ "$PARMETIS_MD5" \ "1566d914d1035ac17b73fe9bc0eed02a" \ "$SUPERLU_DIST_MD5" \ -"SKIP" \ $TRILINOS_MD5 \ $MUMPS_MD5 \ "$SUITESPARSE_MD5" \ @@ -332,7 +332,6 @@ ARCHIVE_URLS=("$ARPACK_NG_URL" \ "https://bitbucket.org/petsc/pkg-parmetis/get/v$PARMETIS_VERSION.tar.gz" \ "http://downloads.sourceforge.net/project/sparse/sparse/sparse1.4b/sparse1.4b.tar.gz" \ "http://codeload.github.com/xiaoyeli/superlu_dist/tar.gz/v$SUPERLU_DIST_VERSION" \ -"http://www.netlib.org/cgi-bin/netlibfiles.tgz?format=tgz\\&filename=y12m%2Fy12m.f" \ "https://github.com/trilinos/Trilinos/archive/trilinos-release-$TRILINOS_VERSION_DASH.tar.gz" \ "$MUMPS_URL" \ "https://github.com/DrTimothyAldenDavis/SuiteSparse/archive/refs/tags/v$SUITESPARSE_VERSION.tar.gz" \ @@ -349,7 +348,6 @@ ARCHIVE_DIR_NAMES=("arpack-ng-${ARPACK_NG_VERSION}" \ "parmetis-$PARMETIS_VERSION" \ "sparse" \ "superlu_dist-$SUPERLU_DIST_VERSION" \ -"y12m" \ "Trilinos-trilinos-release-$TRILINOS_VERSION_DASH" \ "MUMPS_$MUMPS_VERSION" \ "SuiteSparse-$SUITESPARSE_VERSION" \ @@ -364,7 +362,6 @@ ARCHIVE_HOMEPAGES=("https://github.com/opencollab/arpack-ng/" \ "http://glaros.dtc.umn.edu/gkhome/metis/metis/overview" \ "http://sparse.sourceforge.net/" \ "https://github.com/xiaoyeli/superlu_dist" \ -"http://doi.org/10.1007/3-540-10874-2" \ "https://trilinos.org/" \ "http://mumps.enseeiht.fr/" \ "http://faculty.cse.tamu.edu/davis/suitesparse.html" \ @@ -379,7 +376,6 @@ ARCHIVE_REAL_NAMES=("arpack-ng" \ "ParMETIS" \ "Sparse" \ "SuperLU_DIST" \ -"y12m" \ "Trilinos" \ "MUMPS" \ "SuiteSparse-$SUITESPARSE_VERSION" \ @@ -572,9 +568,9 @@ function setMathVars { export SCALAPACK_LIBRARY_NAME="scalapack" export SCALAPACK_LIBRARY_NAME_ARG="-L${SCALAPACK_LIBRARY_DIR} -lscalapack" SCALAPACK_INCLUDE_DIR="${GOMA_LIB}/scalapack-$SCALAPACK_VERSION/include" - ARCHIVE_NAMES+=("scalapack-$SCALAPACK_VERSION.tgz") + ARCHIVE_NAMES+=("$SCALAPACK_VERSION.tar.gz") ARCHIVE_MD5SUMS+=("$SCALAPACK_MD5") - ARCHIVE_URLS+=("http://www.netlib.org/scalapack/scalapack-$SCALAPACK_VERSION.tgz") + ARCHIVE_URLS+=("https://github.com/Reference-ScaLAPACK/scalapack/archive/$SCALAPACK_VERSION.tar.gz") ARCHIVE_DIR_NAMES+=("scalapack-$SCALAPACK_VERSION") ARCHIVE_HOMEPAGES+=("http://www.netlib.org/scalapack/") ARCHIVE_REAL_NAMES+=("ScaLAPACK") @@ -593,11 +589,11 @@ function setMathVars { USING_MKLS="OFF" #Add packages that otherwise come preinstalled in intel compiler. ARCHIVE_NAMES+=("lapack-$LAPACK_VERSION.tar.gz" \ - "scalapack-$SCALAPACK_VERSION.tgz") + "$SCALAPACK_VERSION.tar.gz") ARCHIVE_MD5SUMS+=("$LAPACK_MD5" \ "$SCALAPACK_MD5" ) ARCHIVE_URLS+=("https://github.com/Reference-LAPACK/lapack/archive/refs/tags/v$LAPACK_VERSION.tar.gz" \ - "http://www.netlib.org/scalapack/scalapack-$SCALAPACK_VERSION.tgz" ) + "https://github.com/Reference-ScaLAPACK/scalapack/archive/$SCALAPACK_VERSION.tar.gz" ) ARCHIVE_DIR_NAMES+=("lapack-$LAPACK_VERSION" \ "scalapack-$SCALAPACK_VERSION" ) ARCHIVE_HOMEPAGES+=("http://www.netlib.org/lapack/" \ @@ -626,11 +622,11 @@ function setMathVars { USING_MKLS="OFF" #Add packages that otherwise come preinstalled in intel compiler. ARCHIVE_NAMES+=("OpenBLAS-$OPENBLAS_VERSION.tar.gz" \ - "scalapack-$SCALAPACK_VERSION.tgz") + "$SCALAPACK_VERSION.tar.gz") ARCHIVE_MD5SUMS+=("$OPENBLAS_MD5" \ "$SCALAPACK_MD5" ) ARCHIVE_URLS+=("$OPENBLAS_URL" \ - "http://www.netlib.org/scalapack/scalapack-$SCALAPACK_VERSION.tgz" ) + "https://github.com/Reference-ScaLAPACK/scalapack/archive/$SCALAPACK_VERSION.tar.gz" ) ARCHIVE_DIR_NAMES+=("OpenBLAS-$OPENBLAS_VERSION" \ "scalapack-$SCALAPACK_VERSION" ) ARCHIVE_HOMEPAGES+=("https://github.com/xianyi/OpenBLAS/" \ @@ -685,9 +681,9 @@ function setMathVars { SCALAPACK_INCLUDE_DIR="${GOMA_LIB}/scalapack-$SCALAPACK_VERSION/include" export NON_INTEL_BLAS_LIBRARY="${BLAS_LIBRARY_DIR}/lib${BLAS_LIBRARY_NAME}.a" export NON_INTEL_BLAS_LINK="-L${BLAS_LIBRARY_DIR} -l${BLAS_LIBRARY_NAME} ${FORTRAN_LIBS}" - ARCHIVE_NAMES+=("scalapack-$SCALAPACK_VERSION.tgz") + ARCHIVE_NAMES+=("$SCALAPACK_VERSION.tar.gz") ARCHIVE_MD5SUMS+=("$SCALAPACK_MD5" ) - ARCHIVE_URLS+=("http://www.netlib.org/scalapack/scalapack-$SCALAPACK_VERSION.tgz" ) + ARCHIVE_URLS+=("https://github.com/Reference-ScaLAPACK/scalapack/archive/$SCALAPACK_VERSION.tar.gz" ) ARCHIVE_DIR_NAMES+=("scalapack-$SCALAPACK_VERSION" ) ARCHIVE_HOMEPAGES+=("http://www.netlib.org/scalapack/") ARCHIVE_REAL_NAMES+=("ScaLAPACK") @@ -1456,48 +1452,6 @@ fi # Otherwise Goma dynamically links to UMFPACK when intel is sourced and disabled export LD_LIBRARY_PATH="${GOMA_LIB}/SuiteSparse-$SUITESPARSE_VERSION/UMFPACK/Lib:$LD_LIBRARY_PATH" -#make y12m -cd $GOMA_LIB/y12m -if [ -e liby12m.a ] -then - log_echo "y12m already built" -else -# The y12m we use does not include a makefile, so this is a simple one for compiling a static library. - -cat > makefile << EOF -LIB=y12m -FFLAGS=-O -FC=$MPI_F77_COMPILER -OBJ = \ - y12m.o \ - y12mae.o \ - y12maf.o \ - y12mbe.o \ - y12mbf.o \ - y12mce.o \ - y12mcf.o \ - y12mde.o \ - y12mdf.o \ - y12mfe.o \ - y12mge.o \ - y12mhe.o \ - y12cck.o -lib\$(LIB).a: \$(OBJ) - ar ru lib\$(LIB).a \$? - ranlib lib\$(LIB).a -EOF - - make 2>&1 | tee -a $COMPILE_LOG - if [ -e $GOMA_LIB/y12m/liby12m.a ] - then - log_echo "Built y12m" - else - log_echo "Failed to build y12m" - exit 1 - fi -fi - - if [[ "$MATH_LIBRARIES" == "intel" ]] && [[ ! "$SCALAPACK_LIBRARY_NAME" = "scalapack" ]]; then log_echo "Not building scalapack because intel MKL used" else @@ -1511,1379 +1465,14 @@ else mv src-scalapack $GOMA_LIB/scalapack-$SCALAPACK_VERSION/src fi cd $GOMA_LIB/scalapack-$SCALAPACK_VERSION/src - # Use PETSc's parallel make changes -cat > Makefile.objs < Makefile.parallel < SLmake.inc <&1 | tee -a $COMPILE_LOG + cmake --build build -j $MAKE_JOBS 2>&1 | tee -a $COMPILE_LOG + cmake --install build 2>&1 | tee -a $COMPILE_LOG cd $GOMA_LIB if [ -f $GOMA_LIB/scalapack-$SCALAPACK_VERSION/lib/libscalapack.a ]; then log_echo "Build scalapack $SCALAPACK_VERSION" @@ -3161,6 +1750,8 @@ cd $GOMA_LIB/trilinos-$TRILINOS_VERSION-Temp -D Trilinos_ENABLE_Teko:BOOL=ON \ -D Trilinos_GOMA_ENABLE_AMESOS2:BOOL=ON \ -D Trilinos_ENABLE_Belos:BOOL=ON \ +-D Trilinos_ENABLE_Amesos2:BOOL=ON \ +-D Trilinos_ENABLE_Sacado:BOOL=ON \ -D Trilinos_ENABLE_EpetraExt:BOOL=ON \ -D Trilinos_ENABLE_Thyra:BOOL=ON \ -D Trilinos_ENABLE_ThyraTpetraAdapters:BOOL=ON \ @@ -3206,9 +1797,6 @@ cd $GOMA_LIB/trilinos-$TRILINOS_VERSION-Temp -D ParMETIS_LIBRARY_DIRS:PATH="$GOMA_LIB/parmetis-$PARMETIS_VERSION/lib;$GOMA_LIB/metis-$METIS_VERSION/lib" \ -D ParMETIS_INCLUDE_DIRS:PATH="$GOMA_LIB/parmetis-$PARMETIS_VERSION/include;$GOMA_LIB/metis-$METIS_VERSION/include" \ -D TPL_ParMETIS_INCLUDE_DIRS:PATH="$GOMA_LIB/parmetis-$PARMETIS_VERSION/include;$GOMA_LIB/metis-$METIS_VERSION/include" \ - -D TPL_ENABLE_y12m:BOOL=ON \ - -D y12m_LIBRARY_NAMES:STRING="y12m" \ - -D y12m_LIBRARY_DIRS:PATH=$GOMA_LIB/y12m \ -D TPL_ENABLE_MUMPS:BOOL=ON \ -D MUMPS_LIBRARY_NAMES:STRING="dmumps;mumps_common;pord;$BLACS_LIBRARY_NAME" \ -D MUMPS_LIBRARY_DIRS:PATH="$GOMA_LIB/MUMPS_$MUMPS_VERSION/lib;$GOMA_LIB/MUMPS_$MUMPS_VERSION/PORD/lib;$SCALAPACK_LIBRARY_DIR" \ @@ -3229,6 +1817,7 @@ cd $GOMA_LIB/trilinos-$TRILINOS_VERSION-Temp -D Amesos_ENABLE_KLU:BOOL=ON \ -D Amesos_ENABLE_UMFPACK:BOOL=ON \ -D Amesos_ENABLE_MUMPS:BOOL=ON \ +-D Tpetra_INST_INT_INT:BOOL=ON \ $EXTRA_ARGS \ $GOMA_LIB/Trilinos-trilinos-release-$TRILINOS_VERSION_DASH 2>&1 | tee -a $COMPILE_LOG diff --git a/src/bc/rotate.c b/src/bc/rotate.c index dccf9e8a..bb1c1805 100644 --- a/src/bc/rotate.c +++ b/src/bc/rotate.c @@ -797,8 +797,8 @@ void rotate_mesh_eqn(int id, /* Elemental stiffness matrix row index * */ for (j = 0; j < rot->d_vector_n; j++) { double sum_val; - int global_row; - int global_col; + GomaGlobalOrdinal global_row; + GomaGlobalOrdinal global_col; J = rot->d_vector_J[j]; if (Dolphin[pg->imtrx][I][MESH_DISPLACEMENT1] > 0 && @@ -1139,8 +1139,8 @@ void rotate_momentum_eqn(int id, /* Elemental stiffness matrix row ind // Direct translation from MSR for (j = 0; j < rotation[I][eq][kdir]->d_vector_n; j++) { double sum_val; - int global_row; - int global_col; + GomaGlobalOrdinal global_row; + GomaGlobalOrdinal global_col; int ktype, ndof, index_eqn, index_var; J = rotation[I][eq][kdir]->d_vector_J[j]; if (Dolphin[pg->imtrx][I][R_MOMENTUM1] > 0 &&