Skip to content

Commit

Permalink
Add Amesos2, Tpetra matrix format, update legacy build script (goma#456)
Browse files Browse the repository at this point in the history
- Adds Tpetra Matrix format
- Refactors Epetra to use generic form
- Adds Amesos2 solver
- Adds Tpetra based Stratimikos solver
  • Loading branch information
wortiz authored Apr 5, 2024
1 parent eb6ec90 commit 3cac332
Show file tree
Hide file tree
Showing 46 changed files with 1,703 additions and 2,074 deletions.
5 changes: 5 additions & 0 deletions .github/workflows/build-unit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,15 @@ jobs:
runs-on: ubuntu-latest
container: westonortiz/goma-libs

defaults:
run:
shell: bash -l -e {0}

steps:
- uses: actions/checkout@v3
- name: Check versions
run: |
echo $PETSC_DIR
mpirun --version
cmake --version
- name: CMake Configure Goma
Expand Down
69 changes: 58 additions & 11 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,11 @@ else()
endif()
endif()

option(GOMA_MATRIX_GO_LONG_LONG OFF)
if(GOMA_MATRIX_GO_LONG_LONG)
list(APPEND GOMA_COMPILE_DEFINITIONS GOMA_MATRIX_GO_LONG_LONG)
endif()

option(ENABLE_CCACHE OFF)
if(ENABLE_CCACHE)
message(STATUS "Searching for ccache.")
Expand Down Expand Up @@ -108,6 +113,47 @@ if(${stratimikos_package_index} GREATER_EQUAL 0)
endif()
endif()

list(FIND Trilinos_PACKAGE_LIST Tpetra tpetra_package_index)
if(${tpetra_package_index} GREATER_EQUAL 0)
option(ENABLE_TPETRA "ENABLE_TPETRA" ON)
if((NOT ${HAVE_TPETRA_INST_INT_INT}) AND (NOT ${GOMA_MATRIX_GO_LONG_LONG}))
message(
WARNING
"Trilinos not built with Tpetra with int int, enable GOMA_MATRIX_GO_LONG_LONG (experimental), or rebuild Trilinos with Tpetra int int"
)
set(ENABLE_TPETRA OFF)
endif()
if(ENABLE_TPETRA)
option(GOMA_MATRIX_GO_LONG_LONG "GOMA_MATRIX_GO_LONG_LONG" OFF)
if(GOMA_MATRIX_GO_LONG_LONG)
message(
WARNING
"GOMA_MATRIX_GO_LONG_LONG is enabled, using long long for Tpetra this is experimental"
)
list(APPEND GOMA_COMPILE_DEFINITIONS GOMA_MATRIX_GO_LONG_LONG)
endif()
message(STATUS "TRILINOS: Tpetra found, enabling in Goma")
list(APPEND GOMA_COMPILE_DEFINITIONS GOMA_ENABLE_TPETRA)
list(FIND Trilinos_PACKAGE_LIST Amesos2 amesos2_package_index)
if(${amesos2_package_index} GREATER_EQUAL 0)
option(ENABLE_AMESOS2 "ENABLE_AMESOS2" ON)
if(ENABLE_AMESOS2)
message(STATUS "TRILINOS: Amesos2 found, enabling in Goma")
list(APPEND GOMA_COMPILE_DEFINITIONS GOMA_ENABLE_AMESOS2)
endif()
endif()
endif()
endif()

list(FIND Trilinos_PACKAGE_LIST Epetra epetra_package_index)
if(${epetra_package_index} GREATER_EQUAL 0)
option(ENABLE_EPETRA "ENABLE_EPETRA" ON)
if(ENABLE_EPETRA)
message(STATUS "TRILINOS: Epetra found, enabling in Goma")
list(APPEND GOMA_COMPILE_DEFINITIONS GOMA_ENABLE_EPETRA)
endif()
endif()

# Exodus
find_package(SEACASExodus REQUIRED HINTS ${Trilinos_DIR}/../SEACASExodus)
message(STATUS "SEACASExodus_DIR = ${SEACASExodus_DIR}")
Expand Down Expand Up @@ -439,6 +485,9 @@ set(GOMA_INCLUDES
include/el_quality.h
include/exo_conn.h
include/exo_struct.h
include/linalg/sparse_matrix.h
include/linalg/sparse_matrix_tpetra.h
include/linalg/sparse_matrix_epetra.h
include/load_field_variables.h
include/loca_const.h
include/loca_util_const.h
Expand Down Expand Up @@ -536,13 +585,12 @@ set(GOMA_INCLUDES
include/shell_tfmp_struct.h
include/shell_tfmp_util.h
include/sl_amesos_interface.h
include/sl_amesos2_interface.h
include/sl_aux.h
include/sl_auxutil.h
include/sl_aztecoo_interface.h
include/sl_eggroll_def.h
include/sl_eggroll.h
include/sl_epetra_interface.h
include/sl_epetra_util.h
include/sl_lu.h
include/sl_petsc.h
include/sl_petsc_complex.h
Expand Down Expand Up @@ -591,6 +639,9 @@ set(GOMA_SOURCES
src/el_quality.c
src/exo_conn.c
src/globals.c
src/linalg/sparse_matrix.cpp
src/linalg/sparse_matrix_tpetra.cpp
src/linalg/sparse_matrix_epetra.cpp
src/load_field_variables.c
src/loca_bord.c
src/loca_eigen_c2f.F
Expand Down Expand Up @@ -684,6 +735,7 @@ set(GOMA_SOURCES
src/bc/rotate_coordinates.c
src/shell_tfmp_util.c
src/sl_amesos_interface.cpp
src/sl_amesos2_interface.cpp
src/sl_aux.c
src/sl_auxutil.c
src/sl_aztecoo_interface.cpp
Expand All @@ -694,8 +746,6 @@ set(GOMA_SOURCES
src/sl_eggroll05.c
src/sl_eggrollutil.c
src/sl_eggrollwrap.c
src/sl_epetra_interface.cpp
src/sl_epetra_util.cpp
src/sl_leastsquares.cpp
src/sl_lu.c
src/sl_lu_fill.c
Expand Down Expand Up @@ -948,14 +998,11 @@ if(ENABLE_PETSC AND PETSC_FOUND)
${PETSC_STATIC_LIBRARY_DIRS})
endif()

if ($Trilinos_VERSION VERSION_LESS 14.0.0)
set(GOMA_TRILINOS_LIBRARIES
${Trilinos_LIBRARIES}
${Trilinos_TPL_LIBRARIES}
${Trilinos_EXTRA_LD_FLAGS})
if($Trilinos_VERSION VERSION_LESS 14.0.0)
set(GOMA_TRILINOS_LIBRARIES ${Trilinos_LIBRARIES} ${Trilinos_TPL_LIBRARIES}
${Trilinos_EXTRA_LD_FLAGS})
else()
set(GOMA_TRILINOS_LIBRARIES
Trilinos::all_libs)
set(GOMA_TRILINOS_LIBRARIES Trilinos::all_libs)
endif()

set(GOMA_TPL_LIBRARIES
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ vbr
selected when an Aztec iterative solver is chosen.
epetra
Compressed Sparse Row format using the Epetra library from Trilinos
tpetra
FECRSMatrix Compressed Sparse Row format using the Tpetra library from Trilinos
currently only works using Stratimikos and Amesos2

------------
Examples
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -98,13 +98,29 @@ amesos
Of these four options, we currently recommend **mumps**.
All options can be run in parallel.
amesos2
Allows access to direct solver options implemented in parallel. Please see
the user-notes below for Goma build options that must be exercised. With
this option, you must add an additional input card to specify the parallel
direct solvers:

::

Amesos2 Solver Package = {SuperLUDist | Mumps | KLU2}
# Optional Amesos2 File to set parameters for the solver
# xml or yaml supported through Trilinos
# Amesos2 File = params_file.(xml | yaml)
Of these four options, we currently recommend **Mumps**.
All options can be run in parallel.
stratimikos
Interface to Trilino's Stratimikos package
requires:

::

Matrix storage format = epetra
# Or tpetra
Allows block solvers, see also ref:`Stratimikos File`
petsc
Expand Down
10 changes: 10 additions & 0 deletions include/dp_comm.h
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,16 @@ EXTERN void exchange_dof(Comm_Ex *, /* cx - ptr to communications exchange info
double *, /* x - local processor dof-based vector */
int);

EXTERN void exchange_dof_int(Comm_Ex *, /* cx - ptr to communications exchange info */
Dpi *, /* dpi - distributed processing info */
int *, /* x - local processor dof-based vector */
int);

EXTERN void exchange_dof_long_long(Comm_Ex *, /* cx - ptr to communications exchange info */
Dpi *, /* dpi - distributed processing info */
long long *, /* x - local processor dof-based vector */
int);

EXTERN void exchange_node(Comm_Ex *cx, /* cx - ptr to communications exchange info */
Dpi *d, /* dpi - distributed processing info */
double *a); /* x - local processor node-based vector */
Expand Down
117 changes: 117 additions & 0 deletions include/linalg/sparse_matrix.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
#ifndef GOMA_SPARSE_MATRIX
#define GOMA_SPARSE_MATRIX

#ifdef __cplusplus
extern "C" {
#endif
#ifdef GOMA_MATRIX_GO_LONG_LONG
typedef long long GomaGlobalOrdinal;
#define MPI_GOMA_ORDINAL MPI_LONG_LONG
#else
typedef int GomaGlobalOrdinal;
#define MPI_GOMA_ORDINAL MPI_INT
#endif

#ifdef DISABLE_CPP
#define CPP_ALREADY_DISABLED
#endif
#define DISABLE_CPP
#include "dp_types.h"
#include "exo_struct.h"
#include "mm_eh.h"
#include "rf_node_const.h"
#include "sl_util_structs.h"
#ifndef CPP_ALREADY_DISABLED
#undef DISABLE_CPP
#endif

enum GomaSparseMatrixType {
GOMA_SPARSE_MATRIX_TYPE_EPETRA,
GOMA_SPARSE_MATRIX_TYPE_AZTEC_MRS,
GOMA_SPARSE_MATRIX_TYPE_TPETRA,
GOMA_SPARSE_MATRIX_TYPE_PETSC
};

struct g_GomaSparseMatrix {
enum GomaSparseMatrixType type;
void *data;
GomaGlobalOrdinal *global_ids;
GomaGlobalOrdinal n_rows;
GomaGlobalOrdinal n_cols;
GomaGlobalOrdinal nnz;
// Create matrix with given rows and columns
// coo_rows and coo_cols are the COO representation of the matrix ordered by row
// local_nnz is the number of non-zero entries in the local partition
// max_nz_per_row is the maximum number of non-zero entries in any row
goma_error (*create_graph)(struct g_GomaSparseMatrix *matrix,
GomaGlobalOrdinal n_rows,
GomaGlobalOrdinal *row_list,
GomaGlobalOrdinal n_cols,
GomaGlobalOrdinal *col_list,
GomaGlobalOrdinal local_nnz,
GomaGlobalOrdinal max_nz_per_row,
GomaGlobalOrdinal *coo_rows,
GomaGlobalOrdinal *coo_cols);
// optional, run after create graph to finalize structure
goma_error (*complete_graph)(struct g_GomaSparseMatrix *matrix);
// Insert values into matrix row replacing existing values
goma_error (*insert_row_values)(struct g_GomaSparseMatrix *matrix,
GomaGlobalOrdinal global_row,
GomaGlobalOrdinal num_entries,
double *values,
GomaGlobalOrdinal *indices);
// Add values into a matrix row, sums existing values
goma_error (*sum_into_row_values)(struct g_GomaSparseMatrix *matrix,
GomaGlobalOrdinal global_row,
GomaGlobalOrdinal num_entries,
double *values,
GomaGlobalOrdinal *indices);
// set matrix non-zeros to specified scalar value (commonly re-zero for next assembly)
goma_error (*put_scalar)(struct g_GomaSparseMatrix *matrix, double scalar);
// row sum scaling, compute row sum scale, scale matrix and b, and return scaling vector
goma_error (*row_sum_scaling)(struct g_GomaSparseMatrix *matrix, double *b, double *scale);
// Zeros a global row
goma_error (*zero_global_row)(struct g_GomaSparseMatrix *matrix, GomaGlobalOrdinal global_row);
// Zeros a global row and sets diagonal to 1.0
goma_error (*zero_global_row_set_diag)(struct g_GomaSparseMatrix *matrix,
GomaGlobalOrdinal global_row);
// delete the allocated matrix;
goma_error (*destroy)(struct g_GomaSparseMatrix *matrix);
};
typedef struct g_GomaSparseMatrix *GomaSparseMatrix;

goma_error GomaSparseMatrix_CreateFromFormat(GomaSparseMatrix *matrix, char *matrix_format);
goma_error GomaSparseMatrix_Create(GomaSparseMatrix *matrix, enum GomaSparseMatrixType type);
// populates:
// - global_ids
// - n_rows
// - n_cols
goma_error GomaSparseMatrix_SetProblemGraph(
GomaSparseMatrix matrix,
int num_internal_dofs,
int num_boundary_dofs,
int num_external_dofs,
int local_nodes,
NODE_INFO_STRUCT **Nodes,
int MaxVarPerNode,
int *Matilda,
int Inter_Mask[MAX_NUM_MATRICES][MAX_VARIABLE_TYPES][MAX_VARIABLE_TYPES],
Exo_DB *exo,
Dpi *dpi,
Comm_Ex *cx,
int imtrx,
int Debug_Flag,
struct GomaLinearSolverData *ams);

goma_error GomaSparseMatrix_LoadLec(GomaSparseMatrix matrix,
int ielem,
struct Local_Element_Contributions *lec,
double resid_vector[]);

goma_error GomaSparseMatrix_Destroy(GomaSparseMatrix *matrix);

#ifdef __cplusplus
}
#endif

#endif // GOMA_SPARSE_MATRIX
65 changes: 65 additions & 0 deletions include/linalg/sparse_matrix_epetra.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
#ifndef GOMA_SPARSE_MATRIX_EPETRA
#define GOMA_SPARSE_MATRIX_EPETRA
#ifdef GOMA_ENABLE_EPETRA
#ifdef __cplusplus
#include "Teuchos_RCP.hpp"
#include <Epetra_CrsGraph.h>
#include <Epetra_CrsMatrix.h>
#include <Epetra_Map.h>

#include "linalg/sparse_matrix.h"

using LO = int;
using GO = GomaGlobalOrdinal;

struct EpetraSparseMatrix {
Teuchos::RCP<Epetra_CrsMatrix> matrix;
Teuchos::RCP<Epetra_Map> row_map;
Teuchos::RCP<Epetra_CrsGraph> crs_graph;
EpetraSparseMatrix() = default;
};

extern "C" {
#endif

goma_error GomaSparseMatrix_Epetra_Create(GomaSparseMatrix *matrix);

goma_error g_epetra_create_graph(GomaSparseMatrix matrix,
GomaGlobalOrdinal n_rows,
GomaGlobalOrdinal *row_list,
GomaGlobalOrdinal n_cols,
GomaGlobalOrdinal *col_list,
GomaGlobalOrdinal local_nnz,
GomaGlobalOrdinal max_per_row,
GomaGlobalOrdinal *coo_rows,
GomaGlobalOrdinal *coo_cols);

goma_error g_epetra_complete_graph(GomaSparseMatrix matrix);

goma_error g_epetra_insert_row_values(GomaSparseMatrix matrix,
GomaGlobalOrdinal global_row,
GomaGlobalOrdinal num_entries,
double *values,
GomaGlobalOrdinal *indices);

goma_error g_epetra_sum_into_row_values(GomaSparseMatrix matrix,
GomaGlobalOrdinal global_row,
GomaGlobalOrdinal num_entries,
double *values,
GomaGlobalOrdinal *indices);

goma_error g_epetra_put_scalar(GomaSparseMatrix matrix, double scalar);

goma_error g_epetra_row_sum_scaling(GomaSparseMatrix matrix, double *b, double *scale);

goma_error g_epetra_zero_row(GomaSparseMatrix matrix, GomaGlobalOrdinal global_row);

goma_error g_epetra_zero_row_set_diag(GomaSparseMatrix matrix, GomaGlobalOrdinal global_row);

goma_error g_epetra_destroy(GomaSparseMatrix matrix);

#ifdef __cplusplus
}
#endif
#endif
#endif // GOMA_SPARSE_MATRIX_EPETRA
Loading

0 comments on commit 3cac332

Please sign in to comment.