From 4c6db24a9e536205ee3910062094830ed2ef4a7f Mon Sep 17 00:00:00 2001 From: Mohd Afeef Badri <52162083+mohd-afeef-badri@users.noreply.github.com> Date: Fri, 7 Feb 2025 21:21:40 +0100 Subject: [PATCH] command line params for laplace (#219) * command line params for laplace --- modules/laplace/CMakeLists.txt | 28 ++--- modules/laplace/Fem.axl | 9 +- modules/laplace/FemModule.cc | 109 +++++++++++++++--- modules/laplace/FemModule.h | 16 ++- modules/laplace/inputs/L-shape.3D.bsr.arc | 37 ------ .../inputs/L-shape.3D.bsr.atomicFree.arc | 37 ------ ...ntDirichlet.bsr.atomicFree.hypreDirect.arc | 49 -------- ...ect.arc => PointDirichlet.hypreDirect.arc} | 1 - 8 files changed, 123 insertions(+), 163 deletions(-) delete mode 100644 modules/laplace/inputs/L-shape.3D.bsr.arc delete mode 100644 modules/laplace/inputs/L-shape.3D.bsr.atomicFree.arc delete mode 100644 modules/laplace/inputs/PointDirichlet.bsr.atomicFree.hypreDirect.arc rename modules/laplace/inputs/{PointDirichlet.bsr.hypreDirect.arc => PointDirichlet.hypreDirect.arc} (98%) diff --git a/modules/laplace/CMakeLists.txt b/modules/laplace/CMakeLists.txt index b13ac46f..566006cd 100644 --- a/modules/laplace/CMakeLists.txt +++ b/modules/laplace/CMakeLists.txt @@ -49,23 +49,25 @@ endif() enable_testing() +add_test(NAME [laplace]pointDirichlet COMMAND Laplace inputs/PointDirichlet.arc) + if(FEMUTILS_HAS_SOLVER_BACKEND_PETSC) add_test(NAME [laplace]2D_Dirichlet COMMAND Laplace inputs/ring.arc) add_test(NAME [laplace]3D_Dirichlet COMMAND Laplace inputs/L-shape.3D.arc) -endif() + add_test(NAME [laplace]2D_Dirichlet_petsc_monitor COMMAND Laplace -A,petsc_flags=-ksp_monitor inputs/ring.arc) + add_test(NAME [laplace]2D_parse_and_exit COMMAND Laplace -A,assemble_linear_system=FALSE -A,solve_linear_system=FALSE -A,cross_validation=FALSE inputs/ring.arc) -add_test(NAME [laplace]pointDirichlet COMMAND Laplace inputs/PointDirichlet.arc) + if(FEMUTILS_HAS_PARALLEL_SOLVER AND MPIEXEC_EXECUTABLE) + add_test(NAME [laplace]2D_Dirichlet_2p COMMAND ${MPIEXEC_EXECUTABLE} -n 2 ./Laplace inputs/ring.arc) + add_test(NAME [laplace]3D_Dirichlet_3D_2p COMMAND ${MPIEXEC_EXECUTABLE} -n 2 ./Laplace inputs/L-shape.3D.arc) -if(FEMUTILS_HAS_SOLVER_BACKEND_HYPRE) - arcanefem_add_gpu_test(NAME [laplace]pointDirichlet_bsr_hypreDirect COMMAND ./Laplace ARGS inputs/PointDirichlet.bsr.atomicFree.hypreDirect.arc) - arcanefem_add_gpu_test(NAME [laplace]pointDirichlet_bsr_atomicFree_hypreDirect COMMAND ./Laplace ARGS inputs/PointDirichlet.bsr.atomicFree.hypreDirect.arc) - endif() - -# If parallel part is available, add some tests -if(FEMUTILS_HAS_PARALLEL_SOLVER AND MPIEXEC_EXECUTABLE) - add_test(NAME [laplace]2D_Dirichlet_4pe COMMAND ${MPIEXEC_EXECUTABLE} -n 4 ./Laplace inputs/ring.arc) - add_test(NAME [laplace]3D_Dirichlet_3D_4pe COMMAND ${MPIEXEC_EXECUTABLE} -n 4 ./Laplace inputs/L-shape.3D.arc) - if(FEMTEST_HAS_GMSH_TEST) - add_test(NAME [laplace]pointDirichlet_10k_4pe COMMAND ${MPIEXEC_EXECUTABLE} -n 4 ./Laplace inputs/PointDirichlet-refined.arc) + if(FEMTEST_HAS_GMSH_TEST) + add_test(NAME [laplace]pointDirichlet_10k_2p COMMAND ${MPIEXEC_EXECUTABLE} -n 2 ./Laplace inputs/PointDirichlet-refined.arc) + endif() endif() endif() + +if(FEMUTILS_HAS_SOLVER_BACKEND_HYPRE) + arcanefem_add_gpu_test(NAME [laplace]pointDirichlet_bsr_hypreDirect COMMAND ./Laplace ARGS -A,matrix_format=BSR inputs/PointDirichlet.hypreDirect.arc) + arcanefem_add_gpu_test(NAME [laplace]pointDirichlet_bsr_atomicFree_hypreDirect COMMAND ./Laplace ARGS -A,matrix_format=AF-BSR inputs/PointDirichlet.hypreDirect.arc) +endif() \ No newline at end of file diff --git a/modules/laplace/Fem.axl b/modules/laplace/Fem.axl index 7ad0dee3..714203f6 100644 --- a/modules/laplace/Fem.axl +++ b/modules/laplace/Fem.axl @@ -20,12 +20,11 @@ Type of mesh provided to the solver - - Use bsr matrix format - - - Use atomic free bsr matrix format + + + Which matrix format to use DOK|BSR|AF-BSR. + diff --git a/modules/laplace/FemModule.cc b/modules/laplace/FemModule.cc index cb26f1fe..96c0d726 100644 --- a/modules/laplace/FemModule.cc +++ b/modules/laplace/FemModule.cc @@ -30,10 +30,8 @@ startInit() m_dofs_on_nodes.initialize(mesh(), 1); m_dof_family = m_dofs_on_nodes.dofFamily(); - if (options()->bsr() || options()->bsrAtomicFree()) { - auto use_csr_in_linear_system = options()->linearSystem.serviceName() == "HypreLinearSystem"; - m_bsr_format.initialize(defaultMesh(), use_csr_in_linear_system, options()->bsrAtomicFree()); - } + m_matrix_format = options()->matrixFormat(); + _handleCommandLineFlags(); elapsedTime = platform::getRealTime() - elapsedTime; _printArcaneFemTime("[ArcaneFem-Timer] initialize", elapsedTime); @@ -63,16 +61,18 @@ compute() m_linear_system.reset(); m_linear_system.setLinearSystemFactory(options()->linearSystem()); m_linear_system.initialize(subDomain(), acceleratorMng()->defaultRunner(), m_dofs_on_nodes.dofFamily(), "Solver"); - // Test for adding parameters for PETSc. - { - StringList string_list; - string_list.add("-ksp_monitor"); - CommandLineArguments args(string_list); - m_linear_system.setSolverCommandLineArguments(args); - } - if (options()->bsr() || options()->bsrAtomicFree()) + if (m_petsc_flags != NULL) + _setPetscFlagsFromCommandline(); + + if (m_matrix_format == "BSR" || m_matrix_format == "AF-BSR") { + auto use_csr_in_linear_system = options()->linearSystem.serviceName() == "HypreLinearSystem"; + if (m_matrix_format == "BSR") + m_bsr_format.initialize(mesh(), use_csr_in_linear_system, 0); + else + m_bsr_format.initialize(mesh(), use_csr_in_linear_system, 1); m_bsr_format.computeSparsity(); + } _doStationarySolve(); @@ -98,11 +98,20 @@ void FemModule:: _doStationarySolve() { _getMaterialParameters(); - _assembleBilinearOperator(); - _assembleLinearOperator(); - _solve(); - _updateVariables(); - _validateResults(); + + if(m_assemble_linear_system){ + _assembleBilinearOperator(); + _assembleLinearOperator(); + } + + if(m_solve_linear_system){ + _solve(); + _updateVariables(); + } + + if(m_cross_validation){ + _validateResults(); + } } /*---------------------------------------------------------------------------*/ @@ -137,7 +146,7 @@ _assembleLinearOperator() info() << "[ArcaneFem-Info] Started module _assembleLinearOperator()"; Real elapsedTime = platform::getRealTime(); - if (options()->bsr || options()->bsrAtomicFree()) + if (m_matrix_format == "BSR" || m_matrix_format == "AF-BSR") m_bsr_format.toLinearSystem(m_linear_system); VariableDoFReal& rhs_values(m_linear_system.rhsVariable()); // Temporary variable to keep values for the RHS @@ -173,7 +182,7 @@ _assembleBilinearOperator() info() << "[ArcaneFem-Info] Started module _assembleBilinearOperator()"; Real elapsedTime = platform::getRealTime(); - if (options()->bsr() || options()->bsrAtomicFree()) { + if (m_matrix_format == "BSR" || m_matrix_format == "AF-BSR") { UnstructuredMeshConnectivityView m_connectivity_view(mesh()); auto cn_cv = m_connectivity_view.cellNode(); auto m_queue = subDomain()->acceleratorMng()->defaultQueue(); @@ -326,6 +335,68 @@ _validateResults() _printArcaneFemTime("[ArcaneFem-Timer] result-validation", elapsedTime); } +/*---------------------------------------------------------------------------*/ +/** + * @brief Function to set PETSc flags from commandline + */ +/*---------------------------------------------------------------------------*/ + +void FemModule:: +_setPetscFlagsFromCommandline() +{ + StringList string_list; + std::string petsc_flags_std = m_petsc_flags.localstr(); + // Use a string stream to split the string by spaces + std::istringstream iss(petsc_flags_std); + String token; + while (iss >> token) { + string_list.add(token); + } + + CommandLineArguments args(string_list); + m_linear_system.setSolverCommandLineArguments(args); +} + +/*---------------------------------------------------------------------------*/ +/** + * @brief Function to hande commandline flags + */ +/*---------------------------------------------------------------------------*/ + +void FemModule:: +_handleCommandLineFlags() +{ + info() << "[ArcaneFem-Module] _handleCommandLineFlags()"; + ParameterList parameter_list = this->subDomain()->application()->applicationInfo().commandLineArguments().parameters(); + + if (parameter_list.getParameterOrNull("assemble_linear_system") == "FALSE") { + m_assemble_linear_system = false; + info() << "[ArcaneFem-Info] Linear system not assembled (assemble_linear_system = FALSE)"; + } + + if (parameter_list.getParameterOrNull("solve_linear_system") == "FALSE") { + m_solve_linear_system = false; + m_cross_validation = false; + info() << "[ArcaneFem-Info] Linear system assembled but not solved (solve_linear_system = FALSE)"; + } + + if (parameter_list.getParameterOrNull("cross_validation") == "FALSE") { + m_cross_validation = false; + info() << "[ArcaneFem-Info] Cross validation disabled (cross_validation = FALSE)"; + } + + m_petsc_flags = parameter_list.getParameterOrNull("petsc_flags"); + if (m_petsc_flags != NULL) { + info() << "[ArcaneFem-Info] PETSc flags the user provided will be used (petsc_flags != NULL)"; + } + + String matrix_format_from_commandline = parameter_list.getParameterOrNull("matrix_format"); + if (matrix_format_from_commandline != NULL){ + m_matrix_format = matrix_format_from_commandline; + info() << "[ArcaneFem-Info] Using commandline format for matrix format (matrix_format != NULL)"; + } +} + /*---------------------------------------------------------------------------*/ /** * @brief Function to prints the execution time `value` of phase `label` diff --git a/modules/laplace/FemModule.h b/modules/laplace/FemModule.h index 99048dce..6d3ae654 100644 --- a/modules/laplace/FemModule.h +++ b/modules/laplace/FemModule.h @@ -15,6 +15,9 @@ /*---------------------------------------------------------------------------*/ /*---------------------------------------------------------------------------*/ +#include +#include +#include #include #include #include @@ -25,7 +28,6 @@ #include #include -// GPU includes #include "arcane/accelerator/core/IAcceleratorMng.h" #include "arcane/accelerator/VariableViews.h" @@ -37,7 +39,6 @@ #include "FemDoFsOnNodes.h" #include "ArcaneFemFunctions.h" -// GPU includes #include "ArcaneFemFunctionsGpu.h" #include "BSRFormat.h" /*---------------------------------------------------------------------------*/ @@ -86,6 +87,15 @@ class FemModule FemDoFsOnNodes m_dofs_on_nodes; BSRFormat<1> m_bsr_format; + String m_petsc_flags; + String m_matrix_format = "DOK"; + + bool m_assemble_linear_system = true; + bool m_solve_linear_system = true; + bool m_cross_validation = true; + bool m_use_bsr_matrix_format = false; + bool m_use_atomic_free_bsr_matrix_format = false; + void _doStationarySolve(); void _getMaterialParameters(); void _solve(); @@ -93,6 +103,8 @@ class FemModule void _updateVariables(); void _validateResults(); + void _handleCommandLineFlags(); + void _setPetscFlagsFromCommandline(); void _printArcaneFemTime(const String label, const Real value); FixedMatrix<3, 3> _computeElementMatrixTria3(Cell cell); diff --git a/modules/laplace/inputs/L-shape.3D.bsr.arc b/modules/laplace/inputs/L-shape.3D.bsr.arc deleted file mode 100644 index 4982ee5c..00000000 --- a/modules/laplace/inputs/L-shape.3D.bsr.arc +++ /dev/null @@ -1,37 +0,0 @@ - - - - Sample - LaplaceLoop - - - - 1 - - - U - - - - - - meshes/L-shape-3D.msh - - - - - check/test_3D_L-shape.txt - TETRA4 - - - bot - 50.0 - - - bc - 10.0 - - - true - - diff --git a/modules/laplace/inputs/L-shape.3D.bsr.atomicFree.arc b/modules/laplace/inputs/L-shape.3D.bsr.atomicFree.arc deleted file mode 100644 index 0dca4d62..00000000 --- a/modules/laplace/inputs/L-shape.3D.bsr.atomicFree.arc +++ /dev/null @@ -1,37 +0,0 @@ - - - - Sample - LaplaceLoop - - - - 1 - - - U - - - - - - meshes/L-shape-3D.msh - - - - - check/test_3D_L-shape.txt - TETRA4 - - - bot - 50.0 - - - bc - 10.0 - - - true - - diff --git a/modules/laplace/inputs/PointDirichlet.bsr.atomicFree.hypreDirect.arc b/modules/laplace/inputs/PointDirichlet.bsr.atomicFree.hypreDirect.arc deleted file mode 100644 index 9cba73f9..00000000 --- a/modules/laplace/inputs/PointDirichlet.bsr.atomicFree.hypreDirect.arc +++ /dev/null @@ -1,49 +0,0 @@ - - - - Sample - LaplaceLoop - - - - 1 - - - U - - - - - - meshes/plancher.msh - - - - - check/test3_results.txt - - - topLeftCorner - 50.0 - - - topRightCorner - 20.0 - - - botLeftCorner - 20.0 - - - botRightCorner - 50.0 - - - true - - 0. - 1e-15 - 0.25 - - - diff --git a/modules/laplace/inputs/PointDirichlet.bsr.hypreDirect.arc b/modules/laplace/inputs/PointDirichlet.hypreDirect.arc similarity index 98% rename from modules/laplace/inputs/PointDirichlet.bsr.hypreDirect.arc rename to modules/laplace/inputs/PointDirichlet.hypreDirect.arc index bb62ca1f..5478fe8f 100644 --- a/modules/laplace/inputs/PointDirichlet.bsr.hypreDirect.arc +++ b/modules/laplace/inputs/PointDirichlet.hypreDirect.arc @@ -39,7 +39,6 @@ 50.0 - true 0. 1e-15