More updates to perf. tests

- Add SFAD16 build - reduce blake tests to two nodes - run blake tests regardless of build job status - updates to configs/submission files - Add AIS structured cases and fix sfad type - update ant-8-30km regression values - rebless enthalpy cases (see sandialabs/Albany#966) - increase tolerance on velocity cases (see sandialabs/Albany#712)
sandialabs · Nov 12, 2024 · 751e82e · 751e82e
1 parent 38d75be
commit 751e82e
Show file tree

Hide file tree

Showing 23 changed files with 67 additions and 50 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -104,6 +104,13 @@ IF (DEFINED SFAD12_EXE_DIR)
 ELSE()
   MESSAGE(FATAL_ERROR "SFAD12_EXE_DIR not set!")
 ENDIF()
+IF (DEFINED SFAD16_EXE_DIR)
+  MESSAGE("-- SFAD16_EXE_DIR set: ${SFAD16_EXE_DIR}.")
+  SET(SFAD16ExePath ${SFAD16_EXE_DIR}/src/Albany)
+  SET(SFAD16ExeAnalysisPath ${SFAD16_EXE_DIR}/src/AlbanyAnalysis)
+ELSE()
+  MESSAGE(FATAL_ERROR "SFAD16_EXE_DIR not set!")
+ENDIF()
 IF (DEFINED SFAD24_EXE_DIR)
   MESSAGE("-- SFAD24_EXE_DIR set: ${SFAD24_EXE_DIR}.")
   SET(SFAD24ExePath ${SFAD24_EXE_DIR}/src/Albany) 
@@ -119,9 +126,11 @@ ENDIF()
 
 # Define Albany executables, procs and args
 IF(ALIPT_BUILD_BLAKE)
-  SET(ALBANY_NUM_PROCS 384)
-  # A delay between tests is needed to allow all ranks to unbind
-  SET(ALBANY_MPI_CONFIG "sleep 60 && mpirun -n ${ALBANY_NUM_PROCS} --map-by core")
+  # Note: This causes issues when there are multiple failing tests which don't fail in a clean way and not enough ports are open.
+  # A delay between tests is needed to allow all ranks to unbind.
+  #SET(ALBANY_NUM_PROCS 384)
+  SET(ALBANY_NUM_PROCS 192)
+  SET(ALBANY_MPI_CONFIG "mpirun -n ${ALBANY_NUM_PROCS} --map-by core")
 ELSEIF(ALIPT_BUILD_WEAVER)
   #SET(ALBANY_NUM_PROCS 8) # TempFix: cannot launch multiple nodes on weaver
   SET(ALBANY_NUM_PROCS 4)
@@ -137,9 +146,11 @@ ENDIF()
 SET(ALBANY_BASH "bash" "-lc")
 SET(ALBANY_SFAD6_EXE "${ALBANY_MPI_CONFIG} ${SFAD6ExePath}")
 SET(ALBANY_SFAD12_EXE "${ALBANY_MPI_CONFIG} ${SFAD12ExePath}")
+SET(ALBANY_SFAD16_EXE "${ALBANY_MPI_CONFIG} ${SFAD16ExePath}")
 SET(ALBANY_SFAD24_EXE "${ALBANY_MPI_CONFIG} ${SFAD24ExePath}")
 SET(ALBANY_ANALYSIS_SFAD6_EXE "${ALBANY_MPI_CONFIG} ${SFAD6ExeAnalysisPath}")
 SET(ALBANY_ANALYSIS_SFAD12_EXE "${ALBANY_MPI_CONFIG} ${SFAD12ExeAnalysisPath}")
+SET(ALBANY_ANALYSIS_SFAD16_EXE "${ALBANY_MPI_CONFIG} ${SFAD16ExeAnalysisPath}")
 SET(ALBANY_ANALYSIS_SFAD24_EXE "${ALBANY_MPI_CONFIG} ${SFAD24ExeAnalysisPath}")
 
 enable_testing()

diff --git a/cdash/blake/batch.openmpi.bash.dummy → cdash/blake/batch.openmpi.bash b/cdash/blake/batch.openmpi.bash.dummy → cdash/blake/batch.openmpi.bash
@@ -17,14 +17,16 @@
 ##SBATCH [email protected]
 ##SBATCH --exclude=node03,node06,node10,node27
 #SBATCH --exclusive
-#SBATCH --dependency=afterok:XXX
+#SBATCH --output=log-batch.out
+#SBATCH --error=log-batch.err
 
 nodes=$SLURM_JOB_NUM_NODES           # Number of nodes - the number of nodes you have requested (for a list of SLURM environment variables see "man sbatch")
 cores=96                             # Number MPI processes to run on each node (a.k.a. PPN)
                                      # tlcc2 has 16 cores per node
 
 unset http_proxy
 unset https_proxy
+cd /home/projects/albany/nightlyCDashAlbanyBlake/aliPerfTests/nightlyCDash
 bash nightly_cron_script_ali_perf_tests_blake_run.sh
 bash process_results_ctest.sh 
 bash nightly_cron_script_ali_perf_tests_blake_bzip2_save.sh >& nightly_log_blakeALIPerfTests_saveResults.txt

diff --git a/doc/blake_gcc_modules.sh b/doc/blake_gcc_modules.sh
@@ -2,7 +2,6 @@
 module purge
 module load git cmake
 module load python/3.10.10
-export PATH=${HOME}/.local/bin:${PATH} # Append local python install bin to path for pip3
 module load gcc/12.2.0
 module load boost/1.82.0
 module load metis/5.1.0
@@ -15,4 +14,5 @@ module load superlu-dist/8.1.2
 module load parallel-netcdf/1.12.3
 module load netcdf-c/4.9.2
 module load netcdf-fortran/4.6.0
+source /home/projects/albany/tpls/python/gcc/12.2.0/openmpi/4.1.5/gcc-env/bin/activate # activate python env
 module list
diff --git a/doc/do-configure-blake b/doc/do-configure-blake
@@ -27,6 +27,7 @@ cmake \
   -D TRILINOS_DIR:FILEPATH=/home/projects/albany/nightlyCDashTrilinosBlake/build-gcc/TrilinosReleaseInstallGcc \
   -D SFAD6_EXE_DIR:FILEPATH=/home/projects/albany/nightlyCDashAlbanyBlake/build-gcc/AlbBuildGccSFad6 \
   -D SFAD12_EXE_DIR:FILEPATH=/home/projects/albany/nightlyCDashAlbanyBlake/build-gcc/AlbBuildGccSFad12 \
+  -D SFAD16_EXE_DIR:FILEPATH=/home/projects/albany/nightlyCDashAlbanyBlake/build-gcc/AlbBuildGccSFad16 \
   -D SFAD24_EXE_DIR:FILEPATH=/home/projects/albany/nightlyCDashAlbanyBlake/build-gcc/AlbBuildGccSFad24 \
   -D MESH_FILE_DIR:FILEPATH=/home/projects/albany/ali-perf-tests-meshes \
   -D CMAKE_BUILD_TYPE:STRING=RELEASE \

diff --git a/doc/do-configure-weaver b/doc/do-configure-weaver
@@ -27,6 +27,7 @@ cmake \
   -D TRILINOS_DIR:FILEPATH=/home/projects/albany/nightlyCDashWeaver/build/TrilinosInstall \
   -D SFAD6_EXE_DIR:FILEPATH=/home/projects/albany/nightlyCDashWeaver/build/AlbBuildSFad6 \
   -D SFAD12_EXE_DIR:FILEPATH=/home/projects/albany/nightlyCDashWeaver/build/AlbBuildSFad12 \
+  -D SFAD16_EXE_DIR:FILEPATH=/home/projects/albany/nightlyCDashWeaver/build/AlbBuildSFad16 \
   -D SFAD24_EXE_DIR:FILEPATH=/home/projects/albany/nightlyCDashWeaver/build/AlbBuildSFad24 \
   -D MESH_FILE_DIR:FILEPATH=/home/projects/albany/ali-perf-tests-meshes \
   -D CMAKE_BUILD_TYPE:STRING=RELEASE \

diff --git a/doc/submit_blake.sh b/doc/submit_blake.sh
@@ -3,21 +3,24 @@
 #SBATCH --job-name=ALIPT
 #SBATCH --output=ALIPT.%j.out
 #SBATCH --error=ALIPT.%j.err
-#SBATCH --nodes=4
-#SBATCH --ntasks=384
+#SBATCH --nodes=2
+#SBATCH --ntasks=192
 #SBATCH --ntasks-per-node=96
 #SBATCH --ntasks-per-socket=48
 #SBATCH --time=04:00:00
 #SBATCH --exclusive
 ## SBATCH --exclude=node03,node06,node10,node27,node15
 
 # Load modules
-source ${HOME}/bin/blake_gcc_modules.sh
+source blake_gcc_modules.sh
+#source blake_intel_modules.sh
 
 # Env variables
 export KOKKOS_TOOLS_LIBS=/home/projects/albany/nightlyCDashAlbanyBlake/kokkos-tools/profiling/space-time-stack-mem-only/kp_space_time_stack.so
+#export KOKKOS_TOOLS_LIBS=/home/projects/albany/nightlyCDashAlbanyBlake/kokkos-tools/profiling/space-time-stack-mem-only/intel/kp_space_time_stack.so
 #unset KOKKOS_TOOLS_LIBS
 
 # Run
+#ctest -V --timeout 240
 ctest -V
 
diff --git a/doc/submit_weaver.sh b/doc/submit_weaver.sh
@@ -1,28 +1,27 @@
 #!/bin/bash -login
 
-#BSUB -J GISRun
-#BSUB -o GISRun.%J.out
-#BSUB -e GISRun.%J.err
-#BSUB -q rhel7W
-## BSUB -q dev
-## BSUB -m "weaver2"
-#BSUB -n 8
+#BSUB -J ALIPT
+#BSUB -o ALIPT.%J.out
+#BSUB -e ALIPT.%J.err
+#BSUB -q rhel8
+#BSUB -n 4
 #BSUB -R "span[ptile=4]"
 #BSUB -W 08:00
 #BSUB -x
+#BSUB -gpu num=4
 
 # Limit disk usage for large files
 ulimit -c 0
 
 # Load modules
-source $HOME/bin/weaver_modules.sh
+source weaver_modules.sh
 
 # Env variables
-export KOKKOS_PROFILE_LIBRARY=/home/projects/albany/nightlyCDashWeaver/kokkos-tools/profiling/space-time-stack-mem-only/kp_space_time_stack.so
-#unset KOKKOS_PROFILE_LIBRARY
+export KOKKOS_TOOLS_LIBS=/home/projects/albany/nightlyCDashWeaver/kokkos-tools/profiling/space-time-stack-mem-only/kp_space_time_stack.so
+#unset KOKKOS_TOOLS_LIBS
 export CUDA_LAUNCH_BLOCKING=1
 #unset CUDA_LAUNCH_BLOCKING
-export TPETRA_ASSUME_CUDA_AWARE_MPI=0
+export TPETRA_ASSUME_GPU_AWARE_MPI=0
 
 # Run
 ctest -V

diff --git a/doc/weaver_modules.sh b/doc/weaver_modules.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
-source /projects/ppc64le-pwr9-rhel8/legacy-env.sh
 module purge
+source /projects/ppc64le-pwr9-rhel8/legacy-env.sh
 module load git/2.31.1 cmake/3.24.2
 module load cuda/11.2.2/gcc/8.3.1
 module load openmpi/4.1.1/gcc/8.3.1/cuda/11.2.2
@@ -14,9 +14,7 @@ module load parmetis/4.0.3/gcc/8.3.1/openmpi/4.1.1
 module load boost/1.70.0/gcc/8.3.1
 module load superlu/5.3.0/gcc/8.3.1
 module load ucx/1.12.1/gcc/8.3.1
-module load python
 module list
 
 export CUDA_MANAGED_FORCE_DEVICE_ALLOC=1
 export CUDA_LAUNCH_BLOCKING=1
-
diff --git a/perf_tests/CMakeLists.txt b/perf_tests/CMakeLists.txt
@@ -2,14 +2,16 @@ IF (ALIPT_BUILD_BLAKE)
   add_subdirectory(ant-2-10km)
   add_subdirectory(ant-4-20km)
   add_subdirectory(ant-8-30km)
+  add_subdirectory(ant-struct)
   add_subdirectory(green-1-10km)
   add_subdirectory(green-3-20km)
   add_subdirectory(humboldt-1-10km)
   add_subdirectory(thwaites-1-10km)
 ELSEIF (ALIPT_BUILD_WEAVER)
-  add_subdirectory(ant-4-20km)
+  #add_subdirectory(ant-4-20km) # Too much memory (1 node)
   add_subdirectory(ant-8-30km)
-  add_subdirectory(green-1-10km)
+  #add_subdirectory(ant-struct) # Too much memory (1 node)
+  #add_subdirectory(green-1-10km) # Too much memory (1 node)
   add_subdirectory(green-3-20km)
   add_subdirectory(humboldt-1-10km)
 ENDIF()

diff --git a/perf_tests/ant-8-30km/input_vel_cheby.yaml b/perf_tests/ant-8-30km/input_vel_cheby.yaml
@@ -73,7 +73,7 @@ ANONYMOUS:
 
 # Final solution comparison
   Regression For Response 0:
-    Test Value: -7.114764808893e+00
+    Test Value: -7.516807281563e+00
     Relative Tolerance: 1.0e-05
     Absolute Tolerance: 1.0e-03
 

diff --git a/perf_tests/ant-8-30km/input_vel_lsgs.yaml b/perf_tests/ant-8-30km/input_vel_lsgs.yaml
@@ -73,7 +73,7 @@ ANONYMOUS:
 
 # Final solution comparison
   Regression For Response 0:
-    Test Value: -7.114764808893e+00
+    Test Value: -7.516807281619e+00
     Relative Tolerance: 1.0e-05
     Absolute Tolerance: 1.0e-03
 

diff --git a/perf_tests/ant-struct/CMakeLists.txt b/perf_tests/ant-struct/CMakeLists.txt
@@ -1,9 +1,6 @@
 IF (ALIPT_BUILD_BLAKE)
   add_subdirectory(ant-16km)
   add_subdirectory(ant-8km)
-  add_subdirectory(ant-4km)
-ELSEIF (ALIPT_BUILD_WEAVER)
-  add_subdirectory(ant-16km)
-  add_subdirectory(ant-8km)
+  #add_subdirectory(ant-4km) # Too much memory (2 nodes)
 ENDIF()
 
diff --git a/perf_tests/ant-struct/ant-16km/CMakeLists.txt b/perf_tests/ant-struct/ant-16km/CMakeLists.txt
@@ -22,13 +22,13 @@ execute_process(COMMAND ${ALBANY_SEACAS_DECOMP} --processors ${ALBANY_NUM_PROCS}
 get_filename_component(testName ${CMAKE_CURRENT_SOURCE_DIR} NAME)
 
 # Add test to populate mesh with ascii inputs
-add_test(${testName}_pop_np${ALBANY_NUM_PROCS} ${ALBANY_BASH} "${ALBANY_SFAD12_EXE} input_pop.yaml ${ALBANY_EXTRA_ARGS}")
+add_test(${testName}_pop_np${ALBANY_NUM_PROCS} ${ALBANY_BASH} "${ALBANY_SFAD16_EXE} input_pop.yaml ${ALBANY_EXTRA_ARGS}")
 
 # Add tests for performance
-add_test(${testName}_vel_cheby_np${ALBANY_NUM_PROCS} ${ALBANY_BASH} "${ALBANY_SFAD12_EXE} input_vel_cheby.yaml ${ALBANY_EXTRA_ARGS}")
+add_test(${testName}_vel_cheby_np${ALBANY_NUM_PROCS} ${ALBANY_BASH} "${ALBANY_SFAD16_EXE} input_vel_cheby.yaml ${ALBANY_EXTRA_ARGS}")
 
 IF (NOT ALBANY_ENABLE_CUDA)
-  add_test(${testName}_vel_lsgs_np${ALBANY_NUM_PROCS} ${ALBANY_BASH} "${ALBANY_SFAD12_EXE} input_vel_lsgs.yaml ${ALBANY_EXTRA_ARGS}")
+  add_test(${testName}_vel_lsgs_np${ALBANY_NUM_PROCS} ${ALBANY_BASH} "${ALBANY_SFAD16_EXE} input_vel_lsgs.yaml ${ALBANY_EXTRA_ARGS}")
 ENDIF()
 
 # Add dependencies

diff --git a/perf_tests/ant-struct/ant-1km/CMakeLists.txt b/perf_tests/ant-struct/ant-1km/CMakeLists.txt
@@ -22,13 +22,13 @@ execute_process(COMMAND ${ALBANY_SEACAS_DECOMP} --processors ${ALBANY_NUM_PROCS}
 get_filename_component(testName ${CMAKE_CURRENT_SOURCE_DIR} NAME)
 
 # Add test to populate mesh with ascii inputs
-add_test(${testName}_pop_np${ALBANY_NUM_PROCS} ${ALBANY_BASH} "${ALBANY_SFAD12_EXE} input_pop.yaml ${ALBANY_EXTRA_ARGS}")
+add_test(${testName}_pop_np${ALBANY_NUM_PROCS} ${ALBANY_BASH} "${ALBANY_SFAD16_EXE} input_pop.yaml ${ALBANY_EXTRA_ARGS}")
 
 # Add tests for performance
-add_test(${testName}_vel_cheby_np${ALBANY_NUM_PROCS} ${ALBANY_BASH} "${ALBANY_SFAD12_EXE} input_vel_cheby.yaml ${ALBANY_EXTRA_ARGS}")
+add_test(${testName}_vel_cheby_np${ALBANY_NUM_PROCS} ${ALBANY_BASH} "${ALBANY_SFAD16_EXE} input_vel_cheby.yaml ${ALBANY_EXTRA_ARGS}")
 
 IF (NOT ALBANY_ENABLE_CUDA)
-  add_test(${testName}_vel_lsgs_np${ALBANY_NUM_PROCS} ${ALBANY_BASH} "${ALBANY_SFAD12_EXE} input_vel_lsgs.yaml ${ALBANY_EXTRA_ARGS}")
+  add_test(${testName}_vel_lsgs_np${ALBANY_NUM_PROCS} ${ALBANY_BASH} "${ALBANY_SFAD16_EXE} input_vel_lsgs.yaml ${ALBANY_EXTRA_ARGS}")
 ENDIF()
 
 # Add dependencies

diff --git a/perf_tests/ant-struct/ant-2km/CMakeLists.txt b/perf_tests/ant-struct/ant-2km/CMakeLists.txt
@@ -22,13 +22,13 @@ execute_process(COMMAND ${ALBANY_SEACAS_DECOMP} --processors ${ALBANY_NUM_PROCS}
 get_filename_component(testName ${CMAKE_CURRENT_SOURCE_DIR} NAME)
 
 # Add test to populate mesh with ascii inputs
-add_test(${testName}_pop_np${ALBANY_NUM_PROCS} ${ALBANY_BASH} "${ALBANY_SFAD12_EXE} input_pop.yaml ${ALBANY_EXTRA_ARGS}")
+add_test(${testName}_pop_np${ALBANY_NUM_PROCS} ${ALBANY_BASH} "${ALBANY_SFAD16_EXE} input_pop.yaml ${ALBANY_EXTRA_ARGS}")
 
 # Add tests for performance
-add_test(${testName}_vel_cheby_np${ALBANY_NUM_PROCS} ${ALBANY_BASH} "${ALBANY_SFAD12_EXE} input_vel_cheby.yaml ${ALBANY_EXTRA_ARGS}")
+add_test(${testName}_vel_cheby_np${ALBANY_NUM_PROCS} ${ALBANY_BASH} "${ALBANY_SFAD16_EXE} input_vel_cheby.yaml ${ALBANY_EXTRA_ARGS}")
 
 IF (NOT ALBANY_ENABLE_CUDA)
-  add_test(${testName}_vel_lsgs_np${ALBANY_NUM_PROCS} ${ALBANY_BASH} "${ALBANY_SFAD12_EXE} input_vel_lsgs.yaml ${ALBANY_EXTRA_ARGS}")
+  add_test(${testName}_vel_lsgs_np${ALBANY_NUM_PROCS} ${ALBANY_BASH} "${ALBANY_SFAD16_EXE} input_vel_lsgs.yaml ${ALBANY_EXTRA_ARGS}")
 ENDIF()
 
 # Add dependencies

diff --git a/perf_tests/ant-struct/ant-4km/CMakeLists.txt b/perf_tests/ant-struct/ant-4km/CMakeLists.txt
@@ -22,13 +22,13 @@ execute_process(COMMAND ${ALBANY_SEACAS_DECOMP} --processors ${ALBANY_NUM_PROCS}
 get_filename_component(testName ${CMAKE_CURRENT_SOURCE_DIR} NAME)
 
 # Add test to populate mesh with ascii inputs
-add_test(${testName}_pop_np${ALBANY_NUM_PROCS} ${ALBANY_BASH} "${ALBANY_SFAD12_EXE} input_pop.yaml ${ALBANY_EXTRA_ARGS}")
+add_test(${testName}_pop_np${ALBANY_NUM_PROCS} ${ALBANY_BASH} "${ALBANY_SFAD16_EXE} input_pop.yaml ${ALBANY_EXTRA_ARGS}")
 
 # Add tests for performance
-add_test(${testName}_vel_cheby_np${ALBANY_NUM_PROCS} ${ALBANY_BASH} "${ALBANY_SFAD12_EXE} input_vel_cheby.yaml ${ALBANY_EXTRA_ARGS}")
+add_test(${testName}_vel_cheby_np${ALBANY_NUM_PROCS} ${ALBANY_BASH} "${ALBANY_SFAD16_EXE} input_vel_cheby.yaml ${ALBANY_EXTRA_ARGS}")
 
 IF (NOT ALBANY_ENABLE_CUDA)
-  add_test(${testName}_vel_lsgs_np${ALBANY_NUM_PROCS} ${ALBANY_BASH} "${ALBANY_SFAD12_EXE} input_vel_lsgs.yaml ${ALBANY_EXTRA_ARGS}")
+  add_test(${testName}_vel_lsgs_np${ALBANY_NUM_PROCS} ${ALBANY_BASH} "${ALBANY_SFAD16_EXE} input_vel_lsgs.yaml ${ALBANY_EXTRA_ARGS}")
 ENDIF()
 
 # Add dependencies

diff --git a/perf_tests/ant-struct/ant-8km/CMakeLists.txt b/perf_tests/ant-struct/ant-8km/CMakeLists.txt
@@ -22,13 +22,13 @@ execute_process(COMMAND ${ALBANY_SEACAS_DECOMP} --processors ${ALBANY_NUM_PROCS}
 get_filename_component(testName ${CMAKE_CURRENT_SOURCE_DIR} NAME)
 
 # Add test to populate mesh with ascii inputs
-add_test(${testName}_pop_np${ALBANY_NUM_PROCS} ${ALBANY_BASH} "${ALBANY_SFAD12_EXE} input_pop.yaml ${ALBANY_EXTRA_ARGS}")
+add_test(${testName}_pop_np${ALBANY_NUM_PROCS} ${ALBANY_BASH} "${ALBANY_SFAD16_EXE} input_pop.yaml ${ALBANY_EXTRA_ARGS}")
 
 # Add tests for performance
-add_test(${testName}_vel_cheby_np${ALBANY_NUM_PROCS} ${ALBANY_BASH} "${ALBANY_SFAD12_EXE} input_vel_cheby.yaml ${ALBANY_EXTRA_ARGS}")
+add_test(${testName}_vel_cheby_np${ALBANY_NUM_PROCS} ${ALBANY_BASH} "${ALBANY_SFAD16_EXE} input_vel_cheby.yaml ${ALBANY_EXTRA_ARGS}")
 
 IF (NOT ALBANY_ENABLE_CUDA)
-  add_test(${testName}_vel_lsgs_np${ALBANY_NUM_PROCS} ${ALBANY_BASH} "${ALBANY_SFAD12_EXE} input_vel_lsgs.yaml ${ALBANY_EXTRA_ARGS}")
+  add_test(${testName}_vel_lsgs_np${ALBANY_NUM_PROCS} ${ALBANY_BASH} "${ALBANY_SFAD16_EXE} input_vel_lsgs.yaml ${ALBANY_EXTRA_ARGS}")
 ENDIF()
 
 # Add dependencies

diff --git a/perf_tests/green-3-20km/input_ent_gs.yaml b/perf_tests/green-3-20km/input_ent_gs.yaml
@@ -69,7 +69,7 @@ ANONYMOUS:
 
 # Final solution comparison
   Regression For Response 0:
-    Test Value: -7.152645593729e+00
+    Test Value: -7.183957424818e+00
     Relative Tolerance: 1.00000000000000008e-05
     Absolute Tolerance: 1.00000000000000002e-03
 

diff --git a/perf_tests/green-3-20km/input_vel_cheby.yaml b/perf_tests/green-3-20km/input_vel_cheby.yaml
@@ -83,7 +83,8 @@ ANONYMOUS:
   Regression For Response 0:
     Test Value: -7.005509894455e+00
     Relative Tolerance: 1.0e-05
-    Absolute Tolerance: 1.0e-03
+    Absolute Tolerance: 3.0e-03 # see https://github.com/sandialabs/Albany/issues/712
+    #Absolute Tolerance: 1.0e-03
 
 # Discretization Description
   Discretization: 

diff --git a/perf_tests/green-3-20km/input_vel_lsgs.yaml b/perf_tests/green-3-20km/input_vel_lsgs.yaml
@@ -83,7 +83,8 @@ ANONYMOUS:
   Regression For Response 0:
     Test Value: -7.005509894455e+00
     Relative Tolerance: 1.0e-05
-    Absolute Tolerance: 1.0e-03
+    Absolute Tolerance: 3.0e-03 # see https://github.com/sandialabs/Albany/issues/712
+    #Absolute Tolerance: 1.0e-03
 
 # Discretization Description
   Discretization: 

diff --git a/perf_tests/humboldt-1-10km/input_ent_lsgs.yaml b/perf_tests/humboldt-1-10km/input_ent_lsgs.yaml
@@ -69,7 +69,7 @@ ANONYMOUS:
 
 # Final solution comparison
   Regression For Response 0:
-    Test Value: -5.291207356861e+00
+    Test Value: -5.038193461993e+00
     Relative Tolerance: 1.0e-05
     Absolute Tolerance: 1.0e-03
 

diff --git a/perf_tests/thwaites-1-10km/input_cop_if2.yaml b/perf_tests/thwaites-1-10km/input_cop_if2.yaml
@@ -325,5 +325,6 @@ ANONYMOUS:
                   Prec Type: Amesos2
                   Ifpack2 Settings:
                     Amesos2: {}
+                    #Amesos2 solver name: klu # does not converge
                     Amesos2 solver name: superlu
 ...
diff --git a/perf_tests/thwaites-1-10km/input_ent_lsgs.yaml b/perf_tests/thwaites-1-10km/input_ent_lsgs.yaml
@@ -69,7 +69,7 @@ ANONYMOUS:
 
 # Final solution comparison
   Regression For Response 0:
-    Test Value: -1.049701482625e+01
+    Test Value: -1.051159402092e+01
     Relative Tolerance: 1.0e-05
     Absolute Tolerance: 1.0e-03