Docker/Singularity build of PETSc / FiniteVolume integration.

PETSc 3.7.7 and mpich 3.1.3 are proper CMake dependencies and
   are statically linked into FinteVolume_PETSc_x64
Dockerfile-PETSc builds only PETSc FiniteVolume (useful for
   development).
Dockerfile builds PETSc and non-PETSc solvers - used within
   vcell project as basis for the vcell-batch Singularity image.

CMakeLists.txt

@@ -35,11 +35,15 @@ elseif (LINUX)
 endif()
 
 project(numerics)
-cmake_minimum_required(VERSION 3.0)
+cmake_minimum_required(VERSION 3.3)
 enable_language(CXX)
 enable_language(C)
 enable_language(Fortran)
 
+if(POLICY CMP0057)
+  cmake_policy(SET CMP0057 NEW)
+endif()
+
 set(SVERSION "unversionedbuild")
 message("svnversion ${SVERSION}")
 add_definitions(-DSVNVERSION=svn${SVERSION})
@@ -49,7 +53,7 @@ add_definitions(-DSVNVERSION=svn${SVERSION})
 set(RULE_MESSAGES OFF)
 set(CMAKE_COLOR_MAKEFILE OFF)
 set(_ECLIPSE_VERSION "4.4")
-set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/")
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/modules/")
 
 set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
@@ -80,6 +84,7 @@ include (FindZLIB)
 
 option(OPTION_TARGET_MESSAGING "Messaging (requires libcurl)" off)
 option(OPTION_TARGET_PARALLEL "Parallel Solver (only chombo for now)" off)
+option(OPTION_TARGET_PETSC "PETSc Solver (only FV for now)" off)
 option(OPTION_TARGET_DOCS "Generate Doxygen documentation" on)
 option(OPTION_TARGET_CHOMBO2D_SOLVER "Chombo2D Solver" off)
 option(OPTION_TARGET_CHOMBO3D_SOLVER "Chombo3D Solver" off)
@@ -205,6 +210,10 @@ if (OPTION_TARGET_MESSAGING)
 	add_definitions(-DUSE_MESSAGING)
 endif()
 
+if (OPTION_TARGET_PETSC)
+	find_package(PETSc REQUIRED)
+endif()
+
 ######################################
 #
 # Add subdirectories

Dockerfile

@@ -2,16 +2,34 @@ FROM ubuntu:xenial as build
 
 RUN apt-get -y update && apt-get install -y apt-utils && \
     apt-get install -y -qq -o=Dpkg::Use-Pty=0 build-essential gfortran zlib1g-dev \
-    libhdf5-dev libcurl4-openssl-dev libboost-dev cmake
+    libhdf5-dev libcurl4-openssl-dev libboost-dev cmake wget python
+
+#
+# build PETSc with mpich for static linking
+#
+RUN mkdir /usr/local/petsc && \
+    cd /usr/local/petsc && \
+    wget http://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc-3.7.7.tar.gz && \
+    tar xzf petsc-3.7.7.tar.gz && \
+    cd petsc-3.7.7 && \
+    ./configure --with-shared-libraries=0 --download-fblaslapack=1 --with-debugging=1 --download-mpich && \
+    make PETSC_DIR=/usr/local/petsc/petsc-3.7.7 PETSC_ARCH=arch-linux2-c-debug all
+
+ENV PETSC_DIR=/usr/local/petsc/petsc-3.7.7 \
+    PETSC_ARCH=arch-linux2-c-debug
 
 COPY . /vcellroot
 
-RUN mkdir build && mkdir build/bin
+#
+# build most solvers, and FiniteVolume without PETSc (FiniteVolume_x64)
+#
+RUN mkdir -p /vcellroot/build/bin
 WORKDIR /vcellroot/build
 
 RUN /usr/bin/cmake \
     -DOPTION_TARGET_MESSAGING=ON \
     -DOPTION_TARGET_PARALLEL=OFF \
+    -DOPTION_TARGET_PETSC=OFF \
     -DOPTION_TARGET_CHOMBO2D_SOLVER=OFF \
     -DOPTION_TARGET_CHOMBO3D_SOLVER=OFF \
     -DOPTION_TARGET_SMOLDYN_SOLVER=ON \
@@ -24,12 +42,36 @@ RUN /usr/bin/cmake \
     .. && \
     make
 
+#
+# build FiniteVolume with PETSc (FiniteVolume_PETSc_x64)
+#
+RUN mkdir -p /vcellroot/build_PETSc/bin
+WORKDIR /vcellroot/build_PETSc
+
+RUN /usr/bin/cmake \
+    -DOPTION_TARGET_MESSAGING=ON \
+    -DOPTION_TARGET_PARALLEL=OFF \
+    -DOPTION_TARGET_PETSC=ON \
+    -DOPTION_TARGET_CHOMBO2D_SOLVER=OFF \
+    -DOPTION_TARGET_CHOMBO3D_SOLVER=OFF \
+    -DOPTION_TARGET_SMOLDYN_SOLVER=OFF \
+    -DOPTION_TARGET_FV_SOLVER=ON \
+    -DOPTION_TARGET_STOCHASTIC_SOLVER=OFF \
+    -DOPTION_TARGET_NFSIM_SOLVER=OFF \
+    -DOPTION_TARGET_MOVINGBOUNDARY_SOLVER=OFF \
+    -DOPTION_TARGET_SUNDIALS_SOLVER=OFF \
+    -DOPTION_TARGET_HY3S_SOLVERS=OFF \
+    .. && \
+    make
+
+
 FROM ubuntu:xenial
 
 RUN apt-get -y update && apt-get install -y apt-utils && \
     apt-get install -y -qq -o=Dpkg::Use-Pty=0 gcc gfortran zlib1g \
     libhdf5-10 libhdf5-cpp-11 libcurl4-openssl-dev 
 
 COPY --from=build /vcellroot/build/bin /vcellbin
+COPY --from=build /vcellroot/build_PETSc/bin/FiniteVolume_PETSc_x64 /vcellbin/
 WORKDIR /vcellbin
 ENV PATH=/vcellbin:$PATH

Dockerfile-PETSc

@@ -0,0 +1,44 @@
+FROM ubuntu:xenial as build_PETSc
+
+RUN apt-get -y update && apt-get install -y apt-utils && \
+    apt-get install -y -qq -o=Dpkg::Use-Pty=0 build-essential gfortran zlib1g-dev \
+    libhdf5-dev libcurl4-openssl-dev libboost-dev cmake wget python
+
+RUN mkdir /usr/local/petsc && \
+    cd /usr/local/petsc && \
+    wget http://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc-3.7.7.tar.gz && \
+    tar xzf petsc-3.7.7.tar.gz && \
+    cd petsc-3.7.7 && \
+    ./configure --with-shared-libraries=0 --download-fblaslapack=1 --with-debugging=1 --download-mpich && \
+    make PETSC_DIR=/usr/local/petsc/petsc-3.7.7 PETSC_ARCH=arch-linux2-c-debug all
+
+ENV PETSC_DIR=/usr/local/petsc/petsc-3.7.7 \
+    PETSC_ARCH=arch-linux2-c-debug
+
+# to test installation of PETSc
+# make PETSC_DIR=/usr/local/petsc/petsc-3.7.7 PETSC_ARCH=arch-linux2-c-debug test
+# make PETSC_DIR=/usr/local/petsc/petsc-3.7.7 PETSC_ARCH=arch-linux2-c-debug streams
+
+
+COPY . /vcellroot
+
+RUN mkdir -p /vcellroot/build_PETSc/bin
+WORKDIR /vcellroot/build_PETSc
+
+RUN /usr/bin/cmake \
+    -DOPTION_TARGET_MESSAGING=ON \
+    -DOPTION_TARGET_PARALLEL=OFF \
+    -DOPTION_TARGET_PETSC=ON \
+    -DOPTION_TARGET_CHOMBO2D_SOLVER=OFF \
+    -DOPTION_TARGET_CHOMBO3D_SOLVER=OFF \
+    -DOPTION_TARGET_SMOLDYN_SOLVER=OFF \
+    -DOPTION_TARGET_FV_SOLVER=ON \
+    -DOPTION_TARGET_STOCHASTIC_SOLVER=OFF \
+    -DOPTION_TARGET_NFSIM_SOLVER=OFF \
+    -DOPTION_TARGET_MOVINGBOUNDARY_SOLVER=OFF \
+    -DOPTION_TARGET_SUNDIALS_SOLVER=OFF \
+    -DOPTION_TARGET_HY3S_SOLVERS=OFF \
+    .. && \
+    make
+
+

VCell/CMakeLists.txt

@@ -86,6 +86,9 @@ set(HEADER_FILES
 	include/VCELL/VolumeVarContextExpression.h
 	include/VCELL/VolumeVariable.h	
 )
+if (OPTION_TARGET_PETSC)
+	list(APPEND HEADER_FILES  include/VCELL/PetscPdeScheduler.h)
+endif()
 
 set(SRC_FILES 
 	src/AlgebraicSystem.cpp
@@ -165,6 +168,10 @@ set(SRC_FILES
 	src/VolumeVarContextExpression.cpp
 	src/VolumeVariable.cpp	
 )
+if (OPTION_TARGET_PETSC)
+	list(APPEND SRC_FILES  src/PetscPdeScheduler.cpp)
+endif()
+
 
 set (PCGWRAPPER_SRC_FILES
 	src/pcgwrapper.f
@@ -212,16 +219,28 @@ add_library(pcgwrapper STATIC ${PCGWRAPPER_SRC_FILES})
 target_link_libraries(pcgwrapper PCGPack blas)
 
 add_library(vcell STATIC ${SRC_FILES} ${HEADER_FILES})
-target_link_libraries(vcell pcgwrapper sundials ExpressionParser smoldyn_static vcellsmoldynbridge ${HDF5_LIBRARIES} qhull vcellmessaging vcellziputils ${ZLIB_LIBRARIES})
-
-target_include_directories(vcell PUBLIC 
-  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
-  ${HDF5_INCLUDE_DIRS}
-  $<INSTALL_INTERFACE:include>  # <prefix>/include
-)
+if (OPTION_TARGET_PETSC)
+	target_compile_definitions(vcell PUBLIC -DVCELL_PETSC)
+	target_link_libraries(vcell pcgwrapper sundials ${PETSC_LIBRARIES} ExpressionParser smoldyn_static vcellsmoldynbridge ${HDF5_LIBRARIES} qhull vcellmessaging vcellziputils ${ZLIB_LIBRARIES})
+	target_include_directories(vcell PUBLIC 
+	  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+	  ${HDF5_INCLUDE_DIRS} ${PETSC_INCLUDES}
+	  $<INSTALL_INTERFACE:include>  # <prefix>/include
+	)
+else()
+	target_link_libraries(vcell pcgwrapper sundials ExpressionParser smoldyn_static vcellsmoldynbridge ${HDF5_LIBRARIES} qhull vcellmessaging vcellziputils ${ZLIB_LIBRARIES})
+	target_include_directories(vcell PUBLIC 
+	  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+	  ${HDF5_INCLUDE_DIRS}
+	  $<INSTALL_INTERFACE:include>  # <prefix>/include
+	)
+endif()
 
 if (OPTION_TARGET_FV_SOLVER)
 	set(EXE_FILE FiniteVolume)
+	if (OPTION_TARGET_PETSC) 
+		set(EXE_FILE ${EXE_FILE}_PETSc)
+	endif()
 	if (ARCH_64bit) 
 		set(EXE_FILE ${EXE_FILE}_x64)
 	endif()

VCell/include/VCELL/PetscPdeScheduler.h

@@ -39,6 +39,7 @@ class PetscPdeScheduler : public Scheduler
 	Vec oldY;
 	SNES snes;
 	Mat Pmat;
+	double internalDt;
 
 	void computeNonZeros(int* nnz);
 	void buildJ_Volume(double t, double* yinput, Mat Pmat);

VCell/src/PetscPdeScheduler.cpp

@@ -70,6 +70,8 @@ PetscPdeScheduler::PetscPdeScheduler(Simulation *sim) : Scheduler(sim)
 	neighborStatePointValues = 0;
 
 	diffCoeffs = 0;
+
+	internalDt = sim->getDT_sec();
 }
 
 
@@ -479,18 +481,26 @@ PetscErrorCode PetscPdeScheduler::initPetscSolver() {
 	}
 }
 
+static int numInternalIter = 32;
+static int linitsHi = 45;
+static int linitsLo = 20;
+static int maxInternalIter = numInternalIter;
+static double maxInteralIterTime = 0;
 PetscErrorCode PetscPdeScheduler::solve() {
-	static string METHOD = "PetscPdeScheduler::solve";
-	cout << "Entry " << METHOD << endl;
+	//static string METHOD = "PetscPdeScheduler::solve";
+	//cout << "Entry " << METHOD << endl;
 
-	int nonlinits, linits;
-	VecCopy(vecY, oldY);
-//	snes_buildPmat(vecY, Pmat);
-	SNESSolve(snes, oldY, vecY);
+	internalDt = simulation->getDT_sec() / numInternalIter;
+	for (int iter = 0; iter < numInternalIter; ++ iter)
+	{
+		VecCopy(vecY, oldY);
+		SNESSolve(snes, oldY, vecY);
 
+		currentTime += internalDt;
+	}
+	int nonlinits, linits;
 	KSP ksp;
 	PC pc;
-	// collect information
 	SNESGetKSP(snes, &ksp);
 	KSPGetPC(ksp, &pc);
 	SNESGetIterationNumber(snes, &nonlinits);
@@ -505,29 +515,29 @@ PetscErrorCode PetscPdeScheduler::solve() {
 	PCFactorGetLevels(pc, &factorLevel);
 	PetscBool reuse;
 	PCGetReusePreconditioner(pc, &reuse);
-	PetscPrintf(PETSC_COMM_WORLD, "pctype %s, factor level %d, pc reuse %d, gmres restart %d, "
-			"nonlinits %D, linits %D\n", pctype, factorLevel, reuse, restart, nonlinits, linits);
-	currentTime = currentTime + simulation->getDT_sec();
+	PetscPrintf(PETSC_COMM_WORLD, "\nt %f, internalits %d, internal-dt %f, pctype %s, factor level %d, pc reuse %d, gmres restart %d, "
+			"nonlinits %D, linits %D\n", currentTime, numInternalIter, internalDt, pctype, factorLevel, reuse, restart, nonlinits, linits);
+	updateSolutions();
 
-	/*
-	PetscInt  steps, nonlinits, linits, snesfails, rejects;
-	TSConvergedReason reason;
-	PetscReal ftime;
-
-	TSGetTimeStepNumber(ts,&steps);
-	TSGetSNESFailures(ts, &snesfails);
-  TSGetStepRejections(ts, &rejects);
-	TSGetSNESIterations(ts, &nonlinits);
-	TSGetKSPIterations(ts, &linits);
-	TSGetSolveTime(ts,&ftime);
-	TSGetConvergedReason(ts, &reason);
-
-	PetscPrintf(PETSC_COMM_WORLD, "%s at steps %D (%D rejected, %D SNES fails), ftime %g, nonlinits %D, linits %D\n",
-			TSConvergedReasons[reason], steps, rejects, snesfails,(double)ftime, nonlinits, linits);
-*/
+	if (linits > linitsHi)
+	{
+		numInternalIter *= 2;
+	}
+	else if (linits < linitsLo && numInternalIter >= 2)
+	{
+		numInternalIter = numInternalIter / 2;
+	}
+	if (maxInternalIter < numInternalIter)
+	{
+		maxInternalIter = std::max(maxInternalIter, numInternalIter);
+		maxInteralIterTime = currentTime;
+	}
 
-	updateSolutions();
-	cout << "Exit " << METHOD << endl;
+	if (std::abs(SimTool::getInstance()->getEndTime() - simulation->getTime_sec()) < epsilon)
+	{
+		PetscPrintf(PETSC_COMM_WORLD, "\nmaxInternalIter:%d, maxInteralIterTime:%f", maxInternalIter, maxInteralIterTime);
+	}
+	//cout << "Exit " << METHOD << endl;
 }
 
 void PetscPdeScheduler::updateSolutions() {
@@ -2141,8 +2151,7 @@ PetscErrorCode PetscPdeScheduler::snes_function(SNES snes, Vec Y, Vec F)
 	VecRestoreArrayRead(Y, (const PetscScalar**)&yinput);
 	VecRestoreArray(F, (PetscScalar**)&f);
 
-	double dt = simulation->getDT_sec();
-	VecAXPBY(F, 1.0 , -dt, Y);
+	VecAXPBY(F, 1.0 , -internalDt, Y);
 
 	return 0;
 }
@@ -2377,8 +2386,8 @@ PetscErrorCode PetscPdeScheduler::snes_buildPmat(Vec Y, Mat Pmat)
 	double *yinput;
 	VecGetArrayRead(Y, (const double**)&yinput);
 
-	buildPmat_Volume(currentTime, simulation->getDT_sec(), yinput, Pmat);
-	buildPmat_Membrane(currentTime, simulation->getDT_sec(), yinput, Pmat);
+	buildPmat_Volume(currentTime, internalDt, yinput, Pmat);
+	buildPmat_Membrane(currentTime, internalDt, yinput, Pmat);
 
 	// volume region variables, diagonal = 1.0;
 	if (simulation->getNumVolRegionVariables() != 0) {
@@ -2439,7 +2448,7 @@ PetscErrorCode PetscPdeScheduler::snes_jacobian(SNES snes, Vec Y, Mat Amat, Mat
 	MatView(Pmat, viewer);
 	PetscViewerDestroy(&viewer);
 */
-	cout << " # snes_jacobian called " << count << endl;
+	//cout << " # snes_jacobian called " << count << endl;
 	return 0;
 }
 

VCell/src/Simulation.cpp

@@ -17,7 +17,7 @@
 #include <VCELL/SerialScheduler.h>
 #include <VCELL/SundialsPdeScheduler.h>
 #ifdef VCELL_PETSC
-#include <VCELL/PetscPdeScheduler.h>
+  #include <VCELL/PetscPdeScheduler.h>
 #endif
 #include <VCELL/PostProcessingBlock.h>
 
@@ -188,7 +188,7 @@ void Simulation::initSimulation()
 #ifdef VCELL_PETSC
 			_scheduler = new PetscPdeScheduler(this);
 #else
-			throw "VCELL_PETSC_SOLVER is not supported  yet";
+			throw "VCELL_PETSC_SOLVER must be built with VCELL_PETSC flag";
 #endif
 		} else {
 			_scheduler = new SerialScheduler(this);

cmake/modules/CorrectWindowsPaths.cmake

@@ -0,0 +1,13 @@
+# CorrectWindowsPaths - this module defines one macro
+#
+# CONVERT_CYGWIN_PATH( PATH )
+#  This uses the command cygpath (provided by cygwin) to convert
+#  unix-style paths into paths useable by cmake on windows
+
+macro (CONVERT_CYGWIN_PATH _path)
+  if (WIN32)
+    EXECUTE_PROCESS(COMMAND cygpath.exe -m ${${_path}}
+      OUTPUT_VARIABLE ${_path})
+    string (STRIP ${${_path}} ${_path})
+  endif (WIN32)
+endmacro (CONVERT_CYGWIN_PATH)