Add --mpi option (Closes #216)

... for manual choice of MPI or not. Previous launchedWithMpi heuristics are getting too complicated.

Argument mandatory for simulator. Optional for optimization. (Ease of implementation)

Cleanup output

Avoid errors if run with --help

Split CLI option parsing into pre-MPI and post-MPI

doc/snakemake_workflow.md

@@ -23,7 +23,7 @@ provided in `snakemake/config.schema.yaml`.
 After that you can run the full pipeline with:
 
     cd snakemake
-    snakemake --configfile parpe_optimize_petab_steadystate.yaml postprocess
+    snakemake --configfile parpe_optimize_petab_steadystate.yaml -- postprocess
 
 This generate C++ code of the model, build model specific binaries for
 parameter estimation, run parameters, and process the results.

examples/parpeamici/steadystate/main_simulator.cpp

@@ -1,22 +1,54 @@
+#include <parpecommon/parpeConfig.h>
+
 #include "steadyStateMultiConditionDataprovider.h"
 
 #include <parpeamici/standaloneSimulator.h>
 #include <parpecommon/misc.h>
+#include <parpecommon/parpeConfig.h>
 
 #include <cstdio> // remove
 #include <iostream>
+#include <stdexcept>
+
+#ifdef PARPE_ENABLE_MPI
+#include <mpi.h>
+#endif
 
 std::unique_ptr<amici::Model> getModel();
 
+void printUsage() {
+    std::cerr<<"Error: wrong number of arguments.\n";
+    std::cerr<<"Usage: ... CONDITION_FILE_NAME CONDITION_FILE_PATH "
+               "[PARAMETER_FILE_NAME PARAMETER_FILE_PATH] "
+               "OUTFILENAME OUTFILEPATH "
+               "--at-optimum|--along-trajectory "
+               "--mpi|--nompi\n";
+    // |--parameter-matrix=PATH-UNSUPPORTED
+}
+
 int main(int argc, char **argv) {
     int status = EXIT_SUCCESS;
 
-    parpe::initMpiIfNeeded(&argc, &argv);
+    if(argc != 7 && argc != 9) {
+        printUsage();
+        return EXIT_FAILURE;
+    }
+
+
+    if(std::string(argv[argc -1]) == "--mpi") {
+#ifdef PARPE_ENABLE_MPI
+        MPI_Init(&argc, &argv);
+#else
+        throw std::runtime_error("parPE was built without MPI support.");
+#endif
+    } else if(std::string(argv[argc -1]) == "--nompi") {
+        ;
+    } else {
+        printUsage();
+        return EXIT_FAILURE;
+    }
 
-    switch(argc)
-    {
-    case 6:
-    {
+    if(argc == 7) {
         std::string dataFileName = argv[1];
         std::string dataFilePath = argv[2];
         std::string resultFileName = argv[3];
@@ -32,10 +64,7 @@ int main(int argc, char **argv) {
                                      dataFileName, dataFilePath,
                                      dataFileName, dataFilePath,
                                      resultFileName, resultPath);
-        break;
-    }
-    case 8:
-    {
+    } else if(argc == 9) {
         // simulate on test set: need optimizer result and test set data as inputs
         std::string conditionFileName = argv[1];
         std::string conditionFilePath = argv[2];
@@ -66,12 +95,6 @@ int main(int argc, char **argv) {
                                      conditionFileName, conditionFilePath,
                                      parameterFileName, parameterFilePath,
                                      resultFileName, resultPath);
-        break;
-    }
-    default:
-        std::cerr<<"Error: wrong number of arguments.\n";
-        std::cerr<<"Usage: ... CONDITION_FILE_NAME CONDITION_FILE_PATH [PARAMETER_FILE_NAME PARAMETER_FILE_PATH] OUTFILENAME OUTFILEPATH --at-optimum|--along-trajectory\n"; // |--parameter-matrix=PATH-UNSUPPORTED
-        status = EXIT_FAILURE;
     }
 
     parpe::finalizeMpiIfNeeded();

examples/parpeamici/steadystate/run-examples.sh

@@ -34,7 +34,7 @@ rm -rf example_steadystate_multi-test-optimize/
 rm -f simulate1.h5
 ./example_steadystate_multi_simulator \
   example_steadystate_multi-test-optimize/_rank00000.h5 / simulate1.h5 / \
-  --at-optimum 2>&1 > test.log
+  --at-optimum --nompi 2>&1 > test.log
 (! grep ERR test.log)
 (! grep WRN test.log)
 (! grep exception test.log)
@@ -48,7 +48,7 @@ test -f simulate1.h5
 # Run optimization with default settings
 
 rm -rf example_steadystate_multi-test-optimize/
-${MPIEXEC} ./example_steadystate_multi \
+${MPIEXEC} ./example_steadystate_multi --mpi \
   -o example_steadystate_multi-test-optimize/ ${HDF5_FILE} 2>&1 >> test.log
 (! grep ERR test.log)
 (! grep WRN test.log)
@@ -58,7 +58,7 @@ ${MPIEXEC} ./example_steadystate_multi \
 rm -f simulate2.h5
 ${MPIEXEC} ./example_steadystate_multi_simulator \
   example_steadystate_multi-test-optimize/_rank00000.h5 / simulate2.h5 / \
-  --along-trajectory 2>&1 >> test.log
+  --along-trajectory --mpi 2>&1 >> test.log
 (! grep ERR test.log)
 (! grep WRN test.log)
 (! grep exception test.log)
@@ -67,11 +67,10 @@ test -f simulate2.h5
 
 # Simulate on test set
 
-
 rm -f simulate3.h5
 ${MPIEXEC} ./example_steadystate_multi_simulator \
   ${HDF5_FILE_TEST} / example_steadystate_multi-test-optimize/_rank00000.h5 / \
-  simulate3.h5 / --at-optimum
+  simulate3.h5 / --at-optimum --mpi
 h5dump -d /multistarts/0/ySim/3 simulate3.h5 # test dataset exists
 (! grep ERR test.log)
 (! grep WRN test.log)

include/parpeamici/optimizationApplication.h

@@ -35,7 +35,8 @@ class OptimizationApplication {
 
     /**
      * @brief User-provided problem initialization.
-     * Must set OptimizationApplication::problem, OptimizationApplication::multiStartOptimization and should set
+     * Must set OptimizationApplication::problem,
+     * OptimizationApplication::multiStartOptimization and should set
      * OptimizationApplication::resultWriter
      * @param inFileArgument
      * @param outFileArgument
@@ -45,14 +46,15 @@ class OptimizationApplication {
 
     /**
      * @brief Start the optimization run. Must only be called once.
+     * Initializes MPI if not already done.
+     * Must be called before any other functions.
      * @return status code; 0 on success
      */
     int run(int argc, char **argv);
 
     /**
      * @brief This is run by the MPI rank 0 process when started with multiple
      * processes.
-     * @return
      */
     virtual void runMaster();
 
@@ -105,14 +107,26 @@ class OptimizationApplication {
     static void initMPI(int *argc, char ***argv);
 
     /**
-     * @brief Parse command line Options.
-     * Must be called before any other functions.
-     * Initializes MPI if not already done.
+     * @brief Parse command line options before MPI_INIT is potentially called.
+     *
+     * Used e.g. to print usage information without first initialization MPI.
+     *
+     * Argv may contain extra MPI arguments.
+     * @param argc
+     * @param argv
+     * @return
+     */
+    virtual int parseCliOptionsPreMpiInit(int argc, char **argv);
+
+    /**
+     * @brief Parse command line options after MPI_Init is called.
+     *
+     * Any MPI-related CLI arguments will be removed here.
      * @param argc
      * @param argv
      * @return
      */
-    virtual int parseOptions(int argc, char **argv);
+    virtual int parseCliOptionsPostMpiInit(int argc, char **argv);
 
     /**
      * @brief Print CLI usage
@@ -134,12 +148,13 @@ class OptimizationApplication {
 
 protected:
     // command line option parsing
-    const char *shortOptions = "dhvt:o:s:";
-    struct option const longOptions[8] = {
+    const char *shortOptions = "dhvmt:o:s:";
+    struct option const longOptions[9] = {
         {"debug", no_argument, NULL, 'd'},
         {"print-worklist", no_argument, NULL, 'p'},
         {"help", no_argument, NULL, 'h'},
         {"version", no_argument, NULL, 'v'},
+        {"mpi", no_argument, NULL, 'm'},
         {"task", required_argument, NULL, 't'},
         {"outfile-prefix", required_argument, NULL, 'o'},
         {"first-start-idx", required_argument, NULL, 's'},
@@ -158,9 +173,10 @@ class OptimizationApplication {
     // the need to be filled in by sub
     std::unique_ptr<MultiStartOptimizationProblem> multiStartOptimizationProblem;
     std::unique_ptr<OptimizationProblem> problem;
-    hid_t file_id;
+    hid_t file_id = 0;
     OperationType operationType = OperationType::parameterEstimation;
     LoadBalancerMaster loadBalancer;
+    bool withMPI = false;
 };
 
 

include/parpecommon/misc.h

@@ -112,6 +112,14 @@ void fillArrayRandomDoubleSameInterval(double min, double max, gsl::span<double>
 int getMpiRank();
 int getMpiCommSize();
 int getMpiActive();
+
+/**
+ * @brief Was application launched by mpiexec?
+ *
+ * Make an educated guess if the application was launched with mpiexec
+ * or similar and therefore require MPI_INIT.
+ * @return True if probably launched by mpiexec
+ */
 bool launchedWithMpi();
 
 void initMpiIfNeeded(int *argc, char ***argv);

include/parpeoptimization/optimizationOptions.h

@@ -91,7 +91,7 @@ Optimizer* optimizerFactory(optimizerName optimizer);
 /**
  * @brief Print list of supported optimizers
  */
-void printAvailableOptimizers();
+void printAvailableOptimizers(std::string prefix = "");
 } // namespace parpe
 
 #endif // OPTIMIZATIONOPTIONS_H

src/parpeamici/optimizationApplication.cpp

@@ -39,17 +39,21 @@ int OptimizationApplication::init(int argc, char **argv) {
     if(std::getenv("PARPE_NO_DEBUG"))
         minimumLogLevel = LOGLVL_INFO;
 
+    int status = parseCliOptionsPreMpiInit(argc, argv);
+    if(status)
+        return status;
+
     // install signal handler for backtrace on error
     sigaction(SIGSEGV, &act, &oldact);
     sigaction(SIGHUP, &act, nullptr);
 
-    if(launchedWithMpi() && !getMpiActive())
+    if(withMPI && !getMpiActive())
         initMPI(&argc, &argv);
 
     printMPIInfo();
     initHDF5Mutex();
 
-    int status = parseOptions(argc, argv);
+    status = parseCliOptionsPostMpiInit(argc, argv);
     if(status)
         return status;
 
@@ -70,7 +74,33 @@ void OptimizationApplication::runMultiStarts()
     optimizer.run();
 }
 
-int OptimizationApplication::parseOptions(int argc, char **argv) {
+int OptimizationApplication::parseCliOptionsPreMpiInit(int argc, char **argv)
+{
+    int c;
+
+    while (true) {
+        int optionIndex = 0;
+        c = getopt_long(argc, argv, shortOptions, longOptions, &optionIndex);
+
+        if (c == -1)
+            break; // no more options
+
+        switch (c) {
+        case 'm':
+            withMPI = true;
+            break;
+        case 'h':
+            printUsage(argv[0]);
+            return 1;
+        }
+    }
+    return 0;
+}
+
+int OptimizationApplication::parseCliOptionsPostMpiInit(int argc, char **argv) {
+    // restart from first argument
+    optind = 1;
+
     int c;
 
     while (true) {
@@ -98,8 +128,8 @@ int OptimizationApplication::parseOptions(int argc, char **argv) {
             printf("Version: %s\n", PARPE_VERSION);
             return 1;
         case 'h':
-            printUsage(argv[0]);
-            return 1;
+        case 'm':
+            continue;
         default:
             printf("Unrecognized option: %c\n", c);
             exit(EXIT_FAILURE);
@@ -120,18 +150,19 @@ int OptimizationApplication::parseOptions(int argc, char **argv) {
 
 void OptimizationApplication::printUsage(char * const argZero)
 {
-    printf("Usage: %s [OPTION]... FILE\n", argZero);
-    printf("FILE: HDF5 data file");
+    printf("Usage: %s [OPTION]... FILE\n\n", argZero);
+    printf("FILE: HDF5 data file\n\n");
     printf("Options: \n"
-           "  -o, --outfile-prefix Prefix for result files (path + "
+           "  -o, --outfile-prefix  Prefix for result files (path + "
            "filename)\n"
-           "  -t, --task    What to do? Parameter estimation (default) "
+           "  -t, --task            What to do? Parameter estimation (default) "
            "or check gradient ('gradient_check')\n"
-           "  -s, --first-start-idx Starting point index for first optimization"
-           "  -h, --help    Print this help text\n"
-           "  -v, --version Print version info\n");
+           "  -s, --first-start-idx Starting point index for first optimization\n"
+           "  -m, --mpi             Enable MPI (default: off)\n"
+           "  -h, --help            Print this help text\n"
+           "  -v, --version         Print version info\n");
     printf("\nSupported optimizers:\n");
-    printAvailableOptimizers();
+    printAvailableOptimizers("  ");
 }
 
 void OptimizationApplication::logParPEVersion(hid_t file_id) const
@@ -327,7 +358,8 @@ bool OptimizationApplication::isWorker() { return getMpiRank() > 0; }
 OptimizationApplication::~OptimizationApplication() {
     // objects must be destroyed before MPI_Finalize is called
     // and Hdf5 mutex is destroyed
-    closeHDF5File(file_id);
+    if(file_id)
+        closeHDF5File(file_id);
     problem.reset(nullptr);
 #ifdef PARPE_ENABLE_MPI
     if(getMpiActive())

src/parpecommon/misc.cpp

@@ -279,7 +279,7 @@ void initMpiIfNeeded(int *argc, char ***argv)
 void finalizeMpiIfNeeded()
 {
 #ifdef PARPE_ENABLE_MPI
-    if(parpe::launchedWithMpi())
+    if(parpe::getMpiActive())
         MPI_Finalize();
 #endif
 }