Setup docker image for use with charliecloud (#182)

* Create Dockerfile for generating parPE docker image
* Documentation and examples
* Document Snakemake workflow for model import, optimization, ... 
* Add schema file for Snakemake config file

Closes #180

Details:
* Add basic Dockerfile adapted from ci image

* Cleanup image, allow mpi as root in docker, update deprecated apt options

* Disable valgrind on docker build

* Basic charliecloud example doc

* Add Snakemake config file schema and validate

* Add documentation stub on using snakemake optimization workflow

* cleanup

* Update base image to Ubuntu19

* Update install_parpe.sh

* Fix >2 arguments to run_in_venv.sh

* Add snakemake as parPE python package requirement

* Add example for optimizing a PEtab model using the parPE docker image and charliecloud

container/charliecloud/parpe_base/90forceyes

@@ -0,0 +1,2 @@
+APT::Get::Assume-Yes "true";
+APT::Get::allow-downgrades "true";

container/charliecloud/parpe_base/Dockerfile

@@ -0,0 +1,9 @@
+FROM ubuntu:19.04
+
+COPY . /u18
+
+RUN /u18/install.sh && rm -rf /tmp && mkdir /tmp
+
+ENV BASH_ENV "/etc/drydock/.env"
+
+RUN /u18/install_parpe.sh

container/charliecloud/parpe_base/config

@@ -0,0 +1,2 @@
+Host *
+	StrictHostKeyChecking no

container/charliecloud/parpe_base/gbl_env.sh

@@ -0,0 +1,3 @@
+#!/bin/bash -e
+
+export DEBIAN_FRONTEND=noninteractive

container/charliecloud/parpe_base/install.sh

@@ -0,0 +1,118 @@
+#!/bin/bash -e
+
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get clean && apt-get update
+apt-get install -y apt-utils
+
+echo "================ Installing locales ======================="
+apt-get install -q locales
+
+dpkg-divert --local --rename --add /sbin/initctl
+locale-gen en_US en_US.UTF-8
+dpkg-reconfigure locales
+
+echo "HOME=$HOME"
+cd /u18
+
+echo "================= Updating package lists ==================="
+apt-get update
+
+echo "================= Adding some global settings ==================="
+mv gbl_env.sh /etc/profile.d/
+mkdir -p "$HOME/.ssh/"
+mv config "$HOME/.ssh/"
+mv 90forceyes /etc/apt/apt.conf.d/
+touch "$HOME/.ssh/known_hosts"
+mkdir -p /etc/drydock
+
+echo "================= Installing basic packages ==================="
+apt-get install -q \
+  build-essential \
+  curl \
+  gcc \
+  clang \
+  gettext \
+  libxml2-dev \
+  libxslt1-dev \
+  make \
+  nano \
+  openssh-client \
+  openssl \
+  software-properties-common \
+  sudo  \
+  texinfo \
+  zip \
+  unzip \
+  wget \
+  rsync \
+  psmisc \
+  netcat-openbsd \
+  vim \
+  python-lxml
+
+echo "================= Installing Python packages ==================="
+apt-get install -q -y \
+  python-pip \
+  python-dev \
+  python3-pip \
+  python3-dev
+
+echo "================= Installing Git ==================="
+add-apt-repository ppa:git-core/ppa -y
+apt-get update
+apt-get install -q -y git
+
+
+#echo "================= Adding JQ 1.5.1 ==================="
+#apt-get install -q jq
+
+
+#echo "================ Adding ansible 2.4.3.0 ===================="
+#sudo pip install -q 'ansible==2.4.3.0'
+
+#echo "================ Adding boto 2.48.0 ======================="
+#sudo pip install -q 'boto==2.48.0'
+
+#echo "================ Adding boto3 ======================="
+#sudo pip install -q 'boto3==1.6.16'
+
+#export PK_VERSION=1.2.2
+#echo "================ Adding packer $PK_VERSION  ===================="
+#export PK_FILE=packer_"$PK_VERSION"_linux_amd64.zip
+
+#echo "Fetching packer"
+#echo "-----------------------------------"
+#rm -rf /tmp/packer
+#mkdir -p /tmp/packer
+#wget -nv https://releases.hashicorp.com/packer/$PK_VERSION/$PK_FILE
+#unzip -o $PK_FILE -d /tmp/packer
+#sudo chmod +x /tmp/packer/packer
+#mv /tmp/packer/packer /usr/bin/packer
+
+#echo "Added packer successfully"
+#echo "-----------------------------------"
+
+#echo "================= Adding awscli 1.14.64 ============"
+#sudo pip install -q 'awscli==1.14.64'
+
+#echo "================= parPE requirements ============"
+apt-get install gfortran libmpich-dev libatlas-base-dev libboost-all-dev libhdf5-dev cmake libceres-dev coinor-libipopt-dev swig3.0 python3-venv hdf5-tools libpython-dev
+# for setuptools to find:
+ln -s /usr/bin/swig3.0 /usr/bin/swig
+python3 -m pip install --upgrade pip
+pip3 install -U setuptools pkgconfig wheel
+
+# echo "================= Intalling Shippable CLIs ================="
+# 
+# git clone https://github.com/Shippable/node.git nodeRepo
+# ./nodeRepo/shipctl/x86_64/Ubuntu_16.04/install.sh
+# rm -rf nodeRepo
+# 
+# echo "Installed Shippable CLIs successfully"
+# echo "-------------------------------------"
+
+echo "================= Cleaning package lists ==================="
+apt-get clean
+apt-get autoclean
+apt-get autoremove

container/charliecloud/parpe_base/install_parpe.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+set -e
+
+cd
+
+# Clone clean repository
+git clone --single-branch --branch feature_charlie --depth=1 https://github.com/ICB-DCM/parPE.git
+
+cd parPE
+export PARPE_BASE=$(pwd)
+
+# Build dependencies
+
+# Install AMICI
+export AMICI_PATH=${PARPE_BASE}/deps/AMICI/
+cd "${AMICI_PATH}" && scripts/buildSuiteSparse.sh && scripts/buildSundials.sh && scripts/buildCpputest.sh #&& scripts/buildAmici.sh
+mkdir -p "${AMICI_PATH}"/build && cd "${AMICI_PATH}"/build
+CPPUTEST_BUILD_DIR=${AMICI_PATH}/ThirdParty/cpputest-master/build/
+cmake -DCMAKE_BUILD_TYPE=Debug -DENABLE_PYTHON=ON -DBUILD_TESTS=OFF -DCppUTest_DIR="${CPPUTEST_BUILD_DIR}" .. && make -j12
+
+#- cd $PARPE_BASE/ThirdParty && ./downloadPackages.sh
+#- cd $PARPE_BASE/ThirdParty && ./installCeres.sh
+
+# For google-test for parPE tests
+cd "${PARPE_BASE}" && ThirdParty/installGoogleTest.sh
+
+# build parPE
+pip install -r "${PARPE_BASE}"/python/requirements.txt
+cd "${PARPE_BASE}"
+mkdir -p build
+cd build
+CC=mpicc CXX=mpiCC cmake \
+      -DIPOPT_INCLUDE_DIRS=/usr/include/coin/ \
+      -DIPOPT_LIBRARIES=/usr/lib/libipopt.so \
+      -DCERES_LIBRARIES="/usr/lib/libceres.so;/usr/lib/x86_64-linux-gnu/libglog.so;/usr/lib/x86_64-linux-gnu/libgflags.so" \
+      -DCERES_INCLUDE_DIRS="/usr/include/;/usr/include/eigen3" \
+      -DMPI_INCLUDE_DIRS=/usr/include/openmpi-x86_64/ \
+      -DBUILD_TESTS=ON \
+      ..
+make -j12 VERBOSE=1
+
+# run tests
+cd "${PARPE_BASE}"/build && CTEST_OUTPUT_ON_FAILURE=1 make test
+
+# valgrind
+#CTEST_OUTPUT_ON_FAILURE=1 make ExperimentalMemCheck; cat Testing/Temporary/MemoryChecker.*.log

doc/parpe_with_charliecloud.md

@@ -0,0 +1,135 @@
+# Using parPE with charliecloud
+
+This document is an extension to [petab_model_import.md](petab_model_import.md),
+describing how to perform parameter estimation using parPE inside a docker
+container under charliecloud. 
+[Charliecloud](https://hpc.github.io/charliecloud/) is a tool that "provides
+user-defined software stacks (UDSS) for high-performance computing (HPC)
+centers". We focus on charliecloud here, but most of the steps are easily
+adapted for use with plain docker.
+
+## parPE base docker image
+
+The next two sections describe how to build or download the default parPE
+base docker image. The following section demonstrates how to perform parameter
+estimation for an example model included in the the image.
+
+
+### Generating parPE base docker image
+
+This will create the parPE base image *from parPE from github*
+(takes about 10'):
+
+    cd container/charliecloud/parpe_base
+    ch-build -t parpe .
+
+Export image to charliecloud archive in the current directory:
+
+    ch-docker2tar parpe:latest .
+
+
+### Using a provided docker image
+
+Instead of building the docker image yourself, you can download a Ubuntu-based
+parPE image from [dockerhub](https://hub.docker.com/r/dweindl/parpe) using:
+
+    docker pull dweindl/parpe:latest
+
+**Developer note:**
+To update the image on dockerhub, run:
+
+    sudo docker images # check for IMAGE ID
+    sudo docker tag $IMAGE_ID dweindl/parpe:latest
+    sudo docker push dweindl/parpe:latest 
+
+(TODO: auto-deploy after CI)
+
+## Running the example model
+
+This section show how to perform parameter estimation for an example model
+included in the the image. The example here assume that you are using
+interactive cluster access with SLURM. It should be easy to adapt the example
+code to use a different submission system or to create a batch job.
+
+### In an interactive session
+
+Start an interactive session. With SLURM, e.g. 
+`srun -p serial_fed28 --pty bash`
+
+On the compute node:
+
+    CHARLIE_DEST_DIR=/var/tmp
+    CHARLIE_TAR=parpe\:latest.tar.gz
+    OUTPUT_DIR=parpe_test # where results will be written to
+    ch-tar2dir "${CHARLIE_TAR}" "${CHARLIE_DEST_DIR}"
+    mkdir -p "${OUTPUT_DIR}"
+    ch-run -b "${OUTPUT_DIR}":/mnt/ "${CHARLIE_DEST_DIR}"/parpe\:latest/ -- \
+        mpirun /root/parPE/build/examples/parpeamici/steadystate/example_steadystate_multi \
+        -o /mnt/pe-results/ \
+        /root/parPE/build/examples/parpeamici/steadystate/steadystate_scaled-prefix/src/steadystate_scaled/example_data.h5
+
+The results will be written to `${OUTPUT_DIR}`.
+
+
+## Optimizing a PEtab model
+
+1. Generate / fetch a parPE docker image as described above
+
+1. Generate charliecloud image (potentially requires `sudo`)
+
+    `ch-docker2tar parpe:latest .`
+
+1. Extract charliecloud image (adapt paths as needed)
+
+    ```
+    CHARLIE_DEST_DIR=/var/tmp
+    CHARLIE_TAR=parpe\:latest.tar.gz
+    ch-tar2dir "${CHARLIE_TAR}" "${CHARLIE_DEST_DIR}"
+    ```
+
+1. Place your PEtab files into `$PETAB_DIR`
+
+   For testing you can download an example model via:
+
+    ```    
+    wget "https://raw.githubusercontent.com/LeonardSchmiester/Benchmark-Models/hackathon/hackathon_contributions_new_data_format/Zheng_PNAS2012/model_Zheng_PNAS2012.xml"
+    wget "https://raw.githubusercontent.com/LeonardSchmiester/Benchmark-Models/hackathon/hackathon_contributions_new_data_format/Zheng_PNAS2012/measurementData_Zheng_PNAS2012.tsv"
+    wget "https://raw.githubusercontent.com/LeonardSchmiester/Benchmark-Models/hackathon/hackathon_contributions_new_data_format/Zheng_PNAS2012/experimentalCondition_Zheng_PNAS2012.tsv"
+    wget "https://raw.githubusercontent.com/LeonardSchmiester/Benchmark-Models/hackathon/hackathon_contributions_new_data_format/Zheng_PNAS2012/parameters_Zheng_PNAS2012.tsv"
+    ```
+
+1. Create Snakemake YAML config for those files
+
+    For the example files above, run:
+    
+    ```
+    cat > parpe_optimize_petab.yaml << EOF
+   petab:
+        sbml_file: 'model_Zheng_PNAS2012.xml'
+        measurement_file: 'measurementData_Zheng_PNAS2012.tsv'
+        condition_file: 'experimentalCondition_Zheng_PNAS2012.tsv'
+        parameter_file: 'parameters_Zheng_PNAS2012.tsv'
+    
+    model_name: 'Zheng_PNAS2012'
+    amici_build_dir: '/root/parPE/deps/AMICI/build'
+    amici_src_dir: '/root/parPE/deps/AMICI/'
+    parpe_build_dir: '/root/parPE/build/'
+    EOF
+    ```
+
+1. Run the Snakemake workflow, e.g. as
+
+    ```    
+    PETAB_DIR=Zheng_PNAS2012 # where results will be written to
+    SNAKEMAKE_CONFIG=/mnt/parpe_optimize_petab.yaml
+    mkdir -p "${PETAB_DIR}"
+    ch-run -b "${PETAB_DIR}":/mnt/ --no-home -c /mnt/ \
+        --unset-env=AMICI_ROOT \
+        "${CHARLIE_DEST_DIR}"/parpe\:latest/ -- \
+        /root/parPE/misc/run_in_venv.sh /root/parPE/build/venv snakemake \
+        -s /root/parPE/snakemake/Snakefile \
+        --configfile "${SNAKEMAKE_CONFIG}" -- postprocess
+    ```
+
+    For more details on this workflow, see
+    [../snakemake/Snakefile](../snakemake/Snakefile).

doc/snakemake_workflow.md

@@ -0,0 +1,39 @@
+# Using parPE with Snakemake
+
+*parPE* comes with some rudimentary 
+[Snakemake](https://snakemake.readthedocs.io/en/stable/) workflows.
+
+
+## Parameter estimation
+
+A very basic Snakefile for parameter estimation is located in `snakemake/`.
+This workflow is adapted to a specific model via a YAML configuration file.
+An example is shown in 
+[`snakemake/parpe_optimize_petab_steadystate.yaml`](../snakemake/parpe_optimize_petab_steadystate.yaml).
+
+
+### An example
+
+If parPE was build with examples, this workflow can be run after adapting
+some paths in the example config file. `petab.root` needs to be adapted to
+reflect your parPE build directory, and probably you want to change the output
+directories as well. The documented schema for this configuration file is
+provided in `snakemake/config.schema.yaml`.
+
+After that you can run the full pipeline with:
+
+    cd snakemake
+    snakemake --configfile parpe_optimize_petab_steadystate.yaml postprocess
+
+This generate C++ code of the model, build model specific binaries for
+parameter estimation, run parameters, and process the results.
+After successful completion, you should see a file `test.png` in your working
+directory showing the optimization trajectory (as well as other files, once
+this gets extended).
+
+The different snakemake targets are explained in the Snakefile and can be
+accessed using `snakemake --list` from the directory in which the Snakefile
+is located.
+
+To use this workflow with a different PEtab problem, you should only need to
+adapt the configuration file. 

examples/parpeamici/steadystate/run-examples.sh

@@ -8,7 +8,8 @@ set -x
 HDF5_FILE=$1
 HDF5_FILE_TEST=$2
 MPIEXEC="mpiexec --oversubscribe -n 5"
-test -v SHIPPABLE && MPIEXEC="${MPIEXEC} --allow-run-as-root"
+# Allow running as root in docker
+grep docker /proc/1/cgroup -qa && MPIEXEC="${MPIEXEC} --allow-run-as-root"
 
 rm -f test.log
 

misc/run_in_venv.sh

@@ -4,7 +4,7 @@
 set -e
 
 # invalid options:
-if [[ $# -lt 2 ]] || [[ $# -gt 2 ]]; then
+if [[ $# -lt 2 ]]; then
     echo "Run a given command in the given Python virtual environment"
     echo "USAGE: $(basename "$0") VENV_DIR COMMAND [ARG ...]"
     exit 1;

python/setup.py

@@ -15,6 +15,7 @@
                       'amici>=0.10.10',
                       'h5py',
                       'python-libsbml>=5.17.0',
-                      'jinja2'
+                      'jinja2',
+                      'snakemake'
                       ],
 )