Merge pull request #201 from ICB-DCM/develop

Release 0.4.0

* Enabled [Zenodo](https://zenodo.org/) integration
* Update AMICI to 0.10.13
* Setup docker image for use with charliecloud (#182)
* Cleanup misc/generateHDF5DataFileFromText.py, remove obsolete warning
* Formatting
* Update reference; add funding section to README
* Add parPE logo (#195)
* Show only interesting lines from /proc/self/status
* Update FAQ
* Disable building tests by default
* CLI option to show supported optimizers (Closes #169)
* Enable CLI starting point index selection for OptimizationApplication
* Add script to read final parameter vectors from parPE (multi-start) optimization result file and write to TSV file
* Add AMICI version and script invocation to generated hdf5 file
* Add gradient check to snakefile
* Fix: Ensure rdata->numstepsB has elements before access (segfault with forward sensitivities)
* Fix parameter indexing for generating example data
* Add doc/README.md
* Document HDF5 output file format (Closes #175)
* Add worfklow figure
* Add ../doc/petab_model_import.md describing PEtab model import (Closes #168)
* Pass H5File as const&, not as Id which will implicitely call H5File which requires HDF5 mutex locking.
* Extend, split, update steadystate example notebooks (#171)
* Ensure hdf5 mutex lock when using AMICI hdf5 helper functions (Closes #167)
* Keep fixed parameters out of /parameters/parameterNames in HDF5 file, which should only contain optimization parameters
* Update readme: Add coinhsl instructions
* Fix HDF5 file generation after PEtab update: do not add AMICI fixed parameters to optmization parameters
* Add scaffold for generation of LoadLeveler job files
* Add snakemake scaffold for PEtab...analysis
* Update PEtab parameter mapping to allow for condition specific scaling etc.
* Update benchmark collection scripts, add Snakefile
* Add basic data analysis Snakefile
* Update plotting functions and example notebook
* Use condition-specific parameter scales (#161)
* Remove obsolete test files
* Remove cpputest remnants
* Update Ipopt install script to 3.12.12, cleanup
* Remove work-around for non-threadsafe adjoints
* Gradient check tolerance relative to objective function values to reduce number of test failures (Closes #80)
* Reenable CERES parameter_tolerance option
* PEtab: Add support for non-constant parameter overrides in condition table (Closes #158); support non-gaussian noise
* Use HDF5 C++ interface; cleanup.

CMakeLists.txt

@@ -35,7 +35,7 @@ split_version(${PARPE_VERSION} libname major minor patch)
 
 
 # <Build options>
-set(BUILD_TESTS TRUE CACHE BOOL "Build tests? (requires googletest)")
+set(BUILD_TESTS FALSE CACHE BOOL "Build tests? (requires googletest)")
 set(BUILD_EXAMPLES TRUE CACHE BOOL "Build examples?")
 set(BUILD_DOCS FALSE CACHE BOOL "Generate documentation?")
 set(BUILD_PYTHON_MODULE FALSE CACHE BOOL "Build Python module?")

README.md

@@ -104,9 +104,23 @@ the C++ interface see [`examples/`](examples/) and `*/tests`.
 
 parPE is being used or has been used in the following projects:
 
-- Leonard Schmiester, Yannik Schälte, Fabian Fröhlich, Jan Hasenauer, Daniel Weindl.
+- Leonard Schmiester, Yannik Schälte, Fabian Fröhlich, Jan Hasenauer,
+  Daniel Weindl.
   *Efficient parameterization of large-scale dynamic models based on relative measurements*.
-  bioRxiv 2018.
-  [doi:10.1101/579045](https://www.biorxiv.org/content/10.1101/579045v1)
+  Bioinformatics, btz581, [doi:10.1093/bioinformatics/btz581](https://doi.org/10.1093/bioinformatics/btz581)
+  (preprint: [doi:10.1101/579045](https://www.biorxiv.org/content/10.1101/579045v1)).
 
 - [CanPathPro](http://canpathpro.eu/)
+
+
+## Funding
+
+parPE has been developed within research projects receiving external funding:
+
+* Through  the  European  Union's  Horizon  2020  research  and innovation
+  programme under grant agreement no. 686282
+  ([CanPathPro](http://canpathpro.eu/)).
+
+* Computer resources for testing parPE have been provided among others by the 
+  Gauss Centre for Supercomputing / Leibniz Supercomputing Centre under grant
+  pr62li.

buildShippable.sh

@@ -13,6 +13,7 @@ CC=mpicc CXX=mpiCC cmake \
       -DCERES_INCLUDE_DIRS="/usr/include/;/usr/include/eigen3" \
       -DMPI_INCLUDE_DIRS=/usr/include/openmpi-x86_64/ \
       -DGCOVR_REPORT=TRUE \
+      -DBUILD_TESTS=TRUE \
       ..
 
 make -j12 VERBOSE=1

container/charliecloud/parpe_base/90forceyes

@@ -0,0 +1,2 @@
+APT::Get::Assume-Yes "true";
+APT::Get::allow-downgrades "true";

container/charliecloud/parpe_base/Dockerfile

@@ -0,0 +1,9 @@
+FROM ubuntu:19.04
+
+COPY . /u18
+
+RUN /u18/install.sh && rm -rf /tmp && mkdir /tmp
+
+ENV BASH_ENV "/etc/drydock/.env"
+
+RUN /u18/install_parpe.sh

container/charliecloud/parpe_base/config

@@ -0,0 +1,2 @@
+Host *
+	StrictHostKeyChecking no

container/charliecloud/parpe_base/gbl_env.sh

@@ -0,0 +1,3 @@
+#!/bin/bash -e
+
+export DEBIAN_FRONTEND=noninteractive

container/charliecloud/parpe_base/install.sh

@@ -0,0 +1,118 @@
+#!/bin/bash -e
+
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get clean && apt-get update
+apt-get install -y apt-utils
+
+echo "================ Installing locales ======================="
+apt-get install -q locales
+
+dpkg-divert --local --rename --add /sbin/initctl
+locale-gen en_US en_US.UTF-8
+dpkg-reconfigure locales
+
+echo "HOME=$HOME"
+cd /u18
+
+echo "================= Updating package lists ==================="
+apt-get update
+
+echo "================= Adding some global settings ==================="
+mv gbl_env.sh /etc/profile.d/
+mkdir -p "$HOME/.ssh/"
+mv config "$HOME/.ssh/"
+mv 90forceyes /etc/apt/apt.conf.d/
+touch "$HOME/.ssh/known_hosts"
+mkdir -p /etc/drydock
+
+echo "================= Installing basic packages ==================="
+apt-get install -q \
+  build-essential \
+  curl \
+  gcc \
+  clang \
+  gettext \
+  libxml2-dev \
+  libxslt1-dev \
+  make \
+  nano \
+  openssh-client \
+  openssl \
+  software-properties-common \
+  sudo  \
+  texinfo \
+  zip \
+  unzip \
+  wget \
+  rsync \
+  psmisc \
+  netcat-openbsd \
+  vim \
+  python-lxml
+
+echo "================= Installing Python packages ==================="
+apt-get install -q -y \
+  python-pip \
+  python-dev \
+  python3-pip \
+  python3-dev
+
+echo "================= Installing Git ==================="
+add-apt-repository ppa:git-core/ppa -y
+apt-get update
+apt-get install -q -y git
+
+
+#echo "================= Adding JQ 1.5.1 ==================="
+#apt-get install -q jq
+
+
+#echo "================ Adding ansible 2.4.3.0 ===================="
+#sudo pip install -q 'ansible==2.4.3.0'
+
+#echo "================ Adding boto 2.48.0 ======================="
+#sudo pip install -q 'boto==2.48.0'
+
+#echo "================ Adding boto3 ======================="
+#sudo pip install -q 'boto3==1.6.16'
+
+#export PK_VERSION=1.2.2
+#echo "================ Adding packer $PK_VERSION  ===================="
+#export PK_FILE=packer_"$PK_VERSION"_linux_amd64.zip
+
+#echo "Fetching packer"
+#echo "-----------------------------------"
+#rm -rf /tmp/packer
+#mkdir -p /tmp/packer
+#wget -nv https://releases.hashicorp.com/packer/$PK_VERSION/$PK_FILE
+#unzip -o $PK_FILE -d /tmp/packer
+#sudo chmod +x /tmp/packer/packer
+#mv /tmp/packer/packer /usr/bin/packer
+
+#echo "Added packer successfully"
+#echo "-----------------------------------"
+
+#echo "================= Adding awscli 1.14.64 ============"
+#sudo pip install -q 'awscli==1.14.64'
+
+#echo "================= parPE requirements ============"
+apt-get install gfortran libmpich-dev libatlas-base-dev libboost-all-dev libhdf5-dev cmake libceres-dev coinor-libipopt-dev swig3.0 python3-venv hdf5-tools libpython-dev
+# for setuptools to find:
+ln -s /usr/bin/swig3.0 /usr/bin/swig
+python3 -m pip install --upgrade pip
+pip3 install -U setuptools pkgconfig wheel
+
+# echo "================= Intalling Shippable CLIs ================="
+# 
+# git clone https://github.com/Shippable/node.git nodeRepo
+# ./nodeRepo/shipctl/x86_64/Ubuntu_16.04/install.sh
+# rm -rf nodeRepo
+# 
+# echo "Installed Shippable CLIs successfully"
+# echo "-------------------------------------"
+
+echo "================= Cleaning package lists ==================="
+apt-get clean
+apt-get autoclean
+apt-get autoremove

container/charliecloud/parpe_base/install_parpe.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+set -e
+
+cd
+
+# Clone clean repository
+git clone --single-branch --branch feature_charlie --depth=1 https://github.com/ICB-DCM/parPE.git
+
+cd parPE
+export PARPE_BASE=$(pwd)
+
+# Build dependencies
+
+# Install AMICI
+export AMICI_PATH=${PARPE_BASE}/deps/AMICI/
+cd "${AMICI_PATH}" && scripts/buildSuiteSparse.sh && scripts/buildSundials.sh && scripts/buildCpputest.sh #&& scripts/buildAmici.sh
+mkdir -p "${AMICI_PATH}"/build && cd "${AMICI_PATH}"/build
+CPPUTEST_BUILD_DIR=${AMICI_PATH}/ThirdParty/cpputest-master/build/
+cmake -DCMAKE_BUILD_TYPE=Debug -DENABLE_PYTHON=ON -DBUILD_TESTS=OFF -DCppUTest_DIR="${CPPUTEST_BUILD_DIR}" .. && make -j12
+
+#- cd $PARPE_BASE/ThirdParty && ./downloadPackages.sh
+#- cd $PARPE_BASE/ThirdParty && ./installCeres.sh
+
+# For google-test for parPE tests
+cd "${PARPE_BASE}" && ThirdParty/installGoogleTest.sh
+
+# build parPE
+pip install -r "${PARPE_BASE}"/python/requirements.txt
+cd "${PARPE_BASE}"
+mkdir -p build
+cd build
+CC=mpicc CXX=mpiCC cmake \
+      -DIPOPT_INCLUDE_DIRS=/usr/include/coin/ \
+      -DIPOPT_LIBRARIES=/usr/lib/libipopt.so \
+      -DCERES_LIBRARIES="/usr/lib/libceres.so;/usr/lib/x86_64-linux-gnu/libglog.so;/usr/lib/x86_64-linux-gnu/libgflags.so" \
+      -DCERES_INCLUDE_DIRS="/usr/include/;/usr/include/eigen3" \
+      -DMPI_INCLUDE_DIRS=/usr/include/openmpi-x86_64/ \
+      -DBUILD_TESTS=ON \
+      ..
+make -j12 VERBOSE=1
+
+# run tests
+cd "${PARPE_BASE}"/build && CTEST_OUTPUT_ON_FAILURE=1 make test
+
+# valgrind
+#CTEST_OUTPUT_ON_FAILURE=1 make ExperimentalMemCheck; cat Testing/Temporary/MemoryChecker.*.log

deps/AMICI/.cppcheck-exitcode-suppressions

@@ -0,0 +1 @@
+alloca:*:*

deps/AMICI/.github/workflows/sbml-semantic-test-suite.yml

@@ -0,0 +1,24 @@
+name: SBML semantic test suite
+on:
+  push:
+    branches:
+      - develop
+      - master
+      - test_action
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+      with:
+        fetch-depth: 1
+    - name: apt
+      run: sudo apt-get install -y swig3.0 libatlas-base-dev && sudo ln -s /usr/bin/swig3.0 /usr/bin/swig
+    - run: AMICI_PARALLEL_COMPILE=2 ./scripts/installAmiciSource.sh
+    - run: AMICI_PARALLEL_COMPILE=2 ./scripts/run-SBMLTestsuite.sh

deps/AMICI/.gitignore

@@ -160,3 +160,4 @@ ThirdParty/SuiteSparse/lib/*
 ThirdParty/SuiteSparse/share/*
 ThirdParty/SuperLU_MT_3.1/
 ThirdParty/superlu_mt_3.1.tar.gz
+ThirdParty/swig-*

deps/AMICI/.gitrepo

@@ -5,8 +5,8 @@
 ;
 [subrepo]
 	remote = [email protected]:ICB-DCM/AMICI.git
-	branch = reuse_solver
-	commit = d5e2c2ca9d676a6e04f9777728bf4655ea517e1e
-	parent = 863a6f1a6a610631a1548b065132da18d8c9e2f5
+	branch = v0.10.13
+	commit = 76162dfc09d74d439939ab92eac7a2f7b80f0873
+	parent = 3fe186d2f265eb58466b7b0e25c56bc9c3a71abf
 	cmdver = 0.4.0
 	method = merge

deps/AMICI/.travis.yml

@@ -9,7 +9,7 @@ matrix:
   fast_finish: true
   include:
     - os: linux
-      dist: xenial
+      dist: bionic
       language: python
       python: 3.7
       compiler: gcc
@@ -23,7 +23,6 @@ matrix:
             - libboost-serialization-dev
             - swig3.0
             - g++-5
-            - cppcheck
             - libc6-dbg
       env:
         - ENABLE_GCOV_COVERAGE=TRUE
@@ -44,7 +43,7 @@ matrix:
           tags: true
 
     - os: osx
-      osx_image: xcode10.1
+      osx_image: xcode11
       language: minimal
       compiler: clang
       env:
@@ -55,6 +54,8 @@ matrix:
             - hdf5
             - swig
             - gcc
+            - cppcheck
+          update: true
       before_install:
         - travis_wait brew link --overwrite gcc # fix linker warning regarding /usr/local/include/c++
         - export -f travis_fold travis_nanoseconds travis_time_start travis_time_finish
@@ -63,7 +64,7 @@ matrix:
         - cd $BASE_DIR # cd to base dir for correct relative path in deploy
 
     - os: osx
-      osx_image: xcode10.1
+      osx_image: xcode11
       language: minimal
       compiler: clang
       env:
@@ -76,6 +77,7 @@ matrix:
             - doxygen
             - ragel
             - graphviz
+          update: true
       before_install:
         - export PATH=/Users/travis/Library/Python/3.7/bin:/Library/TeX/texbin:$PATH
         - export -f travis_fold travis_nanoseconds travis_time_start travis_time_finish
@@ -115,10 +117,9 @@ script:
     - if [[ "$TRAVIS_OS_NAME" != "linux" ]] && [[ "$CI_MODE" == "test" ]]; then $FOLD python-tests ./scripts/run-python-tests.sh; fi
     - if [[ "$TRAVIS_OS_NAME" != "linux" ]] && [[ "$CI_MODE" == "test" ]]; then $FOLD cmake ./tests/testCMakeCompilation.sh; fi
     - if [[ "$CI_MODE" == "deploy" ]]; then $FOLD doxygen ./scripts/run-doxygen.sh; fi
-    - if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CI_MODE" == "test" ]]; then $FOLD cppcheck ./scripts/run-cppcheck.sh; fi
+    - if [[ "$TRAVIS_OS_NAME" != "linux" ]] && [[ "$CI_MODE" == "test" ]]; then $FOLD cppcheck ./scripts/run-cppcheck.sh; fi
     - if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CI_MODE" == "test" ]]; then $FOLD valgrind ./scripts/run-valgrind.sh; fi
     - if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CI_MODE" == "test" ]]; then $FOLD codecov ./scripts/run-codecov.sh; fi
-    - if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$TRAVIS_EVENT_TYPE" == "cron" ]]; then $FOLD sbmltestsuite ./scripts/run-SBMLTestsuite.sh; fi
 
 after_failure:
     - $FOLD ls -alR